From 2a492cc355c0f0cb67a259e5385cf8f1b7099e83 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 30 Aug 2026 17:37:30 +0100 Subject: [PATCH] hidrive: stop copying every upload chunk a second time Upload chunks are buffered into a bytes.Reader and then, when transfer accounting is active (always for a real copy), re-wrapped in accounting before being handed to cachedReader. cachedReader only recognised a bare *bytes.Reader, so the accounted chunk fell through to readers.NewRepeatableReader, which copied the whole chunk again into an append-grown slice. Every chunk (and the upload-cutoff prefix of every file) therefore cost roughly twice its size in memory. Look through the accounting wrapper when deciding whether the reader is already a seekable buffer and, if so, seek the buffer underneath while still reading through the accounting, so retries rewind without a copy. --- backend/hidrive/helpers.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/backend/hidrive/helpers.go b/backend/hidrive/helpers.go index b7a522257..e7afc8a6d 100644 --- a/backend/hidrive/helpers.go +++ b/backend/hidrive/helpers.go @@ -839,17 +839,24 @@ func createHiDriveScopes(role string, access string) []string { return []string{} } +// accountedReadSeeker reads through any accounting wrapped around a +// buffered reader while seeking the buffer underneath it, +// so a retry can rewind the buffer without copying it. +type accountedReadSeeker struct { + io.Reader + io.Seeker +} + // cachedReader returns a version of the reader that caches its contents and // can therefore be reset using Seek. +// +// Readers which are already seekable buffers are used as they are, +// even when wrapped in accounting. func cachedReader(reader io.Reader) io.ReadSeeker { - bytesReader, ok := reader.(*bytes.Reader) - if ok { - return bytesReader - } - - repeatableReader, ok := reader.(*readers.RepeatableReader) - if ok { - return repeatableReader + unwrapped, _ := accounting.UnWrap(reader) + switch unwrapped.(type) { + case *bytes.Reader, *readers.RepeatableReader: + return accountedReadSeeker{Reader: reader, Seeker: unwrapped.(io.Seeker)} } return readers.NewRepeatableReader(reader)