huaweidrive: reuse resumable-upload chunk buffers via the global pool

Each resumable upload allocated a fresh chunk-sized buffer (8 MiB by
default, up to 64 MiB), so bulk transfers of large files churned
allocations and GC.

Buffer each chunk in a pool.RW from the global page pool instead so
the memory is reused across uploads and bounded by rclone's central
memory management.

The pool.RW is seekable, so the chunk is rewound at the start of each
retry rather than re-wrapped. lib/rest wraps request bodies in
readers.NoCloser so the transport cannot return the pages to the pool
between attempts - the Content-Length is set through rest.Opts because
the wrapped body is not a *bytes.Reader net/http can measure. A source
that runs dry before its declared size is reported as an unexpected EOF
rather than sending the short chunk.
This commit is contained in:
Nick Craig-Wood
2026-09-01 14:21:52 +01:00
parent 57eb80f182
commit 204f817100
+23 -11
View File
@@ -1916,17 +1916,22 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
// single upload size according to the Huawei Drive API
chunkSize := min(max(int64(o.fs.opt.ChunkSize), 256*1024), 64*1024*1024)
buf := make([]byte, chunkSize)
var offset int64
for offset < size {
n, err := io.ReadFull(in, buf)
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
// Buffer the chunk in pooled memory so it can be re-sent on retry
rw := pool.NewRW(pool.Global())
n, err := io.CopyN(rw, in, min(size-offset, chunkSize))
if err != nil {
// io.EOF here means the source ran out before the declared size
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
_ = rw.Close()
return fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)
}
chunk := buf[:n]
end := offset + int64(n) - 1
end := offset + n - 1
// Upload this chunk using the original client with OAuth authentication
chunkOpts := rest.Opts{
@@ -1934,17 +1939,20 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
RootURL: location,
Path: "", // Empty path since RootURL contains the full URL
ExtraHeaders: map[string]string{
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
"Content-Length": strconv.Itoa(n),
"Content-Type": mimeType,
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
"Content-Type": mimeType,
},
ContentLength: &n,
Body: rw,
}
var info *api.File
var chunkResp *http.Response
err = o.fs.pacer.Call(func() (bool, error) {
chunkOpts.Body = bytes.NewReader(chunk)
var err error
_, err := rw.Seek(0, io.SeekStart)
if err != nil {
return false, err
}
chunkResp, err = srv.CallJSON(ctx, &chunkOpts, nil, &info)
// 308 means continue uploading (incomplete)
if chunkResp != nil && chunkResp.StatusCode == 308 {
@@ -1958,11 +1966,15 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
}
return shouldRetry(ctx, chunkResp, err)
})
closeErr := rw.Close()
if err != nil {
return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err)
}
if closeErr != nil {
return closeErr
}
offset += int64(n)
offset += n
// If we got file info back, we're done
if info != nil && info.ID != "" {