huaweidrive: build small uploads in pooled memory instead of a growing buffer

Files below upload_cutoff (up to 20 MiB) were assembled into a bytes.Buffer
which grows by doubling, so each upload allocated roughly twice its size
and threw it away afterwards, churning the GC on bulk transfers. Write the
multipart/related body into a pool.RW from the global page pool instead so
the memory is reused across uploads and bounded by rclone's memory
management.

The pool.RW is seekable, so the same body is rewound at the start of each
retry rather than being re-wrapped. lib/rest already wraps request bodies
in readers.NoCloser so the transport cannot free the pages between
attempts; the Content-Length is passed explicitly because the wrapped body
is no longer a *bytes.Reader net/http can measure.

The FsPutRetry integration test covers the retry of a failed upload
request and checks the buffers are returned to the pool.
This commit is contained in:
Nick Craig-Wood
2026-09-01 14:21:52 +01:00
parent bd217a1082
commit 57eb80f182
+14 -7
View File
@@ -33,6 +33,7 @@ import (
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"golang.org/x/oauth2"
@@ -1741,9 +1742,10 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
metadata["autoRename"] = 3
}
// Create multipart form
var buf bytes.Buffer
writer := multipart.NewWriter(&buf)
// Build the multipart body in pooled memory so it can be re-sent on retry
rw := pool.NewRW(pool.Global())
defer fs.CheckClose(rw, &err)
writer := multipart.NewWriter(rw)
// Add metadata part - use exact format from Huawei docs
metadataWriter, err := writer.CreatePart(textproto.MIMEHeader{
@@ -1789,15 +1791,17 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
return fmt.Errorf("failed to close multipart writer: %w", err)
}
bodyBytes := buf.Bytes()
bodySize := rw.Size()
opts := rest.Opts{
Method: "POST",
Parameters: url.Values{
"uploadType": []string{"multipart"},
"fields": []string{"*"},
},
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
RootURL: o.fs.uploadURL,
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
ContentLength: &bodySize,
RootURL: o.fs.uploadURL,
Body: rw,
}
if o.id != "" {
@@ -1811,7 +1815,10 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
var info *api.File
err = o.fs.pacer.Call(func() (bool, error) {
opts.Body = bytes.NewReader(bodyBytes)
_, err := rw.Seek(0, io.SeekStart)
if err != nil {
return false, err
}
resp, err := srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(ctx, resp, err)
})