huaweidrive: build small uploads in pooled memory instead of a growing buffer
Files below upload_cutoff (up to 20 MiB) were assembled into a bytes.Buffer which grows by doubling, so each upload allocated roughly twice its size and threw it away afterwards, churning the GC on bulk transfers. Write the multipart/related body into a pool.RW from the global page pool instead so the memory is reused across uploads and bounded by rclone's memory management. The pool.RW is seekable, so the same body is rewound at the start of each retry rather than being re-wrapped. lib/rest already wraps request bodies in readers.NoCloser so the transport cannot free the pages between attempts; the Content-Length is passed explicitly because the wrapped body is no longer a *bytes.Reader net/http can measure. The FsPutRetry integration test covers the retry of a failed upload request and checks the buffers are returned to the pool.
This commit is contained in:
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -1741,9 +1742,10 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
metadata["autoRename"] = 3
|
||||
}
|
||||
|
||||
// Create multipart form
|
||||
var buf bytes.Buffer
|
||||
writer := multipart.NewWriter(&buf)
|
||||
// Build the multipart body in pooled memory so it can be re-sent on retry
|
||||
rw := pool.NewRW(pool.Global())
|
||||
defer fs.CheckClose(rw, &err)
|
||||
writer := multipart.NewWriter(rw)
|
||||
|
||||
// Add metadata part - use exact format from Huawei docs
|
||||
metadataWriter, err := writer.CreatePart(textproto.MIMEHeader{
|
||||
@@ -1789,15 +1791,17 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
return fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
bodyBytes := buf.Bytes()
|
||||
bodySize := rw.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Parameters: url.Values{
|
||||
"uploadType": []string{"multipart"},
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
RootURL: o.fs.uploadURL,
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
ContentLength: &bodySize,
|
||||
RootURL: o.fs.uploadURL,
|
||||
Body: rw,
|
||||
}
|
||||
|
||||
if o.id != "" {
|
||||
@@ -1811,7 +1815,10 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
|
||||
var info *api.File
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
opts.Body = bytes.NewReader(bodyBytes)
|
||||
_, err := rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user