From b6b3a0a485f6f8f9c06c852aaeb79d7d3976c8de Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 30 Aug 2026 17:37:43 +0100 Subject: [PATCH] pikpak: share upload chunk buffers with the global memory pool The multipart uploader kept its own private buffer pool, a copy of the one in lib/pool with identical settings, so its chunk memory was never shared with the rest of rclone. Pages cached here were invisible to other backends and vice versa, costing up to 64 MiB of extra idle cache. Allocate chunks with multipart.NewRW, the global pool used by the other backends instead. --- backend/pikpak/multipart.go | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/backend/pikpak/multipart.go b/backend/pikpak/multipart.go index ad453b00a..f6c25e0ca 100644 --- a/backend/pikpak/multipart.go +++ b/backend/pikpak/multipart.go @@ -7,7 +7,6 @@ import ( "sort" "strings" "sync" - "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -18,38 +17,11 @@ import ( "github.com/rclone/rclone/fs/chunksize" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/multipart" "github.com/rclone/rclone/lib/pacer" - "github.com/rclone/rclone/lib/pool" "golang.org/x/sync/errgroup" ) -const ( - bufferSize = 1024 * 1024 // default size of the pages used in the reader - bufferCacheSize = 64 // max number of buffers to keep in cache - bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long -) - -// bufferPool is a global pool of buffers -var ( - bufferPool *pool.Pool - bufferPoolOnce sync.Once -) - -// get a buffer pool -func getPool() *pool.Pool { - bufferPoolOnce.Do(func() { - ci := fs.GetConfig(context.Background()) - // Initialise the buffer pool when used - bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap) - }) - return bufferPool -} - -// NewRW gets a pool.RW using the multipart pool -func NewRW() *pool.RW { - return pool.NewRW(getPool()) -} - // Upload does a multipart upload in parallel func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) { // make concurrency machinery @@ -80,7 +52,7 @@ func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) { for partNum := int64(0); !finished; partNum++ { // Get a block of memory from the pool and token which limits concurrency. tokens.Get() - rw := NewRW() + rw := multipart.NewRW() if acc != nil { rw.SetAccounting(acc.AccountRead) }