pikpak: share upload chunk buffers with the global memory pool

The multipart uploader kept its own private buffer pool, a copy of the
one in lib/pool with identical settings, so its chunk memory was never
shared with the rest of rclone. Pages cached here were invisible to
other backends and vice versa, costing up to 64 MiB of extra idle cache.

Allocate chunks with multipart.NewRW, the global pool used by the
other backends instead.
This commit is contained in:
Nick Craig-Wood
2026-09-01 14:21:52 +01:00
parent 337f762c79
commit b6b3a0a485
+2 -30
View File
@@ -7,7 +7,6 @@ import (
"sort"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -18,38 +17,11 @@ import (
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
"golang.org/x/sync/errgroup"
)
const (
bufferSize = 1024 * 1024 // default size of the pages used in the reader
bufferCacheSize = 64 // max number of buffers to keep in cache
bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long
)
// bufferPool is a global pool of buffers
var (
bufferPool *pool.Pool
bufferPoolOnce sync.Once
)
// get a buffer pool
func getPool() *pool.Pool {
bufferPoolOnce.Do(func() {
ci := fs.GetConfig(context.Background())
// Initialise the buffer pool when used
bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)
})
return bufferPool
}
// NewRW gets a pool.RW using the multipart pool
func NewRW() *pool.RW {
return pool.NewRW(getPool())
}
// Upload does a multipart upload in parallel
func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
// make concurrency machinery
@@ -80,7 +52,7 @@ func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
for partNum := int64(0); !finished; partNum++ {
// Get a block of memory from the pool and token which limits concurrency.
tokens.Get()
rw := NewRW()
rw := multipart.NewRW()
if acc != nil {
rw.SetAccounting(acc.AccountRead)
}