pikpak: share upload chunk buffers with the global memory pool
The multipart uploader kept its own private buffer pool, a copy of the one in lib/pool with identical settings, so its chunk memory was never shared with the rest of rclone. Pages cached here were invisible to other backends and vice versa, costing up to 64 MiB of extra idle cache. Allocate chunks with multipart.NewRW, the global pool used by the other backends instead.
This commit is contained in:
@@ -7,7 +7,6 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
@@ -18,38 +17,11 @@ import (
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
bufferSize = 1024 * 1024 // default size of the pages used in the reader
|
||||
bufferCacheSize = 64 // max number of buffers to keep in cache
|
||||
bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long
|
||||
)
|
||||
|
||||
// bufferPool is a global pool of buffers
|
||||
var (
|
||||
bufferPool *pool.Pool
|
||||
bufferPoolOnce sync.Once
|
||||
)
|
||||
|
||||
// get a buffer pool
|
||||
func getPool() *pool.Pool {
|
||||
bufferPoolOnce.Do(func() {
|
||||
ci := fs.GetConfig(context.Background())
|
||||
// Initialise the buffer pool when used
|
||||
bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)
|
||||
})
|
||||
return bufferPool
|
||||
}
|
||||
|
||||
// NewRW gets a pool.RW using the multipart pool
|
||||
func NewRW() *pool.RW {
|
||||
return pool.NewRW(getPool())
|
||||
}
|
||||
|
||||
// Upload does a multipart upload in parallel
|
||||
func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
|
||||
// make concurrency machinery
|
||||
@@ -80,7 +52,7 @@ func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
|
||||
for partNum := int64(0); !finished; partNum++ {
|
||||
// Get a block of memory from the pool and token which limits concurrency.
|
||||
tokens.Get()
|
||||
rw := NewRW()
|
||||
rw := multipart.NewRW()
|
||||
if acc != nil {
|
||||
rw.SetAccounting(acc.AccountRead)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user