From 468eccb122628c9ae1d17229d46967ff3abed410 Mon Sep 17 00:00:00 2001 From: Rayan Salhab Date: Thu, 27 Aug 2026 14:07:18 +0300 Subject: [PATCH] accounting: fix bwlimit burst overflow - fixes #9820 Co-authored-by: cyphercodes --- fs/accounting/token_bucket.go | 18 +++++++++++++----- fs/accounting/token_bucket_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/fs/accounting/token_bucket.go b/fs/accounting/token_bucket.go index 00b36d344..264d881fa 100644 --- a/fs/accounting/token_bucket.go +++ b/fs/accounting/token_bucket.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "sync" "time" @@ -59,18 +60,25 @@ func (bs *buckets) _setOff() { } const defaultMaxBurstSize = 4 * 1024 * 1024 // must be bigger than the biggest request +const tokenBucketBurstScale = (256 * 1024 * 1024) / defaultMaxBurstSize -// make a new empty token bucket with the bandwidth given -func newEmptyTokenBucket(bandwidth fs.SizeSuffix) *rate.Limiter { +func tokenBucketBurst(bandwidth fs.SizeSuffix) int { // Relate maxBurstSize to bandwidth limit // 4M gives 2.5 Gb/s on Windows // Use defaultMaxBurstSize up to 2GBit/s (256MiB/s) then scale - maxBurstSize := max((bandwidth*defaultMaxBurstSize)/(256*1024*1024), defaultMaxBurstSize) + maxBurstSize := max(bandwidth/tokenBucketBurstScale, defaultMaxBurstSize) + maxBurstSize = min(maxBurstSize, fs.SizeSuffix(math.MaxInt)) + return int(maxBurstSize) +} + +// make a new empty token bucket with the bandwidth given +func newEmptyTokenBucket(bandwidth fs.SizeSuffix) *rate.Limiter { + maxBurstSize := tokenBucketBurst(bandwidth) // fs.Debugf(nil, "bandwidth=%v maxBurstSize=%v", bandwidth, maxBurstSize) - tb := rate.NewLimiter(rate.Limit(bandwidth), int(maxBurstSize)) + tb := rate.NewLimiter(rate.Limit(bandwidth), maxBurstSize) if tb != nil { // empty the bucket - err := tb.WaitN(context.Background(), int(maxBurstSize)) + err := tb.WaitN(context.Background(), maxBurstSize) if err != nil { fs.Errorf(nil, "Failed to empty token bucket: %v", err) } diff --git a/fs/accounting/token_bucket_test.go b/fs/accounting/token_bucket_test.go index 2c3e3bcce..bb7388891 100644 --- a/fs/accounting/token_bucket_test.go +++ b/fs/accounting/token_bucket_test.go @@ -2,14 +2,38 @@ package accounting import ( "context" + "math" "testing" + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/rc" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/time/rate" ) +func TestTokenBucketBurstScalesLargeBandwidthWithoutOverflow(t *testing.T) { + bandwidth := 4 * fs.Tebi + want := bandwidth / tokenBucketBurstScale + if want > fs.SizeSuffix(math.MaxInt) { + want = fs.SizeSuffix(math.MaxInt) + } + + tb := newEmptyTokenBucket(bandwidth) + require.NotNil(t, tb) + assert.Equal(t, rate.Limit(bandwidth), tb.Limit()) + assert.Equal(t, int(want), tb.Burst()) +} + +func TestTokenBucketBurstCapsAtMaxInt(t *testing.T) { + want := fs.SizeSuffix(fs.SizeSuffixMaxValue / tokenBucketBurstScale) + if want > fs.SizeSuffix(math.MaxInt) { + want = fs.SizeSuffix(math.MaxInt) + } + + assert.Equal(t, int(want), tokenBucketBurst(fs.SizeSuffixMaxValue)) +} + func TestRcBwLimit(t *testing.T) { call := rc.Calls.Get("core/bwlimit") assert.NotNil(t, call)