Files
rclone/fs/accounting/accounting_test.go
T
Nick Craig-Wood 67e5f435c6 accounting: fix rcat/copyurl for files.com
The files.com integration tests for rcat/copyurl were failing because
fs/account.Account was declaring a ReadAt method when the underlying
handle did not support it. The files.com SDK decided to use the ReadAt
method to speed transfers up which failed.

ReadAt and Seek methods were added in this commit to support the
archive command:

409dc75328 accounting: add io.Seeker/io.ReaderAt support to accounting.Account

This fixes the problem by adding new methods to the Account object
WithSeeker/WithReaderAt/WithReadAtSeeker which produce an object with
the desired methods or errors if it isn't possible.

This stops Account advertising things it can't do which is bad Go
practice.
2026-04-18 17:48:03 +01:00

639 lines
16 KiB
Go

package accounting
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
"unicode/utf8"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/asyncreader"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/readers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Check it satisfies the interfaces
var (
_ io.ReadCloser = &Account{}
_ io.WriterTo = &Account{}
_ io.Reader = &accountStream{}
_ Accounter = &Account{}
_ Accounter = &accountStream{}
)
func TestNewAccountSizeName(t *testing.T) {
ctx := context.Background()
in := io.NopCloser(bytes.NewBuffer([]byte{1}))
stats := NewStats(ctx)
acc := newAccountSizeName(context.Background(), stats, in, 1, "test")
assert.Equal(t, in, acc.in)
assert.Equal(t, acc, stats.inProgress.get("test"))
err := acc.Close()
assert.NoError(t, err)
assert.Equal(t, acc, stats.inProgress.get("test"))
acc.Done()
assert.Nil(t, stats.inProgress.get("test"))
assert.False(t, acc.HasBuffer())
}
func TestAccountWithBuffer(t *testing.T) {
ctx := context.Background()
in := io.NopCloser(bytes.NewBuffer([]byte{1}))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, -1, "test")
assert.False(t, acc.HasBuffer())
acc.WithBuffer()
assert.True(t, acc.HasBuffer())
// should have a buffer for an unknown size
_, ok := acc.in.(*asyncreader.AsyncReader)
require.True(t, ok)
assert.NoError(t, acc.Close())
acc = newAccountSizeName(ctx, stats, in, 1, "test")
acc.WithBuffer()
// should not have a buffer for a small size
_, ok = acc.in.(*asyncreader.AsyncReader)
require.False(t, ok)
assert.NoError(t, acc.Close())
}
func TestAccountGetUpdateReader(t *testing.T) {
ctx := context.Background()
test := func(doClose bool) func(t *testing.T) {
return func(t *testing.T) {
in := io.NopCloser(bytes.NewBuffer([]byte{1}))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, 1, "test")
assert.Equal(t, in, acc.GetReader())
assert.Equal(t, acc, stats.inProgress.get("test"))
if doClose {
// close the account before swapping it out
require.NoError(t, acc.Close())
}
in2 := io.NopCloser(bytes.NewBuffer([]byte{1}))
acc.UpdateReader(ctx, in2)
assert.Equal(t, in2, acc.GetReader())
assert.Equal(t, acc, stats.inProgress.get("test"))
assert.NoError(t, acc.Close())
}
}
t.Run("NoClose", test(false))
t.Run("Close", test(true))
}
func TestAccountRead(t *testing.T) {
ctx := context.Background()
in := io.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, 1, "test")
assert.True(t, acc.values.start.IsZero())
acc.values.mu.Lock()
assert.Equal(t, int64(0), acc.values.lpBytes)
assert.Equal(t, int64(0), acc.values.bytes)
acc.values.mu.Unlock()
assert.Equal(t, int64(0), stats.bytes)
var buf = make([]byte, 2)
n, err := acc.Read(buf)
assert.NoError(t, err)
assert.Equal(t, 2, n)
assert.Equal(t, []byte{1, 2}, buf[:n])
assert.False(t, acc.values.start.IsZero())
acc.values.mu.Lock()
assert.Equal(t, int64(2), acc.values.lpBytes)
assert.Equal(t, int64(2), acc.values.bytes)
acc.values.mu.Unlock()
assert.Equal(t, int64(2), stats.bytes)
n, err = acc.Read(buf)
assert.NoError(t, err)
assert.Equal(t, 1, n)
assert.Equal(t, []byte{3}, buf[:n])
n, err = acc.Read(buf)
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, n)
assert.NoError(t, acc.Close())
}
func testAccountWriteTo(t *testing.T, withBuffer bool) {
ctx := context.Background()
buf := make([]byte, 2*asyncreader.BufferSize+1)
for i := range buf {
buf[i] = byte(i % 251)
}
in := io.NopCloser(bytes.NewBuffer(buf))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(buf)), "test")
if withBuffer {
acc = acc.WithBuffer()
}
assert.True(t, acc.values.start.IsZero())
acc.values.mu.Lock()
assert.Equal(t, int64(0), acc.values.lpBytes)
assert.Equal(t, int64(0), acc.values.bytes)
acc.values.mu.Unlock()
assert.Equal(t, int64(0), stats.bytes)
var out bytes.Buffer
n, err := acc.WriteTo(&out)
assert.NoError(t, err)
assert.Equal(t, int64(len(buf)), n)
assert.Equal(t, buf, out.Bytes())
assert.False(t, acc.values.start.IsZero())
acc.values.mu.Lock()
assert.Equal(t, int64(len(buf)), acc.values.lpBytes)
assert.Equal(t, int64(len(buf)), acc.values.bytes)
acc.values.mu.Unlock()
assert.Equal(t, int64(len(buf)), stats.bytes)
assert.NoError(t, acc.Close())
}
func TestAccountWriteTo(t *testing.T) {
testAccountWriteTo(t, false)
}
func TestAccountWriteToWithBuffer(t *testing.T) {
testAccountWriteTo(t, true)
}
func TestAccountString(t *testing.T) {
ctx := context.Background()
in := io.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, 3, "test")
// FIXME not an exhaustive test!
assert.Equal(t, "test: 0% / 3 B, 0 B/s, -", strings.TrimSpace(acc.String()))
var buf = make([]byte, 2)
n, err := acc.Read(buf)
assert.NoError(t, err)
assert.Equal(t, 2, n)
assert.Equal(t, "test: 66% / 3 B, 0 B/s, -", strings.TrimSpace(acc.String()))
assert.NoError(t, acc.Close())
}
// Test the Accounter interface methods on Account and accountStream
func TestAccountAccounter(t *testing.T) {
ctx := context.Background()
in := io.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, 3, "test")
assert.True(t, in == acc.OldStream())
in2 := io.NopCloser(bytes.NewBuffer([]byte{2, 3, 4}))
acc.SetStream(in2)
assert.True(t, in2 == acc.OldStream())
r := acc.WrapStream(in)
as, ok := r.(Accounter)
require.True(t, ok)
assert.True(t, in == as.OldStream())
assert.True(t, in2 == acc.OldStream())
accs, ok := r.(*accountStream)
require.True(t, ok)
assert.Equal(t, acc, accs.acc)
assert.True(t, in == accs.in)
// Check Read on the accountStream
var buf = make([]byte, 2)
n, err := r.Read(buf)
assert.NoError(t, err)
assert.Equal(t, 2, n)
assert.Equal(t, []byte{1, 2}, buf[:n])
// Test that we can get another accountstream out
in3 := io.NopCloser(bytes.NewBuffer([]byte{3, 1, 2}))
r2 := as.WrapStream(in3)
as2, ok := r2.(Accounter)
require.True(t, ok)
assert.True(t, in3 == as2.OldStream())
assert.True(t, in2 == acc.OldStream())
accs2, ok := r2.(*accountStream)
require.True(t, ok)
assert.Equal(t, acc, accs2.acc)
assert.True(t, in3 == accs2.in)
// Test we can set this new accountStream
as2.SetStream(in)
assert.True(t, in == as2.OldStream())
// Test UnWrap on accountStream
unwrapped, wrap := UnWrap(r2)
assert.True(t, unwrapped == in)
r3 := wrap(in2)
assert.True(t, in2 == r3.(Accounter).OldStream())
// TestUnWrap on a normal io.Reader
unwrapped, wrap = UnWrap(in2)
assert.True(t, unwrapped == in2)
assert.True(t, wrap(in3) == in3)
}
func TestAccountMaxTransfer(t *testing.T) {
ctx := context.Background()
ci := fs.GetConfig(ctx)
old := ci.MaxTransfer
oldMode := ci.CutoffMode
ci.MaxTransfer = 15
defer func() {
ci.MaxTransfer = old
ci.CutoffMode = oldMode
}()
in := io.NopCloser(bytes.NewBuffer(make([]byte, 100)))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, 1, "test")
var b = make([]byte, 10)
n, err := acc.Read(b)
assert.Equal(t, 10, n)
assert.NoError(t, err)
n, err = acc.Read(b)
assert.Equal(t, 5, n)
assert.Equal(t, ErrorMaxTransferLimitReachedFatal, err)
n, err = acc.Read(b)
assert.Equal(t, 0, n)
assert.Equal(t, ErrorMaxTransferLimitReachedFatal, err)
assert.True(t, fserrors.IsFatalError(err))
ci.CutoffMode = fs.CutoffModeSoft
stats = NewStats(ctx)
acc = newAccountSizeName(ctx, stats, in, 1, "test")
n, err = acc.Read(b)
assert.Equal(t, 10, n)
assert.NoError(t, err)
n, err = acc.Read(b)
assert.Equal(t, 10, n)
assert.NoError(t, err)
n, err = acc.Read(b)
assert.Equal(t, 10, n)
assert.NoError(t, err)
}
func TestAccountMaxTransferWriteTo(t *testing.T) {
ctx := context.Background()
ci := fs.GetConfig(ctx)
old := ci.MaxTransfer
oldMode := ci.CutoffMode
ci.MaxTransfer = 15
defer func() {
ci.MaxTransfer = old
ci.CutoffMode = oldMode
}()
in := io.NopCloser(readers.NewPatternReader(1024))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, 1, "test")
var b bytes.Buffer
n, err := acc.WriteTo(&b)
assert.Equal(t, int64(15), n)
assert.Equal(t, ErrorMaxTransferLimitReachedFatal, err)
}
func TestAccountReadCtx(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
in := io.NopCloser(bytes.NewBuffer(make([]byte, 100)))
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, 1, "test")
var b = make([]byte, 10)
n, err := acc.Read(b)
assert.Equal(t, 10, n)
assert.NoError(t, err)
cancel()
n, err = acc.Read(b)
assert.Equal(t, 0, n)
assert.Equal(t, context.Canceled, err)
}
func TestShortenName(t *testing.T) {
for _, test := range []struct {
in string
size int
want string
}{
{"", 0, ""},
{"abcde", 10, "abcde"},
{"abcde", 0, "abcde"},
{"abcde", -1, "abcde"},
{"abcde", 5, "abcde"},
{"abcde", 4, "ab…e"},
{"abcde", 3, "a…e"},
{"abcde", 2, "a…"},
{"abcde", 1, "…"},
{"abcdef", 6, "abcdef"},
{"abcdef", 5, "ab…ef"},
{"abcdef", 4, "ab…f"},
{"abcdef", 3, "a…f"},
{"abcdef", 2, "a…"},
{"áßcdèf", 1, "…"},
{"áßcdè", 5, "áßcdè"},
{"áßcdè", 4, "áß…è"},
{"áßcdè", 3, "á…è"},
{"áßcdè", 2, "á…"},
{"áßcdè", 1, "…"},
{"áßcdèł", 6, "áßcdèł"},
{"áßcdèł", 5, "áß…èł"},
{"áßcdèł", 4, "áß…ł"},
{"áßcdèł", 3, "á…ł"},
{"áßcdèł", 2, "á…"},
{"áßcdèł", 1, "…"},
} {
t.Run(fmt.Sprintf("in=%q, size=%d", test.in, test.size), func(t *testing.T) {
got := shortenName(test.in, test.size)
assert.Equal(t, test.want, got)
if test.size > 0 {
assert.True(t, utf8.RuneCountInString(got) <= test.size, "too big")
}
})
}
}
// Test readers for WithSeeker/WithReaderAt tests
type testSeeker struct {
*bytes.Buffer
}
func newTestSeeker(data []byte) *testSeeker {
return &testSeeker{bytes.NewBuffer(data)}
}
func (ts *testSeeker) Close() error { return nil }
func (ts *testSeeker) Seek(offset int64, whence int) (int64, error) {
// Simple implementation for testing
switch whence {
case io.SeekStart:
ts.Buffer = bytes.NewBuffer(ts.Buffer.Bytes()[offset:])
return offset, nil
default:
return 0, fmt.Errorf("seek whence %d not implemented", whence)
}
}
type testReadAtSeeker struct {
*bytes.Reader
}
func newTestReadAtSeeker(data []byte) *testReadAtSeeker {
return &testReadAtSeeker{bytes.NewReader(data)}
}
func (tras *testReadAtSeeker) Close() error { return nil }
type testReaderAt struct {
data []byte
pos int
}
func newTestReaderAt(data []byte) *testReaderAt {
return &testReaderAt{data: data}
}
func (tra *testReaderAt) Close() error { return nil }
func (tra *testReaderAt) Read(p []byte) (n int, err error) {
if tra.pos >= len(tra.data) {
return 0, io.EOF
}
n = copy(p, tra.data[tra.pos:])
tra.pos += n
if tra.pos >= len(tra.data) {
err = io.EOF
}
return n, err
}
func (tra *testReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
if off < 0 || off >= int64(len(tra.data)) {
return 0, io.EOF
}
n = copy(p, tra.data[off:])
if n < len(p) {
err = io.EOF
}
return n, err
}
type testNoSeekNoReadAt struct {
*bytes.Buffer
}
func newTestNoSeekNoReadAt(data []byte) *testNoSeekNoReadAt {
return &testNoSeekNoReadAt{bytes.NewBuffer(data)}
}
func (tns *testNoSeekNoReadAt) Close() error { return nil }
func TestAccountWithSeeker(t *testing.T) {
ctx := context.Background()
testData := []byte{1, 2, 3, 4, 5}
t.Run("Success", func(t *testing.T) {
in := newTestSeeker(testData)
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
seeker, err := acc.WithSeeker()
assert.NoError(t, err)
// Test that it implements the expected interfaces
var _ io.Reader = seeker
var _ io.Seeker = seeker
// Test seeking functionality
offset, err := seeker.Seek(2, io.SeekStart)
assert.NoError(t, err)
assert.Equal(t, int64(2), offset)
// Test reading after seek
buf := make([]byte, 2)
n, err := seeker.Read(buf)
assert.NoError(t, err)
assert.Equal(t, 2, n)
assert.Equal(t, []byte{3, 4}, buf)
assert.NoError(t, acc.Close())
})
t.Run("NoSeeker", func(t *testing.T) {
in := newTestNoSeekNoReadAt(testData)
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
_, err := acc.WithSeeker()
assert.Error(t, err)
assert.ErrorIs(t, err, errors.ErrUnsupported)
assert.Contains(t, err.Error(), "Seek not implemented for")
assert.NoError(t, acc.Close())
})
}
func TestAccountWithReaderAt(t *testing.T) {
ctx := context.Background()
testData := []byte{1, 2, 3, 4, 5}
t.Run("Success", func(t *testing.T) {
in := newTestReaderAt(testData)
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
readerAt, err := acc.WithReaderAt()
assert.NoError(t, err)
// Test that it implements the expected interfaces
var _ io.Reader = readerAt
var _ io.ReaderAt = readerAt
// Test ReadAt functionality
buf := make([]byte, 2)
n, err := readerAt.ReadAt(buf, 2)
assert.NoError(t, err)
assert.Equal(t, 2, n)
assert.Equal(t, []byte{3, 4}, buf)
// Test that accounting is updated
acc.values.mu.Lock()
bytes := acc.values.bytes
acc.values.mu.Unlock()
assert.Equal(t, int64(2), bytes)
assert.NoError(t, acc.Close())
})
t.Run("NoReaderAt", func(t *testing.T) {
in := newTestNoSeekNoReadAt(testData)
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
_, err := acc.WithReaderAt()
assert.Error(t, err)
assert.ErrorIs(t, err, errors.ErrUnsupported)
assert.Contains(t, err.Error(), "ReadAt not implemented for")
assert.NoError(t, acc.Close())
})
}
func TestAccountWithReadAtSeeker(t *testing.T) {
ctx := context.Background()
testData := []byte{1, 2, 3, 4, 5}
t.Run("Success", func(t *testing.T) {
// Use testReadAtSeeker which implements ReadAt, Seek, and Close
in := newTestReadAtSeeker(testData)
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
readAtSeeker, err := acc.WithReadAtSeeker()
assert.NoError(t, err)
// Test that it implements the expected interfaces
var _ io.Reader = readAtSeeker
var _ io.ReaderAt = readAtSeeker
var _ io.Seeker = readAtSeeker
// Test ReadAt functionality
buf := make([]byte, 2)
n, err := readAtSeeker.ReadAt(buf, 2)
assert.NoError(t, err)
assert.Equal(t, 2, n)
assert.Equal(t, []byte{3, 4}, buf)
// Test Seek functionality
offset, err := readAtSeeker.Seek(1, io.SeekStart)
assert.NoError(t, err)
assert.Equal(t, int64(1), offset)
// Test regular Read after seek
n, err = readAtSeeker.Read(buf)
assert.NoError(t, err)
assert.Equal(t, 2, n)
assert.Equal(t, []byte{2, 3}, buf)
assert.NoError(t, acc.Close())
})
t.Run("NoReadAt", func(t *testing.T) {
in := newTestSeeker(testData) // Has Seek but not ReadAt
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
_, err := acc.WithReadAtSeeker()
assert.Error(t, err)
assert.ErrorIs(t, err, errors.ErrUnsupported)
assert.NoError(t, acc.Close())
})
t.Run("NoSeeker", func(t *testing.T) {
// Create a reader that has ReadAt but not Seek interface
in := &struct {
io.ReadCloser
io.ReaderAt
}{
ReadCloser: io.NopCloser(bytes.NewReader(testData)),
ReaderAt: bytes.NewReader(testData),
}
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
_, err := acc.WithReadAtSeeker()
assert.Error(t, err)
assert.ErrorIs(t, err, errors.ErrUnsupported)
assert.NoError(t, acc.Close())
})
t.Run("NoSeekOrReadAt", func(t *testing.T) {
in := newTestNoSeekNoReadAt(testData)
stats := NewStats(ctx)
acc := newAccountSizeName(ctx, stats, in, int64(len(testData)), "test")
_, err := acc.WithReadAtSeeker()
assert.Error(t, err)
assert.ErrorIs(t, err, errors.ErrUnsupported)
assert.NoError(t, acc.Close())
})
}