local: speed up default checksummed copies by writing in larger blocks

When copying to the local backend with checksums enabled (the default),
rclone hashed the incoming data by wrapping the source reader in an
io.TeeReader. TeeReader has no WriteTo method, so io.Copy could not use
the source's fast path and fell back to its generic 32 KiB buffer loop.
The same wrapping also stopped the destination *os.File using
copy_file_range, since the source was no longer a raw fd.

This meant checksummed copies were written in 32 KiB chunks whereas
--ignore-checksum copies were written in much larger blocks (typically
1 MiB). On filesystems where small writes are expensive, such as FUSE
mounts like LucidLink, this made a big difference: copying a 100 MiB
file took 3203 x 32 KiB writes in 3.6s, and now takes 108 x ~1 MiB
writes in 0.47s.
This commit is contained in:
Dominik Sander
2026-08-21 17:40:53 +01:00
committed by GitHub
parent be7f9b38b0
commit e2352201d1
2 changed files with 30 additions and 3 deletions
+26
View File
@@ -499,6 +499,32 @@ func TestHashOnUpdate(t *testing.T) {
assert.Equal(t, "45685e95985e20822fb2538a522a5ccf", md5)
}
// Test the hash cached by Update matches a HashesOption hint passed by the caller
func TestHashOnUpdateWithHashOption(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
const filePath = "file.txt"
when := time.Now()
r.WriteFile(filePath, "x", when)
f := r.Flocal.(*Fs)
o, err := f.NewObject(ctx, filePath)
require.NoError(t, err)
b := bytes.NewBufferString("content")
src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)
options := []fs.OpenOption{&fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}}
require.NoError(t, o.Update(ctx, b, src, options...))
gotContent, err := os.ReadFile(filepath.Join(f.root, filePath))
require.NoError(t, err)
assert.Equal(t, "content", string(gotContent))
md5, err := o.Hash(ctx, hash.MD5)
require.NoError(t, err)
assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)
}
// Test hashes on deleting an object
func TestHashOnDelete(t *testing.T) {
ctx := context.Background()