From e2352201d112f36f05abc5dc91768000fb23f927 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Fri, 21 Aug 2026 18:40:53 +0200 Subject: [PATCH] local: speed up default checksummed copies by writing in larger blocks When copying to the local backend with checksums enabled (the default), rclone hashed the incoming data by wrapping the source reader in an io.TeeReader. TeeReader has no WriteTo method, so io.Copy could not use the source's fast path and fell back to its generic 32 KiB buffer loop. The same wrapping also stopped the destination *os.File using copy_file_range, since the source was no longer a raw fd. This meant checksummed copies were written in 32 KiB chunks whereas --ignore-checksum copies were written in much larger blocks (typically 1 MiB). On filesystems where small writes are expensive, such as FUSE mounts like LucidLink, this made a big difference: copying a 100 MiB file took 3203 x 32 KiB writes in 3.6s, and now takes 108 x ~1 MiB writes in 0.47s. --- backend/local/local.go | 7 ++++--- backend/local/local_internal_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index cd0cc3890..a6cf3f2eb 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -1650,12 +1650,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op out = nopWriterCloser{&symlinkData} } - // Calculate the hash of the object we are reading as we go along + // Calculate the hash of the object as we write it + var w io.Writer = out if hasher != nil { - in = io.TeeReader(in, hasher) + w = io.MultiWriter(w, hasher) } - _, err = io.Copy(out, in) + _, err = io.Copy(w, in) closeErr := out.Close() if err == nil { err = closeErr diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index 761aa8ea6..25adcdf80 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -499,6 +499,32 @@ func TestHashOnUpdate(t *testing.T) { assert.Equal(t, "45685e95985e20822fb2538a522a5ccf", md5) } +// Test the hash cached by Update matches a HashesOption hint passed by the caller +func TestHashOnUpdateWithHashOption(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + const filePath = "file.txt" + when := time.Now() + r.WriteFile(filePath, "x", when) + f := r.Flocal.(*Fs) + + o, err := f.NewObject(ctx, filePath) + require.NoError(t, err) + + b := bytes.NewBufferString("content") + src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f) + options := []fs.OpenOption{&fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}} + require.NoError(t, o.Update(ctx, b, src, options...)) + + gotContent, err := os.ReadFile(filepath.Join(f.root, filePath)) + require.NoError(t, err) + assert.Equal(t, "content", string(gotContent)) + + md5, err := o.Hash(ctx, hash.MD5) + require.NoError(t, err) + assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5) +} + // Test hashes on deleting an object func TestHashOnDelete(t *testing.T) { ctx := context.Background()