From 7dd42c93cf253e4be14da0f1b6df5b778386fc5a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 13 Jul 2026 13:34:46 +0100 Subject: [PATCH] fstests: check backends return accurate object fingerprints after writes Layers which wrap backends, such as the hasher backend and the VFS cache, use fs.Fingerprint to detect whether an object has changed. This only works if the object a backend hands back from a write operation is identical to the object a fresh listing returns. If a backend keeps the source's nanosecond precision modtime in memory while the server stores milliseconds, or omits a hash the server knows, every fingerprint comparison fails and cached hashes or files are silently invalidated. The existing tests compared modtimes within f.Precision() so they could not detect these sub-precision divergences. Add a checkFingerprint helper which asserts the fast and slow fs.Fingerprint of the object returned from a write operation exactly match those of the object read back from the remote, and wire it into all the write paths which return or mutate an object: Put (all variants), PutStream, large and streamed uploads, Update, SetModTime, server side Copy and Move, and the metadata mutating Copy/Move subtests. Rework the ObjectOpenFingerprint test to use the same helper against a fully read reference object, with subtests reporting which kind of read is being checked. --- fstest/fstests/fstests.go | 77 +++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 52f44153b..c03db1749 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -146,7 +146,22 @@ func retry(t *testing.T, what string, f func() error) { require.NoError(t, err, what) } -// check interface +// checkFingerprint checks that the fingerprint of the in-memory +// object (memory) exactly matches the fingerprint of the same object +// read back from the remote (reloaded). +func checkFingerprint(ctx context.Context, t *testing.T, memory, reloaded fs.Object) { + t.Helper() + for _, test := range []struct { + fast bool + name string + }{ + {fast: false, name: "slow"}, + {fast: true, name: "fast"}, + } { + memoryFingerprint, reloadedFingerprint := fs.Fingerprint(ctx, memory, test.fast), fs.Fingerprint(ctx, reloaded, test.fast) + assert.Equal(t, memoryFingerprint, reloadedFingerprint, "%s fingerprint of object in memory (%v) should exactly match fingerprint read back from the remote (%v)", test.name, memoryFingerprint, reloadedFingerprint) + } +} // PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove // @@ -198,8 +213,10 @@ func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *f } file.Check(t, obj, f.Precision()) // Re-read the object and check again + liveObj := obj obj = fstest.NewObject(ctx, t, f, file.Path) file.Check(t, obj, f.Precision()) + checkFingerprint(ctx, t, liveObj, obj) } return obj } @@ -253,8 +270,10 @@ func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, file.Check(t, obj, f.Precision()) // Re-read the object and check again + liveObj := obj obj = fstest.NewObject(ctx, t, f, file.Path) file.Check(t, obj, f.Precision()) + checkFingerprint(ctx, t, liveObj, obj) // Download the object and check it is OK downloadHash := hash.NewMultiHasher() @@ -1276,6 +1295,7 @@ func Run(t *testing.T, opt *Opt) { // Check dst lightly - list above has checked ModTime/Hashes assert.Equal(t, file2Copy.Path, dst.Remote()) + checkFingerprint(ctx, t, dst, fstest.NewObject(ctx, t, f, file2Copy.Path)) // check that mutating dst does not mutate src if !strings.Contains(fs.ConfigStringFull(f), "copy_is_hardlink") { @@ -1332,8 +1352,10 @@ func Run(t *testing.T, opt *Opt) { // Check metadata is correct fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet) + liveObj := oDst oDst = fstest.NewObject(ctx, t, f, dstName) fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet) + checkFingerprint(ctx, t, liveObj, oDst) // Remove test files require.NoError(t, oSrc.Remove(ctx)) @@ -1371,6 +1393,7 @@ func Run(t *testing.T, opt *Opt) { fstest.CheckListing(t, f, []fstest.Item{file1, file2Move}) // Check dst lightly - list above has checked ModTime/Hashes assert.Equal(t, file2Move.Path, dst.Remote()) + checkFingerprint(ctx, t, dst, fstest.NewObject(ctx, t, f, file2Move.Path)) // 1: file name.txt // 2: other.txt @@ -1440,8 +1463,10 @@ func Run(t *testing.T, opt *Opt) { // Check metadata is correct fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet) + liveObj := newO newO = fstest.NewObject(ctx, t, f, newName) fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet) + checkFingerprint(ctx, t, liveObj, newO) // Remove test file require.NoError(t, newO.Remove(ctx)) @@ -1763,6 +1788,7 @@ func Run(t *testing.T, opt *Opt) { require.NoError(t, err) file1.ModTime = newModTime file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision()) + checkFingerprint(ctx, t, obj, fstest.NewObject(ctx, t, f, file1.Path)) // And make a new object and read it from there too TestObjectModTime(t) }) @@ -1827,33 +1853,34 @@ func Run(t *testing.T, opt *Opt) { obj := fstest.NewObject(ctx, t, f, file1.Path) // Read the whole object first so any lazily fetched hash - // is populated, then take that as the reference. + // is populated, then read a reference object to compare + // fingerprints against. _ = ReadObject(ctx, t, obj, -1) - fastWant := fs.Fingerprint(ctx, obj, true) - slowWant := fs.Fingerprint(ctx, obj, false) - - checkFingerprint := func(what string) { - assert.Equal(t, fastWant, fs.Fingerprint(ctx, obj, true), "fast fingerprint changed after %s", what) - assert.Equal(t, slowWant, fs.Fingerprint(ctx, obj, false), "slow fingerprint changed after %s", what) - } + reference := fstest.NewObject(ctx, t, f, file1.Path) + _ = ReadObject(ctx, t, reference, -1) // Opening the object in various ways must not change the fingerprint - _ = ReadObject(ctx, t, obj, -1) - checkFingerprint("full read") - - _ = ReadObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}) - checkFingerprint("seek read") - - _ = ReadObject(ctx, t, obj, -1, &fs.RangeOption{Start: 5, End: 15}) - checkFingerprint("range read") - - _ = ReadObject(ctx, t, obj, -1, &fs.RangeOption{Start: 80, End: -1}) - checkFingerprint("range read to end") + for _, test := range []struct { + name string + options []fs.OpenOption + }{ + {name: "full read"}, + {name: "seek read", options: []fs.OpenOption{&fs.SeekOption{Offset: 50}}}, + {name: "range read", options: []fs.OpenOption{&fs.RangeOption{Start: 5, End: 15}}}, + {name: "range read to end", options: []fs.OpenOption{&fs.RangeOption{Start: 80, End: -1}}}, + } { + t.Run(test.name, func(t *testing.T) { + _ = ReadObject(ctx, t, obj, -1, test.options...) + checkFingerprint(ctx, t, obj, reference) + }) + } // A freshly fetched object must have the same fingerprint too - obj = fstest.NewObject(ctx, t, f, file1.Path) - _ = ReadObject(ctx, t, obj, -1) - checkFingerprint("refresh") + t.Run("refresh", func(t *testing.T) { + obj := fstest.NewObject(ctx, t, f, file1.Path) + _ = ReadObject(ctx, t, obj, -1) + checkFingerprint(ctx, t, obj, reference) + }) }) // TestObjectUpdate tests that Update works @@ -1880,8 +1907,10 @@ func Run(t *testing.T, opt *Opt) { file1.Check(t, obj, f.Precision()) // Re-read the object and check again + liveObj := obj obj = fstest.NewObject(ctx, t, f, file1.Path) file1.Check(t, obj, f.Precision()) + checkFingerprint(ctx, t, liveObj, obj) // check contents correct assert.Equal(t, contents, ReadObject(ctx, t, obj, -1), "contents of updated file1 differ") @@ -2377,8 +2406,10 @@ func Run(t *testing.T, opt *Opt) { file.Size = int64(contentSize) // use correct size when checking file.Check(t, obj, f.Precision()) // Re-read the object and check again + liveObj := obj obj = fstest.NewObject(ctx, t, f, file.Path) file.Check(t, obj, f.Precision()) + checkFingerprint(ctx, t, liveObj, obj) require.NoError(t, obj.Remove(ctx)) }) }