fstests: check backends return accurate object fingerprints after writes

Layers which wrap backends, such as the hasher backend and the VFS
cache, use fs.Fingerprint to detect whether an object has changed.
This only works if the object a backend hands back from a write
operation is identical to the object a fresh listing returns. If a
backend keeps the source's nanosecond precision modtime in memory
while the server stores milliseconds, or omits a hash the server
knows, every fingerprint comparison fails and cached hashes or files
are silently invalidated.

The existing tests compared modtimes within f.Precision() so they
could not detect these sub-precision divergences.

Add a checkFingerprint helper which asserts the fast and slow
fs.Fingerprint of the object returned from a write operation exactly
match those of the object read back from the remote, and wire it into
all the write paths which return or mutate an object: Put (all
variants), PutStream, large and streamed uploads, Update, SetModTime,
server side Copy and Move, and the metadata mutating Copy/Move
subtests.

Rework the ObjectOpenFingerprint test to use the same helper against
a fully read reference object, with subtests reporting which kind of
read is being checked.

fstest/fstests/fstests.go

@@ -146,7 +146,22 @@ func retry(t *testing.T, what string, f func() error) {
 	require.NoError(t, err, what)
 }
 
-// check interface
+// checkFingerprint checks that the fingerprint of the in-memory
+// object (memory) exactly matches the fingerprint of the same object
+// read back from the remote (reloaded).
+func checkFingerprint(ctx context.Context, t *testing.T, memory, reloaded fs.Object) {
+	t.Helper()
+	for _, test := range []struct {
+		fast bool
+		name string
+	}{
+		{fast: false, name: "slow"},
+		{fast: true, name: "fast"},
+	} {
+		memoryFingerprint, reloadedFingerprint := fs.Fingerprint(ctx, memory, test.fast), fs.Fingerprint(ctx, reloaded, test.fast)
+		assert.Equal(t, memoryFingerprint, reloadedFingerprint, "%s fingerprint of object in memory (%v) should exactly match fingerprint read back from the remote (%v)", test.name, memoryFingerprint, reloadedFingerprint)
+	}
+}
 
 // PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove
 //
@@ -198,8 +213,10 @@ func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *f
 		}
 		file.Check(t, obj, f.Precision())
 		// Re-read the object and check again
+		liveObj := obj
 		obj = fstest.NewObject(ctx, t, f, file.Path)
 		file.Check(t, obj, f.Precision())
+		checkFingerprint(ctx, t, liveObj, obj)
 	}
 	return obj
 }
@@ -253,8 +270,10 @@ func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item,
 	file.Check(t, obj, f.Precision())
 
 	// Re-read the object and check again
+	liveObj := obj
 	obj = fstest.NewObject(ctx, t, f, file.Path)
 	file.Check(t, obj, f.Precision())
+	checkFingerprint(ctx, t, liveObj, obj)
 
 	// Download the object and check it is OK
 	downloadHash := hash.NewMultiHasher()
@@ -1276,6 +1295,7 @@ func Run(t *testing.T, opt *Opt) {
 
 				// Check dst lightly - list above has checked ModTime/Hashes
 				assert.Equal(t, file2Copy.Path, dst.Remote())
+				checkFingerprint(ctx, t, dst, fstest.NewObject(ctx, t, f, file2Copy.Path))
 
 				// check that mutating dst does not mutate src
 				if !strings.Contains(fs.ConfigStringFull(f), "copy_is_hardlink") {
@@ -1332,8 +1352,10 @@ func Run(t *testing.T, opt *Opt) {
 
 					// Check metadata is correct
 					fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet)
+					liveObj := oDst
 					oDst = fstest.NewObject(ctx, t, f, dstName)
 					fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet)
+					checkFingerprint(ctx, t, liveObj, oDst)
 
 					// Remove test files
 					require.NoError(t, oSrc.Remove(ctx))
@@ -1371,6 +1393,7 @@ func Run(t *testing.T, opt *Opt) {
 				fstest.CheckListing(t, f, []fstest.Item{file1, file2Move})
 				// Check dst lightly - list above has checked ModTime/Hashes
 				assert.Equal(t, file2Move.Path, dst.Remote())
+				checkFingerprint(ctx, t, dst, fstest.NewObject(ctx, t, f, file2Move.Path))
 				// 1: file name.txt
 				// 2: other.txt
 
@@ -1440,8 +1463,10 @@ func Run(t *testing.T, opt *Opt) {
 
 					// Check metadata is correct
 					fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet)
+					liveObj := newO
 					newO = fstest.NewObject(ctx, t, f, newName)
 					fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet)
+					checkFingerprint(ctx, t, liveObj, newO)
 
 					// Remove test file
 					require.NoError(t, newO.Remove(ctx))
@@ -1763,6 +1788,7 @@ func Run(t *testing.T, opt *Opt) {
 				require.NoError(t, err)
 				file1.ModTime = newModTime
 				file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision())
+				checkFingerprint(ctx, t, obj, fstest.NewObject(ctx, t, f, file1.Path))
 				// And make a new object and read it from there too
 				TestObjectModTime(t)
 			})
@@ -1827,33 +1853,34 @@ func Run(t *testing.T, opt *Opt) {
 				obj := fstest.NewObject(ctx, t, f, file1.Path)
 
 				// Read the whole object first so any lazily fetched hash
-				// is populated, then take that as the reference.
+				// is populated, then read a reference object to compare
+				// fingerprints against.
 				_ = ReadObject(ctx, t, obj, -1)
-				fastWant := fs.Fingerprint(ctx, obj, true)
+				reference := fstest.NewObject(ctx, t, f, file1.Path)
-				slowWant := fs.Fingerprint(ctx, obj, false)
+				_ = ReadObject(ctx, t, reference, -1)
-
-				checkFingerprint := func(what string) {
-					assert.Equal(t, fastWant, fs.Fingerprint(ctx, obj, true), "fast fingerprint changed after %s", what)
-					assert.Equal(t, slowWant, fs.Fingerprint(ctx, obj, false), "slow fingerprint changed after %s", what)
-				}
 
 				// Opening the object in various ways must not change the fingerprint
-				_ = ReadObject(ctx, t, obj, -1)
+				for _, test := range []struct {
-				checkFingerprint("full read")
+					name    string
-
+					options []fs.OpenOption
-				_ = ReadObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50})
+				}{
-				checkFingerprint("seek read")
+					{name: "full read"},
-
+					{name: "seek read", options: []fs.OpenOption{&fs.SeekOption{Offset: 50}}},
-				_ = ReadObject(ctx, t, obj, -1, &fs.RangeOption{Start: 5, End: 15})
+					{name: "range read", options: []fs.OpenOption{&fs.RangeOption{Start: 5, End: 15}}},
-				checkFingerprint("range read")
+					{name: "range read to end", options: []fs.OpenOption{&fs.RangeOption{Start: 80, End: -1}}},
-
+				} {
-				_ = ReadObject(ctx, t, obj, -1, &fs.RangeOption{Start: 80, End: -1})
+					t.Run(test.name, func(t *testing.T) {
-				checkFingerprint("range read to end")
+						_ = ReadObject(ctx, t, obj, -1, test.options...)
+						checkFingerprint(ctx, t, obj, reference)
+					})
+				}
 
 				// A freshly fetched object must have the same fingerprint too
-				obj = fstest.NewObject(ctx, t, f, file1.Path)
+				t.Run("refresh", func(t *testing.T) {
-				_ = ReadObject(ctx, t, obj, -1)
+					obj := fstest.NewObject(ctx, t, f, file1.Path)
-				checkFingerprint("refresh")
+					_ = ReadObject(ctx, t, obj, -1)
+					checkFingerprint(ctx, t, obj, reference)
+				})
 			})
 
 			// TestObjectUpdate tests that Update works
@@ -1880,8 +1907,10 @@ func Run(t *testing.T, opt *Opt) {
 				file1.Check(t, obj, f.Precision())
 
 				// Re-read the object and check again
+				liveObj := obj
 				obj = fstest.NewObject(ctx, t, f, file1.Path)
 				file1.Check(t, obj, f.Precision())
+				checkFingerprint(ctx, t, liveObj, obj)
 
 				// check contents correct
 				assert.Equal(t, contents, ReadObject(ctx, t, obj, -1), "contents of updated file1 differ")
@@ -2377,8 +2406,10 @@ func Run(t *testing.T, opt *Opt) {
 						file.Size = int64(contentSize) // use correct size when checking
 						file.Check(t, obj, f.Precision())
 						// Re-read the object and check again
+						liveObj := obj
 						obj = fstest.NewObject(ctx, t, f, file.Path)
 						file.Check(t, obj, f.Precision())
+						checkFingerprint(ctx, t, liveObj, obj)
 						require.NoError(t, obj.Remove(ctx))
 					})
 				}