f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

operations: check checksums in rcat with known size - fixes #6305

Browse Source 
RcatSize streams the body straight into Put behind an ObjectInfo with no
hashes, so a known-size upload had its size checked but its checksum
never verified, unlike the unknown-size path through Rcat which hashes as
it goes.

The stream can only be read once, so hash it on the way past and compare
with the destination after the upload, as operations.Copy does. A
destination which reports no usable hash is still checked by size, and one
which fails either check has the failed copy removed.
This commit is contained in:
phatlc
2026-08-13 19:49:33 +02:00
committed by Nick Craig-Wood
parent 35f052e29a
commit fc348fcb0b
2 changed files with 123 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/operations/operations.go
+31 -7
View File
@@ -1818,8 +1818,20 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
defer func() {
tr.Done(ctx, err)
}()
body := io.NopCloser(in) // we let the server close the body
in := tr.Account(ctx, body) // account the transfer (no buffering)
// The stream can only be read once, so hash it on the way past
// and compare with the destination afterwards.
hashType, hashOption := CommonHash(ctx, fdst, fdst)
var hasher *hash.MultiHasher
var streamIn io.Reader = in
if hashType != hash.None {
hasher, err = hash.NewMultiHasherTypes(hashOption.Hashes)
if err != nil {
return nil, err
}
streamIn = io.TeeReader(streamIn, hasher)
}
body := io.NopCloser(streamIn) // we let the server close the body
in := tr.Account(ctx, body) // account the transfer (no buffering)
if SkipDestructive(ctx, dstFileName, "upload from pipe") {
// prevents "broken pipe" errors
@@ -1827,7 +1839,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
return nil, err
}
var options []fs.OpenOption
options := []fs.OpenOption{hashOption}
for _, option := range fs.GetConfig(ctx).UploadHeaders {
options = append(options, option)
}
@@ -1839,13 +1851,25 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
return nil, err
}
// Check transfer - the source may have ended before size
// bytes in which case the object will have been truncated
// Verify the upload
if sizeDiffers(ctx, info, obj) {
err = fmt.Errorf("corrupted on transfer: sizes differ src %d vs dst(%s) %d", info.Size(), fdst, obj.Size())
err = fmt.Errorf("corrupted on transfer: sizes differ src %d vs dst(%s) %d", size, obj.Fs(), obj.Size())
} else if hasher != nil {
src := object.NewStaticObjectInfo(dstFileName, modTime, size, true, hasher.Sums(), fdst)
// checkHashes logs and counts errors
same, _, srcHash, dstHash, _ := checkHashes(ctx, src, obj, hashType)
if !same {
err = fmt.Errorf("corrupted on transfer: %v hashes differ src %q vs dst(%s) %q", hashType, srcHash, obj.Fs(), dstHash)
}
}
if err != nil {
err = fs.CountError(ctx, err)
fs.Errorf(obj, "%v", err)
return obj, err
fs.Infof(obj, "Removing failed copy")
if removeErr := obj.Remove(ctx); removeErr != nil {
fs.Infof(obj, "Failed to remove failed copy: %s", removeErr)
}
return nil, err
}
} else {
// Size unknown use Rcat