internetarchive: fix corrupted files being created when the source ends early

If the source supplied fewer bytes than its declared size, the upload
request failed but a retry could report success even though the stored
file was truncated, because the retry re-sent an already exhausted
reader.

Count the bytes actually read from the source and if they do not match
the declared size return an error.

This was found by the new FsPutShortEOF integration test.
This commit is contained in:
Nick Craig-Wood
2026-08-11 19:32:12 +01:00
parent 5d057549b9
commit 7357fb82a9
+9 -1
View File
@@ -30,6 +30,7 @@ import (
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
)
@@ -854,11 +855,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
// make a PUT request at (IAS3)/encoded(:item/:path)
counter := readers.NewCountingReader(in)
var resp *http.Response
opts := rest.Opts{
Method: "PUT",
Path: "/" + url.PathEscape(path.Join(bucket, bucketPath)),
Body: in,
Body: counter,
ContentLength: &size,
ExtraHeaders: headers,
}
@@ -868,6 +870,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return o.fs.shouldRetry(resp, err)
})
// Check the source supplied the number of bytes it declared
// otherwise a truncated file would be stored as a good upload
if err == nil && size >= 0 && int64(counter.BytesRead()) != size {
err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
}
// we can't update/find metadata here as IA will "ingest" uploaded file(s)
// upon uploads. (you can find its progress at https://archive.org/history/ItemNameHere )
// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)