From 7357fb82a9ef29eef7902694a871247398baa177 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 7 Aug 2026 14:02:09 +0100 Subject: [PATCH] internetarchive: fix corrupted files being created when the source ends early If the source supplied fewer bytes than its declared size, the upload request failed but a retry could report success even though the stored file was truncated, because the retry re-sent an already exhausted reader. Count the bytes actually read from the source and if they do not match the declared size return an error. This was found by the new FsPutShortEOF integration test. --- backend/internetarchive/internetarchive.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go index 65f045d22..cfd4e57ac 100644 --- a/backend/internetarchive/internetarchive.go +++ b/backend/internetarchive/internetarchive.go @@ -30,6 +30,7 @@ import ( "github.com/rclone/rclone/lib/encoder" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/random" + "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" ) @@ -854,11 +855,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // make a PUT request at (IAS3)/encoded(:item/:path) + counter := readers.NewCountingReader(in) var resp *http.Response opts := rest.Opts{ Method: "PUT", Path: "/" + url.PathEscape(path.Join(bucket, bucketPath)), - Body: in, + Body: counter, ContentLength: &size, ExtraHeaders: headers, } @@ -868,6 +870,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return o.fs.shouldRetry(resp, err) }) + // Check the source supplied the number of bytes it declared + // otherwise a truncated file would be stored as a good upload + if err == nil && size >= 0 && int64(counter.BytesRead()) != size { + err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF) + } + // we can't update/find metadata here as IA will "ingest" uploaded file(s) // upon uploads. (you can find its progress at https://archive.org/history/ItemNameHere ) // or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)