sia: fix corrupted files being created when the source ends early
If the source supplied fewer bytes than its declared size, the upload request failed but a retry could report success even though the stored file was truncated, because the retry re-sent an already exhausted reader. Count the bytes actually read from the source and if they do not match the declared size, remove the partially uploaded file and return an error. This was found by the new FsPutShortEOF integration test.
This commit is contained in:
+11
-1
@@ -24,6 +24,7 @@ import (
|
|||||||
"github.com/rclone/rclone/fs/hash"
|
"github.com/rclone/rclone/fs/hash"
|
||||||
"github.com/rclone/rclone/lib/encoder"
|
"github.com/rclone/rclone/lib/encoder"
|
||||||
"github.com/rclone/rclone/lib/pacer"
|
"github.com/rclone/rclone/lib/pacer"
|
||||||
|
"github.com/rclone/rclone/lib/readers"
|
||||||
"github.com/rclone/rclone/lib/rest"
|
"github.com/rclone/rclone/lib/rest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -176,11 +177,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
|||||||
// Update the object with the contents of the io.Reader
|
// Update the object with the contents of the io.Reader
|
||||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||||
size := src.Size()
|
size := src.Size()
|
||||||
|
counter := readers.NewCountingReader(in)
|
||||||
var resp *http.Response
|
var resp *http.Response
|
||||||
opts := rest.Opts{
|
opts := rest.Opts{
|
||||||
Method: "POST",
|
Method: "POST",
|
||||||
Path: path.Join("/renter/uploadstream/", o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
|
Path: path.Join("/renter/uploadstream/", o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
|
||||||
Body: in,
|
Body: counter,
|
||||||
ContentLength: &size,
|
ContentLength: &size,
|
||||||
Parameters: url.Values{},
|
Parameters: url.Values{},
|
||||||
}
|
}
|
||||||
@@ -191,6 +193,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
|||||||
return o.fs.shouldRetry(resp, err)
|
return o.fs.shouldRetry(resp, err)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Check the source supplied the number of bytes it declared
|
||||||
|
// otherwise a truncated file would be stored as a good upload.
|
||||||
|
// The partially uploaded file is cleaned up by Put which copes
|
||||||
|
// with it appearing asynchronously after the failed upload.
|
||||||
|
if err == nil && size >= 0 && int64(counter.BytesRead()) != size {
|
||||||
|
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||||
|
}
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = o.readMetaData(ctx)
|
err = o.readMetaData(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user