s3: fix server side copy failing with --s3-no-head-object - fixes #9629
With no_head_object set, NewObject does not read any metadata, so the destination object returned from a server side copy had a size of 0. The size check in operations.Copy then failed with "corrupted on transfer: sizes differ N vs 0" and deleted the newly copied object. This also broke Move and hence renames through rclone mount. Populate the destination object's size and MD5 from the source object when no_head_object is set, as a server side copy produces an object with identical content. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
committed by
Nick Craig-Wood
co-authored by
Claude Fable 5
parent
4af64270cc
commit
6df7b8aba1
@@ -3275,6 +3275,16 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// With NoHeadObject no metadata was read for the new object, so carry
|
||||
// the size and MD5 over from the source as a server-side copy produces
|
||||
// an object with identical content.
|
||||
if f.opt.NoHeadObject {
|
||||
if dstObject, ok := dstObj.(*Object); ok {
|
||||
dstObject.bytes = srcObj.bytes
|
||||
dstObject.md5 = srcObj.md5
|
||||
}
|
||||
}
|
||||
|
||||
// Set Object Lock via separate API calls if requested
|
||||
if f.opt.ObjectLockSetAfterUpload {
|
||||
if dstObject, ok := dstObj.(*Object); ok {
|
||||
|
||||
@@ -144,6 +144,33 @@ func (f *Fs) InternalTestNoHead(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestNoHeadObjectCopy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
contents := random.String(1000)
|
||||
item := fstest.NewItem("test-no-head-object-copy-src", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
src := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
assert.NoError(t, src.Remove(ctx))
|
||||
}()
|
||||
// Set NoHeadObject for this test so the copied object's metadata is not read back
|
||||
f.opt.NoHeadObject = true
|
||||
defer func() {
|
||||
f.opt.NoHeadObject = false
|
||||
}()
|
||||
dst, err := f.Copy(ctx, src, "test-no-head-object-copy-dst")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
assert.NoError(t, dst.Remove(ctx))
|
||||
}()
|
||||
assert.Equal(t, src.Size(), dst.Size())
|
||||
srcHash, err := src.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
dstHash, err := dst.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, srcHash, dstHash)
|
||||
assert.NotEqual(t, "", dstHash)
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestHasChildren(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
contents := random.String(100)
|
||||
@@ -809,6 +836,7 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Metadata", f.InternalTestMetadata)
|
||||
t.Run("NoHead", f.InternalTestNoHead)
|
||||
t.Run("NoHeadObjectCopy", f.InternalTestNoHeadObjectCopy)
|
||||
t.Run("HasChildren", f.InternalTestHasChildren)
|
||||
t.Run("Versions", f.InternalTestVersions)
|
||||
t.Run("ObjectLock", f.InternalTestObjectLock)
|
||||
|
||||
Reference in New Issue
Block a user