diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go index 1ac2d2cba..afa4e2cae 100644 --- a/cmd/serve/s3/backend.go +++ b/cmd/serve/s3/backend.go @@ -25,9 +25,13 @@ var ( emptyPrefix = &gofakes3.Prefix{} ) +// tempObjectPrefix is reserved for serve s3's temporary objects: an object +// whose leaf name starts with it is hidden from S3 listings. +const tempObjectPrefix = ".rclone_temp_" + // putObjectPrefix is prepended to the leaf name of the temporary object a // PutObject upload is written to before it is renamed into place. -const putObjectPrefix = ".rclone_put_object_" +const putObjectPrefix = tempObjectPrefix + "put_" // s3Backend implements the gofacess3.Backend interface to make an S3 // backend for gofakes3. It also implements gofakes3.MultipartBackend so that diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go index a95bb8dee..67e9f21df 100644 --- a/cmd/serve/s3/list.go +++ b/cmd/serve/s3/list.go @@ -8,6 +8,11 @@ import ( "github.com/rclone/rclone/vfs" ) +// legacyMultipartUploadPrefix marked the temporary objects of in-progress +// multipart uploads before the tempObjectPrefix namespace was reserved +// (rclone v1.75); leftovers from an older server are still hidden. +const legacyMultipartUploadPrefix = ".rclone_multipart_upload_" + func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { fp, err := bucketDirPath(bucketName, fdPath) if err != nil { @@ -24,7 +29,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, a object := entry.Name() // Hide the temporary objects of in-progress uploads - if strings.HasPrefix(object, multipartUploadPrefix) || strings.HasPrefix(object, putObjectPrefix) { + if strings.HasPrefix(object, tempObjectPrefix) || strings.HasPrefix(object, legacyMultipartUploadPrefix) { continue } diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go index aa17a71ef..dd38e3c3d 100644 --- a/cmd/serve/s3/multipart.go +++ b/cmd/serve/s3/multipart.go @@ -37,7 +37,7 @@ import ( // multipartUploadPrefix is prepended to the leaf name of the temporary object // a streamed multipart upload is written to before it is moved into place. -const multipartUploadPrefix = ".rclone_multipart_upload_" +const multipartUploadPrefix = tempObjectPrefix + "multipart_" // multipartUpload tracks one in-flight S3 multipart upload that is being // streamed, in part order, into a single PutStream upload to the underlying Fs. diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go index f54605d11..67c3a69bb 100644 --- a/cmd/serve/s3/multipart_test.go +++ b/cmd/serve/s3/multipart_test.go @@ -21,6 +21,7 @@ import ( _ "github.com/rclone/rclone/backend/memory" "github.com/rclone/rclone/cmd/serve/proxy" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/object" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/lib/multipart" "github.com/rclone/rclone/lib/random" @@ -492,6 +493,39 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) { up.mu.Unlock() } +// TestTempObjectsHiddenFromListings checks that the reserved .rclone_temp_ +// prefix, and the multipart prefix used before it was reserved, are hidden +// from S3 listings while remaining visible to rclone itself for cleanup. +func TestTempObjectsHiddenFromListings(t *testing.T) { + core, f, bucket := newMultipartTestServer(t, false) + ctx := context.Background() + + names := []string{ + "visible.bin", + tempObjectPrefix + "anything", + multipartUploadPrefix + "leftover", + putObjectPrefix + "leftover", + legacyMultipartUploadPrefix + "leftover", + } + for _, name := range names { + data := []byte("x") + src := object.NewStaticObjectInfo(path.Join(bucket, name), time.Now(), int64(len(data)), true, nil, f) + _, err := f.Put(ctx, bytes.NewReader(data), src) + require.NoError(t, err) + } + // All the objects really exist on the backing remote... + requireOnly(t, f, bucket, names...) + + // ...but only the visible one appears in an S3 listing. + result, err := core.ListObjects(bucket, "", "", "", 1000) + require.NoError(t, err) + var keys []string + for _, o := range result.Contents { + keys = append(keys, o.Key) + } + assert.Equal(t, []string{"visible.bin"}, keys) +} + // TestMultipartOverwrite checks that a completed multipart upload atomically // replaces an existing object of the same name. func TestMultipartOverwrite(t *testing.T) { diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md index 4d214a0f0..2c9152007 100644 --- a/cmd/serve/s3/serve_s3.md +++ b/cmd/serve/s3/serve_s3.md @@ -103,7 +103,7 @@ renamed into place on success; these remotes need to support a server-side move or copy for this (nearly all do - without move or copy the upload is written directly and a failed PUT may leave a partial object at the key). If `serve s3` is killed part-way through an upload the temporary object (named -with a leading `.rclone_put_object_`) may be left behind; it is hidden from +with a leading `.rclone_temp_put_`) may be left behind; it is hidden from S3 listings but must be removed manually. ### Multipart uploads @@ -173,9 +173,33 @@ also need to support a server-side move or copy. completed object is moved into place with a server-side operation. This is a cheap rename on most such remotes. On these remotes, if `serve s3` is killed part-way through an upload the temporary object - (named with a leading `.rclone_multipart_upload_`) may be left behind; + (named with a leading `.rclone_temp_multipart_`) may be left behind; it is hidden from S3 listings but must be removed manually. +#### Cleaning up temporary objects + +If `serve s3` is killed part-way through an upload it can leave a +temporary object behind, named with a leading `.rclone_temp_`. This +whole prefix is reserved: any object whose name (the last +`/`-separated segment of its key) starts with `.rclone_temp_` is +hidden from S3 listings, so don't give real objects such names - an +existing object with such a name disappears from listings (though it +stays accessible directly by its key: only listings hide reserved +names, `GET`, `HEAD` and `DELETE` of the exact key still work). A +temporary object never holds acknowledged data - uploads whose +temporary object survived were never confirmed to the client - so old +ones are safe to delete: + + rclone delete --min-age 24h --include ".rclone_temp_*" remote:path + +The `--min-age` protects uploads which are still in progress: make sure +it is longer than your longest upload, especially if several `serve s3` +instances share the same remote. + +rclone v1.75 named its temporary multipart objects +`.rclone_multipart_upload_*`; leftovers from an older server are also +hidden from listings and can be cleaned up the same way. + #### Disabling streaming If you pass `--disable-multipart-streaming`, or the remote doesn't