serve s3: reserve the .rclone_temp_ prefix for temporary objects

The temporary objects that uploads are written to before being renamed
into place are now named .rclone_temp_put_* and .rclone_temp_multipart_*,
and the whole .rclone_temp_ prefix is reserved: any object whose name
starts with it is hidden from S3 listings. This gives a single pattern
for cleaning up leftovers from killed servers:

    rclone delete --min-age 24h --include ".rclone_temp_*" remote:path

The .rclone_multipart_upload_* objects rclone v1.75 used are still
hidden from listings so leftovers from an older server stay invisible
to S3 clients.

cmd/serve/s3/backend.go

@@ -25,9 +25,13 @@ var (
 	emptyPrefix = &gofakes3.Prefix{}
 )
 
+// tempObjectPrefix is reserved for serve s3's temporary objects: an object
+// whose leaf name starts with it is hidden from S3 listings.
+const tempObjectPrefix = ".rclone_temp_"
+
 // putObjectPrefix is prepended to the leaf name of the temporary object a
 // PutObject upload is written to before it is renamed into place.
-const putObjectPrefix = ".rclone_put_object_"
+const putObjectPrefix = tempObjectPrefix + "put_"
 
 // s3Backend implements the gofacess3.Backend interface to make an S3
 // backend for gofakes3. It also implements gofakes3.MultipartBackend so that

cmd/serve/s3/list.go

@@ -8,6 +8,11 @@ import (
 	"github.com/rclone/rclone/vfs"
 )
 
+// legacyMultipartUploadPrefix marked the temporary objects of in-progress
+// multipart uploads before the tempObjectPrefix namespace was reserved
+// (rclone v1.75); leftovers from an older server are still hidden.
+const legacyMultipartUploadPrefix = ".rclone_multipart_upload_"
+
 func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {
 	fp, err := bucketDirPath(bucketName, fdPath)
 	if err != nil {
@@ -24,7 +29,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, a
 		object := entry.Name()
 
 		// Hide the temporary objects of in-progress uploads
-		if strings.HasPrefix(object, multipartUploadPrefix) || strings.HasPrefix(object, putObjectPrefix) {
+		if strings.HasPrefix(object, tempObjectPrefix) || strings.HasPrefix(object, legacyMultipartUploadPrefix) {
 			continue
 		}
 

cmd/serve/s3/multipart.go

@@ -37,7 +37,7 @@ import (
 
 // multipartUploadPrefix is prepended to the leaf name of the temporary object
 // a streamed multipart upload is written to before it is moved into place.
-const multipartUploadPrefix = ".rclone_multipart_upload_"
+const multipartUploadPrefix = tempObjectPrefix + "multipart_"
 
 // multipartUpload tracks one in-flight S3 multipart upload that is being
 // streamed, in part order, into a single PutStream upload to the underlying Fs.

cmd/serve/s3/multipart_test.go

@@ -21,6 +21,7 @@ import (
 	_ "github.com/rclone/rclone/backend/memory"
 	"github.com/rclone/rclone/cmd/serve/proxy"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/random"
@@ -492,6 +493,39 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) {
 	up.mu.Unlock()
 }
 
+// TestTempObjectsHiddenFromListings checks that the reserved .rclone_temp_
+// prefix, and the multipart prefix used before it was reserved, are hidden
+// from S3 listings while remaining visible to rclone itself for cleanup.
+func TestTempObjectsHiddenFromListings(t *testing.T) {
+	core, f, bucket := newMultipartTestServer(t, false)
+	ctx := context.Background()
+
+	names := []string{
+		"visible.bin",
+		tempObjectPrefix + "anything",
+		multipartUploadPrefix + "leftover",
+		putObjectPrefix + "leftover",
+		legacyMultipartUploadPrefix + "leftover",
+	}
+	for _, name := range names {
+		data := []byte("x")
+		src := object.NewStaticObjectInfo(path.Join(bucket, name), time.Now(), int64(len(data)), true, nil, f)
+		_, err := f.Put(ctx, bytes.NewReader(data), src)
+		require.NoError(t, err)
+	}
+	// All the objects really exist on the backing remote...
+	requireOnly(t, f, bucket, names...)
+
+	// ...but only the visible one appears in an S3 listing.
+	result, err := core.ListObjects(bucket, "", "", "", 1000)
+	require.NoError(t, err)
+	var keys []string
+	for _, o := range result.Contents {
+		keys = append(keys, o.Key)
+	}
+	assert.Equal(t, []string{"visible.bin"}, keys)
+}
+
 // TestMultipartOverwrite checks that a completed multipart upload atomically
 // replaces an existing object of the same name.
 func TestMultipartOverwrite(t *testing.T) {

cmd/serve/s3/serve_s3.md

@@ -103,7 +103,7 @@ renamed into place on success; these remotes need to support a server-side
 move or copy for this (nearly all do - without move or copy the upload is
 written directly and a failed PUT may leave a partial object at the key). If
 `serve s3` is killed part-way through an upload the temporary object (named
-with a leading `.rclone_put_object_`) may be left behind; it is hidden from
+with a leading `.rclone_temp_put_`) may be left behind; it is hidden from
 S3 listings but must be removed manually.
 
 ### Multipart uploads
@@ -173,9 +173,33 @@ also need to support a server-side move or copy.
   completed object is moved into place with a server-side operation.
   This is a cheap rename on most such remotes. On these remotes, if
   `serve s3` is killed part-way through an upload the temporary object
-  (named with a leading `.rclone_multipart_upload_`) may be left behind;
+  (named with a leading `.rclone_temp_multipart_`) may be left behind;
   it is hidden from S3 listings but must be removed manually.
 
+#### Cleaning up temporary objects
+
+If `serve s3` is killed part-way through an upload it can leave a
+temporary object behind, named with a leading `.rclone_temp_`. This
+whole prefix is reserved: any object whose name (the last
+`/`-separated segment of its key) starts with `.rclone_temp_` is
+hidden from S3 listings, so don't give real objects such names - an
+existing object with such a name disappears from listings (though it
+stays accessible directly by its key: only listings hide reserved
+names, `GET`, `HEAD` and `DELETE` of the exact key still work). A
+temporary object never holds acknowledged data - uploads whose
+temporary object survived were never confirmed to the client - so old
+ones are safe to delete:
+
+    rclone delete --min-age 24h --include ".rclone_temp_*" remote:path
+
+The `--min-age` protects uploads which are still in progress: make sure
+it is longer than your longest upload, especially if several `serve s3`
+instances share the same remote.
+
+rclone v1.75 named its temporary multipart objects
+`.rclone_multipart_upload_*`; leftovers from an older server are also
+hidden from listings and can be cleaned up the same way.
+
 #### Disabling streaming
 
 If you pass `--disable-multipart-streaming`, or the remote doesn't