s3: add backend link command with signed response header overrides

Fixes #7684.

backend/s3/s3.go

@@ -3303,6 +3303,10 @@ func (f *Fs) Hashes() hash.Set {
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
+	return f.publicLink(ctx, remote, expire, &s3.GetObjectInput{})
+}
+
+func (f *Fs) publicLink(ctx context.Context, remote string, expire fs.Duration, req *s3.GetObjectInput) (link string, err error) {
 	if strings.HasSuffix(remote, "/") {
 		return "", fs.ErrorCantShareDirectories
 	}
@@ -3316,11 +3320,10 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		expire = maxExpireDuration
 	}
 	bucket, bucketPath := f.split(remote)
-	httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, &s3.GetObjectInput{
+	req.Bucket = &bucket
-		Bucket:    &bucket,
+	req.Key = &bucketPath
-		Key:       &bucketPath,
+	req.VersionId = o.versionID
-		VersionId: o.versionID,
+	httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, req, s3.WithPresignExpires(time.Duration(expire)))
-	}, s3.WithPresignExpires(time.Duration(expire)))
 	if err != nil {
 		return "", err
 	}
@@ -3328,6 +3331,40 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 }
 
 var commandHelp = []fs.CommandHelp{{
+	Name:  "link",
+	Short: "Generate a signed link with response header overrides.",
+	Long: `This command generates a signed download link for one file, with optional
+overrides for the HTTP response headers. Pass the file path as a separate
+argument, relative to the remote path.
+
+Usage examples:
+
+` + "```console" + `
+rclone backend link s3:bucket path/to/file -o expire=1h
+rclone backend link s3:bucket path/to/file -o response-expires="Thu, 01 Jan 1970 00:00:00 GMT"
+rclone backend link s3:bucket path/to/file -o response-content-disposition='attachment; filename="download.txt"'
+` + "```" + `
+
+The link expires after 7 days by default. Use ` + "`-o expire=1h`" + ` to change
+its lifetime. Durations must be at least 1 second; values above 7 days are
+limited to 7 days, as with ` + "`rclone link`" + `.
+
+The ` + "`response-expires`" + ` option sets the HTTP Expires response header,
+not the lifetime of the signed link. It must be an HTTP date, such as
+` + "`Thu, 01 Jan 1970 00:00:00 GMT`" + `.
+
+The overrides are included in the signature and must not be changed in the
+returned URL. They do not modify the object's stored metadata.`,
+	Opts: map[string]string{
+		"expire":                       "How long the link will be valid (default 7d, maximum 7d).",
+		"response-cache-control":       "Set the Cache-Control response header.",
+		"response-content-disposition": "Set the Content-Disposition response header.",
+		"response-content-encoding":    "Set the Content-Encoding response header.",
+		"response-content-language":    "Set the Content-Language response header.",
+		"response-content-type":        "Set the Content-Type response header.",
+		"response-expires":             "Set the Expires response header to an HTTP date.",
+	},
+}, {
 	Name:  "restore",
 	Short: "Restore objects from GLACIER or INTELLIGENT-TIERING archive tier.",
 	Long: `This command can be used to restore one or more objects from GLACIER to normal
@@ -3555,6 +3592,44 @@ It doesn't return anything.`,
 // otherwise it will be JSON encoded and shown to the user like that
 func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
 	switch name {
+	case "link":
+		if len(arg) != 1 || arg[0] == "" {
+			return nil, errors.New("link requires exactly one file path argument")
+		}
+		req := s3.GetObjectInput{}
+		expire := maxExpireDuration
+		for key, value := range opt {
+			switch key {
+			case "expire":
+				duration, err := fs.ParseDuration(value)
+				if err != nil {
+					return nil, fmt.Errorf("invalid expire: %w", err)
+				}
+				if duration < time.Second {
+					return nil, errors.New("expire must be at least 1 second")
+				}
+				expire = fs.Duration(duration)
+			case "response-cache-control":
+				req.ResponseCacheControl = aws.String(value)
+			case "response-content-disposition":
+				req.ResponseContentDisposition = aws.String(value)
+			case "response-content-encoding":
+				req.ResponseContentEncoding = aws.String(value)
+			case "response-content-language":
+				req.ResponseContentLanguage = aws.String(value)
+			case "response-content-type":
+				req.ResponseContentType = aws.String(value)
+			case "response-expires":
+				date, err := http.ParseTime(value)
+				if err != nil {
+					return nil, fmt.Errorf("invalid response-expires: %w", err)
+				}
+				req.ResponseExpires = &date
+			default:
+				return nil, fmt.Errorf("unknown link option %q", key)
+			}
+		}
+		return f.publicLink(ctx, arg[0], expire, &req)
 	case "restore":
 		req := s3.RestoreObjectInput{
 			//Bucket:         &f.rootBucket,

backend/s3/s3_link_test.go

@@ -0,0 +1,227 @@
+package s3
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newLinkTestFs(t *testing.T) *Fs {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Has("versions") {
+			assert.Equal(t, http.MethodGet, r.Method)
+			assert.Equal(t, "prefix/file", r.URL.Query().Get("prefix"))
+			_, err := io.WriteString(w, `<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+<IsTruncated>false</IsTruncated>
+<Version><Key>prefix/file</Key><VersionId>version+id/1</VersionId><IsLatest>false</IsLatest>
+<LastModified>2024-01-01T00:00:00Z</LastModified><Size>1</Size></Version>
+</ListVersionsResult>`)
+			assert.NoError(t, err)
+			return
+		}
+		assert.Equal(t, http.MethodHead, r.Method)
+		if strings.HasSuffix(r.URL.Path, "/missing") {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Length", "1")
+		w.Header().Set("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT")
+	}))
+	t.Cleanup(server.Close)
+
+	ctx, opt, client := SetupS3Test(t)
+	opt.Endpoint = server.URL
+	opt.ForcePathStyle = true
+	opt.Region = "us-east-1"
+	opt.AccessKeyID = "test-access-key"
+	opt.SecretAccessKey = "test-secret-key"
+	opt.SessionToken = "test-token+/="
+	c, _, err := s3Connection(ctx, opt, client)
+	require.NoError(t, err)
+	f := &Fs{
+		name:  "s3test",
+		opt:   *opt,
+		ctx:   ctx,
+		c:     c,
+		pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
+		cache: bucket.NewCache(),
+	}
+	f.setRoot("bucket/prefix")
+	return f
+}
+
+func linkSignature(t *testing.T, u *url.URL) string {
+	t.Helper()
+	q := u.Query()
+	signingTime, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
+	require.NoError(t, err)
+	q.Del("X-Amz-Signature")
+	unsigned := *u
+	unsigned.RawQuery = q.Encode()
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, unsigned.String(), nil)
+	require.NoError(t, err)
+	signed, _, err := v4.NewSigner().PresignHTTP(context.Background(), aws.Credentials{
+		AccessKeyID:     "test-access-key",
+		SecretAccessKey: "test-secret-key",
+		SessionToken:    "test-token+/=",
+	}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", signingTime, func(opt *v4.SignerOptions) {
+		opt.DisableURIPathEscaping = true
+	})
+	require.NoError(t, err)
+	result, err := url.Parse(signed)
+	require.NoError(t, err)
+	return result.Query().Get("X-Amz-Signature")
+}
+
+func TestCommandLink(t *testing.T) {
+	f := newLinkTestFs(t)
+	remote := "dir/a file +&?%\u2603.txt"
+	overrides := map[string]string{
+		"response-cache-control":       "private, max-age=0",
+		"response-content-disposition": `attachment; filename="a +&?.txt"`,
+		"response-content-encoding":    "identity",
+		"response-content-language":    "en-US",
+		"response-content-type":        "text/plain; charset=utf-8",
+		"response-expires":             "Thu, 01 Jan 1970 00:00:00 GMT",
+	}
+	opts := map[string]string{"expire": "1h"}
+	for k, v := range overrides {
+		opts[k] = v
+	}
+	out, err := f.Command(context.Background(), "link", []string{remote}, opts)
+	require.NoError(t, err)
+	link, ok := out.(string)
+	require.True(t, ok)
+	u, err := url.Parse(link)
+	require.NoError(t, err)
+	assert.Equal(t, "/bucket/prefix/"+remote, u.Path)
+	assert.NotContains(t, u.RawQuery, " ")
+	q := u.Query()
+	assert.Equal(t, "3600", q.Get("X-Amz-Expires"))
+	assert.Equal(t, "test-token+/=", q.Get("X-Amz-Security-Token"))
+	assert.Equal(t, "AWS4-HMAC-SHA256", q.Get("X-Amz-Algorithm"))
+	assert.Equal(t, "host", q.Get("X-Amz-SignedHeaders"))
+	for k, v := range overrides {
+		assert.Equal(t, v, q.Get(k), k)
+	}
+	signature := q.Get("X-Amz-Signature")
+	require.NotEmpty(t, signature)
+	assert.Equal(t, signature, linkSignature(t, u))
+	for k := range overrides {
+		t.Run("Signed/"+k, func(t *testing.T) {
+			tampered := *u
+			changed := u.Query()
+			changed.Set(k, "changed")
+			tampered.RawQuery = changed.Encode()
+			assert.NotEqual(t, signature, linkSignature(t, &tampered))
+		})
+	}
+}
+
+func TestCommandLinkExpire(t *testing.T) {
+	f := newLinkTestFs(t)
+	for _, test := range []struct {
+		name   string
+		opts   map[string]string
+		expire string
+	}{
+		{"Default", nil, "604800"},
+		{"Minimum", map[string]string{"expire": "1s"}, "1"},
+		{"Maximum", map[string]string{"expire": "7d"}, "604800"},
+		{"Clamped", map[string]string{"expire": "8d"}, "604800"},
+		{"Off", map[string]string{"expire": "off"}, "604800"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			out, err := f.Command(context.Background(), "link", []string{"file"}, test.opts)
+			require.NoError(t, err)
+			u, err := url.Parse(out.(string))
+			require.NoError(t, err)
+			assert.Equal(t, test.expire, u.Query().Get("X-Amz-Expires"))
+			for key := range u.Query() {
+				assert.False(t, strings.HasPrefix(key, "response-"))
+			}
+			assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
+		})
+	}
+}
+
+func TestCommandLinkErrors(t *testing.T) {
+	f := newLinkTestFs(t)
+	for _, test := range []struct {
+		name string
+		args []string
+		opts map[string]string
+	}{
+		{"NoPath", nil, nil},
+		{"TooManyPaths", []string{"one", "two"}, nil},
+		{"EmptyPath", []string{""}, nil},
+		{"Directory", []string{"dir/"}, nil},
+		{"Missing", []string{"missing"}, nil},
+		{"UnknownOption", []string{"file"}, map[string]string{"response-unknown": "value"}},
+		{"InvalidDate", []string{"file"}, map[string]string{"response-expires": "tomorrow"}},
+		{"InvalidExpire", []string{"file"}, map[string]string{"expire": "invalid"}},
+		{"EmptyExpire", []string{"file"}, map[string]string{"expire": ""}},
+		{"ZeroExpire", []string{"file"}, map[string]string{"expire": "0"}},
+		{"NegativeExpire", []string{"file"}, map[string]string{"expire": "-1s"}},
+		{"SubsecondExpire", []string{"file"}, map[string]string{"expire": "500ms"}},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			_, err := f.Command(context.Background(), "link", test.args, test.opts)
+			require.Error(t, err)
+			assert.NotErrorIs(t, err, fs.ErrorCommandNotFound)
+		})
+	}
+}
+
+func TestCommandLinkVersionAt(t *testing.T) {
+	f := newLinkTestFs(t)
+	f.opt.VersionAt = fs.Time(time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC))
+	for _, command := range []bool{false, true} {
+		var link string
+		if command {
+			out, err := f.Command(context.Background(), "link", []string{"file"}, map[string]string{
+				"response-content-type": "text/plain",
+			})
+			require.NoError(t, err)
+			link = out.(string)
+		} else {
+			var err error
+			link, err = f.PublicLink(context.Background(), "file", fs.DurationOff, false)
+			require.NoError(t, err)
+		}
+		u, err := url.Parse(link)
+		require.NoError(t, err)
+		assert.Equal(t, "/bucket/prefix/file", u.Path)
+		assert.Equal(t, "version+id/1", u.Query().Get("versionId"))
+		assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
+	}
+}
+
+func TestPublicLink(t *testing.T) {
+	f := newLinkTestFs(t)
+	link, err := f.PublicLink(context.Background(), "file", fs.DurationOff, false)
+	require.NoError(t, err)
+	u, err := url.Parse(link)
+	require.NoError(t, err)
+	assert.Equal(t, "/bucket/prefix/file", u.Path)
+	assert.Equal(t, "604800", u.Query().Get("X-Amz-Expires"))
+	assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
+	_, err = f.PublicLink(context.Background(), "dir/", fs.DurationOff, false)
+	assert.ErrorIs(t, err, fs.ErrorCantShareDirectories)
+	_, err = f.PublicLink(context.Background(), "missing", fs.DurationOff, false)
+	assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}