From 39b487d7f758e3bb7a1f02fbd6b91395b982d688 Mon Sep 17 00:00:00 2001 From: eliotee <17210244+eliotee@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:04:18 +0300 Subject: [PATCH] s3: add backend link command with signed response header overrides Fixes #7684. --- backend/s3/s3.go | 85 +++++++++++++- backend/s3/s3_link_test.go | 227 +++++++++++++++++++++++++++++++++++++ 2 files changed, 307 insertions(+), 5 deletions(-) create mode 100644 backend/s3/s3_link_test.go diff --git a/backend/s3/s3.go b/backend/s3/s3.go index af365c00c..a444eff35 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -3303,6 +3303,10 @@ func (f *Fs) Hashes() hash.Set { // PublicLink generates a public link to the remote path (usually readable by anyone) func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) { + return f.publicLink(ctx, remote, expire, &s3.GetObjectInput{}) +} + +func (f *Fs) publicLink(ctx context.Context, remote string, expire fs.Duration, req *s3.GetObjectInput) (link string, err error) { if strings.HasSuffix(remote, "/") { return "", fs.ErrorCantShareDirectories } @@ -3316,11 +3320,10 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, expire = maxExpireDuration } bucket, bucketPath := f.split(remote) - httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &bucketPath, - VersionId: o.versionID, - }, s3.WithPresignExpires(time.Duration(expire))) + req.Bucket = &bucket + req.Key = &bucketPath + req.VersionId = o.versionID + httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, req, s3.WithPresignExpires(time.Duration(expire))) if err != nil { return "", err } @@ -3328,6 +3331,40 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, } var commandHelp = []fs.CommandHelp{{ + Name: "link", + Short: "Generate a signed link with response header overrides.", + Long: `This command generates a signed download link for one file, with optional +overrides for the HTTP response headers. Pass the file path as a separate +argument, relative to the remote path. + +Usage examples: + +` + "```console" + ` +rclone backend link s3:bucket path/to/file -o expire=1h +rclone backend link s3:bucket path/to/file -o response-expires="Thu, 01 Jan 1970 00:00:00 GMT" +rclone backend link s3:bucket path/to/file -o response-content-disposition='attachment; filename="download.txt"' +` + "```" + ` + +The link expires after 7 days by default. Use ` + "`-o expire=1h`" + ` to change +its lifetime. Durations must be at least 1 second; values above 7 days are +limited to 7 days, as with ` + "`rclone link`" + `. + +The ` + "`response-expires`" + ` option sets the HTTP Expires response header, +not the lifetime of the signed link. It must be an HTTP date, such as +` + "`Thu, 01 Jan 1970 00:00:00 GMT`" + `. + +The overrides are included in the signature and must not be changed in the +returned URL. They do not modify the object's stored metadata.`, + Opts: map[string]string{ + "expire": "How long the link will be valid (default 7d, maximum 7d).", + "response-cache-control": "Set the Cache-Control response header.", + "response-content-disposition": "Set the Content-Disposition response header.", + "response-content-encoding": "Set the Content-Encoding response header.", + "response-content-language": "Set the Content-Language response header.", + "response-content-type": "Set the Content-Type response header.", + "response-expires": "Set the Expires response header to an HTTP date.", + }, +}, { Name: "restore", Short: "Restore objects from GLACIER or INTELLIGENT-TIERING archive tier.", Long: `This command can be used to restore one or more objects from GLACIER to normal @@ -3555,6 +3592,44 @@ It doesn't return anything.`, // otherwise it will be JSON encoded and shown to the user like that func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) { switch name { + case "link": + if len(arg) != 1 || arg[0] == "" { + return nil, errors.New("link requires exactly one file path argument") + } + req := s3.GetObjectInput{} + expire := maxExpireDuration + for key, value := range opt { + switch key { + case "expire": + duration, err := fs.ParseDuration(value) + if err != nil { + return nil, fmt.Errorf("invalid expire: %w", err) + } + if duration < time.Second { + return nil, errors.New("expire must be at least 1 second") + } + expire = fs.Duration(duration) + case "response-cache-control": + req.ResponseCacheControl = aws.String(value) + case "response-content-disposition": + req.ResponseContentDisposition = aws.String(value) + case "response-content-encoding": + req.ResponseContentEncoding = aws.String(value) + case "response-content-language": + req.ResponseContentLanguage = aws.String(value) + case "response-content-type": + req.ResponseContentType = aws.String(value) + case "response-expires": + date, err := http.ParseTime(value) + if err != nil { + return nil, fmt.Errorf("invalid response-expires: %w", err) + } + req.ResponseExpires = &date + default: + return nil, fmt.Errorf("unknown link option %q", key) + } + } + return f.publicLink(ctx, arg[0], expire, &req) case "restore": req := s3.RestoreObjectInput{ //Bucket: &f.rootBucket, diff --git a/backend/s3/s3_link_test.go b/backend/s3/s3_link_test.go new file mode 100644 index 000000000..7a8b01708 --- /dev/null +++ b/backend/s3/s3_link_test.go @@ -0,0 +1,227 @@ +package s3 + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/lib/bucket" + "github.com/rclone/rclone/lib/pacer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newLinkTestFs(t *testing.T) *Fs { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Has("versions") { + assert.Equal(t, http.MethodGet, r.Method) + assert.Equal(t, "prefix/file", r.URL.Query().Get("prefix")) + _, err := io.WriteString(w, ` +false +prefix/fileversion+id/1false +2024-01-01T00:00:00Z1 +`) + assert.NoError(t, err) + return + } + assert.Equal(t, http.MethodHead, r.Method) + if strings.HasSuffix(r.URL.Path, "/missing") { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Length", "1") + w.Header().Set("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT") + })) + t.Cleanup(server.Close) + + ctx, opt, client := SetupS3Test(t) + opt.Endpoint = server.URL + opt.ForcePathStyle = true + opt.Region = "us-east-1" + opt.AccessKeyID = "test-access-key" + opt.SecretAccessKey = "test-secret-key" + opt.SessionToken = "test-token+/=" + c, _, err := s3Connection(ctx, opt, client) + require.NoError(t, err) + f := &Fs{ + name: "s3test", + opt: *opt, + ctx: ctx, + c: c, + pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))), + cache: bucket.NewCache(), + } + f.setRoot("bucket/prefix") + return f +} + +func linkSignature(t *testing.T, u *url.URL) string { + t.Helper() + q := u.Query() + signingTime, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date")) + require.NoError(t, err) + q.Del("X-Amz-Signature") + unsigned := *u + unsigned.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, unsigned.String(), nil) + require.NoError(t, err) + signed, _, err := v4.NewSigner().PresignHTTP(context.Background(), aws.Credentials{ + AccessKeyID: "test-access-key", + SecretAccessKey: "test-secret-key", + SessionToken: "test-token+/=", + }, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", signingTime, func(opt *v4.SignerOptions) { + opt.DisableURIPathEscaping = true + }) + require.NoError(t, err) + result, err := url.Parse(signed) + require.NoError(t, err) + return result.Query().Get("X-Amz-Signature") +} + +func TestCommandLink(t *testing.T) { + f := newLinkTestFs(t) + remote := "dir/a file +&?%\u2603.txt" + overrides := map[string]string{ + "response-cache-control": "private, max-age=0", + "response-content-disposition": `attachment; filename="a +&?.txt"`, + "response-content-encoding": "identity", + "response-content-language": "en-US", + "response-content-type": "text/plain; charset=utf-8", + "response-expires": "Thu, 01 Jan 1970 00:00:00 GMT", + } + opts := map[string]string{"expire": "1h"} + for k, v := range overrides { + opts[k] = v + } + out, err := f.Command(context.Background(), "link", []string{remote}, opts) + require.NoError(t, err) + link, ok := out.(string) + require.True(t, ok) + u, err := url.Parse(link) + require.NoError(t, err) + assert.Equal(t, "/bucket/prefix/"+remote, u.Path) + assert.NotContains(t, u.RawQuery, " ") + q := u.Query() + assert.Equal(t, "3600", q.Get("X-Amz-Expires")) + assert.Equal(t, "test-token+/=", q.Get("X-Amz-Security-Token")) + assert.Equal(t, "AWS4-HMAC-SHA256", q.Get("X-Amz-Algorithm")) + assert.Equal(t, "host", q.Get("X-Amz-SignedHeaders")) + for k, v := range overrides { + assert.Equal(t, v, q.Get(k), k) + } + signature := q.Get("X-Amz-Signature") + require.NotEmpty(t, signature) + assert.Equal(t, signature, linkSignature(t, u)) + for k := range overrides { + t.Run("Signed/"+k, func(t *testing.T) { + tampered := *u + changed := u.Query() + changed.Set(k, "changed") + tampered.RawQuery = changed.Encode() + assert.NotEqual(t, signature, linkSignature(t, &tampered)) + }) + } +} + +func TestCommandLinkExpire(t *testing.T) { + f := newLinkTestFs(t) + for _, test := range []struct { + name string + opts map[string]string + expire string + }{ + {"Default", nil, "604800"}, + {"Minimum", map[string]string{"expire": "1s"}, "1"}, + {"Maximum", map[string]string{"expire": "7d"}, "604800"}, + {"Clamped", map[string]string{"expire": "8d"}, "604800"}, + {"Off", map[string]string{"expire": "off"}, "604800"}, + } { + t.Run(test.name, func(t *testing.T) { + out, err := f.Command(context.Background(), "link", []string{"file"}, test.opts) + require.NoError(t, err) + u, err := url.Parse(out.(string)) + require.NoError(t, err) + assert.Equal(t, test.expire, u.Query().Get("X-Amz-Expires")) + for key := range u.Query() { + assert.False(t, strings.HasPrefix(key, "response-")) + } + assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u)) + }) + } +} + +func TestCommandLinkErrors(t *testing.T) { + f := newLinkTestFs(t) + for _, test := range []struct { + name string + args []string + opts map[string]string + }{ + {"NoPath", nil, nil}, + {"TooManyPaths", []string{"one", "two"}, nil}, + {"EmptyPath", []string{""}, nil}, + {"Directory", []string{"dir/"}, nil}, + {"Missing", []string{"missing"}, nil}, + {"UnknownOption", []string{"file"}, map[string]string{"response-unknown": "value"}}, + {"InvalidDate", []string{"file"}, map[string]string{"response-expires": "tomorrow"}}, + {"InvalidExpire", []string{"file"}, map[string]string{"expire": "invalid"}}, + {"EmptyExpire", []string{"file"}, map[string]string{"expire": ""}}, + {"ZeroExpire", []string{"file"}, map[string]string{"expire": "0"}}, + {"NegativeExpire", []string{"file"}, map[string]string{"expire": "-1s"}}, + {"SubsecondExpire", []string{"file"}, map[string]string{"expire": "500ms"}}, + } { + t.Run(test.name, func(t *testing.T) { + _, err := f.Command(context.Background(), "link", test.args, test.opts) + require.Error(t, err) + assert.NotErrorIs(t, err, fs.ErrorCommandNotFound) + }) + } +} + +func TestCommandLinkVersionAt(t *testing.T) { + f := newLinkTestFs(t) + f.opt.VersionAt = fs.Time(time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)) + for _, command := range []bool{false, true} { + var link string + if command { + out, err := f.Command(context.Background(), "link", []string{"file"}, map[string]string{ + "response-content-type": "text/plain", + }) + require.NoError(t, err) + link = out.(string) + } else { + var err error + link, err = f.PublicLink(context.Background(), "file", fs.DurationOff, false) + require.NoError(t, err) + } + u, err := url.Parse(link) + require.NoError(t, err) + assert.Equal(t, "/bucket/prefix/file", u.Path) + assert.Equal(t, "version+id/1", u.Query().Get("versionId")) + assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u)) + } +} + +func TestPublicLink(t *testing.T) { + f := newLinkTestFs(t) + link, err := f.PublicLink(context.Background(), "file", fs.DurationOff, false) + require.NoError(t, err) + u, err := url.Parse(link) + require.NoError(t, err) + assert.Equal(t, "/bucket/prefix/file", u.Path) + assert.Equal(t, "604800", u.Query().Get("X-Amz-Expires")) + assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u)) + _, err = f.PublicLink(context.Background(), "dir/", fs.DurationOff, false) + assert.ErrorIs(t, err, fs.ErrorCantShareDirectories) + _, err = f.PublicLink(context.Background(), "missing", fs.DurationOff, false) + assert.ErrorIs(t, err, fs.ErrorObjectNotFound) +}