diff --git a/backend/oracleobjectstorage/client.go b/backend/oracleobjectstorage/client.go index 0749c4699..e44b35f22 100644 --- a/backend/oracleobjectstorage/client.go +++ b/backend/oracleobjectstorage/client.go @@ -90,6 +90,12 @@ func modifyClient(ctx context.Context, opt *Options, client *common.BaseClient) if opt.Provider == noAuth { client.Signer = getNoAuthSigner() } + // Set Accept-Encoding: gzip on every request to stop the Go HTTP + // transport transparently decompressing objects. + client.Interceptor = func(request *http.Request) error { + request.Header.Set("Accept-Encoding", "gzip") + return nil + } } // getClient makes http client according to the global options diff --git a/backend/oracleobjectstorage/client_test.go b/backend/oracleobjectstorage/client_test.go new file mode 100644 index 000000000..9ef5c2068 --- /dev/null +++ b/backend/oracleobjectstorage/client_test.go @@ -0,0 +1,105 @@ +//go:build !plan9 && !solaris && !js + +package oracleobjectstorage + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/hash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestGzipEncoding checks the handling of objects stored with +// Content-Encoding: gzip. +// +// By default they should be downloaded as stored, not transparently +// decompressed by the Go runtime. With decompress set they should be +// decompressed on download with size and hash unknown. +func TestGzipEncoding(t *testing.T) { + ctx := context.Background() + + // Gzip compressed test data served with Content-Encoding: gzip + // as if it had been uploaded with that metadata set. + plain := []byte("hello, world - some uncompressed data which is longer than the compressed version") + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err := zw.Write(plain) + require.NoError(t, err) + require.NoError(t, zw.Close()) + compressed := buf.Bytes() + compressedMd5 := md5.Sum(compressed) + + var gotAcceptEncoding string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + gotAcceptEncoding = r.Header.Get("Accept-Encoding") + } + w.Header().Set("Content-Encoding", "gzip") + w.Header().Set("Content-Length", strconv.Itoa(len(compressed))) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-MD5", base64.StdEncoding.EncodeToString(compressedMd5[:])) + w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) + _, _ = w.Write(compressed) + })) + defer srv.Close() + + newFs := func(t *testing.T, extraConfig configmap.Simple) *Fs { + m := configmap.Simple{ + "provider": noAuth, + "namespace": "test", + "endpoint": srv.URL, + } + for k, v := range extraConfig { + m[k] = v + } + fsInfo, err := NewFs(ctx, "TestOOSGzip", "bucket", m) + require.NoError(t, err) + return fsInfo.(*Fs) + } + + readObject := func(t *testing.T, f *Fs) ([]byte, *Object) { + obj, err := f.NewObject(ctx, "test.gz") + require.NoError(t, err) + rc, err := obj.Open(ctx) + require.NoError(t, err) + body, err := io.ReadAll(rc) + require.NoError(t, err) + require.NoError(t, rc.Close()) + return body, obj.(*Object) + } + + t.Run("Default", func(t *testing.T) { + f := newFs(t, configmap.Simple{}) + body, obj := readObject(t, f) + assert.Equal(t, "gzip", gotAcceptEncoding) + assert.Equal(t, compressed, body) + assert.Equal(t, int64(len(compressed)), obj.Size()) + md5sum, err := obj.Hash(ctx, hash.MD5) + require.NoError(t, err) + assert.Equal(t, hex.EncodeToString(compressedMd5[:]), md5sum) + }) + + t.Run("Decompress", func(t *testing.T) { + f := newFs(t, configmap.Simple{"decompress": "true"}) + body, obj := readObject(t, f) + assert.Equal(t, "gzip", gotAcceptEncoding) + assert.Equal(t, plain, body) + assert.Equal(t, int64(-1), obj.Size()) + md5sum, err := obj.Hash(ctx, hash.MD5) + require.NoError(t, err) + assert.Equal(t, "", md5sum) + }) +} diff --git a/backend/oracleobjectstorage/object.go b/backend/oracleobjectstorage/object.go index 74591c56c..b9c6d1b27 100644 --- a/backend/oracleobjectstorage/object.go +++ b/backend/oracleobjectstorage/object.go @@ -21,6 +21,7 @@ import ( "github.com/oracle/oci-go-sdk/v65/objectstorage" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/readers" ) // ------------------------------------------------------------ @@ -110,6 +111,7 @@ func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (err return o.setMetaData( info.ContentLength, info.ContentMd5, + info.ContentEncoding, info.ContentType, info.LastModified, info.StorageTier, @@ -120,6 +122,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (er return o.setMetaData( info.ContentLength, info.ContentMd5, + info.ContentEncoding, info.ContentType, info.LastModified, info.StorageTier, @@ -129,6 +132,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (er func (o *Object) setMetaData( contentLength *int64, contentMd5 *string, + contentEncoding *string, contentType *string, lastModified *common.SDKTime, storageTier any, @@ -169,6 +173,11 @@ func (o *Object) setMetaData( tier := strings.ToLower(fmt.Sprintf("%v", storageTier)) o.storageTier = storageTierMap[tier] } + // If decompressing then size and md5sum are unknown + if o.fs.opt.Decompress && contentEncoding != nil && *contentEncoding == "gzip" { + o.bytes = -1 + o.md5 = "" + } return nil } @@ -362,6 +371,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo if err != nil { return nil, err } + // Decompress body if necessary + if resp.ContentEncoding != nil && *resp.ContentEncoding == "gzip" { + if o.fs.opt.Decompress { + return readers.NewGzipReader(resp.HTTPResponse().Body) + } + o.fs.warnCompressed.Do(func() { + fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --oos-decompress to override") + }) + } if bytes != nil { o.bytes = *bytes } else { diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go index f24a4d156..a1301ab10 100644 --- a/backend/oracleobjectstorage/options.go +++ b/backend/oracleobjectstorage/options.go @@ -72,6 +72,7 @@ type Options struct { SSECustomerKey string `config:"sse_customer_key"` SSECustomerKeyFile string `config:"sse_customer_key_file"` SSECustomerKeySha256 string `config:"sse_customer_key_sha256"` + Decompress bool `config:"decompress"` } func newOptions() []fs.Option { @@ -346,5 +347,20 @@ Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Co Value: sseDefaultAlgorithm, Help: sseDefaultAlgorithm, }}, + }, { + Name: "decompress", + Help: `If set this will decompress gzip encoded objects. + +It is possible to upload objects to object storage with +"Content-Encoding: gzip" set. Normally rclone will download these +files as compressed objects. + +If this flag is set then rclone will decompress these files with +"Content-Encoding: gzip" as they are received. This means that rclone +can't check the size and hash but the file contents will be +decompressed. +`, + Default: false, + Advanced: true, }} } diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go index fca03bbe9..933662697 100644 --- a/backend/oracleobjectstorage/oracleobjectstorage.go +++ b/backend/oracleobjectstorage/oracleobjectstorage.go @@ -10,6 +10,7 @@ import ( "net/http" "path" "strings" + "sync" "time" "github.com/ncw/swift/v2" @@ -76,16 +77,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{ // Fs represents a remote object storage server type Fs struct { - name string // name of this remote - root string // the path we are working on if any - opt Options // parsed config options - ci *fs.ConfigInfo // global config - features *fs.Features // optional features - srv *objectstorage.ObjectStorageClient // the connection to the object storage - rootBucket string // bucket part of root (if any) - rootDirectory string // directory part of root (if any) - cache *bucket.Cache // cache for bucket creation status - pacer *fs.Pacer // To pace the API calls + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + ci *fs.ConfigInfo // global config + features *fs.Features // optional features + srv *objectstorage.ObjectStorageClient // the connection to the object storage + rootBucket string // bucket part of root (if any) + rootDirectory string // directory part of root (if any) + cache *bucket.Cache // cache for bucket creation status + pacer *fs.Pacer // To pace the API calls + warnCompressed sync.Once // warn once about compressed files } // NewFs Initialize backend diff --git a/backend/oracleobjectstorage/oracleobjectstorage_test.go b/backend/oracleobjectstorage/oracleobjectstorage_test.go index f41dfb605..feb215db5 100644 --- a/backend/oracleobjectstorage/oracleobjectstorage_test.go +++ b/backend/oracleobjectstorage/oracleobjectstorage_test.go @@ -3,10 +3,20 @@ package oracleobjectstorage import ( + "bytes" + "compress/gzip" + "context" + "crypto/md5" + "fmt" "testing" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" + "github.com/rclone/rclone/lib/random" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestIntegration runs integration tests against the remote @@ -21,6 +31,61 @@ func TestIntegration(t *testing.T) { }) } +func gz(t *testing.T, s string) string { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err := zw.Write([]byte(s)) + require.NoError(t, err) + err = zw.Close() + require.NoError(t, err) + return buf.String() +} + +func md5sum(t *testing.T, s string) string { + hash := md5.Sum([]byte(s)) + return fmt.Sprintf("%x", hash) +} + +// InternalTestGzipEncoding tests that a file uploaded with +// Content-Encoding: gzip can be downloaded with and without +// decompression. +func (f *Fs) InternalTestGzipEncoding(t *testing.T) { + ctx := context.Background() + original := random.String(1000) + contents := gz(t, original) + item := fstest.NewItem("test-gzip-encoding", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", nil, &fs.HTTPOption{Key: "Content-Encoding", Value: "gzip"}) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + o := obj.(*Object) + + checkDownload := func(wantContents string, wantSize int64, wantHash string) { + gotContents := fstests.ReadObject(ctx, t, o, -1) + assert.Equal(t, wantContents, gotContents) + assert.Equal(t, wantSize, o.Size()) + gotHash, err := o.Hash(ctx, hash.MD5) + require.NoError(t, err) + assert.Equal(t, wantHash, gotHash) + } + + t.Run("NoDecompress", func(t *testing.T) { + checkDownload(contents, int64(len(contents)), md5sum(t, contents)) + }) + t.Run("Decompress", func(t *testing.T) { + f.opt.Decompress = true + defer func() { + f.opt.Decompress = false + }() + checkDownload(original, -1, "") + }) +} + +// InternalTest is called by fstests.Run to extra tests +func (f *Fs) InternalTest(t *testing.T) { + t.Run("GzipEncoding", f.InternalTestGzipEncoding) +} + func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) { return f.setUploadChunkSize(cs) }