oracleobjectstorage: add --oos-decompress flag to download gzip-encoded files

Before this change, if an object compressed with "Content-Encoding: gzip" was downloaded, a length and hash mismatch would occur since the go runtime automatically decompressed the object on download, giving errors like corrupted on transfer: sizes differ This change sets "Accept-Encoding: gzip" on all requests which stops the go runtime decompressing objects on download, so compressed objects are downloaded as-is with intact size and hash information. If --oos-decompress is set then rclone will decompress objects with "Content-Encoding: gzip" as they are received, at the cost of not being able to check the length or the hash of the downloaded object. Fixes #9694
2026-07-30 19:30:08 +01:00
parent 8772c94011
commit b3a41206da
6 changed files with 222 additions and 10 deletions
@@ -90,6 +90,12 @@ func modifyClient(ctx context.Context, opt *Options, client *common.BaseClient)
 	if opt.Provider == noAuth {
 		client.Signer = getNoAuthSigner()
 	}
+	// Set Accept-Encoding: gzip on every request to stop the Go HTTP
+	// transport transparently decompressing objects.
+	client.Interceptor = func(request *http.Request) error {
+		request.Header.Set("Accept-Encoding", "gzip")
+		return nil
+	}
 }

 // getClient makes http client according to the global options
@@ -0,0 +1,105 @@
+//go:build !plan9 && !solaris && !js
+
+package oracleobjectstorage
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestGzipEncoding checks the handling of objects stored with
+// Content-Encoding: gzip.
+//
+// By default they should be downloaded as stored, not transparently
+// decompressed by the Go runtime. With decompress set they should be
+// decompressed on download with size and hash unknown.
+func TestGzipEncoding(t *testing.T) {
+	ctx := context.Background()
+
+	// Gzip compressed test data served with Content-Encoding: gzip
+	// as if it had been uploaded with that metadata set.
+	plain := []byte("hello, world - some uncompressed data which is longer than the compressed version")
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	_, err := zw.Write(plain)
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	compressed := buf.Bytes()
+	compressedMd5 := md5.Sum(compressed)
+
+	var gotAcceptEncoding string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "GET" {
+			gotAcceptEncoding = r.Header.Get("Accept-Encoding")
+		}
+		w.Header().Set("Content-Encoding", "gzip")
+		w.Header().Set("Content-Length", strconv.Itoa(len(compressed)))
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Header().Set("Content-MD5", base64.StdEncoding.EncodeToString(compressedMd5[:]))
+		w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
+		_, _ = w.Write(compressed)
+	}))
+	defer srv.Close()
+
+	newFs := func(t *testing.T, extraConfig configmap.Simple) *Fs {
+		m := configmap.Simple{
+			"provider":  noAuth,
+			"namespace": "test",
+			"endpoint":  srv.URL,
+		}
+		for k, v := range extraConfig {
+			m[k] = v
+		}
+		fsInfo, err := NewFs(ctx, "TestOOSGzip", "bucket", m)
+		require.NoError(t, err)
+		return fsInfo.(*Fs)
+	}
+
+	readObject := func(t *testing.T, f *Fs) ([]byte, *Object) {
+		obj, err := f.NewObject(ctx, "test.gz")
+		require.NoError(t, err)
+		rc, err := obj.Open(ctx)
+		require.NoError(t, err)
+		body, err := io.ReadAll(rc)
+		require.NoError(t, err)
+		require.NoError(t, rc.Close())
+		return body, obj.(*Object)
+	}
+
+	t.Run("Default", func(t *testing.T) {
+		f := newFs(t, configmap.Simple{})
+		body, obj := readObject(t, f)
+		assert.Equal(t, "gzip", gotAcceptEncoding)
+		assert.Equal(t, compressed, body)
+		assert.Equal(t, int64(len(compressed)), obj.Size())
+		md5sum, err := obj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, hex.EncodeToString(compressedMd5[:]), md5sum)
+	})
+
+	t.Run("Decompress", func(t *testing.T) {
+		f := newFs(t, configmap.Simple{"decompress": "true"})
+		body, obj := readObject(t, f)
+		assert.Equal(t, "gzip", gotAcceptEncoding)
+		assert.Equal(t, plain, body)
+		assert.Equal(t, int64(-1), obj.Size())
+		md5sum, err := obj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, "", md5sum)
+	})
+}
@@ -21,6 +21,7 @@ import (
 	"github.com/oracle/oci-go-sdk/v65/objectstorage"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/readers"
 )

 // ------------------------------------------------------------
@@ -110,6 +111,7 @@ func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (err
 	return o.setMetaData(
 		info.ContentLength,
 		info.ContentMd5,
+		info.ContentEncoding,
 		info.ContentType,
 		info.LastModified,
 		info.StorageTier,
@@ -120,6 +122,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (er
 	return o.setMetaData(
 		info.ContentLength,
 		info.ContentMd5,
+		info.ContentEncoding,
 		info.ContentType,
 		info.LastModified,
 		info.StorageTier,
@@ -129,6 +132,7 @@ func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (er
 func (o *Object) setMetaData(
 	contentLength *int64,
 	contentMd5 *string,
+	contentEncoding *string,
 	contentType *string,
 	lastModified *common.SDKTime,
 	storageTier any,
@@ -169,6 +173,11 @@ func (o *Object) setMetaData(
 		tier := strings.ToLower(fmt.Sprintf("%v", storageTier))
 		o.storageTier = storageTierMap[tier]
 	}
+	// If decompressing then size and md5sum are unknown
+	if o.fs.opt.Decompress && contentEncoding != nil && *contentEncoding == "gzip" {
+		o.bytes = -1
+		o.md5 = ""
+	}
 	return nil
 }

@@ -362,6 +371,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 	if err != nil {
 		return nil, err
 	}
+	// Decompress body if necessary
+	if resp.ContentEncoding != nil && *resp.ContentEncoding == "gzip" {
+		if o.fs.opt.Decompress {
+			return readers.NewGzipReader(resp.HTTPResponse().Body)
+		}
+		o.fs.warnCompressed.Do(func() {
+			fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --oos-decompress to override")
+		})
+	}
 	if bytes != nil {
 		o.bytes = *bytes
 	} else {
@@ -72,6 +72,7 @@ type Options struct {
 	SSECustomerKey       string               `config:"sse_customer_key"`
 	SSECustomerKeyFile   string               `config:"sse_customer_key_file"`
 	SSECustomerKeySha256 string               `config:"sse_customer_key_sha256"`
+	Decompress           bool                 `config:"decompress"`
 }

 func newOptions() []fs.Option {
@@ -346,5 +347,20 @@ Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Co
 			Value: sseDefaultAlgorithm,
 			Help:  sseDefaultAlgorithm,
 		}},
+	}, {
+		Name: "decompress",
+		Help: `If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to object storage with
+"Content-Encoding: gzip" set. Normally rclone will download these
+files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be
+decompressed.
+`,
+		Default:  false,
+		Advanced: true,
 	}}
 }
@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"path"
 	"strings"
+	"sync"
 	"time"

 	"github.com/ncw/swift/v2"
@@ -76,16 +77,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{

 // Fs represents a remote object storage server
 type Fs struct {
-	name          string                             // name of this remote
-	root          string                             // the path we are working on if any
-	opt           Options                            // parsed config options
-	ci            *fs.ConfigInfo                     // global config
-	features      *fs.Features                       // optional features
-	srv           *objectstorage.ObjectStorageClient // the connection to the object storage
-	rootBucket    string                             // bucket part of root (if any)
-	rootDirectory string                             // directory part of root (if any)
-	cache         *bucket.Cache                      // cache for bucket creation status
-	pacer         *fs.Pacer                          // To pace the API calls
+	name           string                             // name of this remote
+	root           string                             // the path we are working on if any
+	opt            Options                            // parsed config options
+	ci             *fs.ConfigInfo                     // global config
+	features       *fs.Features                       // optional features
+	srv            *objectstorage.ObjectStorageClient // the connection to the object storage
+	rootBucket     string                             // bucket part of root (if any)
+	rootDirectory  string                             // directory part of root (if any)
+	cache          *bucket.Cache                      // cache for bucket creation status
+	pacer          *fs.Pacer                          // To pace the API calls
+	warnCompressed sync.Once                          // warn once about compressed files
 }

 // NewFs Initialize backend
@@ -3,10 +3,20 @@
 package oracleobjectstorage

 import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/md5"
+	"fmt"
 	"testing"

 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )

 // TestIntegration runs integration tests against the remote
@@ -21,6 +31,61 @@ func TestIntegration(t *testing.T) {
 	})
 }

+func gz(t *testing.T, s string) string {
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	_, err := zw.Write([]byte(s))
+	require.NoError(t, err)
+	err = zw.Close()
+	require.NoError(t, err)
+	return buf.String()
+}
+
+func md5sum(t *testing.T, s string) string {
+	hash := md5.Sum([]byte(s))
+	return fmt.Sprintf("%x", hash)
+}
+
+// InternalTestGzipEncoding tests that a file uploaded with
+// Content-Encoding: gzip can be downloaded with and without
+// decompression.
+func (f *Fs) InternalTestGzipEncoding(t *testing.T) {
+	ctx := context.Background()
+	original := random.String(1000)
+	contents := gz(t, original)
+	item := fstest.NewItem("test-gzip-encoding", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", nil, &fs.HTTPOption{Key: "Content-Encoding", Value: "gzip"})
+	defer func() {
+		assert.NoError(t, obj.Remove(ctx))
+	}()
+	o := obj.(*Object)
+
+	checkDownload := func(wantContents string, wantSize int64, wantHash string) {
+		gotContents := fstests.ReadObject(ctx, t, o, -1)
+		assert.Equal(t, wantContents, gotContents)
+		assert.Equal(t, wantSize, o.Size())
+		gotHash, err := o.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, wantHash, gotHash)
+	}
+
+	t.Run("NoDecompress", func(t *testing.T) {
+		checkDownload(contents, int64(len(contents)), md5sum(t, contents))
+	})
+	t.Run("Decompress", func(t *testing.T) {
+		f.opt.Decompress = true
+		defer func() {
+			f.opt.Decompress = false
+		}()
+		checkDownload(original, -1, "")
+	})
+}
+
+// InternalTest is called by fstests.Run to extra tests
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("GzipEncoding", f.InternalTestGzipEncoding)
+}
+
 func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadChunkSize(cs)
 }