compress: stop buffering the compressibility check output in memory

The compressibility heuristic compressed a 1 MiB sample of each upload
into a bytes.Buffer only to read its length, growing up to ~1 MiB of
garbage per file. Write the sample through a counting io.Discard-style
writer instead so no output buffer is allocated at all.

backend/compress/compress.go

@@ -1125,6 +1125,15 @@ func (o *Object) Remove(ctx context.Context) error {
 	return objErr
 }
 
+// countingDiscard is an io.Writer which discards its input and counts
+// the bytes written
+type countingDiscard int64
+
+func (c *countingDiscard) Write(p []byte) (int, error) {
+	*c += countingDiscard(len(p))
+	return len(p), nil
+}
+
 // ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
 type ReadCloserWrapper struct {
 	io.Reader

backend/compress/gzip_handler.go

@@ -2,7 +2,6 @@ package compress
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/hex"
@@ -23,7 +22,7 @@ type gzipModeHandler struct{}
 // isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
 // the configured threshold
 func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
-	var b bytes.Buffer
+	var b countingDiscard
 	var n int64
 	w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
 	if err != nil {
@@ -37,7 +36,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
 	if err != nil {
 		return false, err
 	}
-	ratio := float64(n) / float64(b.Len())
+	ratio := float64(n) / float64(b)
 	return ratio > minCompressionRatio, nil
 }
 

backend/compress/rcat_internal_test.go

@@ -3,6 +3,7 @@ package compress
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"io"
 	"testing"
 	"time"
@@ -112,3 +113,25 @@ func TestRcat(t *testing.T) {
 		})
 	}
 }
+
+// TestIsCompressible checks the compressibility heuristic on data at
+// both ends of the scale for every compression mode which has one.
+func TestIsCompressible(t *testing.T) {
+	compressible := bytes.Repeat([]byte("compress me "), 4096)
+	incompressible := make([]byte, len(compressible))
+	_, err := io.ReadFull(rand.Reader, incompressible)
+	require.NoError(t, err)
+	for name, handler := range map[string]compressionModeHandler{
+		"gzip": &gzipModeHandler{},
+		"zstd": &zstdModeHandler{},
+	} {
+		t.Run(name, func(t *testing.T) {
+			ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
+			require.NoError(t, err)
+			assert.True(t, ok)
+			ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
+			require.NoError(t, err)
+			assert.False(t, ok)
+		})
+	}
+}

backend/compress/zstd_handler.go

@@ -2,7 +2,6 @@ package compress
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/hex"
@@ -23,7 +22,7 @@ type zstdModeHandler struct{}
 // isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
 // the configured threshold
 func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
-	var b bytes.Buffer
+	var b countingDiscard
 	var n int64
 	w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
 	if err != nil {
@@ -37,7 +36,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
 	if err != nil {
 		return false, err
 	}
-	ratio := float64(n) / float64(b.Len())
+	ratio := float64(n) / float64(b)
 	return ratio > minCompressionRatio, nil
 }