From da868b06f8e02dad5abbcc95f37d1c77017e30a6 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Sun, 30 Aug 2026 17:34:35 +0100
Subject: [PATCH] compress: stop buffering the compressibility check output in
 memory

The compressibility heuristic compressed a 1 MiB sample of each upload
into a bytes.Buffer only to read its length, growing up to ~1 MiB of
garbage per file. Write the sample through a counting io.Discard-style
writer instead so no output buffer is allocated at all.
---
 backend/compress/compress.go           |  9 +++++++++
 backend/compress/gzip_handler.go       |  5 ++---
 backend/compress/rcat_internal_test.go | 23 +++++++++++++++++++++++
 backend/compress/zstd_handler.go       |  5 ++---
 4 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/backend/compress/compress.go b/backend/compress/compress.go
index 34cf1e9ee..3de158897 100644
--- a/backend/compress/compress.go
+++ b/backend/compress/compress.go
@@ -1125,6 +1125,15 @@ func (o *Object) Remove(ctx context.Context) error {
 	return objErr
 }
 
+// countingDiscard is an io.Writer which discards its input and counts
+// the bytes written
+type countingDiscard int64
+
+func (c *countingDiscard) Write(p []byte) (int, error) {
+	*c += countingDiscard(len(p))
+	return len(p), nil
+}
+
 // ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
 type ReadCloserWrapper struct {
 	io.Reader
diff --git a/backend/compress/gzip_handler.go b/backend/compress/gzip_handler.go
index a5abbf7c8..16a0920e0 100644
--- a/backend/compress/gzip_handler.go
+++ b/backend/compress/gzip_handler.go
@@ -2,7 +2,6 @@ package compress
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/hex"
@@ -23,7 +22,7 @@ type gzipModeHandler struct{}
 // isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
 // the configured threshold
 func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
-	var b bytes.Buffer
+	var b countingDiscard
 	var n int64
 	w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
 	if err != nil {
@@ -37,7 +36,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
 	if err != nil {
 		return false, err
 	}
-	ratio := float64(n) / float64(b.Len())
+	ratio := float64(n) / float64(b)
 	return ratio > minCompressionRatio, nil
 }
 
diff --git a/backend/compress/rcat_internal_test.go b/backend/compress/rcat_internal_test.go
index 616ec7e38..b4b5428ba 100644
--- a/backend/compress/rcat_internal_test.go
+++ b/backend/compress/rcat_internal_test.go
@@ -3,6 +3,7 @@ package compress
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"io"
 	"testing"
 	"time"
@@ -112,3 +113,25 @@ func TestRcat(t *testing.T) {
 		})
 	}
 }
+
+// TestIsCompressible checks the compressibility heuristic on data at
+// both ends of the scale for every compression mode which has one.
+func TestIsCompressible(t *testing.T) {
+	compressible := bytes.Repeat([]byte("compress me "), 4096)
+	incompressible := make([]byte, len(compressible))
+	_, err := io.ReadFull(rand.Reader, incompressible)
+	require.NoError(t, err)
+	for name, handler := range map[string]compressionModeHandler{
+		"gzip": &gzipModeHandler{},
+		"zstd": &zstdModeHandler{},
+	} {
+		t.Run(name, func(t *testing.T) {
+			ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
+			require.NoError(t, err)
+			assert.True(t, ok)
+			ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
+			require.NoError(t, err)
+			assert.False(t, ok)
+		})
+	}
+}
diff --git a/backend/compress/zstd_handler.go b/backend/compress/zstd_handler.go
index 43914427b..7cf8d7617 100644
--- a/backend/compress/zstd_handler.go
+++ b/backend/compress/zstd_handler.go
@@ -2,7 +2,6 @@ package compress
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/hex"
@@ -23,7 +22,7 @@ type zstdModeHandler struct{}
 // isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
 // the configured threshold
 func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
-	var b bytes.Buffer
+	var b countingDiscard
 	var n int64
 	w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
 	if err != nil {
@@ -37,7 +36,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
 	if err != nil {
 		return false, err
 	}
-	ratio := float64(n) / float64(b.Len())
+	ratio := float64(n) / float64(b)
 	return ratio > minCompressionRatio, nil
 }