lib/rest: validate ranged responses

Add response validation for calls made with Range open options. Verify
Content-Range, Content-Length, response status, and the complete
representation size before a backend accepts the response body.

Return a shared sentinel when a server ignores a partial range so callers
can avoid retrying the same unsupported request.

fs/fs.go

@@ -53,6 +53,7 @@ var (
 	ErrorFileNameTooLong             = errors.New("file name too long")
 	ErrorCantListRoot                = errors.New("can't list root")
 	ErrorFileTooSmall                = errors.New("file too small for multipart upload")
+	ErrorRangeIgnored                = errors.New("server ignored requested range")
 )
 
 // FileTooSmallError is returned by OpenChunkWriter when a file is below the

lib/rest/headers.go

@@ -1,11 +1,161 @@
 package rest
 
 import (
+	"errors"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
+
+	"github.com/rclone/rclone/fs"
 )
 
+type contentRange struct {
+	start int64
+	end   int64
+	size  int64
+}
+
+func parseContentRange(value string) (contentRange, error) {
+	const prefix = "bytes "
+	if !strings.HasPrefix(value, prefix) {
+		return contentRange{}, fmt.Errorf("doesn't start with %q", prefix)
+	}
+
+	rangeAndSize := strings.Split(value[len(prefix):], "/")
+	if len(rangeAndSize) != 2 {
+		return contentRange{}, errors.New("must contain one '/'")
+	}
+	bounds := strings.Split(rangeAndSize[0], "-")
+	if len(bounds) != 2 {
+		return contentRange{}, errors.New("must contain one '-'")
+	}
+
+	start, err := strconv.ParseInt(bounds[0], 10, 64)
+	if err != nil || start < 0 {
+		return contentRange{}, errors.New("invalid start")
+	}
+	end, err := strconv.ParseInt(bounds[1], 10, 64)
+	if err != nil || end < start {
+		return contentRange{}, errors.New("invalid end")
+	}
+
+	size := int64(-1)
+	if rangeAndSize[1] != "*" {
+		size, err = strconv.ParseInt(rangeAndSize[1], 10, 64)
+		if err != nil || size < 0 || end >= size {
+			return contentRange{}, errors.New("invalid complete length")
+		}
+	}
+
+	return contentRange{start: start, end: end, size: size}, nil
+}
+
+// CheckContentRange checks that a response satisfies a Range open option.
+// The size is the expected size of the complete representation, or -1 if it is
+// unknown. Calls without a Range option are ignored.
+func CheckContentRange(resp *http.Response, options []fs.OpenOption, size int64) error {
+	var requestRange string
+	for _, option := range options {
+		key, value := option.Header()
+		if strings.EqualFold(key, "Range") {
+			requestRange = value
+		}
+	}
+	if requestRange == "" {
+		return nil
+	}
+
+	requested, err := fs.ParseRangeOption(requestRange)
+	if err != nil {
+		return fmt.Errorf("invalid requested range %q: %w", requestRange, err)
+	}
+	if requested.Start < 0 && requested.End < 0 {
+		return fmt.Errorf("invalid requested range %q", requestRange)
+	}
+	if requested.Start >= 0 && requested.End >= 0 && requested.End < requested.Start {
+		return fmt.Errorf("invalid requested range %q", requestRange)
+	}
+	if resp == nil {
+		return errors.New("nil response to range request")
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		responseSize := size
+		if resp.ContentLength >= 0 {
+			if size >= 0 && resp.ContentLength != size {
+				return fmt.Errorf("Content-Length %d does not match expected size %d", resp.ContentLength, size)
+			}
+			responseSize = resp.ContentLength
+		}
+		if responseSize >= 0 {
+			offset, limit := requested.Decode(responseSize)
+			if limit < 0 {
+				limit = responseSize - offset
+			}
+			if offset == 0 && limit >= responseSize {
+				return nil
+			}
+		} else if requested.Start == 0 && requested.End < 0 {
+			return nil
+		}
+		return fmt.Errorf("%w %q", fs.ErrorRangeIgnored, requestRange)
+	}
+	if resp.StatusCode != http.StatusPartialContent {
+		return fmt.Errorf("response status %d does not satisfy requested range %q", resp.StatusCode, requestRange)
+	}
+
+	responseRange := resp.Header.Get("Content-Range")
+	got, err := parseContentRange(responseRange)
+	if err != nil {
+		return fmt.Errorf("invalid Content-Range %q: %w", responseRange, err)
+	}
+
+	if size >= 0 && got.size >= 0 && got.size != size {
+		return fmt.Errorf("Content-Range %q does not match expected size %d", responseRange, size)
+	}
+
+	rangeSize := size
+	if rangeSize < 0 {
+		rangeSize = got.size
+	}
+	expectedStart := requested.Start
+	expectedEnd := requested.End
+	if requested.Start >= 0 {
+		if rangeSize >= 0 && (expectedEnd < 0 || expectedEnd >= rangeSize) {
+			expectedEnd = rangeSize - 1
+		}
+	} else if rangeSize >= 0 {
+		expectedStart = rangeSize - requested.End
+		if expectedStart < 0 {
+			expectedStart = 0
+		}
+		expectedEnd = rangeSize - 1
+	}
+
+	var matches bool
+	if requested.Start < 0 && rangeSize < 0 {
+		matches = got.end-got.start+1 <= requested.End
+	} else {
+		matches = got.start == expectedStart
+		if expectedEnd >= 0 {
+			matches = matches && got.end == expectedEnd
+		}
+	}
+	if !matches {
+		return fmt.Errorf("Content-Range %q does not match requested range %q", responseRange, requestRange)
+	}
+
+	contentLength := got.end - got.start + 1
+	if contentLength <= 0 {
+		return fmt.Errorf("invalid Content-Range %q: length overflows", responseRange)
+	}
+	if resp.ContentLength >= 0 && resp.ContentLength != contentLength {
+		return fmt.Errorf("Content-Length %d does not match Content-Range %q", resp.ContentLength, responseRange)
+	}
+	return nil
+}
+
 // ParseSizeFromHeaders parses HTTP response headers to get the full file size.
 // Returns -1 if the headers did not exist or were invalid.
 func ParseSizeFromHeaders(headers http.Header) (size int64) {

lib/rest/headers_test.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/rclone/rclone/fs"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -39,3 +40,196 @@ func TestParseSizeFromHeaders(t *testing.T) {
 		assert.Equalf(t, testCase.Size, ParseSizeFromHeaders(headers), "%+v", testCase)
 	}
 }
+
+func TestCheckContentRange(t *testing.T) {
+	testCases := []struct {
+		name          string
+		status        int
+		contentRange  string
+		contentLength int64
+		unknownSize   bool
+		options       []fs.OpenOption
+		wantErr       bool
+		wantErrorIs   error
+	}{
+		{
+			name:          "exact range",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-4/10",
+			contentLength: 3,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+		},
+		{
+			name:          "unknown content length",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-4/10",
+			contentLength: -1,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+		},
+		{
+			name:          "unknown complete length",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-4/*",
+			contentLength: 3,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+		},
+		{
+			name:          "open ended range",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-9/10",
+			contentLength: 8,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: -1}},
+		},
+		{
+			name:          "suffix range",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 6-9/10",
+			contentLength: 4,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},
+		},
+		{
+			name:          "suffix range with unknown complete length",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 6-9/*",
+			contentLength: 4,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},
+		},
+		{
+			name:          "suffix range larger than representation",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 0-9/10",
+			contentLength: 10,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: -1, End: 20}},
+		},
+		{
+			name:          "seek option",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-9/10",
+			contentLength: 8,
+			options:       []fs.OpenOption{&fs.SeekOption{Offset: 2}},
+		},
+		{
+			name:          "whole representation range",
+			status:        http.StatusOK,
+			contentLength: 10,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},
+		},
+		{
+			name:          "open ended whole representation range with unknown size",
+			status:        http.StatusOK,
+			contentLength: -1,
+			unknownSize:   true,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 0, End: -1}},
+		},
+		{
+			name:          "partial range ignored",
+			status:        http.StatusOK,
+			contentLength: 10,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+			wantErrorIs:   fs.ErrorRangeIgnored,
+		},
+		{
+			name:          "whole range response has wrong length",
+			status:        http.StatusOK,
+			contentLength: 9,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},
+			wantErr:       true,
+		},
+		{
+			name:          "no range requested",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-4/10",
+			contentLength: 3,
+		},
+		{
+			name:          "missing content range",
+			status:        http.StatusPartialContent,
+			contentLength: 3,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+		},
+		{
+			name:          "wrong unit",
+			status:        http.StatusPartialContent,
+			contentRange:  "items 2-4/10",
+			contentLength: 3,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+		},
+		{
+			name:          "wrong start",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 0-2/10",
+			contentLength: 3,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+		},
+		{
+			name:          "wrong end",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-5/10",
+			contentLength: 4,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+		},
+		{
+			name:          "wrong content length",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-4/10",
+			contentLength: 4,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+		},
+		{
+			name:          "invalid complete length",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-10/10",
+			contentLength: 9,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 10}},
+			wantErr:       true,
+		},
+		{
+			name:          "wrong complete length",
+			status:        http.StatusPartialContent,
+			contentRange:  "bytes 2-4/11",
+			contentLength: 3,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+		},
+		{
+			name:          "unexpected successful status",
+			status:        http.StatusNoContent,
+			contentLength: 0,
+			options:       []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+			wantErr:       true,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			expectedSize := int64(10)
+			if testCase.unknownSize {
+				expectedSize = -1
+			}
+			resp := &http.Response{
+				StatusCode:    testCase.status,
+				Header:        make(http.Header),
+				ContentLength: testCase.contentLength,
+			}
+			if testCase.contentRange != "" {
+				resp.Header.Set("Content-Range", testCase.contentRange)
+			}
+
+			err := CheckContentRange(resp, testCase.options, expectedSize)
+			if testCase.wantErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+			if testCase.wantErrorIs != nil {
+				assert.ErrorIs(t, err, testCase.wantErrorIs)
+			}
+		})
+	}
+}