From 69e5aff2a93bebbf4183a3b433154ebedd6ecf50 Mon Sep 17 00:00:00 2001 From: Morax Date: Thu, 13 Aug 2026 20:22:26 +0200 Subject: [PATCH] lib/rest: validate ranged responses Add response validation for calls made with Range open options. Verify Content-Range, Content-Length, response status, and the complete representation size before a backend accepts the response body. Return a shared sentinel when a server ignores a partial range so callers can avoid retrying the same unsupported request. --- fs/fs.go | 1 + lib/rest/headers.go | 150 ++++++++++++++++++++++++++++++ lib/rest/headers_test.go | 194 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 345 insertions(+) diff --git a/fs/fs.go b/fs/fs.go index 6ca2c679b..ae630b15f 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -53,6 +53,7 @@ var ( ErrorFileNameTooLong = errors.New("file name too long") ErrorCantListRoot = errors.New("can't list root") ErrorFileTooSmall = errors.New("file too small for multipart upload") + ErrorRangeIgnored = errors.New("server ignored requested range") ) // FileTooSmallError is returned by OpenChunkWriter when a file is below the diff --git a/lib/rest/headers.go b/lib/rest/headers.go index 3e52fbdaa..f3bd6bf30 100644 --- a/lib/rest/headers.go +++ b/lib/rest/headers.go @@ -1,11 +1,161 @@ package rest import ( + "errors" + "fmt" "net/http" "strconv" "strings" + + "github.com/rclone/rclone/fs" ) +type contentRange struct { + start int64 + end int64 + size int64 +} + +func parseContentRange(value string) (contentRange, error) { + const prefix = "bytes " + if !strings.HasPrefix(value, prefix) { + return contentRange{}, fmt.Errorf("doesn't start with %q", prefix) + } + + rangeAndSize := strings.Split(value[len(prefix):], "/") + if len(rangeAndSize) != 2 { + return contentRange{}, errors.New("must contain one '/'") + } + bounds := strings.Split(rangeAndSize[0], "-") + if len(bounds) != 2 { + return contentRange{}, errors.New("must contain one '-'") + } + + start, err := strconv.ParseInt(bounds[0], 10, 64) + if err != nil || start < 0 { + return contentRange{}, errors.New("invalid start") + } + end, err := strconv.ParseInt(bounds[1], 10, 64) + if err != nil || end < start { + return contentRange{}, errors.New("invalid end") + } + + size := int64(-1) + if rangeAndSize[1] != "*" { + size, err = strconv.ParseInt(rangeAndSize[1], 10, 64) + if err != nil || size < 0 || end >= size { + return contentRange{}, errors.New("invalid complete length") + } + } + + return contentRange{start: start, end: end, size: size}, nil +} + +// CheckContentRange checks that a response satisfies a Range open option. +// The size is the expected size of the complete representation, or -1 if it is +// unknown. Calls without a Range option are ignored. +func CheckContentRange(resp *http.Response, options []fs.OpenOption, size int64) error { + var requestRange string + for _, option := range options { + key, value := option.Header() + if strings.EqualFold(key, "Range") { + requestRange = value + } + } + if requestRange == "" { + return nil + } + + requested, err := fs.ParseRangeOption(requestRange) + if err != nil { + return fmt.Errorf("invalid requested range %q: %w", requestRange, err) + } + if requested.Start < 0 && requested.End < 0 { + return fmt.Errorf("invalid requested range %q", requestRange) + } + if requested.Start >= 0 && requested.End >= 0 && requested.End < requested.Start { + return fmt.Errorf("invalid requested range %q", requestRange) + } + if resp == nil { + return errors.New("nil response to range request") + } + + if resp.StatusCode == http.StatusOK { + responseSize := size + if resp.ContentLength >= 0 { + if size >= 0 && resp.ContentLength != size { + return fmt.Errorf("Content-Length %d does not match expected size %d", resp.ContentLength, size) + } + responseSize = resp.ContentLength + } + if responseSize >= 0 { + offset, limit := requested.Decode(responseSize) + if limit < 0 { + limit = responseSize - offset + } + if offset == 0 && limit >= responseSize { + return nil + } + } else if requested.Start == 0 && requested.End < 0 { + return nil + } + return fmt.Errorf("%w %q", fs.ErrorRangeIgnored, requestRange) + } + if resp.StatusCode != http.StatusPartialContent { + return fmt.Errorf("response status %d does not satisfy requested range %q", resp.StatusCode, requestRange) + } + + responseRange := resp.Header.Get("Content-Range") + got, err := parseContentRange(responseRange) + if err != nil { + return fmt.Errorf("invalid Content-Range %q: %w", responseRange, err) + } + + if size >= 0 && got.size >= 0 && got.size != size { + return fmt.Errorf("Content-Range %q does not match expected size %d", responseRange, size) + } + + rangeSize := size + if rangeSize < 0 { + rangeSize = got.size + } + expectedStart := requested.Start + expectedEnd := requested.End + if requested.Start >= 0 { + if rangeSize >= 0 && (expectedEnd < 0 || expectedEnd >= rangeSize) { + expectedEnd = rangeSize - 1 + } + } else if rangeSize >= 0 { + expectedStart = rangeSize - requested.End + if expectedStart < 0 { + expectedStart = 0 + } + expectedEnd = rangeSize - 1 + } + + var matches bool + if requested.Start < 0 && rangeSize < 0 { + matches = got.end-got.start+1 <= requested.End + } else { + matches = got.start == expectedStart + if expectedEnd >= 0 { + matches = matches && got.end == expectedEnd + } + } + if !matches { + return fmt.Errorf("Content-Range %q does not match requested range %q", responseRange, requestRange) + } + + contentLength := got.end - got.start + 1 + if contentLength <= 0 { + return fmt.Errorf("invalid Content-Range %q: length overflows", responseRange) + } + if resp.ContentLength >= 0 && resp.ContentLength != contentLength { + return fmt.Errorf("Content-Length %d does not match Content-Range %q", resp.ContentLength, responseRange) + } + return nil +} + // ParseSizeFromHeaders parses HTTP response headers to get the full file size. // Returns -1 if the headers did not exist or were invalid. func ParseSizeFromHeaders(headers http.Header) (size int64) { diff --git a/lib/rest/headers_test.go b/lib/rest/headers_test.go index 23400062d..9901cc56f 100644 --- a/lib/rest/headers_test.go +++ b/lib/rest/headers_test.go @@ -4,6 +4,7 @@ import ( "net/http" "testing" + "github.com/rclone/rclone/fs" "github.com/stretchr/testify/assert" ) @@ -39,3 +40,196 @@ func TestParseSizeFromHeaders(t *testing.T) { assert.Equalf(t, testCase.Size, ParseSizeFromHeaders(headers), "%+v", testCase) } } + +func TestCheckContentRange(t *testing.T) { + testCases := []struct { + name string + status int + contentRange string + contentLength int64 + unknownSize bool + options []fs.OpenOption + wantErr bool + wantErrorIs error + }{ + { + name: "exact range", + status: http.StatusPartialContent, + contentRange: "bytes 2-4/10", + contentLength: 3, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + }, + { + name: "unknown content length", + status: http.StatusPartialContent, + contentRange: "bytes 2-4/10", + contentLength: -1, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + }, + { + name: "unknown complete length", + status: http.StatusPartialContent, + contentRange: "bytes 2-4/*", + contentLength: 3, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + }, + { + name: "open ended range", + status: http.StatusPartialContent, + contentRange: "bytes 2-9/10", + contentLength: 8, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: -1}}, + }, + { + name: "suffix range", + status: http.StatusPartialContent, + contentRange: "bytes 6-9/10", + contentLength: 4, + options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}}, + }, + { + name: "suffix range with unknown complete length", + status: http.StatusPartialContent, + contentRange: "bytes 6-9/*", + contentLength: 4, + options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}}, + }, + { + name: "suffix range larger than representation", + status: http.StatusPartialContent, + contentRange: "bytes 0-9/10", + contentLength: 10, + options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 20}}, + }, + { + name: "seek option", + status: http.StatusPartialContent, + contentRange: "bytes 2-9/10", + contentLength: 8, + options: []fs.OpenOption{&fs.SeekOption{Offset: 2}}, + }, + { + name: "whole representation range", + status: http.StatusOK, + contentLength: 10, + options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}}, + }, + { + name: "open ended whole representation range with unknown size", + status: http.StatusOK, + contentLength: -1, + unknownSize: true, + options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: -1}}, + }, + { + name: "partial range ignored", + status: http.StatusOK, + contentLength: 10, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + wantErrorIs: fs.ErrorRangeIgnored, + }, + { + name: "whole range response has wrong length", + status: http.StatusOK, + contentLength: 9, + options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}}, + wantErr: true, + }, + { + name: "no range requested", + status: http.StatusPartialContent, + contentRange: "bytes 2-4/10", + contentLength: 3, + }, + { + name: "missing content range", + status: http.StatusPartialContent, + contentLength: 3, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + }, + { + name: "wrong unit", + status: http.StatusPartialContent, + contentRange: "items 2-4/10", + contentLength: 3, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + }, + { + name: "wrong start", + status: http.StatusPartialContent, + contentRange: "bytes 0-2/10", + contentLength: 3, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + }, + { + name: "wrong end", + status: http.StatusPartialContent, + contentRange: "bytes 2-5/10", + contentLength: 4, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + }, + { + name: "wrong content length", + status: http.StatusPartialContent, + contentRange: "bytes 2-4/10", + contentLength: 4, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + }, + { + name: "invalid complete length", + status: http.StatusPartialContent, + contentRange: "bytes 2-10/10", + contentLength: 9, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 10}}, + wantErr: true, + }, + { + name: "wrong complete length", + status: http.StatusPartialContent, + contentRange: "bytes 2-4/11", + contentLength: 3, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + }, + { + name: "unexpected successful status", + status: http.StatusNoContent, + contentLength: 0, + options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}}, + wantErr: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + expectedSize := int64(10) + if testCase.unknownSize { + expectedSize = -1 + } + resp := &http.Response{ + StatusCode: testCase.status, + Header: make(http.Header), + ContentLength: testCase.contentLength, + } + if testCase.contentRange != "" { + resp.Header.Set("Content-Range", testCase.contentRange) + } + + err := CheckContentRange(resp, testCase.options, expectedSize) + if testCase.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + if testCase.wantErrorIs != nil { + assert.ErrorIs(t, err, testCase.wantErrorIs) + } + }) + } +}