lib/rest: validate ranged responses
Add response validation for calls made with Range open options. Verify Content-Range, Content-Length, response status, and the complete representation size before a backend accepts the response body. Return a shared sentinel when a server ignores a partial range so callers can avoid retrying the same unsupported request.
This commit is contained in:
Morax
Nick Craig-Wood
@@ -1,11 +1,161 @@
|
||||
package rest
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
type contentRange struct {
|
||||
start int64
|
||||
end int64
|
||||
size int64
|
||||
}
|
||||
|
||||
func parseContentRange(value string) (contentRange, error) {
|
||||
const prefix = "bytes "
|
||||
if !strings.HasPrefix(value, prefix) {
|
||||
return contentRange{}, fmt.Errorf("doesn't start with %q", prefix)
|
||||
}
|
||||
|
||||
rangeAndSize := strings.Split(value[len(prefix):], "/")
|
||||
if len(rangeAndSize) != 2 {
|
||||
return contentRange{}, errors.New("must contain one '/'")
|
||||
}
|
||||
bounds := strings.Split(rangeAndSize[0], "-")
|
||||
if len(bounds) != 2 {
|
||||
return contentRange{}, errors.New("must contain one '-'")
|
||||
}
|
||||
|
||||
start, err := strconv.ParseInt(bounds[0], 10, 64)
|
||||
if err != nil || start < 0 {
|
||||
return contentRange{}, errors.New("invalid start")
|
||||
}
|
||||
end, err := strconv.ParseInt(bounds[1], 10, 64)
|
||||
if err != nil || end < start {
|
||||
return contentRange{}, errors.New("invalid end")
|
||||
}
|
||||
|
||||
size := int64(-1)
|
||||
if rangeAndSize[1] != "*" {
|
||||
size, err = strconv.ParseInt(rangeAndSize[1], 10, 64)
|
||||
if err != nil || size < 0 || end >= size {
|
||||
return contentRange{}, errors.New("invalid complete length")
|
||||
}
|
||||
}
|
||||
|
||||
return contentRange{start: start, end: end, size: size}, nil
|
||||
}
|
||||
|
||||
// CheckContentRange checks that a response satisfies a Range open option.
|
||||
// The size is the expected size of the complete representation, or -1 if it is
|
||||
// unknown. Calls without a Range option are ignored.
|
||||
func CheckContentRange(resp *http.Response, options []fs.OpenOption, size int64) error {
|
||||
var requestRange string
|
||||
for _, option := range options {
|
||||
key, value := option.Header()
|
||||
if strings.EqualFold(key, "Range") {
|
||||
requestRange = value
|
||||
}
|
||||
}
|
||||
if requestRange == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
requested, err := fs.ParseRangeOption(requestRange)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid requested range %q: %w", requestRange, err)
|
||||
}
|
||||
if requested.Start < 0 && requested.End < 0 {
|
||||
return fmt.Errorf("invalid requested range %q", requestRange)
|
||||
}
|
||||
if requested.Start >= 0 && requested.End >= 0 && requested.End < requested.Start {
|
||||
return fmt.Errorf("invalid requested range %q", requestRange)
|
||||
}
|
||||
if resp == nil {
|
||||
return errors.New("nil response to range request")
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
responseSize := size
|
||||
if resp.ContentLength >= 0 {
|
||||
if size >= 0 && resp.ContentLength != size {
|
||||
return fmt.Errorf("Content-Length %d does not match expected size %d", resp.ContentLength, size)
|
||||
}
|
||||
responseSize = resp.ContentLength
|
||||
}
|
||||
if responseSize >= 0 {
|
||||
offset, limit := requested.Decode(responseSize)
|
||||
if limit < 0 {
|
||||
limit = responseSize - offset
|
||||
}
|
||||
if offset == 0 && limit >= responseSize {
|
||||
return nil
|
||||
}
|
||||
} else if requested.Start == 0 && requested.End < 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w %q", fs.ErrorRangeIgnored, requestRange)
|
||||
}
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
return fmt.Errorf("response status %d does not satisfy requested range %q", resp.StatusCode, requestRange)
|
||||
}
|
||||
|
||||
responseRange := resp.Header.Get("Content-Range")
|
||||
got, err := parseContentRange(responseRange)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid Content-Range %q: %w", responseRange, err)
|
||||
}
|
||||
|
||||
if size >= 0 && got.size >= 0 && got.size != size {
|
||||
return fmt.Errorf("Content-Range %q does not match expected size %d", responseRange, size)
|
||||
}
|
||||
|
||||
rangeSize := size
|
||||
if rangeSize < 0 {
|
||||
rangeSize = got.size
|
||||
}
|
||||
expectedStart := requested.Start
|
||||
expectedEnd := requested.End
|
||||
if requested.Start >= 0 {
|
||||
if rangeSize >= 0 && (expectedEnd < 0 || expectedEnd >= rangeSize) {
|
||||
expectedEnd = rangeSize - 1
|
||||
}
|
||||
} else if rangeSize >= 0 {
|
||||
expectedStart = rangeSize - requested.End
|
||||
if expectedStart < 0 {
|
||||
expectedStart = 0
|
||||
}
|
||||
expectedEnd = rangeSize - 1
|
||||
}
|
||||
|
||||
var matches bool
|
||||
if requested.Start < 0 && rangeSize < 0 {
|
||||
matches = got.end-got.start+1 <= requested.End
|
||||
} else {
|
||||
matches = got.start == expectedStart
|
||||
if expectedEnd >= 0 {
|
||||
matches = matches && got.end == expectedEnd
|
||||
}
|
||||
}
|
||||
if !matches {
|
||||
return fmt.Errorf("Content-Range %q does not match requested range %q", responseRange, requestRange)
|
||||
}
|
||||
|
||||
contentLength := got.end - got.start + 1
|
||||
if contentLength <= 0 {
|
||||
return fmt.Errorf("invalid Content-Range %q: length overflows", responseRange)
|
||||
}
|
||||
if resp.ContentLength >= 0 && resp.ContentLength != contentLength {
|
||||
return fmt.Errorf("Content-Length %d does not match Content-Range %q", resp.ContentLength, responseRange)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseSizeFromHeaders parses HTTP response headers to get the full file size.
|
||||
// Returns -1 if the headers did not exist or were invalid.
|
||||
func ParseSizeFromHeaders(headers http.Header) (size int64) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -39,3 +40,196 @@ func TestParseSizeFromHeaders(t *testing.T) {
|
||||
assert.Equalf(t, testCase.Size, ParseSizeFromHeaders(headers), "%+v", testCase)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckContentRange(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
status int
|
||||
contentRange string
|
||||
contentLength int64
|
||||
unknownSize bool
|
||||
options []fs.OpenOption
|
||||
wantErr bool
|
||||
wantErrorIs error
|
||||
}{
|
||||
{
|
||||
name: "exact range",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-4/10",
|
||||
contentLength: 3,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
},
|
||||
{
|
||||
name: "unknown content length",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-4/10",
|
||||
contentLength: -1,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
},
|
||||
{
|
||||
name: "unknown complete length",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-4/*",
|
||||
contentLength: 3,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
},
|
||||
{
|
||||
name: "open ended range",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-9/10",
|
||||
contentLength: 8,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: -1}},
|
||||
},
|
||||
{
|
||||
name: "suffix range",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 6-9/10",
|
||||
contentLength: 4,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},
|
||||
},
|
||||
{
|
||||
name: "suffix range with unknown complete length",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 6-9/*",
|
||||
contentLength: 4,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},
|
||||
},
|
||||
{
|
||||
name: "suffix range larger than representation",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 0-9/10",
|
||||
contentLength: 10,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 20}},
|
||||
},
|
||||
{
|
||||
name: "seek option",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-9/10",
|
||||
contentLength: 8,
|
||||
options: []fs.OpenOption{&fs.SeekOption{Offset: 2}},
|
||||
},
|
||||
{
|
||||
name: "whole representation range",
|
||||
status: http.StatusOK,
|
||||
contentLength: 10,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},
|
||||
},
|
||||
{
|
||||
name: "open ended whole representation range with unknown size",
|
||||
status: http.StatusOK,
|
||||
contentLength: -1,
|
||||
unknownSize: true,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: -1}},
|
||||
},
|
||||
{
|
||||
name: "partial range ignored",
|
||||
status: http.StatusOK,
|
||||
contentLength: 10,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
wantErrorIs: fs.ErrorRangeIgnored,
|
||||
},
|
||||
{
|
||||
name: "whole range response has wrong length",
|
||||
status: http.StatusOK,
|
||||
contentLength: 9,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "no range requested",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-4/10",
|
||||
contentLength: 3,
|
||||
},
|
||||
{
|
||||
name: "missing content range",
|
||||
status: http.StatusPartialContent,
|
||||
contentLength: 3,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong unit",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "items 2-4/10",
|
||||
contentLength: 3,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong start",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 0-2/10",
|
||||
contentLength: 3,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong end",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-5/10",
|
||||
contentLength: 4,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong content length",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-4/10",
|
||||
contentLength: 4,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid complete length",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-10/10",
|
||||
contentLength: 9,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 10}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "wrong complete length",
|
||||
status: http.StatusPartialContent,
|
||||
contentRange: "bytes 2-4/11",
|
||||
contentLength: 3,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unexpected successful status",
|
||||
status: http.StatusNoContent,
|
||||
contentLength: 0,
|
||||
options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
expectedSize := int64(10)
|
||||
if testCase.unknownSize {
|
||||
expectedSize = -1
|
||||
}
|
||||
resp := &http.Response{
|
||||
StatusCode: testCase.status,
|
||||
Header: make(http.Header),
|
||||
ContentLength: testCase.contentLength,
|
||||
}
|
||||
if testCase.contentRange != "" {
|
||||
resp.Header.Set("Content-Range", testCase.contentRange)
|
||||
}
|
||||
|
||||
err := CheckContentRange(resp, testCase.options, expectedSize)
|
||||
if testCase.wantErr {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
if testCase.wantErrorIs != nil {
|
||||
assert.ErrorIs(t, err, testCase.wantErrorIs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user