Leon Brocard
075552367e
The S3 ListObjects response from `rclone serve s3` was sorting object contents by modification time instead of object key. This made the listing order incompatible with S3 clients which expect lexicographic key ordering. In particular, `aws s3 sync` assumes both source and destination iterators are ordered by key. With the old modtime ordering it could misidentify files as missing or outdated and re-download objects that were already up to date. Change the pager to sort returned objects by key and add a regression test which uses keys and modtimes arranged so the old behaviour would fail. Fixes #9002
67 lines
1.5 KiB
Go
67 lines
1.5 KiB
Go
// Package s3 implements a fake s3 server for rclone
|
|
package s3
|
|
|
|
import (
|
|
"sort"
|
|
|
|
"github.com/rclone/gofakes3"
|
|
)
|
|
|
|
// pager splits the object list into smulitply pages.
|
|
func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
|
|
// sort by alphabet
|
|
sort.Slice(list.CommonPrefixes, func(i, j int) bool {
|
|
return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix
|
|
})
|
|
// sort by key name
|
|
sort.Slice(list.Contents, func(i, j int) bool {
|
|
return list.Contents[i].Key < list.Contents[j].Key
|
|
})
|
|
tokens := page.MaxKeys
|
|
if tokens == 0 {
|
|
tokens = 1000
|
|
}
|
|
if page.HasMarker {
|
|
for i, obj := range list.Contents {
|
|
if obj.Key == page.Marker {
|
|
list.Contents = list.Contents[i+1:]
|
|
break
|
|
}
|
|
}
|
|
for i, obj := range list.CommonPrefixes {
|
|
if obj.Prefix == page.Marker {
|
|
list.CommonPrefixes = list.CommonPrefixes[i+1:]
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
response := gofakes3.NewObjectList()
|
|
for _, obj := range list.CommonPrefixes {
|
|
if tokens <= 0 {
|
|
break
|
|
}
|
|
response.AddPrefix(obj.Prefix)
|
|
tokens--
|
|
}
|
|
|
|
for _, obj := range list.Contents {
|
|
if tokens <= 0 {
|
|
break
|
|
}
|
|
response.Add(obj)
|
|
tokens--
|
|
}
|
|
|
|
if len(list.CommonPrefixes)+len(list.Contents) > int(page.MaxKeys) {
|
|
response.IsTruncated = true
|
|
if len(response.Contents) > 0 {
|
|
response.NextMarker = response.Contents[len(response.Contents)-1].Key
|
|
} else {
|
|
response.NextMarker = response.CommonPrefixes[len(response.CommonPrefixes)-1].Prefix
|
|
}
|
|
}
|
|
|
|
return response, nil
|
|
}
|