diff --git a/docs/content/docs.md b/docs/content/docs.md index 4ee271b4c..14d563c32 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -2378,8 +2378,10 @@ describes what aspect is being measured: - `size` - order by the size of the files - `name` - order by the full path of the files - `modtime` - order by the modification date of the files +- `pattern` - order by a list of path glob patterns -This can have a modifier appended with a comma: +The `size`, `name` and `modtime` comparisons can have a modifier appended with +a comma: - `ascending` or `asc` - order so that the smallest (or oldest) is processed first - `descending` or `desc` - order so that the largest (or newest) is processed first @@ -2398,11 +2400,18 @@ processed continuously. If no modifier is supplied then the order is `ascending`. +For `pattern`, append one or more comma-separated patterns. Patterns use the +[filter pattern syntax](/filtering/#patterns). Files matching the first pattern +are processed first, followed by files matching the second pattern, and so on. +Files which do not match a pattern are processed last. Within each group, files +are ordered by full path. Patterns cannot contain a comma. + For example - `--order-by size,desc` - send the largest files first - `--order-by modtime,ascending` - send the oldest files first - `--order-by name` - send the files with alphabetically by path first +- `--order-by 'pattern,*.tar.gz,*.md5'` - send archives before checksum files If the `--order-by` flag is not supplied or it is supplied with an empty string then the default ordering will be used which is as diff --git a/fs/sync/pipe.go b/fs/sync/pipe.go index 124a07780..42d3e9234 100644 --- a/fs/sync/pipe.go +++ b/fs/sync/pipe.go @@ -10,6 +10,7 @@ import ( "github.com/aalpar/deheap" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/fserrors" ) @@ -187,8 +188,8 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) { if orderBy == "" { return nil, fraction, nil } - parts := strings.Split(strings.ToLower(orderBy), ",") - switch parts[0] { + parts := strings.Split(orderBy, ",") + switch strings.ToLower(parts[0]) { case "name": less = func(a, b fs.ObjectPair) bool { return a.Src.Remote() < b.Src.Remote() @@ -202,12 +203,44 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) { ctx := context.Background() return a.Src.ModTime(ctx).Before(b.Src.ModTime(ctx)) } + case "pattern": + if len(parts) < 2 { + return nil, fraction, fmt.Errorf("pattern requires at least one glob") + } + matches := make([]func(string) bool, 0, len(parts)-1) + for _, glob := range parts[1:] { + if glob == "" { + return nil, fraction, fmt.Errorf("empty pattern glob") + } + re, err := filter.GlobPathToRegexp(glob, false) + if err != nil { + return nil, fraction, fmt.Errorf("bad pattern glob %q: %w", glob, err) + } + matches = append(matches, re.MatchString) + } + rank := func(remote string) int { + for i, match := range matches { + if match(remote) { + return i + } + } + return len(matches) + } + less = func(a, b fs.ObjectPair) bool { + aRemote, bRemote := a.Src.Remote(), b.Src.Remote() + aRank, bRank := rank(aRemote), rank(bRemote) + if aRank != bRank { + return aRank < bRank + } + return aRemote < bRemote + } + return less, fraction, nil default: return nil, fraction, fmt.Errorf("unknown --order-by comparison %q", parts[0]) } descending := false if len(parts) > 1 { - switch parts[1] { + switch strings.ToLower(parts[1]) { case "ascending", "asc": case "descending", "desc": descending = true diff --git a/fs/sync/pipe_test.go b/fs/sync/pipe_test.go index 94916aa82..c08ca926c 100644 --- a/fs/sync/pipe_test.go +++ b/fs/sync/pipe_test.go @@ -250,6 +250,41 @@ func TestNewLess(t *testing.T) { assert.Contains(t, err.Error(), "unknown --order-by sort direction") }) + t.Run("patternNeedsGlob", func(t *testing.T) { + _, _, err := newLess("pattern") + require.Error(t, err) + assert.Contains(t, err.Error(), "pattern requires at least one glob") + }) + + t.Run("patternRejectsEmptyGlob", func(t *testing.T) { + _, _, err := newLess("pattern,") + require.Error(t, err) + assert.Contains(t, err.Error(), "empty pattern glob") + }) + + t.Run("patternRejectsBadGlob", func(t *testing.T) { + _, _, err := newLess("pattern,[") + require.Error(t, err) + assert.Contains(t, err.Error(), "bad pattern glob") + }) + + t.Run("patternOrder", func(t *testing.T) { + less, fraction, err := newLess("pattern,*.tar.gz,*.md5") + require.NoError(t, err) + assert.Equal(t, -1, fraction) + + archiveA := fs.ObjectPair{Src: mockobject.New("a/archive.tar.gz")} + archiveB := fs.ObjectPair{Src: mockobject.New("b/archive.tar.gz")} + checksum := fs.ObjectPair{Src: mockobject.New("a/archive.md5")} + unmatched := fs.ObjectPair{Src: mockobject.New("a/README")} + + assert.True(t, less(archiveA, checksum)) + assert.True(t, less(checksum, unmatched)) + assert.False(t, less(unmatched, archiveA)) + assert.True(t, less(archiveA, archiveB)) + assert.False(t, less(archiveB, archiveA)) + }) + var ( obj1 = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone) obj2 = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone)