sync: add pattern-based transfer ordering

Allow --order-by to rank files using comma-separated rclone path globs. Patterns are evaluated in order, unmatched files are placed last, and path ordering makes ties deterministic.

Fixes #3975

docs/content/docs.md

@@ -2378,8 +2378,10 @@ describes what aspect is being measured:
 - `size` - order by the size of the files
 - `name` - order by the full path of the files
 - `modtime` - order by the modification date of the files
+- `pattern` - order by a list of path glob patterns
 
-This can have a modifier appended with a comma:
+The `size`, `name` and `modtime` comparisons can have a modifier appended with
+a comma:
 
 - `ascending` or `asc` - order so that the smallest (or oldest) is processed first
 - `descending` or `desc` - order so that the largest (or newest) is processed first
@@ -2398,11 +2400,18 @@ processed continuously.
 
 If no modifier is supplied then the order is `ascending`.
 
+For `pattern`, append one or more comma-separated patterns. Patterns use the
+[filter pattern syntax](/filtering/#patterns). Files matching the first pattern
+are processed first, followed by files matching the second pattern, and so on.
+Files which do not match a pattern are processed last. Within each group, files
+are ordered by full path. Patterns cannot contain a comma.
+
 For example
 
 - `--order-by size,desc` - send the largest files first
 - `--order-by modtime,ascending` - send the oldest files first
 - `--order-by name` - send the files with alphabetically by path first
+- `--order-by 'pattern,*.tar.gz,*.md5'` - send archives before checksum files
 
 If the `--order-by` flag is not supplied or it is supplied with an
 empty string then the default ordering will be used which is as

fs/sync/pipe.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/aalpar/deheap"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/fserrors"
 )
 
@@ -187,8 +188,8 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
 	if orderBy == "" {
 		return nil, fraction, nil
 	}
-	parts := strings.Split(strings.ToLower(orderBy), ",")
-	switch parts[0] {
+	parts := strings.Split(orderBy, ",")
+	switch strings.ToLower(parts[0]) {
 	case "name":
 		less = func(a, b fs.ObjectPair) bool {
 			return a.Src.Remote() < b.Src.Remote()
@@ -202,12 +203,44 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
 			ctx := context.Background()
 			return a.Src.ModTime(ctx).Before(b.Src.ModTime(ctx))
 		}
+	case "pattern":
+		if len(parts) < 2 {
+			return nil, fraction, fmt.Errorf("pattern requires at least one glob")
+		}
+		matches := make([]func(string) bool, 0, len(parts)-1)
+		for _, glob := range parts[1:] {
+			if glob == "" {
+				return nil, fraction, fmt.Errorf("empty pattern glob")
+			}
+			re, err := filter.GlobPathToRegexp(glob, false)
+			if err != nil {
+				return nil, fraction, fmt.Errorf("bad pattern glob %q: %w", glob, err)
+			}
+			matches = append(matches, re.MatchString)
+		}
+		rank := func(remote string) int {
+			for i, match := range matches {
+				if match(remote) {
+					return i
+				}
+			}
+			return len(matches)
+		}
+		less = func(a, b fs.ObjectPair) bool {
+			aRemote, bRemote := a.Src.Remote(), b.Src.Remote()
+			aRank, bRank := rank(aRemote), rank(bRemote)
+			if aRank != bRank {
+				return aRank < bRank
+			}
+			return aRemote < bRemote
+		}
+		return less, fraction, nil
 	default:
 		return nil, fraction, fmt.Errorf("unknown --order-by comparison %q", parts[0])
 	}
 	descending := false
 	if len(parts) > 1 {
-		switch parts[1] {
+		switch strings.ToLower(parts[1]) {
 		case "ascending", "asc":
 		case "descending", "desc":
 			descending = true

fs/sync/pipe_test.go

@@ -250,6 +250,41 @@ func TestNewLess(t *testing.T) {
 		assert.Contains(t, err.Error(), "unknown --order-by sort direction")
 	})
 
+	t.Run("patternNeedsGlob", func(t *testing.T) {
+		_, _, err := newLess("pattern")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "pattern requires at least one glob")
+	})
+
+	t.Run("patternRejectsEmptyGlob", func(t *testing.T) {
+		_, _, err := newLess("pattern,")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "empty pattern glob")
+	})
+
+	t.Run("patternRejectsBadGlob", func(t *testing.T) {
+		_, _, err := newLess("pattern,[")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "bad pattern glob")
+	})
+
+	t.Run("patternOrder", func(t *testing.T) {
+		less, fraction, err := newLess("pattern,*.tar.gz,*.md5")
+		require.NoError(t, err)
+		assert.Equal(t, -1, fraction)
+
+		archiveA := fs.ObjectPair{Src: mockobject.New("a/archive.tar.gz")}
+		archiveB := fs.ObjectPair{Src: mockobject.New("b/archive.tar.gz")}
+		checksum := fs.ObjectPair{Src: mockobject.New("a/archive.md5")}
+		unmatched := fs.ObjectPair{Src: mockobject.New("a/README")}
+
+		assert.True(t, less(archiveA, checksum))
+		assert.True(t, less(checksum, unmatched))
+		assert.False(t, less(unmatched, archiveA))
+		assert.True(t, less(archiveA, archiveB))
+		assert.False(t, less(archiveB, archiveA))
+	})
+
 	var (
 		obj1  = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone)
 		obj2  = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone)