sync: add pattern-based transfer ordering

Allow --order-by to rank files using comma-separated rclone path globs. Patterns are evaluated in order, unmatched files are placed last, and path ordering makes ties deterministic.

Fixes #3975

fs/sync/pipe.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/aalpar/deheap"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/fserrors"
 )
 
@@ -187,8 +188,8 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
 	if orderBy == "" {
 		return nil, fraction, nil
 	}
-	parts := strings.Split(strings.ToLower(orderBy), ",")
-	switch parts[0] {
+	parts := strings.Split(orderBy, ",")
+	switch strings.ToLower(parts[0]) {
 	case "name":
 		less = func(a, b fs.ObjectPair) bool {
 			return a.Src.Remote() < b.Src.Remote()
@@ -202,12 +203,44 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
 			ctx := context.Background()
 			return a.Src.ModTime(ctx).Before(b.Src.ModTime(ctx))
 		}
+	case "pattern":
+		if len(parts) < 2 {
+			return nil, fraction, fmt.Errorf("pattern requires at least one glob")
+		}
+		matches := make([]func(string) bool, 0, len(parts)-1)
+		for _, glob := range parts[1:] {
+			if glob == "" {
+				return nil, fraction, fmt.Errorf("empty pattern glob")
+			}
+			re, err := filter.GlobPathToRegexp(glob, false)
+			if err != nil {
+				return nil, fraction, fmt.Errorf("bad pattern glob %q: %w", glob, err)
+			}
+			matches = append(matches, re.MatchString)
+		}
+		rank := func(remote string) int {
+			for i, match := range matches {
+				if match(remote) {
+					return i
+				}
+			}
+			return len(matches)
+		}
+		less = func(a, b fs.ObjectPair) bool {
+			aRemote, bRemote := a.Src.Remote(), b.Src.Remote()
+			aRank, bRank := rank(aRemote), rank(bRemote)
+			if aRank != bRank {
+				return aRank < bRank
+			}
+			return aRemote < bRemote
+		}
+		return less, fraction, nil
 	default:
 		return nil, fraction, fmt.Errorf("unknown --order-by comparison %q", parts[0])
 	}
 	descending := false
 	if len(parts) > 1 {
-		switch parts[1] {
+		switch strings.ToLower(parts[1]) {
 		case "ascending", "asc":
 		case "descending", "desc":
 			descending = true

fs/sync/pipe_test.go

@@ -250,6 +250,41 @@ func TestNewLess(t *testing.T) {
 		assert.Contains(t, err.Error(), "unknown --order-by sort direction")
 	})
 
+	t.Run("patternNeedsGlob", func(t *testing.T) {
+		_, _, err := newLess("pattern")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "pattern requires at least one glob")
+	})
+
+	t.Run("patternRejectsEmptyGlob", func(t *testing.T) {
+		_, _, err := newLess("pattern,")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "empty pattern glob")
+	})
+
+	t.Run("patternRejectsBadGlob", func(t *testing.T) {
+		_, _, err := newLess("pattern,[")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "bad pattern glob")
+	})
+
+	t.Run("patternOrder", func(t *testing.T) {
+		less, fraction, err := newLess("pattern,*.tar.gz,*.md5")
+		require.NoError(t, err)
+		assert.Equal(t, -1, fraction)
+
+		archiveA := fs.ObjectPair{Src: mockobject.New("a/archive.tar.gz")}
+		archiveB := fs.ObjectPair{Src: mockobject.New("b/archive.tar.gz")}
+		checksum := fs.ObjectPair{Src: mockobject.New("a/archive.md5")}
+		unmatched := fs.ObjectPair{Src: mockobject.New("a/README")}
+
+		assert.True(t, less(archiveA, checksum))
+		assert.True(t, less(checksum, unmatched))
+		assert.False(t, less(unmatched, archiveA))
+		assert.True(t, less(archiveA, archiveB))
+		assert.False(t, less(archiveB, archiveA))
+	})
+
 	var (
 		obj1  = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone)
 		obj2  = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone)