bisync: hard file/byte delete caps and dual-boot session-name

--max-delete-files and --max-delete-size are independent of the
percent cap. --session-name shares listings/locks across OS path strings.

cmd/bisync/bilib/canonical.go

@@ -64,7 +64,10 @@ func HasHexString(path string) bool {
 }
 
 // BasePath joins the workDir with the SessionName, stripping {hexstring} suffix if necessary
-func BasePath(ctx context.Context, workDir string, fs1, fs2 fs.Fs) string {
+func BasePath(ctx context.Context, workDir string, fs1, fs2 fs.Fs, sessionOverride string) string {
+	if sessionOverride != "" {
+		return filepath.Join(workDir, CanonicalPath(sessionOverride))
+	}
 	suffixedSession := CanonicalPath(FsPath(fs1)) + ".." + CanonicalPath(FsPath(fs2))
 	suffixedBasePath := filepath.Join(workDir, suffixedSession)
 	listing1 := suffixedBasePath + ".path1.lst"

cmd/bisync/cmd.go

@@ -39,6 +39,9 @@ type Options struct {
 	CreateEmptySrcDirs    bool
 	RemoveEmptyDirs       bool
 	MaxDelete             int // percentage from 0 to 100
+	MaxDeleteFiles        int // absolute file cap; 0 disables
+	MaxDeleteSize         fs.SizeSuffix
+	SessionName           string // override listing/lock basename (dual-boot)
 	Force                 bool
 	FiltersFile           string
 	Workdir               string
@@ -131,6 +134,9 @@ func init() {
 	flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, MakeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")
 	flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, MakeHelp("Filename for --check-access (default: {CHECKFILE})"), "")
 	flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "")
+	flags.IntVarP(cmdFlags, &Opt.MaxDeleteFiles, "max-delete-files", "", Opt.MaxDeleteFiles, "Abort if more than this many files would be deleted (0=disabled). Independent of --max-delete percent.", "")
+	flags.FVarP(cmdFlags, &Opt.MaxDeleteSize, "max-delete-size", "", "Abort if deleted bytes would exceed this size (0=disabled)", "")
+	flags.StringVarP(cmdFlags, &Opt.SessionName, "session-name", "", Opt.SessionName, "Override workdir listing/lock basename so dual-boot path strings share state", "")
 	flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "")
 	flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "")
 	flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")

cmd/bisync/delete_budget.go

@@ -0,0 +1,30 @@
+package bisync
+
+import (
+	"fmt"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// deleteBudgetExceeded reports whether a proposed delete set trips percent,
+// absolute file-count, or byte caps. Percent is opt.MaxDelete (0–100).
+// File/byte caps are inactive when <= 0. Force bypasses all caps.
+func deleteBudgetExceeded(opt *Options, oldCount, deleted int, deletedBytes int64) (bool, string) {
+	if opt == nil || opt.Force || deleted <= 0 {
+		return false, ""
+	}
+	if opt.MaxDeleteFiles > 0 && deleted > opt.MaxDeleteFiles {
+		return true, fmt.Sprintf("too many deletes (>%d files, %d of %d)", opt.MaxDeleteFiles, deleted, oldCount)
+	}
+	if int64(opt.MaxDeleteSize) > 0 && deletedBytes > int64(opt.MaxDeleteSize) {
+		return true, fmt.Sprintf("too many deletes by size (>%v, %v of listing)", opt.MaxDeleteSize, fs.SizeSuffix(deletedBytes))
+	}
+	if opt.MaxDelete >= 0 && oldCount > 0 {
+		maxRatio := float64(opt.MaxDelete) / 100.0
+		curRatio := float64(deleted) / float64(oldCount)
+		if curRatio > maxRatio {
+			return true, fmt.Sprintf("too many deletes (>%d%%, %d of %d)", opt.MaxDelete, deleted, oldCount)
+		}
+	}
+	return false, ""
+}

cmd/bisync/delta_list.go

@@ -9,6 +9,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/rclone/rclone/fs"
 )
 
 // errDeltaTooManyDeletes is returned when applying a delta would exceed --max-delete.
@@ -44,11 +46,13 @@ func guardFileCount(oldC, newC int, opt deltaApplyOpts) error {
 	if newC == 0 {
 		return errDeltaListingEmptied
 	}
-	if opt.MaxDelete >= 0 {
+	dropped := oldC - newC
-		dropped := oldC - newC
+	if dropped <= 0 {
-		if dropped > 0 && float64(dropped)/float64(oldC) > float64(opt.MaxDelete)/100.0 {
+		return nil
-			return errDeltaListingCollapsed
+	}
-		}
+	o := &Options{MaxDelete: opt.MaxDelete, MaxDeleteFiles: opt.MaxDeleteFiles, MaxDeleteSize: opt.MaxDeleteSize, Force: opt.Force}
+	if hit, _ := deleteBudgetExceeded(o, oldC, dropped, 0); hit {
+		return errDeltaListingCollapsed
 	}
 	return nil
 }
@@ -90,9 +94,11 @@ type listingDelta struct {
 }
 
 type deltaApplyOpts struct {
-	MaxDelete int
+	MaxDelete      int
-	Force     bool
+	MaxDeleteFiles int
-	Reset     bool
+	MaxDeleteSize  fs.SizeSuffix
+	Force          bool
+	Reset          bool
 }
 
 func cursorPath(listing string) string {
@@ -123,6 +129,7 @@ func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts)
 		}
 	}
 	deletedFiles := 0
+	var deletedBytes int64
 
 	for _, d := range deltas {
 		if d.Deleted {
@@ -154,6 +161,7 @@ func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts)
 			for _, name := range toRemove {
 				if !ls.isDir(name) {
 					deletedFiles++
+					deletedBytes += ls.getSize(name)
 				}
 				ls.remove(name)
 			}
@@ -166,9 +174,9 @@ func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts)
 		ls.put(d.Remote, d.Size, d.ModTime, d.Hash, d.ID, flags)
 	}
 
-	if !opt.Force && opt.MaxDelete >= 0 && oldCount > 0 {
+	if !opt.Force && oldCount > 0 {
-		maxRatio := float64(opt.MaxDelete) / 100.0
+		o := &Options{MaxDelete: opt.MaxDelete, MaxDeleteFiles: opt.MaxDeleteFiles, MaxDeleteSize: opt.MaxDeleteSize}
-		if float64(deletedFiles)/float64(oldCount) > maxRatio {
+		if hit, _ := deleteBudgetExceeded(o, oldCount, deletedFiles, deletedBytes); hit {
 			restore()
 			return errDeltaTooManyDeletes
 		}

cmd/bisync/delta_list_run.go

@@ -45,7 +45,7 @@ func (b *bisyncRun) makeDeltaListing(ctx context.Context) (*fileList, *fileList,
 		err = b.listOneFs(ctx, true)
 		if err == nil {
 			if prior, lerr := b.loadListing(b.listing1); lerr == nil {
-				err = guardListingShrink(prior, b.march.ls1, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, Force: b.opt.Force})
+				err = guardListingShrink(prior, b.march.ls1, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force})
 			}
 		}
 	}
@@ -63,7 +63,7 @@ func (b *bisyncRun) makeDeltaListing(ctx context.Context) (*fileList, *fileList,
 		err = b.listOneFs(ctx, false)
 		if err == nil {
 			if prior, lerr := b.loadListing(b.listing2); lerr == nil {
-				err = guardListingShrink(prior, b.march.ls2, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, Force: b.opt.Force})
+				err = guardListingShrink(prior, b.march.ls2, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force})
 			}
 		}
 	}
@@ -167,8 +167,10 @@ func (b *bisyncRun) refreshDropboxListing(ctx context.Context, dbx *dropbox.Fs,
 	}
 
 	applyOpts := deltaApplyOpts{
-		MaxDelete: b.opt.MaxDelete,
+		MaxDelete:      b.opt.MaxDelete,
-		Force:     b.opt.Force,
+		MaxDeleteFiles: b.opt.MaxDeleteFiles,
+		MaxDeleteSize:  b.opt.MaxDeleteSize,
+		Force:          b.opt.Force,
 	}
 	priorCount := 0
 	hadPrior := prior != nil

cmd/bisync/delta_list_test.go

@@ -121,6 +121,34 @@ func TestLoadCursor_MissingIsEmpty(t *testing.T) {
 	assert.True(t, os.IsNotExist(err))
 }
 
+func TestApplyListingDeltas_MaxDeleteFilesAborts(t *testing.T) {
+	ls := testListing()
+	orig := len(ls.list)
+	err := applyListingDeltas(ls, []listingDelta{
+		{Remote: "gone.txt", Deleted: true},
+		{Remote: "keep.txt", Deleted: true},
+	}, deltaApplyOpts{MaxDelete: 100, MaxDeleteFiles: 1})
+	require.ErrorIs(t, err, errDeltaTooManyDeletes)
+	assert.Equal(t, orig, len(ls.list))
+}
+
+func TestGuardFileCount_MaxDeleteFiles(t *testing.T) {
+	require.ErrorIs(t, guardFileCount(100, 89, deltaApplyOpts{MaxDelete: 50, MaxDeleteFiles: 10}), errDeltaListingCollapsed)
+	require.NoError(t, guardFileCount(100, 95, deltaApplyOpts{MaxDelete: 50, MaxDeleteFiles: 10}))
+}
+
+func TestDeleteBudgetExceeded(t *testing.T) {
+	opt := &Options{MaxDelete: 50, MaxDeleteFiles: 10, MaxDeleteSize: 100}
+	hit, _ := deleteBudgetExceeded(opt, 1000, 11, 0)
+	require.True(t, hit)
+	hit, _ = deleteBudgetExceeded(opt, 1000, 1, 200)
+	require.True(t, hit)
+	hit, _ = deleteBudgetExceeded(&Options{MaxDelete: 1, MaxDeleteFiles: 100}, 1000, 20, 0)
+	require.True(t, hit)
+	hit, _ = deleteBudgetExceeded(&Options{MaxDelete: 50, MaxDeleteFiles: 10, Force: true}, 1000, 500, 1e12)
+	require.False(t, hit)
+}
+
 func TestGuardFileCount(t *testing.T) {
 	opt := deltaApplyOpts{MaxDelete: 50}
 	require.NoError(t, guardFileCount(0, 0, opt))

cmd/bisync/deltas.go

@@ -44,17 +44,18 @@ func (d delta) is(cond delta) bool {
 
 // deltaSet
 type deltaSet struct {
-	deltas     map[string]delta
+	deltas       map[string]delta
-	size       map[string]int64
+	size         map[string]int64
-	time       map[string]time.Time
+	time         map[string]time.Time
-	hash       map[string]string
+	hash         map[string]string
-	opt        *Options
+	opt          *Options
-	fs         fs.Fs  // base filesystem
+	fs           fs.Fs  // base filesystem
-	msg        string // filesystem name for logging
+	msg          string // filesystem name for logging
-	oldCount   int    // original number of files (for "excess deletes" check)
+	oldCount     int    // original number of files (for "excess deletes" check)
-	deleted    int    // number of deleted files (for "excess deletes" check)
+	deleted      int    // number of deleted files (for "excess deletes" check)
-	foundSame  bool   // true if found at least one unchanged file
+	deletedBytes int64
-	checkFiles bilib.Names
+	foundSame    bool // true if found at least one unchanged file
+	checkFiles   bilib.Names
 }
 
 func (ds *deltaSet) empty() bool {
@@ -187,6 +188,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string,
 		if !now.has(file) {
 			b.indent(msg, file, Color(terminal.RedFg, "File was deleted"))
 			ds.deleted++
+			ds.deletedBytes += old.getSize(file)
 			d |= deltaDeleted
 		} else if !now.isDir(file) {
 			// skip dirs here, as we only care if they are new/deleted, not newer/older
@@ -543,20 +545,12 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 
 // excessDeletes checks whether number of deletes is within allowed range
 func (ds *deltaSet) excessDeletes() bool {
-	maxDelete := ds.opt.MaxDelete
+	hit, msg := deleteBudgetExceeded(ds.opt, ds.oldCount, ds.deleted, ds.deletedBytes)
-	maxRatio := float64(maxDelete) / 100.0
+	if !hit {
-	curRatio := 0.0
-	if ds.deleted > 0 && ds.oldCount > 0 {
-		curRatio = float64(ds.deleted) / float64(ds.oldCount)
-	}
-
-	if curRatio <= maxRatio {
 		return false
 	}
-
+	fs.Errorf("Safety abort", "%s on %s %s. Run with --force if desired.",
-	fs.Errorf("Safety abort",
+		msg, ds.msg, quotePath(bilib.FsPath(ds.fs)))
-		"too many deletes (>%d%%, %d of %d) on %s %s. Run with --force if desired.",
-		maxDelete, ds.deleted, ds.oldCount, ds.msg, quotePath(bilib.FsPath(ds.fs)))
 	return true
 }
 

cmd/bisync/operations.go

@@ -112,7 +112,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 	}
 
 	// Produce a unique name for the sync operation
-	b.basePath = bilib.BasePath(ctx, b.workDir, b.fs1, b.fs2)
+	b.basePath = bilib.BasePath(ctx, b.workDir, b.fs1, b.fs2, opt.SessionName)
 	b.listing1 = b.basePath + ".path1.lst"
 	b.listing2 = b.basePath + ".path2.lst"
 	b.newListing1 = b.listing1 + "-new"

cmd/bisync/rc.go

@@ -166,6 +166,22 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 		fs.Debugf("maxLock", "optional parameter is missing. using default value: %v", opt.MaxLock)
 	}
 
+	if n, err := in.GetInt64("maxDeleteFiles"); err == nil {
+		opt.MaxDeleteFiles = int(n)
+	} else if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	if opt.SessionName, err = in.GetString("sessionName"); rc.NotErrParamNotFound(err) {
+		fs.Debugf("sessionName", "optional parameter is missing. using default value: %v", opt.SessionName)
+	}
+	if sz, err := in.GetString("maxDeleteSize"); err == nil && sz != "" {
+		if e := opt.MaxDeleteSize.Set(sz); e != nil {
+			return nil, rc.NewErrParamInvalid(e)
+		}
+	} else if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+
 	fs1, err := rc.GetFsNamed(octx, in, "path1")
 	if err != nil {
 		return nil, err
@@ -184,7 +200,7 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	if opt.Workdir != "" {
 		workDir, _ = filepath.Abs(opt.Workdir)
 	}
-	basePath := bilib.BasePath(ctx, workDir, fs1, fs2)
+	basePath := bilib.BasePath(ctx, workDir, fs1, fs2, opt.SessionName)
 
 	_, _ = log.Writer().Write(output)
 	return rc.Params{