From b957d264dfee4ad192de60e2307718af0383a4c0 Mon Sep 17 00:00:00 2001 From: Federico Justus Denkena Date: Thu, 10 Sep 2026 01:03:08 +0200 Subject: [PATCH] bisync: hard file/byte delete caps and dual-boot session-name --max-delete-files and --max-delete-size are independent of the percent cap. --session-name shares listings/locks across OS path strings. --- cmd/bisync/bilib/canonical.go | 5 ++++- cmd/bisync/cmd.go | 6 ++++++ cmd/bisync/delete_budget.go | 30 ++++++++++++++++++++++++++ cmd/bisync/delta_list.go | 30 ++++++++++++++++---------- cmd/bisync/delta_list_run.go | 10 +++++---- cmd/bisync/delta_list_test.go | 28 ++++++++++++++++++++++++ cmd/bisync/deltas.go | 40 +++++++++++++++-------------------- cmd/bisync/operations.go | 2 +- cmd/bisync/rc.go | 18 +++++++++++++++- 9 files changed, 128 insertions(+), 41 deletions(-) create mode 100644 cmd/bisync/delete_budget.go diff --git a/cmd/bisync/bilib/canonical.go b/cmd/bisync/bilib/canonical.go index dac147670..98735c1c4 100644 --- a/cmd/bisync/bilib/canonical.go +++ b/cmd/bisync/bilib/canonical.go @@ -64,7 +64,10 @@ func HasHexString(path string) bool { } // BasePath joins the workDir with the SessionName, stripping {hexstring} suffix if necessary -func BasePath(ctx context.Context, workDir string, fs1, fs2 fs.Fs) string { +func BasePath(ctx context.Context, workDir string, fs1, fs2 fs.Fs, sessionOverride string) string { + if sessionOverride != "" { + return filepath.Join(workDir, CanonicalPath(sessionOverride)) + } suffixedSession := CanonicalPath(FsPath(fs1)) + ".." + CanonicalPath(FsPath(fs2)) suffixedBasePath := filepath.Join(workDir, suffixedSession) listing1 := suffixedBasePath + ".path1.lst" diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go index 6b7fc3c22..e392918a1 100644 --- a/cmd/bisync/cmd.go +++ b/cmd/bisync/cmd.go @@ -39,6 +39,9 @@ type Options struct { CreateEmptySrcDirs bool RemoveEmptyDirs bool MaxDelete int // percentage from 0 to 100 + MaxDeleteFiles int // absolute file cap; 0 disables + MaxDeleteSize fs.SizeSuffix + SessionName string // override listing/lock basename (dual-boot) Force bool FiltersFile string Workdir string @@ -131,6 +134,9 @@ func init() { flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, MakeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "") flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, MakeHelp("Filename for --check-access (default: {CHECKFILE})"), "") flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "") + flags.IntVarP(cmdFlags, &Opt.MaxDeleteFiles, "max-delete-files", "", Opt.MaxDeleteFiles, "Abort if more than this many files would be deleted (0=disabled). Independent of --max-delete percent.", "") + flags.FVarP(cmdFlags, &Opt.MaxDeleteSize, "max-delete-size", "", "Abort if deleted bytes would exceed this size (0=disabled)", "") + flags.StringVarP(cmdFlags, &Opt.SessionName, "session-name", "", Opt.SessionName, "Override workdir listing/lock basename so dual-boot path strings share state", "") flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "") flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "") flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "") diff --git a/cmd/bisync/delete_budget.go b/cmd/bisync/delete_budget.go new file mode 100644 index 000000000..211af7a70 --- /dev/null +++ b/cmd/bisync/delete_budget.go @@ -0,0 +1,30 @@ +package bisync + +import ( + "fmt" + + "github.com/rclone/rclone/fs" +) + +// deleteBudgetExceeded reports whether a proposed delete set trips percent, +// absolute file-count, or byte caps. Percent is opt.MaxDelete (0–100). +// File/byte caps are inactive when <= 0. Force bypasses all caps. +func deleteBudgetExceeded(opt *Options, oldCount, deleted int, deletedBytes int64) (bool, string) { + if opt == nil || opt.Force || deleted <= 0 { + return false, "" + } + if opt.MaxDeleteFiles > 0 && deleted > opt.MaxDeleteFiles { + return true, fmt.Sprintf("too many deletes (>%d files, %d of %d)", opt.MaxDeleteFiles, deleted, oldCount) + } + if int64(opt.MaxDeleteSize) > 0 && deletedBytes > int64(opt.MaxDeleteSize) { + return true, fmt.Sprintf("too many deletes by size (>%v, %v of listing)", opt.MaxDeleteSize, fs.SizeSuffix(deletedBytes)) + } + if opt.MaxDelete >= 0 && oldCount > 0 { + maxRatio := float64(opt.MaxDelete) / 100.0 + curRatio := float64(deleted) / float64(oldCount) + if curRatio > maxRatio { + return true, fmt.Sprintf("too many deletes (>%d%%, %d of %d)", opt.MaxDelete, deleted, oldCount) + } + } + return false, "" +} diff --git a/cmd/bisync/delta_list.go b/cmd/bisync/delta_list.go index f963cb694..3247ca1f1 100644 --- a/cmd/bisync/delta_list.go +++ b/cmd/bisync/delta_list.go @@ -9,6 +9,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/rclone/rclone/fs" ) // errDeltaTooManyDeletes is returned when applying a delta would exceed --max-delete. @@ -44,11 +46,13 @@ func guardFileCount(oldC, newC int, opt deltaApplyOpts) error { if newC == 0 { return errDeltaListingEmptied } - if opt.MaxDelete >= 0 { - dropped := oldC - newC - if dropped > 0 && float64(dropped)/float64(oldC) > float64(opt.MaxDelete)/100.0 { - return errDeltaListingCollapsed - } + dropped := oldC - newC + if dropped <= 0 { + return nil + } + o := &Options{MaxDelete: opt.MaxDelete, MaxDeleteFiles: opt.MaxDeleteFiles, MaxDeleteSize: opt.MaxDeleteSize, Force: opt.Force} + if hit, _ := deleteBudgetExceeded(o, oldC, dropped, 0); hit { + return errDeltaListingCollapsed } return nil } @@ -90,9 +94,11 @@ type listingDelta struct { } type deltaApplyOpts struct { - MaxDelete int - Force bool - Reset bool + MaxDelete int + MaxDeleteFiles int + MaxDeleteSize fs.SizeSuffix + Force bool + Reset bool } func cursorPath(listing string) string { @@ -123,6 +129,7 @@ func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts) } } deletedFiles := 0 + var deletedBytes int64 for _, d := range deltas { if d.Deleted { @@ -154,6 +161,7 @@ func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts) for _, name := range toRemove { if !ls.isDir(name) { deletedFiles++ + deletedBytes += ls.getSize(name) } ls.remove(name) } @@ -166,9 +174,9 @@ func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts) ls.put(d.Remote, d.Size, d.ModTime, d.Hash, d.ID, flags) } - if !opt.Force && opt.MaxDelete >= 0 && oldCount > 0 { - maxRatio := float64(opt.MaxDelete) / 100.0 - if float64(deletedFiles)/float64(oldCount) > maxRatio { + if !opt.Force && oldCount > 0 { + o := &Options{MaxDelete: opt.MaxDelete, MaxDeleteFiles: opt.MaxDeleteFiles, MaxDeleteSize: opt.MaxDeleteSize} + if hit, _ := deleteBudgetExceeded(o, oldCount, deletedFiles, deletedBytes); hit { restore() return errDeltaTooManyDeletes } diff --git a/cmd/bisync/delta_list_run.go b/cmd/bisync/delta_list_run.go index 4240019e5..67ea56803 100644 --- a/cmd/bisync/delta_list_run.go +++ b/cmd/bisync/delta_list_run.go @@ -45,7 +45,7 @@ func (b *bisyncRun) makeDeltaListing(ctx context.Context) (*fileList, *fileList, err = b.listOneFs(ctx, true) if err == nil { if prior, lerr := b.loadListing(b.listing1); lerr == nil { - err = guardListingShrink(prior, b.march.ls1, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, Force: b.opt.Force}) + err = guardListingShrink(prior, b.march.ls1, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force}) } } } @@ -63,7 +63,7 @@ func (b *bisyncRun) makeDeltaListing(ctx context.Context) (*fileList, *fileList, err = b.listOneFs(ctx, false) if err == nil { if prior, lerr := b.loadListing(b.listing2); lerr == nil { - err = guardListingShrink(prior, b.march.ls2, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, Force: b.opt.Force}) + err = guardListingShrink(prior, b.march.ls2, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force}) } } } @@ -167,8 +167,10 @@ func (b *bisyncRun) refreshDropboxListing(ctx context.Context, dbx *dropbox.Fs, } applyOpts := deltaApplyOpts{ - MaxDelete: b.opt.MaxDelete, - Force: b.opt.Force, + MaxDelete: b.opt.MaxDelete, + MaxDeleteFiles: b.opt.MaxDeleteFiles, + MaxDeleteSize: b.opt.MaxDeleteSize, + Force: b.opt.Force, } priorCount := 0 hadPrior := prior != nil diff --git a/cmd/bisync/delta_list_test.go b/cmd/bisync/delta_list_test.go index 5f6d31c33..d6710cac7 100644 --- a/cmd/bisync/delta_list_test.go +++ b/cmd/bisync/delta_list_test.go @@ -121,6 +121,34 @@ func TestLoadCursor_MissingIsEmpty(t *testing.T) { assert.True(t, os.IsNotExist(err)) } +func TestApplyListingDeltas_MaxDeleteFilesAborts(t *testing.T) { + ls := testListing() + orig := len(ls.list) + err := applyListingDeltas(ls, []listingDelta{ + {Remote: "gone.txt", Deleted: true}, + {Remote: "keep.txt", Deleted: true}, + }, deltaApplyOpts{MaxDelete: 100, MaxDeleteFiles: 1}) + require.ErrorIs(t, err, errDeltaTooManyDeletes) + assert.Equal(t, orig, len(ls.list)) +} + +func TestGuardFileCount_MaxDeleteFiles(t *testing.T) { + require.ErrorIs(t, guardFileCount(100, 89, deltaApplyOpts{MaxDelete: 50, MaxDeleteFiles: 10}), errDeltaListingCollapsed) + require.NoError(t, guardFileCount(100, 95, deltaApplyOpts{MaxDelete: 50, MaxDeleteFiles: 10})) +} + +func TestDeleteBudgetExceeded(t *testing.T) { + opt := &Options{MaxDelete: 50, MaxDeleteFiles: 10, MaxDeleteSize: 100} + hit, _ := deleteBudgetExceeded(opt, 1000, 11, 0) + require.True(t, hit) + hit, _ = deleteBudgetExceeded(opt, 1000, 1, 200) + require.True(t, hit) + hit, _ = deleteBudgetExceeded(&Options{MaxDelete: 1, MaxDeleteFiles: 100}, 1000, 20, 0) + require.True(t, hit) + hit, _ = deleteBudgetExceeded(&Options{MaxDelete: 50, MaxDeleteFiles: 10, Force: true}, 1000, 500, 1e12) + require.False(t, hit) +} + func TestGuardFileCount(t *testing.T) { opt := deltaApplyOpts{MaxDelete: 50} require.NoError(t, guardFileCount(0, 0, opt)) diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index fa5ea785e..912802321 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -44,17 +44,18 @@ func (d delta) is(cond delta) bool { // deltaSet type deltaSet struct { - deltas map[string]delta - size map[string]int64 - time map[string]time.Time - hash map[string]string - opt *Options - fs fs.Fs // base filesystem - msg string // filesystem name for logging - oldCount int // original number of files (for "excess deletes" check) - deleted int // number of deleted files (for "excess deletes" check) - foundSame bool // true if found at least one unchanged file - checkFiles bilib.Names + deltas map[string]delta + size map[string]int64 + time map[string]time.Time + hash map[string]string + opt *Options + fs fs.Fs // base filesystem + msg string // filesystem name for logging + oldCount int // original number of files (for "excess deletes" check) + deleted int // number of deleted files (for "excess deletes" check) + deletedBytes int64 + foundSame bool // true if found at least one unchanged file + checkFiles bilib.Names } func (ds *deltaSet) empty() bool { @@ -187,6 +188,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string, if !now.has(file) { b.indent(msg, file, Color(terminal.RedFg, "File was deleted")) ds.deleted++ + ds.deletedBytes += old.getSize(file) d |= deltaDeleted } else if !now.isDir(file) { // skip dirs here, as we only care if they are new/deleted, not newer/older @@ -543,20 +545,12 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result // excessDeletes checks whether number of deletes is within allowed range func (ds *deltaSet) excessDeletes() bool { - maxDelete := ds.opt.MaxDelete - maxRatio := float64(maxDelete) / 100.0 - curRatio := 0.0 - if ds.deleted > 0 && ds.oldCount > 0 { - curRatio = float64(ds.deleted) / float64(ds.oldCount) - } - - if curRatio <= maxRatio { + hit, msg := deleteBudgetExceeded(ds.opt, ds.oldCount, ds.deleted, ds.deletedBytes) + if !hit { return false } - - fs.Errorf("Safety abort", - "too many deletes (>%d%%, %d of %d) on %s %s. Run with --force if desired.", - maxDelete, ds.deleted, ds.oldCount, ds.msg, quotePath(bilib.FsPath(ds.fs))) + fs.Errorf("Safety abort", "%s on %s %s. Run with --force if desired.", + msg, ds.msg, quotePath(bilib.FsPath(ds.fs))) return true } diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go index d53e23483..2ba9bf90a 100644 --- a/cmd/bisync/operations.go +++ b/cmd/bisync/operations.go @@ -112,7 +112,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) { } // Produce a unique name for the sync operation - b.basePath = bilib.BasePath(ctx, b.workDir, b.fs1, b.fs2) + b.basePath = bilib.BasePath(ctx, b.workDir, b.fs1, b.fs2, opt.SessionName) b.listing1 = b.basePath + ".path1.lst" b.listing2 = b.basePath + ".path2.lst" b.newListing1 = b.listing1 + "-new" diff --git a/cmd/bisync/rc.go b/cmd/bisync/rc.go index c49d0eabb..763b831da 100644 --- a/cmd/bisync/rc.go +++ b/cmd/bisync/rc.go @@ -166,6 +166,22 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) { fs.Debugf("maxLock", "optional parameter is missing. using default value: %v", opt.MaxLock) } + if n, err := in.GetInt64("maxDeleteFiles"); err == nil { + opt.MaxDeleteFiles = int(n) + } else if rc.NotErrParamNotFound(err) { + return nil, err + } + if opt.SessionName, err = in.GetString("sessionName"); rc.NotErrParamNotFound(err) { + fs.Debugf("sessionName", "optional parameter is missing. using default value: %v", opt.SessionName) + } + if sz, err := in.GetString("maxDeleteSize"); err == nil && sz != "" { + if e := opt.MaxDeleteSize.Set(sz); e != nil { + return nil, rc.NewErrParamInvalid(e) + } + } else if rc.NotErrParamNotFound(err) { + return nil, err + } + fs1, err := rc.GetFsNamed(octx, in, "path1") if err != nil { return nil, err @@ -184,7 +200,7 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) { if opt.Workdir != "" { workDir, _ = filepath.Abs(opt.Workdir) } - basePath := bilib.BasePath(ctx, workDir, fs1, fs2) + basePath := bilib.BasePath(ctx, workDir, fs1, fs2, opt.SessionName) _, _ = log.Writer().Write(output) return rc.Params{