diff --git a/Makefile b/Makefile index 07f66379e..1e4127994 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,7 @@ MANUAL.txt: MANUAL.md commanddocs: rclone go generate ./lib/transform + go generate ./cmd/bisync -@rmdir -p '$$HOME/.config/rclone' XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/ @[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1) diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go index dd22e48c5..f1b3367bb 100644 --- a/cmd/bisync/cmd.go +++ b/cmd/bisync/cmd.go @@ -127,14 +127,14 @@ func init() { // and the Command line syntax section of docs/content/bisync.md (it doesn't update automatically) flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.", "") flags.FVarP(cmdFlags, &Opt.ResyncMode, "resync-mode", "", "During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.)", "") - flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, makeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "") - flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "") + flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, MakeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "") + flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, MakeHelp("Filename for --check-access (default: {CHECKFILE})"), "") flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "") flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "") flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "") flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "") flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "") - flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "") + flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, MakeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "") flags.StringVarP(cmdFlags, &Opt.BackupDir1, "backup-dir1", "", Opt.BackupDir1, "--backup-dir for Path1. Must be a non-overlapping path on the same remote.", "") flags.StringVarP(cmdFlags, &Opt.BackupDir2, "backup-dir2", "", Opt.BackupDir2, "--backup-dir for Path2. Must be a non-overlapping path on the same remote.", "") flags.StringVarP(cmdFlags, &Opt.DebugName, "debugname", "", Opt.DebugName, "Debug by tracking one file at various points throughout a bisync run (when -v or -vv)", "") @@ -153,6 +153,7 @@ func init() { flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "") _ = cmdFlags.MarkHidden("debugname") _ = cmdFlags.MarkHidden("localtime") + addRC() } // bisync command definition diff --git a/cmd/bisync/help.go b/cmd/bisync/help.go index a9699a848..84cd55967 100644 --- a/cmd/bisync/help.go +++ b/cmd/bisync/help.go @@ -1,65 +1,77 @@ -package bisync +//go:build none + +// Create the help text for the rc +// +// Run with go generate ./cmd/bisync (defined in rc.go) +package main import ( - "strconv" + "fmt" + "os" "strings" + + "github.com/muesli/reflow/wordwrap" + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/cmd/bisync" + "github.com/rclone/rclone/fs" + "github.com/spf13/pflag" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) -func makeHelp(help string) string { - replacer := strings.NewReplacer( - "|", "`", - "{MAXDELETE}", strconv.Itoa(DefaultMaxDelete), - "{CHECKFILE}", DefaultCheckFilename, - // "{WORKDIR}", DefaultWorkdir, - ) - return replacer.Replace(help) +// Output the help to stdout +func main() { + out := os.Stdout + if len(os.Args) > 1 { + var err error + out, err = os.Create(os.Args[1]) + if err != nil { + fs.Fatalf(nil, "Open output failed: %v", err) + } + defer out.Close() + } + fmt.Fprintf(out, "\n\n") + fmt.Fprint(out, RcHelp()) } -var shortHelp = `Perform bidirectional synchronization between two paths.` - -var rcHelp = makeHelp(`This takes the following parameters - -- path1 - a remote directory string e.g. |drive:path1| -- path2 - a remote directory string e.g. |drive:path2| -- dryRun - dry-run mode -- resync - performs the resync run -- checkAccess - abort if {CHECKFILE} files are not found on both filesystems -- checkFilename - file name for checkAccess (default: {CHECKFILE}) -- maxDelete - abort sync if percentage of deleted files is above - this threshold (default: {MAXDELETE}) -- force - Bypass maxDelete safety check and run the sync -- checkSync - |true| by default, |false| disables comparison of final listings, - |only| will skip sync, only compare listings from the last run -- createEmptySrcDirs - Sync creation and deletion of empty directories. - (Not compatible with --remove-empty-dirs) -- removeEmptyDirs - remove empty directories at the final cleanup step -- filtersFile - read filtering patterns from a file -- ignoreListingChecksum - Do not use checksums for listings -- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync. -- workdir - server directory for history files (default: |~/.cache/rclone/bisync|) -- backupdir1 - --backup-dir for Path1. Must be a non-overlapping path on the same remote. -- backupdir2 - --backup-dir for Path2. Must be a non-overlapping path on the same remote. -- noCleanup - retain working files +// RcHelp returns the rc help +func RcHelp() string { + return wordwrap.String(bisync.MakeHelp(`This takes the following parameters: +- path1 (required) - (string) a remote directory string e.g. ||drive:path1|| +- path2 (required) - (string) a remote directory string e.g. ||drive:path2|| +- dryRun - (bool) dry-run mode +`+GenerateParams()+` See [bisync command help](https://rclone.org/commands/rclone_bisync/) and [full bisync description](https://rclone.org/bisync/) -for more information.`) +for more information. +`), 76) +} -var longHelp = shortHelp + makeHelp(` +// example: "create-empty-src-dirs" -> "createEmptySrcDirs" +func toCamel(s string) string { + split := strings.Split(s, "-") + builder := strings.Builder{} + for i, word := range split { + if i == 0 { // first word always all lowercase + builder.WriteString(strings.ToLower(word)) + continue + } + builder.WriteString(cases.Title(language.AmericanEnglish).String(word)) + } + return builder.String() +} -[Bisync](https://rclone.org/bisync/) provides a -bidirectional cloud sync solution in rclone. -It retains the Path1 and Path2 filesystem listings from the prior run. -On each successive run it will: - -- list files on Path1 and Path2, and check for changes on each side. - Changes include |New|, |Newer|, |Older|, and |Deleted| files. -- Propagate changes on Path1 to Path2, and vice-versa. - -Bisync is considered an **advanced command**, so use with care. -Make sure you have read and understood the entire [manual](https://rclone.org/bisync) -(especially the [Limitations](https://rclone.org/bisync/#limitations) section) -before using, or data loss can result. Questions can be asked in the -[Rclone Forum](https://forum.rclone.org/). - -See [full bisync description](https://rclone.org/bisync/) for details.`) +// GenerateParams automatically generates the param list from commandDefinition.Flags +func GenerateParams() string { + builder := strings.Builder{} + fn := func(flag *pflag.Flag) { + if flag.Hidden { + return + } + builder.WriteString(fmt.Sprintf("- %s - (%s) %s \n", toCamel(flag.Name), flag.Value.Type(), flag.Usage)) + } + commandDefinition, _, _ := cmd.Root.Find([]string{"bisync"}) + commandDefinition.Flags().VisitAll(fn) + return builder.String() +} diff --git a/cmd/bisync/rc.go b/cmd/bisync/rc.go index 4e50614ae..0cb677910 100644 --- a/cmd/bisync/rc.go +++ b/cmd/bisync/rc.go @@ -1,10 +1,14 @@ +//go:generate go run help.go rc.md package bisync import ( "context" + _ "embed" "errors" "log" "path/filepath" + "strconv" + "strings" "github.com/rclone/rclone/cmd/bisync/bilib" "github.com/rclone/rclone/fs" @@ -12,7 +16,7 @@ import ( "github.com/rclone/rclone/fs/rc" ) -func init() { +func addRC() { rc.Add(rc.Call{ Path: "sync/bisync", AuthRequired: true, @@ -22,6 +26,42 @@ func init() { }) } +//go:embed rc.md +var rcHelp string + +var shortHelp = `Perform bidirectional synchronization between two paths.` + +var longHelp = shortHelp + MakeHelp(` + +[Bisync](https://rclone.org/bisync/) provides a +bidirectional cloud sync solution in rclone. +It retains the Path1 and Path2 filesystem listings from the prior run. +On each successive run it will: + +- list files on Path1 and Path2, and check for changes on each side. + Changes include ||New||, ||Newer||, ||Older||, and ||Deleted|| files. +- Propagate changes on Path1 to Path2, and vice-versa. + +Bisync is considered an **advanced command**, so use with care. +Make sure you have read and understood the entire [manual](https://rclone.org/bisync) +(especially the [Limitations](https://rclone.org/bisync/#limitations) section) +before using, or data loss can result. Questions can be asked in the +[Rclone Forum](https://forum.rclone.org/). + +See [full bisync description](https://rclone.org/bisync/) for details. +`) + +// MakeHelp replaces some dynamic variables for the help docs +func MakeHelp(help string) string { + replacer := strings.NewReplacer( + "||", "`", + "{MAXDELETE}", strconv.Itoa(DefaultMaxDelete), + "{CHECKFILE}", DefaultCheckFilename, + "{WORKDIR}", DefaultWorkdir, + ) + return replacer.Replace(help) +} + func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) { opt := &Options{} octx, ci := fs.AddConfig(ctx) diff --git a/cmd/bisync/rc.md b/cmd/bisync/rc.md new file mode 100644 index 000000000..271d816ea --- /dev/null +++ b/cmd/bisync/rc.md @@ -0,0 +1,61 @@ + + +This takes the following parameters: + +- path1 (required) - (string) a remote directory string e.g. `drive:path1` +- path2 (required) - (string) a remote directory string e.g. `drive:path2` +- dryRun - (bool) dry-run mode +- backupDir1 - (string) --backup-dir for Path1. Must be a non-overlapping path on +the same remote. +- backupDir2 - (string) --backup-dir for Path2. Must be a non-overlapping path on +the same remote. +- checkAccess - (bool) Ensure expected RCLONE_TEST files are found on both +Path1 and Path2 filesystems, else abort. +- checkFilename - (string) Filename for --check-access (default: RCLONE_TEST) +- checkSync - (string) Controls comparison of final listings: true|false|only +(default: true) +- compare - (string) Comma-separated list of bisync-specific compare options ex. +'size,modtime,checksum' (default: 'size,modtime') +- conflictLoser - (ConflictLoserAction) Action to take on the loser of a sync +conflict (when there is a winner) or on both files (when there is no +winner): , num, pathname, delete (default: num) +- conflictResolve - (string) Automatically resolve conflicts by preferring the +version that is: none, path1, path2, newer, older, larger, smaller (default: +none) +- conflictSuffix - (string) Suffix to use when renaming a --conflict-loser. Can +be either one string or two comma-separated strings to assign different +suffixes to Path1/Path2. (default: 'conflict') +- createEmptySrcDirs - (bool) Sync creation and deletion of empty directories. +(Not compatible with --remove-empty-dirs) +- downloadHash - (bool) Compute hash by downloading when otherwise +unavailable. (warning: may be slow and use lots of data!) +- filtersFile - (string) Read filtering patterns from a file +- force - (bool) Bypass --max-delete safety check and run the sync. Consider +using with --verbose +- ignoreListingChecksum - (bool) Do not use checksums for listings (add --ignore- +checksum to additionally skip post-copy checksum checks) +- maxLock - (Duration) Consider lock files older than this to be expired +(default: 0 (never expire)) (minimum: 2m) +- noCleanup - (bool) Retain working files (useful for troubleshooting and +testing). +- noSlowHash - (bool) Ignore listing checksums only on backends where they are +slow +- recover - (bool) Automatically recover from interruptions without requiring -- +resync. +- removeEmptyDirs - (bool) Remove ALL empty directories at the final cleanup +step. +- resilient - (bool) Allow future runs to retry after certain less-serious +errors, instead of requiring --resync. +- resync - (bool) Performs the resync run. Equivalent to --resync-mode path1. +Consider using --verbose or --dry-run first. +- resyncMode - (string) During resync, prefer the version that is: path1, +path2, newer, older, larger, smaller (default: path1 if --resync, otherwise +none for no resync.) +- slowHashSyncOnly - (bool) Ignore slow checksums for listings and deltas, but +still consider them during sync calls. +- workdir - (string) Use custom working dir - useful for testing. (default: +~/.cache/rclone/bisync) + +See [bisync command help](https://rclone.org/commands/rclone_bisync/) +and [full bisync description](https://rclone.org/bisync/) +for more information. diff --git a/go.mod b/go.mod index 5cbd60bf2..61b5edd6b 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/minio/minio-go/v7 v7.0.98 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/sys/mountinfo v0.7.2 + github.com/muesli/reflow v0.3.0 github.com/ncw/swift/v2 v2.0.5 github.com/oracle/oci-go-sdk/v65 v65.108.2 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index e969577c3..0515fa7e4 100644 --- a/go.sum +++ b/go.sum @@ -499,6 +499,7 @@ github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stg github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= @@ -523,6 +524,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8= @@ -598,6 +601,8 @@ github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGr github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI= github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=