From af382608c4a714ee7254ce941ac24f8c9e034c40 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 1 Sep 2026 11:57:12 +0100 Subject: [PATCH] rc: add sync report parameters to sync/sync, sync/copy and sync/move - fixes #9846 The sync report flags (--combined, --missing-on-src, --missing-on-dst, --match, --differ, --error and --dest-after) were only wired up in the CLI commands so there was no way to get these reports over the rc or from librclone. This adds boolean parameters of the same names as operations/check (combined, missingOnSrc, missingOnDst, match, differ, error and destAfter) to sync/sync, sync/copy and sync/move. Each requested report is returned as an array of strings in the output, just as operations/check does. All default to off so existing callers see no change in the output. To share the code between the CLI and the rc the report writer helper from operations/check is exported as operations.RcReportWriter, the lsf defaults for --dest-after are moved into operations.NewSyncLoggerOpt and the listing setup and --no-traverse warnings from operationsflags.ConfigureLoggers into LoggerOpt.Init. --- fs/operations/logger.go | 46 +++++++++++++++- .../operationsflags/operationsflags.go | 37 ++++--------- fs/operations/rc.go | 45 +++++++++------- fs/sync/rc.go | 51 ++++++++++++++++-- fs/sync/rc_test.go | 52 +++++++++++++++++-- 5 files changed, 176 insertions(+), 55 deletions(-) diff --git a/fs/operations/logger.go b/fs/operations/logger.go index 0d482206f..f9fb41f0e 100644 --- a/fs/operations/logger.go +++ b/fs/operations/logger.go @@ -302,12 +302,54 @@ func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err err return winner } +// NewSyncLoggerOpt returns a LoggerOpt with no report writers set and +// the default listing options for the DestAfter report. +func NewSyncLoggerOpt() LoggerOpt { + return LoggerOpt{ + Format: "p", + Separator: ";", + DirSlash: true, + HashType: hash.MD5, + FilesOnly: true, + } +} + +// Init prepares opt for logging a sync to fdst once its report writers +// have been set. +// +// It configures the DestAfter listing from the lsf options and warns +// about reports which --no-traverse prevents being complete. cmdFlags +// may be nil if the options did not come from command line flags. +func (opt *LoggerOpt) Init(ctx context.Context, fdst fs.Fs, cmdFlags *pflag.FlagSet) { + if opt.TimeFormat == "max" { + opt.TimeFormat = FormatForLSFPrecision(fdst.Precision()) + } + opt.SetListFormat(ctx, cmdFlags) + opt.NewListJSON(ctx, fdst, "") + + ci := fs.GetConfig(ctx) + if ci.NoTraverse && opt.Combined != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n") + } + if ci.NoTraverse && opt.MissingOnSrc != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n") + } + if ci.NoTraverse && opt.DestAfter != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n") + } +} + // SetListFormat sets opt.ListFormat for destAfter +// +// cmdFlags may be nil if the options did not come from command line flags. // TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) { // Work out if the separatorFlag was supplied or not - separatorFlag := cmdFlags.Lookup("separator") - separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed + separatorFlagSupplied := false + if cmdFlags != nil { + separatorFlag := cmdFlags.Lookup("separator") + separatorFlagSupplied = separatorFlag != nil && separatorFlag.Changed + } // Default the separator to , if using CSV if opt.Csv && !separatorFlagSupplied { opt.Separator = "," diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go index 054ed96bc..e2cfda034 100644 --- a/fs/operations/operationsflags/operationsflags.go +++ b/fs/operations/operationsflags/operationsflags.go @@ -11,7 +11,6 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" - "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -61,16 +60,17 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync") // lsf flags for destAfter - flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync") - flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "") - flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync") - flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync") - opt.HashType = hash.MD5 + def := operations.NewSyncLoggerOpt() + flags.StringVarP(cmdFlags, &opt.Format, "format", "F", def.Format, "Output format - see lsf help for details", "Sync") + flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", def.TimeFormat, "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "") + flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", def.Separator, "Separator for the items in the format", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", def.DirSlash, "Append a slash to directory names", "Sync") + opt.HashType = def.HashType flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync") - flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync") - flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync") - flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync") - flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync") + flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", def.FilesOnly, "Only list files", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", def.DirsOnly, "Only list directories", "Sync") + flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", def.Csv, "Output in CSV format", "Sync") + flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", def.Absolute, "Put a leading / in front of path names", "Sync") // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "") } @@ -78,12 +78,6 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) { closers := []io.Closer{} - if opt.TimeFormat == "max" { - opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision()) - } - opt.SetListFormat(ctx, command.Flags()) - opt.NewListJSON(ctx, fdst, "") - open := func(name string, pout *io.Writer) error { if name == "" { return nil @@ -132,16 +126,7 @@ func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, o } } - ci := fs.GetConfig(ctx) - if ci.NoTraverse && opt.Combined != nil { - fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n") - } - if ci.NoTraverse && opt.MissingOnSrc != nil { - fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n") - } - if ci.NoTraverse && opt.DestAfter != nil { - fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n") - } + opt.Init(ctx, fdst, command.Flags()) return close, nil } diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 27a8d5c9e..4c0828127 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -764,6 +764,25 @@ func (s stringWriter) Write(p []byte) (n int, err error) { return len(p), nil } +// RcReportWriter returns a writer for the report called name. +// +// The report is enabled if in[name] is true, or if in[name] is absent +// and Default is true. When enabled, each line written to the returned +// writer is collected as a string in out[name], otherwise nil is +// returned to disable the report. +func RcReportWriter(in rc.Params, out rc.Params, name string, Default bool) io.Writer { + active, err := in.GetBool(name) + if err != nil { + active = Default + } + if !active { + return nil + } + result := []string{} + out[name] = &result + return stringWriter{&result} +} + // Check two directories func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) { srcFs, err := rc.GetFsNamed(ctx, in, "srcFs") @@ -822,26 +841,12 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) { } out = rc.Params{} - - getOutput := func(name string, Default bool) io.Writer { - active, err := in.GetBool(name) - if err != nil { - active = Default - } - if !active { - return nil - } - result := []string{} - out[name] = &result - return stringWriter{&result} - } - - opt.Combined = getOutput("combined", false) - opt.MissingOnSrc = getOutput("missingOnSrc", true) - opt.MissingOnDst = getOutput("missingOnDst", true) - opt.Match = getOutput("match", false) - opt.Differ = getOutput("differ", true) - opt.Error = getOutput("error", true) + opt.Combined = RcReportWriter(in, out, "combined", false) + opt.MissingOnSrc = RcReportWriter(in, out, "missingOnSrc", true) + opt.MissingOnDst = RcReportWriter(in, out, "missingOnDst", true) + opt.Match = RcReportWriter(in, out, "match", false) + opt.Differ = RcReportWriter(in, out, "differ", true) + opt.Error = RcReportWriter(in, out, "error", true) if checkFileHash != "" { out["hashType"] = checkFileHashType.String() diff --git a/fs/sync/rc.go b/fs/sync/rc.go index 52a2626e8..fe2b67021 100644 --- a/fs/sync/rc.go +++ b/fs/sync/rc.go @@ -3,6 +3,8 @@ package sync import ( "context" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/rc" ) @@ -23,7 +25,27 @@ func init() { - srcFs - a remote name string e.g. "drive:src" for the source - dstFs - a remote name string e.g. "drive:dst" for the destination - createEmptySrcDirs - create empty src directories on destination if set -` + moveHelp + ` +` + moveHelp + `- combined - make a combined report of changes (default false) +- missingOnSrc - report all files missing from the source (default false) +- missingOnDst - report all files missing from the destination (default false) +- match - report all matching files (default false) +- differ - report all non-matching files (default false) +- error - report all files with errors (hashing or reading) (default false) +- destAfter - report all files that exist on the destination post-` + name + ` (default false) + +Returns: + +- combined - array of strings of combined report of changes +- missingOnSrc - array of strings of all files missing from the source +- missingOnDst - array of strings of all files missing from the destination +- match - array of strings of all matching files +- differ - array of strings of all non-matching files +- error - array of strings of all files with errors (hashing or reading) +- destAfter - array of strings of all files that exist on the destination post-` + name + ` + +Each report is only returned if its parameter is set to true. If the +operation fails the reports are only available if it was run with +` + "`_async`" + `, as part of the job output. See the [` + name + `](/commands/rclone_` + name + `/) command for more information on the above.`, }) @@ -44,17 +66,38 @@ func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Para if rc.NotErrParamNotFound(err) { return nil, err } + ctx, out = rcLogger(ctx, in, dstFs) switch name { case "sync": - return nil, Sync(ctx, dstFs, srcFs, createEmptySrcDirs) + return out, Sync(ctx, dstFs, srcFs, createEmptySrcDirs) case "copy": - return nil, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs) + return out, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs) case "move": deleteEmptySrcDirs, err := in.GetBool("deleteEmptySrcDirs") if rc.NotErrParamNotFound(err) { return nil, err } - return nil, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs) + return out, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs) } panic("unknown rcSyncCopyMove type") } + +// rcLogger returns ctx with a sync logger which collects the reports +// requested in in into the returned out, which is empty if none were. +func rcLogger(ctx context.Context, in rc.Params, fdst fs.Fs) (context.Context, rc.Params) { + out := rc.Params{} + opt := operations.NewSyncLoggerOpt() + opt.Combined = operations.RcReportWriter(in, out, "combined", false) + opt.MissingOnSrc = operations.RcReportWriter(in, out, "missingOnSrc", false) + opt.MissingOnDst = operations.RcReportWriter(in, out, "missingOnDst", false) + opt.Match = operations.RcReportWriter(in, out, "match", false) + opt.Differ = operations.RcReportWriter(in, out, "differ", false) + opt.Error = operations.RcReportWriter(in, out, "error", false) + opt.DestAfter = operations.RcReportWriter(in, out, "destAfter", false) + if len(out) == 0 { + return ctx, out + } + opt.LoggerFn = operations.NewDefaultLoggerFn(&opt) + opt.Init(ctx, fdst, nil) + return operations.WithSyncLogger(ctx, opt), out +} diff --git a/fs/sync/rc_test.go b/fs/sync/rc_test.go index ad5145633..aeaae4bcd 100644 --- a/fs/sync/rc_test.go +++ b/fs/sync/rc_test.go @@ -2,6 +2,7 @@ package sync import ( "context" + "sort" "testing" "github.com/rclone/rclone/fs/cache" @@ -41,7 +42,7 @@ func TestRcCopy(t *testing.T) { } out, err := call.Fn(context.Background(), in) require.NoError(t, err) - assert.Equal(t, rc.Params(nil), out) + assert.Equal(t, rc.Params{}, out) r.CheckLocalItems(t, file1, file2) r.CheckRemoteItems(t, file1, file2, file3) @@ -65,7 +66,7 @@ func TestRcMove(t *testing.T) { } out, err := call.Fn(context.Background(), in) require.NoError(t, err) - assert.Equal(t, rc.Params(nil), out) + assert.Equal(t, rc.Params{}, out) r.CheckLocalItems(t) r.CheckRemoteItems(t, file1, file2, file3) @@ -89,8 +90,53 @@ func TestRcSync(t *testing.T) { } out, err := call.Fn(context.Background(), in) require.NoError(t, err) - assert.Equal(t, rc.Params(nil), out) + assert.Equal(t, rc.Params{}, out) r.CheckLocalItems(t, file1, file2) r.CheckRemoteItems(t, file1, file2) } + +// sync/copy: check the reports are returned when requested +func TestRcCopyReports(t *testing.T) { + r, call := rcNewRun(t, "sync/copy") + r.Mkdir(context.Background(), r.Fremote) + + file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1) + file2 := r.WriteFile("subdir/file2", "file2 contents", t2) + file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3) + file4 := r.WriteFile("file4", "file4 contents", t1) + file4dst := r.WriteObject(context.Background(), "file4", "different contents", t1) + + r.CheckLocalItems(t, file1, file2, file4) + r.CheckRemoteItems(t, file1, file3, file4dst) + + in := rc.Params{ + "srcFs": r.LocalName, + "dstFs": r.FremoteName, + "combined": true, + "missingOnSrc": true, + "missingOnDst": true, + "match": true, + "differ": true, + "destAfter": true, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + + sorted := func(name string) []string { + result, ok := out[name].(*[]string) + require.True(t, ok, name) + sort.Strings(*result) + return *result + } + assert.Equal(t, []string{"* file4", "+ subdir/file2", "- subdir/subsubdir/file3", "= file1"}, sorted("combined")) + assert.Equal(t, []string{"subdir/subsubdir/file3"}, sorted("missingOnSrc")) + assert.Equal(t, []string{"subdir/file2"}, sorted("missingOnDst")) + assert.Equal(t, []string{"file1"}, sorted("match")) + assert.Equal(t, []string{"file4"}, sorted("differ")) + assert.Equal(t, []string{"file1", "file4", "subdir/file2", "subdir/subsubdir/file3"}, sorted("destAfter")) + assert.NotContains(t, out, "error") + + r.CheckLocalItems(t, file1, file2, file4) + r.CheckRemoteItems(t, file1, file2, file3, file4) +}