rc: add sync report parameters to sync/sync, sync/copy and sync/move - fixes #9846

The sync report flags (--combined, --missing-on-src, --missing-on-dst,
--match, --differ, --error and --dest-after) were only wired up in the
CLI commands so there was no way to get these reports over the rc or
from librclone.

This adds boolean parameters of the same names as operations/check
(combined, missingOnSrc, missingOnDst, match, differ, error and
destAfter) to sync/sync, sync/copy and sync/move. Each requested
report is returned as an array of strings in the output, just as
operations/check does. All default to off so existing callers see no
change in the output.

To share the code between the CLI and the rc the report writer helper
from operations/check is exported as operations.RcReportWriter, the
lsf defaults for --dest-after are moved into
operations.NewSyncLoggerOpt and the listing setup and --no-traverse
warnings from operationsflags.ConfigureLoggers into LoggerOpt.Init.

fs/operations/logger.go

@@ -302,12 +302,54 @@ func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err err
 	return winner
 }
 
+// NewSyncLoggerOpt returns a LoggerOpt with no report writers set and
+// the default listing options for the DestAfter report.
+func NewSyncLoggerOpt() LoggerOpt {
+	return LoggerOpt{
+		Format:    "p",
+		Separator: ";",
+		DirSlash:  true,
+		HashType:  hash.MD5,
+		FilesOnly: true,
+	}
+}
+
+// Init prepares opt for logging a sync to fdst once its report writers
+// have been set.
+//
+// It configures the DestAfter listing from the lsf options and warns
+// about reports which --no-traverse prevents being complete. cmdFlags
+// may be nil if the options did not come from command line flags.
+func (opt *LoggerOpt) Init(ctx context.Context, fdst fs.Fs, cmdFlags *pflag.FlagSet) {
+	if opt.TimeFormat == "max" {
+		opt.TimeFormat = FormatForLSFPrecision(fdst.Precision())
+	}
+	opt.SetListFormat(ctx, cmdFlags)
+	opt.NewListJSON(ctx, fdst, "")
+
+	ci := fs.GetConfig(ctx)
+	if ci.NoTraverse && opt.Combined != nil {
+		fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
+	}
+	if ci.NoTraverse && opt.MissingOnSrc != nil {
+		fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
+	}
+	if ci.NoTraverse && opt.DestAfter != nil {
+		fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
+	}
+}
+
 // SetListFormat sets opt.ListFormat for destAfter
+//
+// cmdFlags may be nil if the options did not come from command line flags.
 // TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from
 func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) {
 	// Work out if the separatorFlag was supplied or not
-	separatorFlag := cmdFlags.Lookup("separator")
-	separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed
+	separatorFlagSupplied := false
+	if cmdFlags != nil {
+		separatorFlag := cmdFlags.Lookup("separator")
+		separatorFlagSupplied = separatorFlag != nil && separatorFlag.Changed
+	}
 	// Default the separator to , if using CSV
 	if opt.Csv && !separatorFlagSupplied {
 		opt.Separator = ","

fs/operations/operationsflags/operationsflags.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -61,16 +60,17 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt
 	flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync")
 
 	// lsf flags for destAfter
-	flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")
-	flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "")
-	flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")
-	flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")
-	opt.HashType = hash.MD5
+	def := operations.NewSyncLoggerOpt()
+	flags.StringVarP(cmdFlags, &opt.Format, "format", "F", def.Format, "Output format - see lsf help for details", "Sync")
+	flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", def.TimeFormat, "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "")
+	flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", def.Separator, "Separator for the items in the format", "Sync")
+	flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", def.DirSlash, "Append a slash to directory names", "Sync")
+	opt.HashType = def.HashType
 	flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")
-	flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")
-	flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")
-	flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")
-	flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
+	flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", def.FilesOnly, "Only list files", "Sync")
+	flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", def.DirsOnly, "Only list directories", "Sync")
+	flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", def.Csv, "Output in CSV format", "Sync")
+	flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", def.Absolute, "Put a leading / in front of path names", "Sync")
 	// flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
 }
 
@@ -78,12 +78,6 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt
 func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) {
 	closers := []io.Closer{}
 
-	if opt.TimeFormat == "max" {
-		opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
-	}
-	opt.SetListFormat(ctx, command.Flags())
-	opt.NewListJSON(ctx, fdst, "")
-
 	open := func(name string, pout *io.Writer) error {
 		if name == "" {
 			return nil
@@ -132,16 +126,7 @@ func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, o
 		}
 	}
 
-	ci := fs.GetConfig(ctx)
-	if ci.NoTraverse && opt.Combined != nil {
-		fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
-	}
-	if ci.NoTraverse && opt.MissingOnSrc != nil {
-		fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
-	}
-	if ci.NoTraverse && opt.DestAfter != nil {
-		fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
-	}
+	opt.Init(ctx, fdst, command.Flags())
 
 	return close, nil
 }

fs/operations/rc.go

@@ -764,6 +764,25 @@ func (s stringWriter) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+// RcReportWriter returns a writer for the report called name.
+//
+// The report is enabled if in[name] is true, or if in[name] is absent
+// and Default is true. When enabled, each line written to the returned
+// writer is collected as a string in out[name], otherwise nil is
+// returned to disable the report.
+func RcReportWriter(in rc.Params, out rc.Params, name string, Default bool) io.Writer {
+	active, err := in.GetBool(name)
+	if err != nil {
+		active = Default
+	}
+	if !active {
+		return nil
+	}
+	result := []string{}
+	out[name] = &result
+	return stringWriter{&result}
+}
+
 // Check two directories
 func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	srcFs, err := rc.GetFsNamed(ctx, in, "srcFs")
@@ -822,26 +841,12 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	}
 
 	out = rc.Params{}
-
-	getOutput := func(name string, Default bool) io.Writer {
-		active, err := in.GetBool(name)
-		if err != nil {
-			active = Default
-		}
-		if !active {
-			return nil
-		}
-		result := []string{}
-		out[name] = &result
-		return stringWriter{&result}
-	}
-
-	opt.Combined = getOutput("combined", false)
-	opt.MissingOnSrc = getOutput("missingOnSrc", true)
-	opt.MissingOnDst = getOutput("missingOnDst", true)
-	opt.Match = getOutput("match", false)
-	opt.Differ = getOutput("differ", true)
-	opt.Error = getOutput("error", true)
+	opt.Combined = RcReportWriter(in, out, "combined", false)
+	opt.MissingOnSrc = RcReportWriter(in, out, "missingOnSrc", true)
+	opt.MissingOnDst = RcReportWriter(in, out, "missingOnDst", true)
+	opt.Match = RcReportWriter(in, out, "match", false)
+	opt.Differ = RcReportWriter(in, out, "differ", true)
+	opt.Error = RcReportWriter(in, out, "error", true)
 
 	if checkFileHash != "" {
 		out["hashType"] = checkFileHashType.String()

fs/sync/rc.go

@@ -3,6 +3,8 @@ package sync
 import (
 	"context"
 
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/rc"
 )
 
@@ -23,7 +25,27 @@ func init() {
 - srcFs - a remote name string e.g. "drive:src" for the source
 - dstFs - a remote name string e.g. "drive:dst" for the destination
 - createEmptySrcDirs - create empty src directories on destination if set
-` + moveHelp + `
+` + moveHelp + `- combined - make a combined report of changes (default false)
+- missingOnSrc - report all files missing from the source (default false)
+- missingOnDst - report all files missing from the destination (default false)
+- match - report all matching files (default false)
+- differ - report all non-matching files (default false)
+- error - report all files with errors (hashing or reading) (default false)
+- destAfter - report all files that exist on the destination post-` + name + ` (default false)
+
+Returns:
+
+- combined - array of strings of combined report of changes
+- missingOnSrc - array of strings of all files missing from the source
+- missingOnDst - array of strings of all files missing from the destination
+- match - array of strings of all matching files
+- differ - array of strings of all non-matching files
+- error - array of strings of all files with errors (hashing or reading)
+- destAfter - array of strings of all files that exist on the destination post-` + name + `
+
+Each report is only returned if its parameter is set to true. If the
+operation fails the reports are only available if it was run with
+` + "`_async`" + `, as part of the job output.
 
 See the [` + name + `](/commands/rclone_` + name + `/) command for more information on the above.`,
 		})
@@ -44,17 +66,38 @@ func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Para
 	if rc.NotErrParamNotFound(err) {
 		return nil, err
 	}
+	ctx, out = rcLogger(ctx, in, dstFs)
 	switch name {
 	case "sync":
-		return nil, Sync(ctx, dstFs, srcFs, createEmptySrcDirs)
+		return out, Sync(ctx, dstFs, srcFs, createEmptySrcDirs)
 	case "copy":
-		return nil, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs)
+		return out, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs)
 	case "move":
 		deleteEmptySrcDirs, err := in.GetBool("deleteEmptySrcDirs")
 		if rc.NotErrParamNotFound(err) {
 			return nil, err
 		}
-		return nil, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)
+		return out, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs)
 	}
 	panic("unknown rcSyncCopyMove type")
 }
+
+// rcLogger returns ctx with a sync logger which collects the reports
+// requested in in into the returned out, which is empty if none were.
+func rcLogger(ctx context.Context, in rc.Params, fdst fs.Fs) (context.Context, rc.Params) {
+	out := rc.Params{}
+	opt := operations.NewSyncLoggerOpt()
+	opt.Combined = operations.RcReportWriter(in, out, "combined", false)
+	opt.MissingOnSrc = operations.RcReportWriter(in, out, "missingOnSrc", false)
+	opt.MissingOnDst = operations.RcReportWriter(in, out, "missingOnDst", false)
+	opt.Match = operations.RcReportWriter(in, out, "match", false)
+	opt.Differ = operations.RcReportWriter(in, out, "differ", false)
+	opt.Error = operations.RcReportWriter(in, out, "error", false)
+	opt.DestAfter = operations.RcReportWriter(in, out, "destAfter", false)
+	if len(out) == 0 {
+		return ctx, out
+	}
+	opt.LoggerFn = operations.NewDefaultLoggerFn(&opt)
+	opt.Init(ctx, fdst, nil)
+	return operations.WithSyncLogger(ctx, opt), out
+}

fs/sync/rc_test.go

@@ -2,6 +2,7 @@ package sync
 
 import (
 	"context"
+	"sort"
 	"testing"
 
 	"github.com/rclone/rclone/fs/cache"
@@ -41,7 +42,7 @@ func TestRcCopy(t *testing.T) {
 	}
 	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
-	assert.Equal(t, rc.Params(nil), out)
+	assert.Equal(t, rc.Params{}, out)
 
 	r.CheckLocalItems(t, file1, file2)
 	r.CheckRemoteItems(t, file1, file2, file3)
@@ -65,7 +66,7 @@ func TestRcMove(t *testing.T) {
 	}
 	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
-	assert.Equal(t, rc.Params(nil), out)
+	assert.Equal(t, rc.Params{}, out)
 
 	r.CheckLocalItems(t)
 	r.CheckRemoteItems(t, file1, file2, file3)
@@ -89,8 +90,53 @@ func TestRcSync(t *testing.T) {
 	}
 	out, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
-	assert.Equal(t, rc.Params(nil), out)
+	assert.Equal(t, rc.Params{}, out)
 
 	r.CheckLocalItems(t, file1, file2)
 	r.CheckRemoteItems(t, file1, file2)
 }
+
+// sync/copy: check the reports are returned when requested
+func TestRcCopyReports(t *testing.T) {
+	r, call := rcNewRun(t, "sync/copy")
+	r.Mkdir(context.Background(), r.Fremote)
+
+	file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
+	file2 := r.WriteFile("subdir/file2", "file2 contents", t2)
+	file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3)
+	file4 := r.WriteFile("file4", "file4 contents", t1)
+	file4dst := r.WriteObject(context.Background(), "file4", "different contents", t1)
+
+	r.CheckLocalItems(t, file1, file2, file4)
+	r.CheckRemoteItems(t, file1, file3, file4dst)
+
+	in := rc.Params{
+		"srcFs":        r.LocalName,
+		"dstFs":        r.FremoteName,
+		"combined":     true,
+		"missingOnSrc": true,
+		"missingOnDst": true,
+		"match":        true,
+		"differ":       true,
+		"destAfter":    true,
+	}
+	out, err := call.Fn(context.Background(), in)
+	require.NoError(t, err)
+
+	sorted := func(name string) []string {
+		result, ok := out[name].(*[]string)
+		require.True(t, ok, name)
+		sort.Strings(*result)
+		return *result
+	}
+	assert.Equal(t, []string{"* file4", "+ subdir/file2", "- subdir/subsubdir/file3", "= file1"}, sorted("combined"))
+	assert.Equal(t, []string{"subdir/subsubdir/file3"}, sorted("missingOnSrc"))
+	assert.Equal(t, []string{"subdir/file2"}, sorted("missingOnDst"))
+	assert.Equal(t, []string{"file1"}, sorted("match"))
+	assert.Equal(t, []string{"file4"}, sorted("differ"))
+	assert.Equal(t, []string{"file1", "file4", "subdir/file2", "subdir/subsubdir/file3"}, sorted("destAfter"))
+	assert.NotContains(t, out, "error")
+
+	r.CheckLocalItems(t, file1, file2, file4)
+	r.CheckRemoteItems(t, file1, file2, file3, file4)
+}