diff --git a/fs/operations/logger.go b/fs/operations/logger.go index 0d482206f..f9fb41f0e 100644 --- a/fs/operations/logger.go +++ b/fs/operations/logger.go @@ -302,12 +302,54 @@ func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err err return winner } +// NewSyncLoggerOpt returns a LoggerOpt with no report writers set and +// the default listing options for the DestAfter report. +func NewSyncLoggerOpt() LoggerOpt { + return LoggerOpt{ + Format: "p", + Separator: ";", + DirSlash: true, + HashType: hash.MD5, + FilesOnly: true, + } +} + +// Init prepares opt for logging a sync to fdst once its report writers +// have been set. +// +// It configures the DestAfter listing from the lsf options and warns +// about reports which --no-traverse prevents being complete. cmdFlags +// may be nil if the options did not come from command line flags. +func (opt *LoggerOpt) Init(ctx context.Context, fdst fs.Fs, cmdFlags *pflag.FlagSet) { + if opt.TimeFormat == "max" { + opt.TimeFormat = FormatForLSFPrecision(fdst.Precision()) + } + opt.SetListFormat(ctx, cmdFlags) + opt.NewListJSON(ctx, fdst, "") + + ci := fs.GetConfig(ctx) + if ci.NoTraverse && opt.Combined != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n") + } + if ci.NoTraverse && opt.MissingOnSrc != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n") + } + if ci.NoTraverse && opt.DestAfter != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n") + } +} + // SetListFormat sets opt.ListFormat for destAfter +// +// cmdFlags may be nil if the options did not come from command line flags. // TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) { // Work out if the separatorFlag was supplied or not - separatorFlag := cmdFlags.Lookup("separator") - separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed + separatorFlagSupplied := false + if cmdFlags != nil { + separatorFlag := cmdFlags.Lookup("separator") + separatorFlagSupplied = separatorFlag != nil && separatorFlag.Changed + } // Default the separator to , if using CSV if opt.Csv && !separatorFlagSupplied { opt.Separator = "," diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go index 054ed96bc..e2cfda034 100644 --- a/fs/operations/operationsflags/operationsflags.go +++ b/fs/operations/operationsflags/operationsflags.go @@ -11,7 +11,6 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" - "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -61,16 +60,17 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync") // lsf flags for destAfter - flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync") - flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "") - flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync") - flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync") - opt.HashType = hash.MD5 + def := operations.NewSyncLoggerOpt() + flags.StringVarP(cmdFlags, &opt.Format, "format", "F", def.Format, "Output format - see lsf help for details", "Sync") + flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", def.TimeFormat, "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "") + flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", def.Separator, "Separator for the items in the format", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", def.DirSlash, "Append a slash to directory names", "Sync") + opt.HashType = def.HashType flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync") - flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync") - flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync") - flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync") - flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync") + flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", def.FilesOnly, "Only list files", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", def.DirsOnly, "Only list directories", "Sync") + flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", def.Csv, "Output in CSV format", "Sync") + flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", def.Absolute, "Put a leading / in front of path names", "Sync") // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "") } @@ -78,12 +78,6 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) { closers := []io.Closer{} - if opt.TimeFormat == "max" { - opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision()) - } - opt.SetListFormat(ctx, command.Flags()) - opt.NewListJSON(ctx, fdst, "") - open := func(name string, pout *io.Writer) error { if name == "" { return nil @@ -132,16 +126,7 @@ func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, o } } - ci := fs.GetConfig(ctx) - if ci.NoTraverse && opt.Combined != nil { - fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n") - } - if ci.NoTraverse && opt.MissingOnSrc != nil { - fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n") - } - if ci.NoTraverse && opt.DestAfter != nil { - fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n") - } + opt.Init(ctx, fdst, command.Flags()) return close, nil } diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 27a8d5c9e..4c0828127 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -764,6 +764,25 @@ func (s stringWriter) Write(p []byte) (n int, err error) { return len(p), nil } +// RcReportWriter returns a writer for the report called name. +// +// The report is enabled if in[name] is true, or if in[name] is absent +// and Default is true. When enabled, each line written to the returned +// writer is collected as a string in out[name], otherwise nil is +// returned to disable the report. +func RcReportWriter(in rc.Params, out rc.Params, name string, Default bool) io.Writer { + active, err := in.GetBool(name) + if err != nil { + active = Default + } + if !active { + return nil + } + result := []string{} + out[name] = &result + return stringWriter{&result} +} + // Check two directories func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) { srcFs, err := rc.GetFsNamed(ctx, in, "srcFs") @@ -822,26 +841,12 @@ func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) { } out = rc.Params{} - - getOutput := func(name string, Default bool) io.Writer { - active, err := in.GetBool(name) - if err != nil { - active = Default - } - if !active { - return nil - } - result := []string{} - out[name] = &result - return stringWriter{&result} - } - - opt.Combined = getOutput("combined", false) - opt.MissingOnSrc = getOutput("missingOnSrc", true) - opt.MissingOnDst = getOutput("missingOnDst", true) - opt.Match = getOutput("match", false) - opt.Differ = getOutput("differ", true) - opt.Error = getOutput("error", true) + opt.Combined = RcReportWriter(in, out, "combined", false) + opt.MissingOnSrc = RcReportWriter(in, out, "missingOnSrc", true) + opt.MissingOnDst = RcReportWriter(in, out, "missingOnDst", true) + opt.Match = RcReportWriter(in, out, "match", false) + opt.Differ = RcReportWriter(in, out, "differ", true) + opt.Error = RcReportWriter(in, out, "error", true) if checkFileHash != "" { out["hashType"] = checkFileHashType.String() diff --git a/fs/sync/rc.go b/fs/sync/rc.go index 52a2626e8..fe2b67021 100644 --- a/fs/sync/rc.go +++ b/fs/sync/rc.go @@ -3,6 +3,8 @@ package sync import ( "context" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/rc" ) @@ -23,7 +25,27 @@ func init() { - srcFs - a remote name string e.g. "drive:src" for the source - dstFs - a remote name string e.g. "drive:dst" for the destination - createEmptySrcDirs - create empty src directories on destination if set -` + moveHelp + ` +` + moveHelp + `- combined - make a combined report of changes (default false) +- missingOnSrc - report all files missing from the source (default false) +- missingOnDst - report all files missing from the destination (default false) +- match - report all matching files (default false) +- differ - report all non-matching files (default false) +- error - report all files with errors (hashing or reading) (default false) +- destAfter - report all files that exist on the destination post-` + name + ` (default false) + +Returns: + +- combined - array of strings of combined report of changes +- missingOnSrc - array of strings of all files missing from the source +- missingOnDst - array of strings of all files missing from the destination +- match - array of strings of all matching files +- differ - array of strings of all non-matching files +- error - array of strings of all files with errors (hashing or reading) +- destAfter - array of strings of all files that exist on the destination post-` + name + ` + +Each report is only returned if its parameter is set to true. If the +operation fails the reports are only available if it was run with +` + "`_async`" + `, as part of the job output. See the [` + name + `](/commands/rclone_` + name + `/) command for more information on the above.`, }) @@ -44,17 +66,38 @@ func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Para if rc.NotErrParamNotFound(err) { return nil, err } + ctx, out = rcLogger(ctx, in, dstFs) switch name { case "sync": - return nil, Sync(ctx, dstFs, srcFs, createEmptySrcDirs) + return out, Sync(ctx, dstFs, srcFs, createEmptySrcDirs) case "copy": - return nil, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs) + return out, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs) case "move": deleteEmptySrcDirs, err := in.GetBool("deleteEmptySrcDirs") if rc.NotErrParamNotFound(err) { return nil, err } - return nil, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs) + return out, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs) } panic("unknown rcSyncCopyMove type") } + +// rcLogger returns ctx with a sync logger which collects the reports +// requested in in into the returned out, which is empty if none were. +func rcLogger(ctx context.Context, in rc.Params, fdst fs.Fs) (context.Context, rc.Params) { + out := rc.Params{} + opt := operations.NewSyncLoggerOpt() + opt.Combined = operations.RcReportWriter(in, out, "combined", false) + opt.MissingOnSrc = operations.RcReportWriter(in, out, "missingOnSrc", false) + opt.MissingOnDst = operations.RcReportWriter(in, out, "missingOnDst", false) + opt.Match = operations.RcReportWriter(in, out, "match", false) + opt.Differ = operations.RcReportWriter(in, out, "differ", false) + opt.Error = operations.RcReportWriter(in, out, "error", false) + opt.DestAfter = operations.RcReportWriter(in, out, "destAfter", false) + if len(out) == 0 { + return ctx, out + } + opt.LoggerFn = operations.NewDefaultLoggerFn(&opt) + opt.Init(ctx, fdst, nil) + return operations.WithSyncLogger(ctx, opt), out +} diff --git a/fs/sync/rc_test.go b/fs/sync/rc_test.go index ad5145633..aeaae4bcd 100644 --- a/fs/sync/rc_test.go +++ b/fs/sync/rc_test.go @@ -2,6 +2,7 @@ package sync import ( "context" + "sort" "testing" "github.com/rclone/rclone/fs/cache" @@ -41,7 +42,7 @@ func TestRcCopy(t *testing.T) { } out, err := call.Fn(context.Background(), in) require.NoError(t, err) - assert.Equal(t, rc.Params(nil), out) + assert.Equal(t, rc.Params{}, out) r.CheckLocalItems(t, file1, file2) r.CheckRemoteItems(t, file1, file2, file3) @@ -65,7 +66,7 @@ func TestRcMove(t *testing.T) { } out, err := call.Fn(context.Background(), in) require.NoError(t, err) - assert.Equal(t, rc.Params(nil), out) + assert.Equal(t, rc.Params{}, out) r.CheckLocalItems(t) r.CheckRemoteItems(t, file1, file2, file3) @@ -89,8 +90,53 @@ func TestRcSync(t *testing.T) { } out, err := call.Fn(context.Background(), in) require.NoError(t, err) - assert.Equal(t, rc.Params(nil), out) + assert.Equal(t, rc.Params{}, out) r.CheckLocalItems(t, file1, file2) r.CheckRemoteItems(t, file1, file2) } + +// sync/copy: check the reports are returned when requested +func TestRcCopyReports(t *testing.T) { + r, call := rcNewRun(t, "sync/copy") + r.Mkdir(context.Background(), r.Fremote) + + file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1) + file2 := r.WriteFile("subdir/file2", "file2 contents", t2) + file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3) + file4 := r.WriteFile("file4", "file4 contents", t1) + file4dst := r.WriteObject(context.Background(), "file4", "different contents", t1) + + r.CheckLocalItems(t, file1, file2, file4) + r.CheckRemoteItems(t, file1, file3, file4dst) + + in := rc.Params{ + "srcFs": r.LocalName, + "dstFs": r.FremoteName, + "combined": true, + "missingOnSrc": true, + "missingOnDst": true, + "match": true, + "differ": true, + "destAfter": true, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + + sorted := func(name string) []string { + result, ok := out[name].(*[]string) + require.True(t, ok, name) + sort.Strings(*result) + return *result + } + assert.Equal(t, []string{"* file4", "+ subdir/file2", "- subdir/subsubdir/file3", "= file1"}, sorted("combined")) + assert.Equal(t, []string{"subdir/subsubdir/file3"}, sorted("missingOnSrc")) + assert.Equal(t, []string{"subdir/file2"}, sorted("missingOnDst")) + assert.Equal(t, []string{"file1"}, sorted("match")) + assert.Equal(t, []string{"file4"}, sorted("differ")) + assert.Equal(t, []string{"file1", "file4", "subdir/file2", "subdir/subsubdir/file3"}, sorted("destAfter")) + assert.NotContains(t, out, "error") + + r.CheckLocalItems(t, file1, file2, file4) + r.CheckRemoteItems(t, file1, file2, file3, file4) +}