From f998f5bb999b235658f87f52c6c06964abf53b1f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 11 Aug 2026 19:42:07 +0100 Subject: [PATCH] operations: show directory operations in the --progress display Syncs which update lots of directories (eg to sftp) could spend a long time setting directory modification times, making directories or removing directories with no feedback in the --progress display or stats, making rclone appear to have hung. This shows directory operations (setting modtime, updating metadata, making and removing directories) in the Checking section of the stats and counts them as checks, in the same way file deletes are shown. --- fs/operations/operations.go | 38 +++++++++++++++++++++-- fs/operations/operations_internal_test.go | 24 ++++++++++++++ fs/operations/operations_test.go | 8 +++++ fs/sync/sync.go | 1 - 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index b3b1ba5df..656e052af 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1057,12 +1057,16 @@ func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error { } // Mkdir makes a destination directory or container -func Mkdir(ctx context.Context, f fs.Fs, dir string) error { +func Mkdir(ctx context.Context, f fs.Fs, dir string) (err error) { + tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory") + defer func() { + tr.Done(ctx, err) + }() if SkipDestructive(ctx, fs.LogDirName(f, dir), "make directory") { return nil } fs.Infof(fs.LogDirName(f, dir), "Making directory") - err := f.Mkdir(ctx, dir) + err = f.Mkdir(ctx, dir) if err != nil { err = fs.CountError(ctx, err) return err @@ -1079,6 +1083,10 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadat if do == nil { return nil, Mkdir(ctx, f, dir) } + tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory") + defer func() { + tr.Done(ctx, err) + }() logName := fs.LogDirName(f, dir) if SkipDestructive(ctx, logName, "make directory") { return nil, nil @@ -1132,6 +1140,11 @@ func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) ( // TryRmdir removes a container but not if not empty. It doesn't // count errors but may return one. func TryRmdir(ctx context.Context, f fs.Fs, dir string) error { + tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "removing directory") + defer func() { + // Pass nil error to Done as TryRmdir doesn't count errors + tr.Done(ctx, nil) + }() accounting.Stats(ctx).DeletedDirs(1) if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") { return nil @@ -2685,6 +2698,15 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any { return f } +// Return the best way of describing the directory as a DirEntry for +// the progress display, describing the root directory as the Fs. +func dirTransferEntry(f fs.Fs, dst fs.Directory, dir string) fs.DirEntry { + if dst != nil && dst.Remote() != "" { + return dst + } + return fs.NewDir(fmt.Sprint(fs.LogDirName(f, dir)), time.Time{}) +} + // CopyDirMetadata copies the src directory to dst or f if nil. If dst is nil then it uses // dir as the name of the new directory. // @@ -2693,6 +2715,12 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any { func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory, err error) { ci := fs.GetConfig(ctx) logName := dirName(f, dst, dir) + tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "updating metadata") + defer func() { + // Count the error before Done so it is only counted once + err = fs.CountError(ctx, err) + tr.Done(ctx, err) + }() if SkipDestructive(ctx, logName, "update directory metadata") { return nil, nil } @@ -2754,6 +2782,12 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, m fs.Debugf(logName, "Skipping set directory modification time as --no-update-dir-modtime is set") return nil, nil } + tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "setting modtime") + defer func() { + // Count the error before Done so it is only counted once + err = fs.CountError(ctx, err) + tr.Done(ctx, err) + }() if SkipDestructive(ctx, logName, "set directory modification time") { return nil, nil } diff --git a/fs/operations/operations_internal_test.go b/fs/operations/operations_internal_test.go index fd85a9819..331481db1 100644 --- a/fs/operations/operations_internal_test.go +++ b/fs/operations/operations_internal_test.go @@ -10,7 +10,9 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fstest/mockfs" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSizeDiffers(t *testing.T) { @@ -41,3 +43,25 @@ func TestSizeDiffers(t *testing.T) { assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize)) } } + +func TestDirTransferEntry(t *testing.T) { + ctx := context.Background() + f, err := mockfs.NewFs(ctx, "mock", "", nil) + require.NoError(t, err) + dst := fs.NewDir("existing dir", time.Time{}) + root := fs.NewDir("", time.Time{}) + + for _, test := range []struct { + dst fs.Directory + dir string + want string + }{ + {nil, "new dir", "new dir"}, + {nil, "", fmt.Sprint(f)}, + {dst, "ignored", "existing dir"}, + {root, "", fmt.Sprint(f)}, + } { + got := dirTransferEntry(f, test.dst, test.dir) + assert.Equal(t, test.want, got.Remote(), fmt.Sprintf("dst=%v, dir=%q", test.dst, test.dir)) + } +} diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 727972c30..0b6081d57 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -37,6 +37,7 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/object" @@ -66,9 +67,11 @@ func TestMkdir(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) + checks := accounting.GlobalStats().GetChecks() err := operations.Mkdir(ctx, r.Fremote, "") require.NoError(t, err) fstest.CheckListing(t, r.Fremote, []fstest.Item{}) + assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "Mkdir should be counted as a check") err = operations.Mkdir(ctx, r.Fremote, "") require.NoError(t, err) @@ -1961,17 +1964,22 @@ func TestSetDirModTime(t *testing.T) { ci.NoUpdateDirModTime = false // First try with the directory not existing - should return an error + errs := accounting.GlobalStats().GetErrors() newDst, err = operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2) require.Error(t, err) require.Nil(t, newDst) + assert.Equal(t, errs+1, accounting.GlobalStats().GetErrors(), "the error should be counted") + assert.True(t, fserrors.IsCounted(err), "the returned error should be marked as counted") // Then try with the directory existing require.NoError(t, r.Fremote.Mkdir(ctx, name)) existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name) + checks := accounting.GlobalStats().GetChecks() newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2) require.NoError(t, err) require.NotNil(t, newDst) + assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "SetDirModTime should be counted as a check") // Check the returned directory and one read from the listing // The modtime will only be correct on newDst if it had a SetModTime method diff --git a/fs/sync/sync.go b/fs/sync/sync.go index d53ac741c..096b43a8f 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -1217,7 +1217,6 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error { _, err = operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime) } if err != nil { - err = fs.CountError(ctx, err) fs.Errorf(item.dir, "Failed to update directory timestamp or metadata: %v", err) errCount.Add(err) }