f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

operations: show directory operations in the --progress display

Browse Source 
Syncs which update lots of directories (eg to sftp) could spend a long
time setting directory modification times, making directories or
removing directories with no feedback in the --progress display or
stats, making rclone appear to have hung.

This shows directory operations (setting modtime, updating metadata,
making and removing directories) in the Checking section of the stats
and counts them as checks, in the same way file deletes are shown.
This commit is contained in:
Nick Craig-Wood
2026-09-05 12:14:46 +01:00
parent 7fe402585e
commit f998f5bb99
4 changed files with 68 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/operations/operations.go
+36 -2
View File
@@ -1057,12 +1057,16 @@ func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {
}
// Mkdir makes a destination directory or container
func Mkdir(ctx context.Context, f fs.Fs, dir string) error {
func Mkdir(ctx context.Context, f fs.Fs, dir string) (err error) {
tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory")
defer func() {
tr.Done(ctx, err)
}()
if SkipDestructive(ctx, fs.LogDirName(f, dir), "make directory") {
return nil
}
fs.Infof(fs.LogDirName(f, dir), "Making directory")
err := f.Mkdir(ctx, dir)
err = f.Mkdir(ctx, dir)
if err != nil {
err = fs.CountError(ctx, err)
return err
@@ -1079,6 +1083,10 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadat
if do == nil {
return nil, Mkdir(ctx, f, dir)
}
tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory")
defer func() {
tr.Done(ctx, err)
}()
logName := fs.LogDirName(f, dir)
if SkipDestructive(ctx, logName, "make directory") {
return nil, nil
@@ -1132,6 +1140,11 @@ func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) (
// TryRmdir removes a container but not if not empty. It doesn't
// count errors but may return one.
func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {
tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "removing directory")
defer func() {
// Pass nil error to Done as TryRmdir doesn't count errors
tr.Done(ctx, nil)
}()
accounting.Stats(ctx).DeletedDirs(1)
if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") {
return nil
@@ -2685,6 +2698,15 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any {
return f
}
// Return the best way of describing the directory as a DirEntry for
// the progress display, describing the root directory as the Fs.
func dirTransferEntry(f fs.Fs, dst fs.Directory, dir string) fs.DirEntry {
if dst != nil && dst.Remote() != "" {
return dst
}
return fs.NewDir(fmt.Sprint(fs.LogDirName(f, dir)), time.Time{})
}
// CopyDirMetadata copies the src directory to dst or f if nil. If dst is nil then it uses
// dir as the name of the new directory.
//
@@ -2693,6 +2715,12 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any {
func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory, err error) {
ci := fs.GetConfig(ctx)
logName := dirName(f, dst, dir)
tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "updating metadata")
defer func() {
// Count the error before Done so it is only counted once
err = fs.CountError(ctx, err)
tr.Done(ctx, err)
}()
if SkipDestructive(ctx, logName, "update directory metadata") {
return nil, nil
}
@@ -2754,6 +2782,12 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, m
fs.Debugf(logName, "Skipping set directory modification time as --no-update-dir-modtime is set")
return nil, nil
}
tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "setting modtime")
defer func() {
// Count the error before Done so it is only counted once
err = fs.CountError(ctx, err)
tr.Done(ctx, err)
}()
if SkipDestructive(ctx, logName, "set directory modification time") {
return nil, nil
}
fs/operations/operations_internal_test.go
+24
View File
@@ -10,7 +10,9 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest/mockfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSizeDiffers(t *testing.T) {
@@ -41,3 +43,25 @@ func TestSizeDiffers(t *testing.T) {
assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))
}
}
func TestDirTransferEntry(t *testing.T) {
ctx := context.Background()
f, err := mockfs.NewFs(ctx, "mock", "", nil)
require.NoError(t, err)
dst := fs.NewDir("existing dir", time.Time{})
root := fs.NewDir("", time.Time{})
for _, test := range []struct {
dst fs.Directory
dir string
want string
}{
{nil, "new dir", "new dir"},
{nil, "", fmt.Sprint(f)},
{dst, "ignored", "existing dir"},
{root, "", fmt.Sprint(f)},
} {
got := dirTransferEntry(f, test.dst, test.dir)
assert.Equal(t, test.want, got.Remote(), fmt.Sprintf("dst=%v, dir=%q", test.dst, test.dir))
}
}
fs/operations/operations_test.go
+8
View File
@@ -37,6 +37,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
@@ -66,9 +67,11 @@ func TestMkdir(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
checks := accounting.GlobalStats().GetChecks()
err := operations.Mkdir(ctx, r.Fremote, "")
require.NoError(t, err)
fstest.CheckListing(t, r.Fremote, []fstest.Item{})
assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "Mkdir should be counted as a check")
err = operations.Mkdir(ctx, r.Fremote, "")
require.NoError(t, err)
@@ -1961,17 +1964,22 @@ func TestSetDirModTime(t *testing.T) {
ci.NoUpdateDirModTime = false
// First try with the directory not existing - should return an error
errs := accounting.GlobalStats().GetErrors()
newDst, err = operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)
require.Error(t, err)
require.Nil(t, newDst)
assert.Equal(t, errs+1, accounting.GlobalStats().GetErrors(), "the error should be counted")
assert.True(t, fserrors.IsCounted(err), "the returned error should be marked as counted")
// Then try with the directory existing
require.NoError(t, r.Fremote.Mkdir(ctx, name))
existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)
checks := accounting.GlobalStats().GetChecks()
newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)
require.NoError(t, err)
require.NotNil(t, newDst)
assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "SetDirModTime should be counted as a check")
// Check the returned directory and one read from the listing
// The modtime will only be correct on newDst if it had a SetModTime method
fs/sync/sync.go
-1
View File
@@ -1217,7 +1217,6 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
_, err = operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
}
if err != nil {
err = fs.CountError(ctx, err)
fs.Errorf(item.dir, "Failed to update directory timestamp or metadata: %v", err)
errCount.Add(err)
}