From 6afb98b758fc1a50a6f983c53a6d6c98514b3e86 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 11 Aug 2026 19:44:48 +0100 Subject: [PATCH] accounting: add "Updated dirs" count of directory modtime/metadata updates to the stats Syncs to backends which preserve directory modification times (eg sftp, local) can update the modtime or metadata on many directories. This count makes that work visible in the stats output, the core/stats rc and the prometheus metrics (as dirs_updated_total). --- fs/accounting/prometheus.go | 7 +++++++ fs/accounting/stats.go | 14 ++++++++++++++ fs/accounting/stats_groups.go | 2 ++ fs/operations/operations.go | 7 +++++++ fs/operations/operations_test.go | 15 +++++++++++++++ 5 files changed, 45 insertions(+) diff --git a/fs/accounting/prometheus.go b/fs/accounting/prometheus.go index 5ae07b749..50397c28a 100644 --- a/fs/accounting/prometheus.go +++ b/fs/accounting/prometheus.go @@ -18,6 +18,7 @@ type RcloneCollector struct { transferredFiles *prometheus.Desc deletes *prometheus.Desc deletedDirs *prometheus.Desc + updatedDirs *prometheus.Desc renames *prometheus.Desc listed *prometheus.Desc fatalError *prometheus.Desc @@ -56,6 +57,10 @@ func NewRcloneCollector(ctx context.Context) *RcloneCollector { "Total number of directories deleted", nil, nil, ), + updatedDirs: prometheus.NewDesc(namespace+"dirs_updated_total", + "Total number of directories updated", + nil, nil, + ), renames: prometheus.NewDesc(namespace+"files_renamed_total", "Total number of files renamed", nil, nil, @@ -84,6 +89,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.transferredFiles ch <- c.deletes ch <- c.deletedDirs + ch <- c.updatedDirs ch <- c.renames ch <- c.listed ch <- c.fatalError @@ -102,6 +108,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(c.transferredFiles, prometheus.CounterValue, float64(s.transfers)) ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes)) ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs)) + ch <- prometheus.MustNewConstMetric(c.updatedDirs, prometheus.CounterValue, float64(s.updatedDirs)) ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames)) ch <- prometheus.MustNewConstMetric(c.listed, prometheus.CounterValue, float64(s.listed)) ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError)) diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index bdb92abf7..f86c809d2 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -55,6 +55,7 @@ type StatsInfo struct { deletes int64 deletesSize int64 deletedDirs int64 + updatedDirs int64 inProgress *inProgress startedTransfers []*Transfer // currently active transfers oldTimeRanges timeRanges // a merged list of time ranges for the transfers @@ -128,6 +129,7 @@ func (s *StatsInfo) RemoteStats(short bool) (out rc.Params, err error) { out["transfers"] = s.transfers out["deletes"] = s.deletes out["deletedDirs"] = s.deletedDirs + out["updatedDirs"] = s.updatedDirs out["renames"] = s.renames out["listed"] = s.listed out["elapsedTime"] = time.Since(s.startTime).Seconds() @@ -489,6 +491,9 @@ func (s *StatsInfo) String() string { if s.deletes != 0 || s.deletedDirs != 0 { _, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs), %s (freed)\n", s.deletes, s.deletedDirs, fs.SizeSuffix(s.deletesSize).ByteUnit()) } + if s.updatedDirs != 0 { + _, _ = fmt.Fprintf(buf, "Updated dirs: %10d\n", s.updatedDirs) + } if s.renames != 0 { _, _ = fmt.Fprintf(buf, "Renamed: %10d\n", s.renames) } @@ -692,6 +697,14 @@ func (s *StatsInfo) DeletedDirs(deletedDirs int64) int64 { return s.deletedDirs } +// UpdatedDirs updates the stats for updatedDirs +func (s *StatsInfo) UpdatedDirs(updatedDirs int64) int64 { + s.mu.Lock() + defer s.mu.Unlock() + s.updatedDirs += updatedDirs + return s.updatedDirs +} + // Renames updates the stats for renames func (s *StatsInfo) Renames(renames int64) int64 { s.mu.Lock() @@ -723,6 +736,7 @@ func (s *StatsInfo) ResetCounters() { s.deletes = 0 s.deletesSize = 0 s.deletedDirs = 0 + s.updatedDirs = 0 s.renames = 0 s.listed = 0 s.startedTransfers = nil diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go index 503ddcefa..5463658de 100644 --- a/fs/accounting/stats_groups.go +++ b/fs/accounting/stats_groups.go @@ -111,6 +111,7 @@ Returns the following values: "totalTransfers": total number of transfers in the group, "transferTime" : total time spent on running jobs, "transfers": number of transferred files, + "updatedDirs": number of directories updated (modtime or metadata set), "transferring": an array of currently active file transfers: [ { @@ -405,6 +406,7 @@ func (sg *statsGroups) sum(ctx context.Context) *StatsInfo { sum.deletes += stats.deletes sum.deletesSize += stats.deletesSize sum.deletedDirs += stats.deletedDirs + sum.updatedDirs += stats.updatedDirs sum.inProgress.merge(stats.inProgress) sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...) sum.oldTimeRanges = append(sum.oldTimeRanges, stats.oldTimeRanges...) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 656e052af..6cae43104 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1097,6 +1097,7 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadat err = fs.CountError(ctx, err) return nil, err } + accounting.Stats(ctx).UpdatedDirs(1) if mtime, ok := metadata["mtime"]; ok { fs.Infof(logName, "Made directory with metadata (mtime=%s)", mtime) } else { @@ -1132,6 +1133,9 @@ func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) ( // The directory was created with Mkdir then we should try to set the time if do := f.Features().DirSetModTime; do != nil { err = do(ctx, dir, modTime) + if err == nil { + accounting.Stats(ctx).UpdatedDirs(1) + } } fs.Infof(logName, "Made directory with modification time %v", modTime) return newDst, err @@ -2763,6 +2767,7 @@ func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, if err != nil { return nil, err } + accounting.Stats(ctx).UpdatedDirs(1) fs.Infof(logName, "Updated directory metadata") return newDst, nil } @@ -2805,6 +2810,7 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, m } else if err != nil { return dst, err } else { + accounting.Stats(ctx).UpdatedDirs(1) fs.Infof(logName, "Set directory modification time (using SetModTime)") return dst, nil } @@ -2817,6 +2823,7 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, m if err != nil { return dst, err } + accounting.Stats(ctx).UpdatedDirs(1) fs.Infof(logName, "Set directory modification time (using DirSetModTime)") return dst, nil } diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 0b6081d57..85afb5eb8 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1880,9 +1880,11 @@ func TestMkdirMetadata(t *testing.T) { t.Skip("Skipping test as remote does not support MkdirMetadata") } + updatedDirs := statsUpdatedDirs(t) newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata) require.NoError(t, err) require.NotNil(t, newDst) + assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "MkdirMetadata should be counted as an updated dir") require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported") @@ -1898,8 +1900,10 @@ func TestMkdirModTime(t *testing.T) { if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil { t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata") } + updatedDirs := statsUpdatedDirs(t) newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2) require.NoError(t, err) + assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "MkdirModTime should be counted as an updated dir") // Check the returned directory and one read from the listing // newDst may be nil here depending on how the modtime was set @@ -1909,6 +1913,13 @@ func TestMkdirModTime(t *testing.T) { fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) } +// statsUpdatedDirs reads the updatedDirs stat from the global stats +func statsUpdatedDirs(t *testing.T) int64 { + out, err := accounting.GlobalStats().RemoteStats(true) + require.NoError(t, err) + return out["updatedDirs"].(int64) +} + func TestCopyDirMetadata(t *testing.T) { const nameNonExistent = "non existent directory" const nameExistent = "existing directory" @@ -1926,9 +1937,11 @@ func TestCopyDirMetadata(t *testing.T) { require.NotNil(t, newSrc) // First try with the directory not existing + updatedDirs := statsUpdatedDirs(t) newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc) require.NoError(t, err) require.NotNil(t, newDst) + assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "CopyDirMetadata should be counted as an updated dir") // Check the returned directory and one read from the listing fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) @@ -1976,10 +1989,12 @@ func TestSetDirModTime(t *testing.T) { existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name) checks := accounting.GlobalStats().GetChecks() + updatedDirs := statsUpdatedDirs(t) newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2) require.NoError(t, err) require.NotNil(t, newDst) assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "SetDirModTime should be counted as a check") + assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "SetDirModTime should be counted as an updated dir") // Check the returned directory and one read from the listing // The modtime will only be correct on newDst if it had a SetModTime method