f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

accounting: add "Updated dirs" count of directory modtime/metadata updates to the stats

Browse Source 
Syncs to backends which preserve directory modification times (eg
sftp, local) can update the modtime or metadata on many directories.
This count makes that work visible in the stats output, the core/stats
rc and the prometheus metrics (as dirs_updated_total).
This commit is contained in:
Nick Craig-Wood
2026-09-05 12:14:46 +01:00
parent f998f5bb99
commit 6afb98b758
5 changed files with 45 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/accounting/prometheus.go
+7
View File
@@ -18,6 +18,7 @@ type RcloneCollector struct {
transferredFiles *prometheus.Desc
deletes *prometheus.Desc
deletedDirs *prometheus.Desc
updatedDirs *prometheus.Desc
renames *prometheus.Desc
listed *prometheus.Desc
fatalError *prometheus.Desc
@@ -56,6 +57,10 @@ func NewRcloneCollector(ctx context.Context) *RcloneCollector {
"Total number of directories deleted",
nil, nil,
),
updatedDirs: prometheus.NewDesc(namespace+"dirs_updated_total",
"Total number of directories updated",
nil, nil,
),
renames: prometheus.NewDesc(namespace+"files_renamed_total",
"Total number of files renamed",
nil, nil,
@@ -84,6 +89,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.transferredFiles
ch <- c.deletes
ch <- c.deletedDirs
ch <- c.updatedDirs
ch <- c.renames
ch <- c.listed
ch <- c.fatalError
@@ -102,6 +108,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.transferredFiles, prometheus.CounterValue, float64(s.transfers))
ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes))
ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs))
ch <- prometheus.MustNewConstMetric(c.updatedDirs, prometheus.CounterValue, float64(s.updatedDirs))
ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames))
ch <- prometheus.MustNewConstMetric(c.listed, prometheus.CounterValue, float64(s.listed))
ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError))
fs/accounting/stats.go
+14
View File
@@ -55,6 +55,7 @@ type StatsInfo struct {
deletes int64
deletesSize int64
deletedDirs int64
updatedDirs int64
inProgress *inProgress
startedTransfers []*Transfer // currently active transfers
oldTimeRanges timeRanges // a merged list of time ranges for the transfers
@@ -128,6 +129,7 @@ func (s *StatsInfo) RemoteStats(short bool) (out rc.Params, err error) {
out["transfers"] = s.transfers
out["deletes"] = s.deletes
out["deletedDirs"] = s.deletedDirs
out["updatedDirs"] = s.updatedDirs
out["renames"] = s.renames
out["listed"] = s.listed
out["elapsedTime"] = time.Since(s.startTime).Seconds()
@@ -489,6 +491,9 @@ func (s *StatsInfo) String() string {
if s.deletes != 0 || s.deletedDirs != 0 {
_, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs), %s (freed)\n", s.deletes, s.deletedDirs, fs.SizeSuffix(s.deletesSize).ByteUnit())
}
if s.updatedDirs != 0 {
_, _ = fmt.Fprintf(buf, "Updated dirs: %10d\n", s.updatedDirs)
}
if s.renames != 0 {
_, _ = fmt.Fprintf(buf, "Renamed: %10d\n", s.renames)
}
@@ -692,6 +697,14 @@ func (s *StatsInfo) DeletedDirs(deletedDirs int64) int64 {
return s.deletedDirs
}
// UpdatedDirs updates the stats for updatedDirs
func (s *StatsInfo) UpdatedDirs(updatedDirs int64) int64 {
s.mu.Lock()
defer s.mu.Unlock()
s.updatedDirs += updatedDirs
return s.updatedDirs
}
// Renames updates the stats for renames
func (s *StatsInfo) Renames(renames int64) int64 {
s.mu.Lock()
@@ -723,6 +736,7 @@ func (s *StatsInfo) ResetCounters() {
s.deletes = 0
s.deletesSize = 0
s.deletedDirs = 0
s.updatedDirs = 0
s.renames = 0
s.listed = 0
s.startedTransfers = nil
fs/accounting/stats_groups.go
+2
View File
@@ -111,6 +111,7 @@ Returns the following values:
"totalTransfers": total number of transfers in the group,
"transferTime" : total time spent on running jobs,
"transfers": number of transferred files,
"updatedDirs": number of directories updated (modtime or metadata set),
"transferring": an array of currently active file transfers:
[
{
@@ -405,6 +406,7 @@ func (sg *statsGroups) sum(ctx context.Context) *StatsInfo {
sum.deletes += stats.deletes
sum.deletesSize += stats.deletesSize
sum.deletedDirs += stats.deletedDirs
sum.updatedDirs += stats.updatedDirs
sum.inProgress.merge(stats.inProgress)
sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...)
sum.oldTimeRanges = append(sum.oldTimeRanges, stats.oldTimeRanges...)
fs/operations/operations.go
+7
View File
@@ -1097,6 +1097,7 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadat
err = fs.CountError(ctx, err)
return nil, err
}
accounting.Stats(ctx).UpdatedDirs(1)
if mtime, ok := metadata["mtime"]; ok {
fs.Infof(logName, "Made directory with metadata (mtime=%s)", mtime)
} else {
@@ -1132,6 +1133,9 @@ func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) (
// The directory was created with Mkdir then we should try to set the time
if do := f.Features().DirSetModTime; do != nil {
err = do(ctx, dir, modTime)
if err == nil {
accounting.Stats(ctx).UpdatedDirs(1)
}
}
fs.Infof(logName, "Made directory with modification time %v", modTime)
return newDst, err
@@ -2763,6 +2767,7 @@ func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string,
if err != nil {
return nil, err
}
accounting.Stats(ctx).UpdatedDirs(1)
fs.Infof(logName, "Updated directory metadata")
return newDst, nil
}
@@ -2805,6 +2810,7 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, m
} else if err != nil {
return dst, err
} else {
accounting.Stats(ctx).UpdatedDirs(1)
fs.Infof(logName, "Set directory modification time (using SetModTime)")
return dst, nil
}
@@ -2817,6 +2823,7 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, m
if err != nil {
return dst, err
}
accounting.Stats(ctx).UpdatedDirs(1)
fs.Infof(logName, "Set directory modification time (using DirSetModTime)")
return dst, nil
}
fs/operations/operations_test.go
+15
View File
@@ -1880,9 +1880,11 @@ func TestMkdirMetadata(t *testing.T) {
t.Skip("Skipping test as remote does not support MkdirMetadata")
}
updatedDirs := statsUpdatedDirs(t)
newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata)
require.NoError(t, err)
require.NotNil(t, newDst)
assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "MkdirMetadata should be counted as an updated dir")
require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported")
@@ -1898,8 +1900,10 @@ func TestMkdirModTime(t *testing.T) {
if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata")
}
updatedDirs := statsUpdatedDirs(t)
newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2)
require.NoError(t, err)
assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "MkdirModTime should be counted as an updated dir")
// Check the returned directory and one read from the listing
// newDst may be nil here depending on how the modtime was set
@@ -1909,6 +1913,13 @@ func TestMkdirModTime(t *testing.T) {
fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
}
// statsUpdatedDirs reads the updatedDirs stat from the global stats
func statsUpdatedDirs(t *testing.T) int64 {
out, err := accounting.GlobalStats().RemoteStats(true)
require.NoError(t, err)
return out["updatedDirs"].(int64)
}
func TestCopyDirMetadata(t *testing.T) {
const nameNonExistent = "non existent directory"
const nameExistent = "existing directory"
@@ -1926,9 +1937,11 @@ func TestCopyDirMetadata(t *testing.T) {
require.NotNil(t, newSrc)
// First try with the directory not existing
updatedDirs := statsUpdatedDirs(t)
newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc)
require.NoError(t, err)
require.NotNil(t, newDst)
assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "CopyDirMetadata should be counted as an updated dir")
// Check the returned directory and one read from the listing
fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
@@ -1976,10 +1989,12 @@ func TestSetDirModTime(t *testing.T) {
existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)
checks := accounting.GlobalStats().GetChecks()
updatedDirs := statsUpdatedDirs(t)
newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)
require.NoError(t, err)
require.NotNil(t, newDst)
assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "SetDirModTime should be counted as a check")
assert.Equal(t, updatedDirs+1, statsUpdatedDirs(t), "SetDirModTime should be counted as an updated dir")
// Check the returned directory and one read from the listing
// The modtime will only be correct on newDst if it had a SetModTime method