f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: implement directory sync for mod times and metadata

Browse Source 
Directory mod times are synced by default if the backend is capable
and directory metadata is synced if the --metadata flag is provided
and the backend is capable.

This updates the bisync golden tests also which were affected by
--dry-run setting of directory modtimes.

Fixes #6685
This commit is contained in:
Nick Craig-Wood
2024-02-28 16:26:14 +00:00
parent 15579c2195
commit f5f86786b2
23 changed files with 504 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/sync/sync.go
+121 -1
View File
@@ -18,6 +18,8 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/march"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/errcount"
"golang.org/x/sync/errgroup"
)
// ErrorMaxDurationReached defines error when transfer duration is reached
@@ -84,6 +86,20 @@ type syncCopyMove struct {
maxDurationEndTime time.Time // end time if --max-duration is set
logger operations.LoggerFn // LoggerFn used to report the results of a sync (or bisync) to an io.Writer
usingLogger bool // whether we are using logger
setDirMetadata bool // if set we set the directory metadata
setDirModTime bool // if set we set the directory modtimes
setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync
setDirModTimeMu sync.Mutex // protect setDirModTimeMu
setDirModTimes []setDirModTime // directories that need their modtime set
setDirModTimesMaxLevel int // max level of the directories to set
}
// For keeping track of delayed modtime sets
type setDirModTime struct {
dst fs.Directory
dir string
modTime time.Time
level int // the level of the directory, 0 is root
}
type trackRenamesStrategy byte
@@ -136,6 +152,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
modifyWindow: fs.GetModifyWindow(ctx, fsrc, fdst),
trackRenamesCh: make(chan fs.Object, ci.Checkers),
checkFirst: ci.CheckFirst,
setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata,
setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil,
setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite,
}
s.logger, s.usingLogger = operations.GetLogger(ctx)
@@ -966,6 +985,11 @@ func (s *syncCopyMove) run() error {
}
}
// Update modtimes for directories if necessary
if s.setDirModTime && s.setDirModTimeAfter {
s.processError(s.setDelayedDirModTimes(s.ctx))
}
// Prune empty directories
if s.deleteMode != fs.DeleteModeOff {
if s.currentError() != nil && !s.ci.IgnoreErrors {
@@ -1055,6 +1079,96 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
return false
}
// copyDirMetadata copies the src directory modTime or Metadata to dst
// or f if nil. If dst is nil then it uses dir as the name of the new
// directory.
//
// It returns the destination directory if possible. Note that this may
// be nil.
func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) {
var err error
if s.setDirMetadata {
newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)
} else if s.setDirModTime {
if dst == nil {
newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))
} else {
newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
}
} else if dst == nil {
// Create the directory if it doesn't exist
err = operations.Mkdir(ctx, f, dir)
}
// If we need to set modtime after and we created a dir, then save it for later
if s.setDirModTime && s.setDirModTimeAfter && err == nil {
if newDst != nil {
dir = newDst.Remote()
}
level := strings.Count(dir, "/") + 1
// The root directory "" is at the top level
if dir == "" {
level = 0
}
s.setDirModTimeMu.Lock()
// Keep track of the maximum level inserted
if level > s.setDirModTimesMaxLevel {
s.setDirModTimesMaxLevel = level
}
s.setDirModTimes = append(s.setDirModTimes, setDirModTime{
dst: newDst,
dir: dir,
modTime: src.ModTime(ctx),
level: level,
})
s.setDirModTimeMu.Unlock()
fs.Debugf(nil, "Added delayed dir = %q, newDst=%v", dir, newDst)
}
s.processError(err)
if err != nil {
return nil
}
return newDst
}
// Set the modtimes for directories
func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
s.setDirModTimeMu.Lock()
defer s.setDirModTimeMu.Unlock()
// Timestamp all directories at the same level in parallel, deepest first
// We do this by iterating the slice multiple times to save memory
// There could be a lot of directories in this slice.
var errCount = errcount.New()
for level := s.setDirModTimesMaxLevel; level >= 0; level-- {
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(s.ci.Checkers)
for _, item := range s.setDirModTimes {
if item.level != level {
continue
}
// End early if error
if gCtx.Err() != nil {
break
}
item := item
g.Go(func() error {
_, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
if err != nil {
err = fs.CountError(err)
fs.Errorf(item.dir, "Failed to timestamp directory: %v", err)
errCount.Add(err)
}
return nil // don't return errors, just count them
})
}
err := g.Wait()
if err != nil {
return err
}
}
return errCount.Err("failed to set directory modtime")
}
// SrcOnly have an object which is in the source only
func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
if s.deleteMode == fs.DeleteModeOnly {
@@ -1101,6 +1215,9 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
s.srcEmptyDirs[src.Remote()] = src
s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)
s.srcEmptyDirsMu.Unlock()
// Create the directory and make sure the Metadata/ModTime is correct
s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)
return true
default:
panic("Bad object in DirEntries")
@@ -1135,9 +1252,12 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
}
case fs.Directory:
// Do the same thing to the entire contents of the directory
_, ok := dst.(fs.Directory)
dstX, ok := dst.(fs.Directory)
if ok {
s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir)
// Create the directory and make sure the Metadata/ModTime is correct
s.copyDirMetadata(s.ctx, s.fdst, dstX, "", srcX)
// Only record matched (src & dst) empty dirs when performing move
if s.DoMove {
// Record the src directory for deletion
fs/sync/sync_test.go
+161 -4
View File
@@ -7,6 +7,7 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"runtime"
@@ -65,15 +66,88 @@ func TestCopy(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
_, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)
if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
require.NoError(t, err)
}
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
err = CopyDir(ctx, r.Fremote, r.Flocal, false)
require.NoError(t, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file1)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir")
}
func TestCopyMetadata(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.Metadata = true
r := fstest.NewRun(t)
features := r.Fremote.Features()
if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata &&
!features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata {
t.Skip("Skipping as metadata not supported")
}
const content = "hello metadata world!"
const dirPath = "metadata sub dir"
const filePath = dirPath + "/hello metadata world"
fileMetadata := fs.Metadata{
// System metadata supported by all backends
"mtime": t1.Format(time.RFC3339Nano),
// User metadata
"potato": "jersey",
}
dirMetadata := fs.Metadata{
// System metadata supported by all backends
"mtime": t2.Format(time.RFC3339Nano),
// User metadata
"potato": "king edward",
}
// Make the directory with metadata - may fall back to Mkdir
_, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata)
require.NoError(t, err)
// Upload the file with metadata
in := io.NopCloser(bytes.NewBufferString(content))
_, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata)
require.NoError(t, err)
file1 := fstest.NewItem(filePath, content, t1)
// Reset the time of the directory
_, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2)
if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
require.NoError(t, err)
}
ctx = predictDstFromLogger(ctx)
err = CopyDir(ctx, r.Fremote, r.Flocal, false)
require.NoError(t, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file1)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, dirPath)
// Check that the metadata on the directory and file is correct
if features.ReadMetadata {
fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata)
}
if features.ReadDirMetadata {
fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata)
}
}
func TestCopyMissingDirectory(t *testing.T) {
@@ -205,10 +279,15 @@ func TestCopyEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
r.Mkdir(ctx, r.Fremote)
// Set the modtime on "sub dir" to something specific
// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
require.NoError(t, err)
ctx = predictDstFromLogger(ctx)
err = CopyDir(ctx, r.Fremote, r.Flocal, true)
require.NoError(t, err)
@@ -224,6 +303,9 @@ func TestCopyEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
}
// Test move empty directories
@@ -231,8 +313,10 @@ func TestMoveEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir")
subDirT := subDir.ModTime(ctx)
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
@@ -250,6 +334,14 @@ func TestMoveEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir2")
// Note that "sub dir" mod time is updated when file1 is deleted from it
// So check it more manually
got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir")
gotT := got.ModTime(ctx)
fstest.AssertTimeEqualWithPrecision(t, subDir.Remote(), subDirT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal))
}
// Test sync empty directories
@@ -257,8 +349,14 @@ func TestSyncEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
_, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
// Set the modtime on "sub dir" to something specific
// Without this it fails on the CI and in VirtualBox with variances of up to 10mS
_, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
require.NoError(t, err)
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
@@ -276,6 +374,65 @@ func TestSyncEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
}
// Test delayed mod time setting
func TestSyncSetDelayedModTimes(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
if !r.Fremote.Features().DirModTimeUpdatesOnWrite {
t.Skip("Backend doesn't have DirModTimeUpdatesOnWrite set")
}
// Create directories without timestamps
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d1/e1/f1"))
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b2/c1/d1/e1/f1"))
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f1"))
require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f2"))
dirs := []string{
"a1",
"a1/b1",
"a1/b1/c1",
"a1/b1/c1/d1",
"a1/b1/c1/d1/e1",
"a1/b1/c1/d1/e1/f1",
"a1/b1/c1/d2",
"a1/b1/c1/d2/e1",
"a1/b1/c1/d2/e1/f1",
"a1/b1/c1/d2/e1/f2",
"a1/b2",
"a1/b2/c1",
"a1/b2/c1/d1",
"a1/b2/c1/d1/e1",
"a1/b2/c1/d1/e1/f1",
}
r.CheckLocalListing(t, []fstest.Item{}, dirs)
// Timestamp the directories in reverse order
ts := t1
for i := len(dirs) - 1; i >= 0; i-- {
dir := dirs[i]
_, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, ts)
require.NoError(t, err)
ts = ts.Add(time.Minute)
}
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
err := Sync(ctx, r.Fremote, r.Flocal, true)
require.NoError(t, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckRemoteListing(t, []fstest.Item{}, dirs)
// Check that the modtimes of the directories are as expected
r.CheckDirectoryModTimes(t, dirs...)
}
// Test a server-side copy if possible, or the backup path if not