f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement --compare-dest & --copy-dest Fixes #3278

Browse Source
This commit is contained in:
yparitcher
2019-07-22 19:42:29 +01:00
committed by Nick Craig-Wood
parent 266600dba7
commit 8e8b78d7e5
7 changed files with 601 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/config.go
+2
View File
@@ -66,6 +66,8 @@ type ConfigInfo struct {
NoTraverse bool
NoUpdateModTime bool
DataRateUnit string
CompareDest string
CopyDest string
BackupDir string
Suffix string
SuffixKeepExtension bool
fs/config/configflags/configflags.go
+6
View File
@@ -67,6 +67,8 @@ func AddFlags(flagSet *pflag.FlagSet) {
flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "use DIR to server side copy flies from.")
flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Compare dest to DIR also.")
flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.")
flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix to add to changed files.")
flags.BoolVarP(flagSet, &fs.Config.SuffixKeepExtension, "suffix-keep-extension", "", fs.Config.SuffixKeepExtension, "Preserve the extension when using --suffix.")
@@ -152,6 +154,10 @@ func SetFlags() {
log.Fatalf(`Can't use --size-only and --ignore-size together.`)
}
if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" {
log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
}
switch {
case len(fs.Config.StatsOneLineDateFormat) > 0:
fs.Config.StatsOneLineDate = true
fs/operations/operations.go
+136 -9
View File
@@ -96,7 +96,7 @@ func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal b
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum)
return equal(ctx, src, dst, fs.Config.SizeOnly, fs.Config.CheckSum, !fs.Config.NoUpdateModTime)
}
// sizeDiffers compare the size of src and dst taking into account the
@@ -110,7 +110,7 @@ func sizeDiffers(src, dst fs.ObjectInfo) bool {
var checksumWarning sync.Once
func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum bool) bool {
func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, checkSum, UpdateModTime bool) bool {
if sizeDiffers(src, dst) {
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
return false
@@ -169,7 +169,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, sizeOnly, chec
}
// mod time differs but hash is the same to reset mod time if required
if !fs.Config.NoUpdateModTime {
if UpdateModTime {
if fs.Config.DryRun {
fs.Logf(src, "Not updating modification time as --dry-run")
} else {
@@ -1360,6 +1360,115 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
return nil
}
// GetCompareDest sets up --compare-dest
func GetCompareDest() (CompareDest fs.Fs, err error) {
CompareDest, err = cache.Get(fs.Config.CompareDest)
if err != nil {
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", fs.Config.CompareDest, err))
}
return CompareDest, nil
}
// compareDest checks --compare-dest to see if src needs to
// be copied
//
// Returns True if src is in --compare-dest
func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (NoNeedTransfer bool, err error) {
var remote string
if dst == nil {
remote = src.Remote()
} else {
remote = dst.Remote()
}
CompareDestFile, err := CompareDest.NewObject(ctx, remote)
switch err {
case fs.ErrorObjectNotFound:
return false, nil
case nil:
break
default:
return false, err
}
if Equal(ctx, src, CompareDestFile) {
fs.Debugf(src, "Destination found in --compare-dest, skipping")
return true, nil
}
return false, nil
}
// GetCopyDest sets up --copy-dest
func GetCopyDest(fdst fs.Fs) (CopyDest fs.Fs, err error) {
CopyDest, err = cache.Get(fs.Config.CopyDest)
if err != nil {
return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", fs.Config.CopyDest, err))
}
if !SameConfig(fdst, CopyDest) {
return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
}
if CopyDest.Features().Copy == nil {
return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy"))
}
return CopyDest, nil
}
// copyDest checks --copy-dest to see if src needs to
// be copied
//
// Returns True if src was copied from --copy-dest
func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
var remote string
if dst == nil {
remote = src.Remote()
} else {
remote = dst.Remote()
}
CopyDestFile, err := CopyDest.NewObject(ctx, remote)
switch err {
case fs.ErrorObjectNotFound:
return false, nil
case nil:
break
default:
return false, err
}
if equal(ctx, src, CopyDestFile, fs.Config.SizeOnly, fs.Config.CheckSum, false) {
if dst == nil || !Equal(ctx, src, dst) {
if dst != nil && backupDir != nil {
err = MoveBackupDir(ctx, backupDir, dst)
if err != nil {
return false, errors.Wrap(err, "moving to --backup-dir failed")
}
// If successful zero out the dstObj as it is no longer there
dst = nil
}
_, err := Copy(ctx, fdst, dst, remote, CopyDestFile)
if err != nil {
fs.Errorf(src, "Destination found in --copy-dest, error copying")
return false, nil
}
fs.Debugf(src, "Destination found in --copy-dest, using server side copy")
return true, nil
}
fs.Debugf(src, "Unchanged skipping")
return true, nil
}
fs.Debugf(src, "Destination not found in --copy-dest")
return false, nil
}
// CompareOrCopyDest checks --compare-dest and --copy-dest to see if src
// does not need to be copied
//
// Returns True if src does not need to be copied
func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
if fs.Config.CompareDest != "" {
return compareDest(ctx, dst, src, CompareOrCopyDest)
} else if fs.Config.CopyDest != "" {
return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
}
return false, nil
}
// NeedTransfer checks to see if src needs to be copied to dst using
// the current config.
//
@@ -1577,13 +1686,31 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
return err
}
if NeedTransfer(ctx, dstObj, srcObj) {
var backupDir, copyDestDir fs.Fs
if fs.Config.BackupDir != "" || fs.Config.Suffix != "" {
backupDir, err = BackupDir(fdst, fsrc, srcFileName)
if err != nil {
return errors.Wrap(err, "creating Fs for --backup-dir failed")
}
}
if fs.Config.CompareDest != "" {
copyDestDir, err = GetCompareDest()
if err != nil {
return err
}
} else if fs.Config.CopyDest != "" {
copyDestDir, err = GetCopyDest(fdst)
if err != nil {
return err
}
}
NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir)
if err != nil {
return err
}
if !NoNeedTransfer && NeedTransfer(ctx, dstObj, srcObj) {
// If destination already exists, then we must move it into --backup-dir if required
if dstObj != nil && (fs.Config.BackupDir != "" || fs.Config.Suffix != "") {
backupDir, err := BackupDir(fdst, fsrc, srcFileName)
if err != nil {
return errors.Wrap(err, "creating Fs for --backup-dir failed")
}
if dstObj != nil && backupDir != nil {
err = MoveBackupDir(ctx, backupDir, dstObj)
if err != nil {
return errors.Wrap(err, "moving to --backup-dir failed")
fs/operations/operations_test.go
+177
View File
@@ -866,6 +866,183 @@ func TestCopyFileBackupDir(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1old, file1)
}
// Test with CompareDest set
func TestCopyFileCompareDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
fs.Config.CompareDest = r.FremoteName + "/CompareDest"
defer func() {
fs.Config.CompareDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty compare
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty compare
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new compare
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3)
// check empty dest, new compare
file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check new dest, new compare
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check empty dest, old compare
file5b := r.WriteFile("two", "twot3", t3)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5b)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5b.Path, file5b.Path)
require.NoError(t, err)
file5bdst := file5b
file5bdst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
}
// Test with CopyDest set
func TestCopyFileCopyDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
if r.Fremote.Features().Copy == nil {
t.Skip("Skipping test as remote does not support server side copy")
}
fs.Config.CopyDest = r.FremoteName + "/CopyDest"
defer func() {
fs.Config.CopyDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty copy
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1.Path, file1.Path)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty copy
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1b.Path, file1b.Path)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new copy, backup-dir
fs.Config.BackupDir = r.FremoteName + "/BackupDir"
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file1c.Path, file1c.Path)
require.NoError(t, err)
file2dst := file2
file2dst.Path = "dst/one"
file3.Path = "BackupDir/one"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
fs.Config.BackupDir = ""
// check empty dest, new copy
file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
file4dst := file4
file4dst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check new dest, new copy
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file5.Path, file5.Path)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check empty dest, old copy
file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2)
file7 := r.WriteFile("three", "threet3", t3)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6)
fstest.CheckItems(t, r.Flocal, file1c, file5, file7)
err = operations.CopyFile(context.Background(), fdst, r.Flocal, file7.Path, file7.Path)
require.NoError(t, err)
file7dst := file7
file7dst.Path = "dst/three"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst)
}
// testFsInfo is for unit testing fs.Info
type testFsInfo struct {
name string
fs/sync/sync.go
+63 -38
View File
@@ -28,39 +28,40 @@ type syncCopyMove struct {
deleteEmptySrcDirs bool
dir string
// internal state
ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context
noTraverse bool // if set don't traverse the dst
deletersWg sync.WaitGroup // for delete before go routine
deleteFilesCh chan fs.Object // channel to receive deletes if delete before
trackRenames bool // set if we should do server side renames
dstFilesMu sync.Mutex // protect dstFiles
dstFiles map[string]fs.Object // dst files, always filled
srcFiles map[string]fs.Object // src files, only used if deleteBefore
srcFilesChan chan fs.Object // passes src objects
srcFilesResult chan error // error result of src listing
dstFilesResult chan error // error result of dst listing
dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs
dstEmptyDirs map[string]fs.DirEntry // potentially empty directories
srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs
srcEmptyDirs map[string]fs.DirEntry // potentially empty directories
checkerWg sync.WaitGroup // wait for checkers
toBeChecked *pipe // checkers channel
transfersWg sync.WaitGroup // wait for transfers
toBeUploaded *pipe // copiers channel
errorMu sync.Mutex // Mutex covering the errors variables
err error // normal error from copy process
noRetryErr error // error with NoRetry set
fatalErr error // fatal error
commonHash hash.Type // common hash type between src and dst
renameMapMu sync.Mutex // mutex to protect the below
renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames
renamerWg sync.WaitGroup // wait for renamers
toBeRenamed *pipe // renamers channel
trackRenamesWg sync.WaitGroup // wg for background track renames
trackRenamesCh chan fs.Object // objects are pumped in here
renameCheck []fs.Object // accumulate files to check for rename here
backupDir fs.Fs // place to store overwrites/deletes
ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context
noTraverse bool // if set don't traverse the dst
deletersWg sync.WaitGroup // for delete before go routine
deleteFilesCh chan fs.Object // channel to receive deletes if delete before
trackRenames bool // set if we should do server side renames
dstFilesMu sync.Mutex // protect dstFiles
dstFiles map[string]fs.Object // dst files, always filled
srcFiles map[string]fs.Object // src files, only used if deleteBefore
srcFilesChan chan fs.Object // passes src objects
srcFilesResult chan error // error result of src listing
dstFilesResult chan error // error result of dst listing
dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs
dstEmptyDirs map[string]fs.DirEntry // potentially empty directories
srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs
srcEmptyDirs map[string]fs.DirEntry // potentially empty directories
checkerWg sync.WaitGroup // wait for checkers
toBeChecked *pipe // checkers channel
transfersWg sync.WaitGroup // wait for transfers
toBeUploaded *pipe // copiers channel
errorMu sync.Mutex // Mutex covering the errors variables
err error // normal error from copy process
noRetryErr error // error with NoRetry set
fatalErr error // fatal error
commonHash hash.Type // common hash type between src and dst
renameMapMu sync.Mutex // mutex to protect the below
renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames
renamerWg sync.WaitGroup // wait for renamers
toBeRenamed *pipe // renamers channel
trackRenamesWg sync.WaitGroup // wg for background track renames
trackRenamesCh chan fs.Object // objects are pumped in here
renameCheck []fs.Object // accumulate files to check for rename here
compareCopyDest fs.Fs // place to check for files to server side copy
backupDir fs.Fs // place to store overwrites/deletes
}
func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) {
@@ -127,6 +128,19 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
return nil, err
}
}
if fs.Config.CompareDest != "" {
var err error
s.compareCopyDest, err = operations.GetCompareDest()
if err != nil {
return nil, err
}
} else if fs.Config.CopyDest != "" {
var err error
s.compareCopyDest, err = operations.GetCopyDest(fdst)
if err != nil {
return nil, err
}
}
return s, nil
}
@@ -204,7 +218,11 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, wg *sync.WaitGroup) {
accounting.Stats.Checking(src.Remote())
// Check to see if can store this
if src.Storable() {
if operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir)
if err != nil {
s.processError(err)
}
if !NoNeedTransfer && operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
// If files are treated as immutable, fail if destination exists and does not match
if fs.Config.Immutable && pair.Dst != nil {
fs.Errorf(pair.Dst, "Source and destination exist but do not match: immutable file modified")
@@ -764,10 +782,17 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
case s.trackRenamesCh <- x:
}
} else {
// No need to check since doesn't exist
ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil})
if !ok {
return
// Check CompareDest && CopyDest
NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, nil, x, s.compareCopyDest, s.backupDir)
if err != nil {
s.processError(err)
}
if !NoNeedTransfer {
// No need to check since doesn't exist
ok := s.toBeUploaded.Put(s.ctx, fs.ObjectPair{Src: x, Dst: nil})
if !ok {
return
}
}
}
case fs.Directory:
fs/sync/sync_test.go
+190
View File
@@ -1216,6 +1216,196 @@ func TestSyncOverlap(t *testing.T) {
checkErr(Sync(context.Background(), FremoteSync, FremoteSync, false))
}
// Test with CompareDest set
func TestSyncCompareDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
fs.Config.CompareDest = r.FremoteName + "/CompareDest"
defer func() {
fs.Config.CompareDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty compare
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty compare
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new compare
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CompareDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3)
// check empty dest, new compare
file4 := r.WriteObject(context.Background(), "CompareDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check new dest, new compare
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
// check empty dest, old compare
file5b := r.WriteFile("two", "twot3", t3)
fstest.CheckItems(t, r.Fremote, file2, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5b)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file5bdst := file5b
file5bdst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file3, file4, file5bdst)
}
// Test with CopyDest set
func TestSyncCopyDest(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
if r.Fremote.Features().Copy == nil {
t.Skip("Skipping test as remote does not support server side copy")
}
fs.Config.CopyDest = r.FremoteName + "/CopyDest"
defer func() {
fs.Config.CopyDest = ""
}()
fdst, err := fs.NewFs(r.FremoteName + "/dst")
require.NoError(t, err)
// check empty dest, empty copy
file1 := r.WriteFile("one", "one", t1)
fstest.CheckItems(t, r.Flocal, file1)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1dst := file1
file1dst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1dst)
// check old dest, empty copy
file1b := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file1dst)
fstest.CheckItems(t, r.Flocal, file1b)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file1bdst := file1b
file1bdst.Path = "dst/one"
fstest.CheckItems(t, r.Fremote, file1bdst)
// check old dest, new copy, backup-dir
fs.Config.BackupDir = r.FremoteName + "/BackupDir"
file3 := r.WriteObject(context.Background(), "dst/one", "one", t1)
file2 := r.WriteObject(context.Background(), "CopyDest/one", "onet2", t2)
file1c := r.WriteFile("one", "onet2", t2)
fstest.CheckItems(t, r.Fremote, file2, file3)
fstest.CheckItems(t, r.Flocal, file1c)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file2dst := file2
file2dst.Path = "dst/one"
file3.Path = "BackupDir/one"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
fs.Config.BackupDir = ""
// check empty dest, new copy
file4 := r.WriteObject(context.Background(), "CopyDest/two", "two", t2)
file5 := r.WriteFile("two", "two", t2)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4)
fstest.CheckItems(t, r.Flocal, file1c, file5)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file4dst := file4
file4dst.Path = "dst/two"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check new dest, new copy
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst)
// check empty dest, old copy
file6 := r.WriteObject(context.Background(), "CopyDest/three", "three", t2)
file7 := r.WriteFile("three", "threet3", t3)
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6)
fstest.CheckItems(t, r.Flocal, file1c, file5, file7)
accounting.Stats.ResetCounters()
err = Sync(context.Background(), fdst, r.Flocal, false)
require.NoError(t, err)
file7dst := file7
file7dst.Path = "dst/three"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3, file4, file4dst, file6, file7dst)
}
// Test with BackupDir set
func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
r := fstest.NewRun(t)