f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filter: Make --files-from traverse as before unless --no-traverse is set

Browse Source 
In c5ac96e9e7 we made --files-from only read the objects specified and
don't scan directories.

This caused problems with Google drive (very very slow) and B2
(excessive API consumption) so it was decided to make the old
behaviour (traversing the directories) the default with --files-from
and use the existing --no-traverse flag (which has exactly the right
semantics) to enable the new non scanning behaviour.

See: https://forum.rclone.org/t/using-files-from-with-drive-hammers-the-api/8726

Fixes #3102 Fixes #3095
This commit is contained in:
Nick Craig-Wood
2019-04-12 17:16:49 +01:00
parent b05da61c82
commit a28239f005
4 changed files with 32 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/walk/walk.go
+6 -6
View File
@@ -54,12 +54,12 @@ type Func func(path string, entries fs.DirEntries, err error) error
// This is implemented by WalkR if Config.UseUseListR is true
// and f supports it and level > 1, or WalkN otherwise.
//
// If --files-from is set then a DirTree will be constructed with just
// those files in and then walked with WalkR
// If --files-from and --no-traverse is set then a DirTree will be
// constructed with just those files in and then walked with WalkR
//
// NB (f, path) to be replaced by fs.Dir at some point
func Walk(f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
if filter.Active.HaveFilesFrom() {
if fs.Config.NoTraverse && filter.Active.HaveFilesFrom() {
return walkR(f, path, includeAll, maxLevel, fn, filter.Active.MakeListR(f.NewObject))
}
// FIXME should this just be maxLevel < 0 - why the maxLevel > 1
@@ -711,12 +711,12 @@ func walkNDirTree(f fs.Fs, path string, includeAll bool, maxLevel int, listDir l
// This is implemented by WalkR if f supports ListR and level > 1, or
// WalkN otherwise.
//
// If --files-from is set then a DirTree will be constructed with just
// those files in.
// If --files-from and --no-traverse is set then a DirTree will be
// constructed with just those files in.
//
// NB (f, path) to be replaced by fs.Dir at some point
func NewDirTree(f fs.Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
if filter.Active.HaveFilesFrom() {
if fs.Config.NoTraverse && filter.Active.HaveFilesFrom() {
return walkRDirTree(f, path, includeAll, maxLevel, filter.Active.MakeListR(f.NewObject))
}
if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && ListR != nil {