diff --git a/docs/content/docs.md b/docs/content/docs.md index 6971d3513..517b1942a 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -3320,6 +3320,7 @@ For the filtering options - `--include-from` - `--files-from` - `--files-from-raw` +- `--files-from0` - `--min-size` - `--max-size` - `--min-age` diff --git a/docs/content/filtering.md b/docs/content/filtering.md index 8c587c968..399b8da05 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -272,8 +272,8 @@ is included. Any path/file included at that stage is processed by the rclone command. -`--files-from` and `--files-from-raw` flags over-ride and cannot be -combined with other filter options. +`--files-from`, `--files-from-raw` and `--files-from0` flags +over-ride and cannot be combined with other filter options. To see the internal combined rule list, in regular expression form, for a command add the `--dump filters` flag. Running an rclone command @@ -396,8 +396,8 @@ processed in. `--exclude` should not be used with `--include`, `--include-from`, `--filter` or `--filter-from` flags. -`--exclude` has no effect when combined with `--files-from` or -`--files-from-raw` flags. +`--exclude` has no effect when combined with `--files-from`, +`--files-from-raw` or `--files-from0` flags. E.g. `rclone ls remote: --exclude *.bak` excludes all .bak files from listing. @@ -440,8 +440,8 @@ are applied to an rclone command. `--exclude-from` should not be used with `--include`, `--include-from`, `--filter` or `--filter-from` flags. -`--exclude-from` has no effect when combined with `--files-from` or -`--files-from-raw` flags. +`--exclude-from` has no effect when combined with `--files-from`, +`--files-from-raw` or `--files-from0` flags. `--exclude-from` followed by `-` reads filter rules from standard input. @@ -453,8 +453,8 @@ command. This flag can be repeated. See above for the order filter flags are processed in. -`--include` has no effect when combined with `--files-from` or -`--files-from-raw` flags. +`--include` has no effect when combined with `--files-from`, +`--files-from-raw` or `--files-from0` flags. `--include` implies `--exclude **` at the end of an rclone internal filter list. Therefore if you mix `--include` and `--include-from` @@ -513,8 +513,8 @@ flags with `--exclude`, `--exclude-from`, `--filter` or `--filter-from`, you must use include rules for all the files you want in the include statement. For more flexibility use the `--filter-from` flag. -`--include-from` has no effect when combined with `--files-from` or -`--files-from-raw` flags. +`--include-from` has no effect when combined with `--files-from`, +`--files-from-raw` or `--files-from0` flags. `--include-from` followed by `-` reads filter rules from standard input. @@ -531,8 +531,8 @@ implies an `--exclude *` rule which it adds to the bottom of the internal rule list. `--filter...+` does not imply that rule. -`--filter` has no effect when combined with `--files-from` or -`--files-from-raw` flags. +`--filter` has no effect when combined with `--files-from`, +`--files-from-raw` or `--files-from0` flags. `--filter` should not be used with `--include`, `--include-from`, `--exclude` or `--exclude-from` flags. @@ -612,7 +612,7 @@ no others. Other filter flags (`--include`, `--include-from`, `--exclude`, `--exclude-from`, `--filter` and `--filter-from`) are ignored when -`--files-from` is used. +`--files-from`, `--files-from-raw` or `--files-from0` is used. `--files-from` expects a list of files as its input. Leading or trailing whitespace is stripped from the input lines. Lines starting @@ -719,6 +719,21 @@ with `;` or `#` are read without any processing. [rclone lsf](/commands/rclone_l has a compatible format that can be used to export file lists from remotes for input to `--files-from-raw`. +### `--files-from0` - Read NUL separated list of source-file names + +This flag is the same as `--files-from-raw` except that input is +split on NUL (`\0`) characters instead of newlines. This allows +handling filenames that contain embedded newline characters. + +It is similar to the `-print0` option of `find` and the `-0` +option of `xargs`. + +E.g. to copy files listed by `find`: + +```console +find /path -print0 | rclone copy --files-from0 - / remote:path +``` + ### `--ignore-case` - make searches case insensitive By default, rclone filter patterns are case sensitive. The `--ignore-case` @@ -846,7 +861,7 @@ This will stay constant across retries. - Safe to use with `rclone sync`; source and destination selections will match. - **Do not** use with `--delete-excluded`, as this could delete unselected files. -- Ignored if `--files-from` is used. +- Ignored if `--files-from`, `--files-from-raw` or `--files-from0` is used. #### Examples diff --git a/fs/filter/filter.go b/fs/filter/filter.go index ab0310dc1..e00d0e8b3 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -45,6 +45,11 @@ var OptionsInfo = fs.Options{{ Default: []string{}, Help: "Read list of source-file names from file without any processing of lines (use - to read from stdin)", Groups: "Filter", +}, { + Name: "files_from0", + Default: []string{}, + Help: "Read list of source-file names from file using NUL as separator (use - to read from stdin)", + Groups: "Filter", }, { Name: "min_age", Default: fs.DurationOff, @@ -145,6 +150,7 @@ type Options struct { ExcludeFile []string `config:"exclude_if_present"` FilesFrom []string `config:"files_from"` FilesFromRaw []string `config:"files_from_raw"` + FilesFrom0 []string `config:"files_from0"` MetaRules RulesOpt `config:"metadata"` MinAge fs.Duration `config:"min_age"` MaxAge fs.Duration `config:"max_age"` @@ -226,10 +232,10 @@ func NewFilter(opt *Options) (f *Filter, err error) { for _, rule := range f.Opt.FilesFrom { if !inActive { - return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw") + return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw or --files-from0") } f.initAddFile() // init to show --files-from set even if no files within - err := forEachLine(rule, false, func(line string) error { + err := forEachLine(rule, false, false, func(line string) error { return f.AddFile(line) }) if err != nil { @@ -241,10 +247,23 @@ func NewFilter(opt *Options) (f *Filter, err error) { // --files-from-raw can be used with --files-from, hence we do // not need to get the value of f.InActive again if !inActive { - return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from") + return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from or --files-from0") } f.initAddFile() // init to show --files-from set even if no files within - err := forEachLine(rule, true, func(line string) error { + err := forEachLine(rule, true, false, func(line string) error { + return f.AddFile(line) + }) + if err != nil { + return nil, err + } + } + + for _, rule := range f.Opt.FilesFrom0 { + if !inActive { + return nil, fmt.Errorf("the usage of --files-from0 overrides all other filters, it should be used alone or with --files-from or --files-from-raw") + } + f.initAddFile() // init to show --files-from set even if no files within + err := forEachLine(rule, true, true, func(line string) error { return f.AddFile(line) }) if err != nil { diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index 67c8b9bc1..75f8c77e1 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -176,6 +176,57 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) { } } +func TestNewFilterForbiddenMixOfFilesFrom0AndFilterRule(t *testing.T) { + Opt := Opt + + // Set up the input + Opt.FilterRule = []string{"- filter1", "- filter1b"} + Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\x00files2\x00")} + + rm := func(p string) { + err := os.Remove(p) + if err != nil { + t.Logf("error removing %q: %v", p, err) + } + } + // Reset the input + defer func() { + rm(Opt.FilesFrom0[0]) + }() + + _, err := NewFilter(&Opt) + require.Error(t, err) + require.Contains(t, err.Error(), "the usage of --files-from0 overrides all other filters") +} + +func TestNewFilterWithFilesFrom0(t *testing.T) { + Opt := Opt + + // Set up the input: NUL-separated, with an embedded newline in one entry + Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\nmore\x00files2\x00")} + + rm := func(p string) { + err := os.Remove(p) + if err != nil { + t.Logf("error removing %q: %v", p, err) + } + } + // Reset the input + defer func() { + rm(Opt.FilesFrom0[0]) + }() + + f, err := NewFilter(&Opt) + require.NoError(t, err) + assert.Len(t, f.files, 3) + for _, name := range []string{"#comment", "files1\nmore", "files2"} { + _, ok := f.files[name] + if !ok { + t.Errorf("Didn't find file %q in f.files", name) + } + } +} + func TestNewFilterFullExceptFilesFromOpt(t *testing.T) { Opt := Opt @@ -770,7 +821,7 @@ five }() fileName = "-" } - err := forEachLine(fileName, raw, func(s string) error { + err := forEachLine(fileName, raw, false, func(s string) error { lines = append(lines, s) return nil }) @@ -799,6 +850,71 @@ func TestFilterForEachLineStdinWithRaw(t *testing.T) { testFilterForEachLine(t, true, true) } +func testFilterForEachLineNul(t *testing.T, useStdin bool) { + file := testFile(t, "one\x00two\nthree\x00four\x00five\x00") + defer func() { + err := os.Remove(file) + require.NoError(t, err) + }() + lines := []string{} + fileName := file + if useStdin { + in, err := os.Open(file) + require.NoError(t, err) + oldStdin := os.Stdin + os.Stdin = in + defer func() { + os.Stdin = oldStdin + _ = in.Close() + }() + fileName = "-" + } + err := forEachLine(fileName, true, true, func(s string) error { + lines = append(lines, s) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"one", "two\nthree", "four", "five"}, lines) +} + +func TestFilterForEachLineNul(t *testing.T) { + testFilterForEachLineNul(t, false) +} + +func TestFilterForEachLineNulStdin(t *testing.T) { + testFilterForEachLineNul(t, true) +} + +func TestFilterForEachLineNulNoTrailing(t *testing.T) { + file := testFile(t, "one\x00two\x00three") + defer func() { + err := os.Remove(file) + require.NoError(t, err) + }() + var lines []string + err := forEachLine(file, true, true, func(s string) error { + lines = append(lines, s) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"one", "two", "three"}, lines) +} + +func TestFilterForEachLineNulConsecutive(t *testing.T) { + file := testFile(t, "one\x00\x00two\x00") + defer func() { + err := os.Remove(file) + require.NoError(t, err) + }() + var lines []string + err := forEachLine(file, true, true, func(s string) error { + lines = append(lines, s) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"one", "", "two"}, lines) +} + func TestFilterMatchesFromDocs(t *testing.T) { for _, test := range []struct { glob string diff --git a/fs/filter/rules.go b/fs/filter/rules.go index 085a02aa5..e5ee2626e 100644 --- a/fs/filter/rules.go +++ b/fs/filter/rules.go @@ -2,6 +2,7 @@ package filter import ( "bufio" + "bytes" "fmt" "os" "regexp" @@ -114,10 +115,26 @@ func (rs *rules) includeMany(remotes []string) bool { return true } +// scanNul is a split function for a Scanner that returns each NUL-terminated +// sequence of bytes. It correctly handles the final segment even if it +// lacks a trailing NUL. +func scanNul(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + if i := bytes.IndexByte(data, '\x00'); i >= 0 { + return i + 1, data[:i], nil + } + if atEOF { + return len(data), data, nil + } + return 0, nil, nil +} + // forEachLine calls fn on every line in the file pointed to by path // // It ignores empty lines and lines starting with '#' or ';' if raw is false -func forEachLine(path string, raw bool, fn func(string) error) (err error) { +func forEachLine(path string, raw bool, useNulDelimiter bool, fn func(string) error) (err error) { var scanner *bufio.Scanner if path == "-" { scanner = bufio.NewScanner(os.Stdin) @@ -129,6 +146,11 @@ func forEachLine(path string, raw bool, fn func(string) error) (err error) { scanner = bufio.NewScanner(in) defer fs.CheckClose(in, &err) } + + if useNulDelimiter { + scanner.Split(scanNul) + } + for scanner.Scan() { line := scanner.Text() if !raw { @@ -199,7 +221,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) { addImplicitExclude = true } for _, rule := range opt.IncludeFrom { - err := forEachLine(rule, false, func(line string) error { + err := forEachLine(rule, false, false, func(line string) error { return add(true, line) }) if err != nil { @@ -215,7 +237,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) { foundExcludeRule = true } for _, rule := range opt.ExcludeFrom { - err := forEachLine(rule, false, func(line string) error { + err := forEachLine(rule, false, false, func(line string) error { return add(false, line) }) if err != nil { @@ -235,7 +257,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) { } } for _, rule := range opt.FilterFrom { - err := forEachLine(rule, false, func(rule string) error { + err := forEachLine(rule, false, false, func(rule string) error { return addRule(rule, add, clear) }) if err != nil {