filter: add --files-from0 to support NUL-delimited input - fixes #9537

2026-07-02 11:21:21 +01:00
parent c91c4cbbff
commit 761af75a25
5 changed files with 196 additions and 23 deletions
@@ -3320,6 +3320,7 @@ For the filtering options
 - `--include-from`
 - `--files-from`
 - `--files-from-raw`
+- `--files-from0`
 - `--min-size`
 - `--max-size`
 - `--min-age`
@@ -272,8 +272,8 @@ is included.
 Any path/file included at that stage is processed by the rclone
 command.

-`--files-from` and `--files-from-raw` flags over-ride and cannot be
-combined with other filter options.
+`--files-from`, `--files-from-raw` and `--files-from0` flags
+over-ride and cannot be combined with other filter options.

 To see the internal combined rule list, in regular expression form,
 for a command add the `--dump filters` flag. Running an rclone command
@@ -396,8 +396,8 @@ processed in.
 `--exclude` should not be used with `--include`, `--include-from`,
 `--filter` or `--filter-from` flags.

-`--exclude` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--exclude` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.

 E.g. `rclone ls remote: --exclude *.bak` excludes all .bak files
 from listing.
@@ -440,8 +440,8 @@ are applied to an rclone command.
 `--exclude-from` should not be used with `--include`, `--include-from`,
 `--filter` or `--filter-from` flags.

-`--exclude-from` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--exclude-from` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.

 `--exclude-from` followed by `-` reads filter rules from standard input.

@@ -453,8 +453,8 @@ command.
 This flag can be repeated. See above for the order filter flags are
 processed in.

-`--include` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--include` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.

 `--include` implies `--exclude **` at the end of an rclone internal
 filter list. Therefore if you mix `--include` and `--include-from`
@@ -513,8 +513,8 @@ flags with `--exclude`, `--exclude-from`, `--filter` or `--filter-from`,
 you must use include rules for all the files you want in the include
 statement. For more flexibility use the `--filter-from` flag.

-`--include-from` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--include-from` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.

 `--include-from` followed by `-` reads filter rules from standard input.

@@ -531,8 +531,8 @@ implies an `--exclude *` rule which it adds to the bottom of the internal rule
 list. `--filter...+` does not imply
 that rule.

-`--filter` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--filter` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.

 `--filter` should not be used with `--include`, `--include-from`,
 `--exclude` or `--exclude-from` flags.
@@ -612,7 +612,7 @@ no others.

 Other filter flags (`--include`, `--include-from`, `--exclude`,
 `--exclude-from`, `--filter` and `--filter-from`) are ignored when
-`--files-from` is used.
+`--files-from`, `--files-from-raw` or `--files-from0` is used.

 `--files-from` expects a list of files as its input. Leading or
 trailing whitespace is stripped from the input lines. Lines starting
@@ -719,6 +719,21 @@ with `;` or `#` are read without any processing. [rclone lsf](/commands/rclone_l
 has a compatible format that can be used to export file lists from remotes for
 input to `--files-from-raw`.

+### `--files-from0` - Read NUL separated list of source-file names
+
+This flag is the same as `--files-from-raw` except that input is
+split on NUL (`\0`) characters instead of newlines. This allows
+handling filenames that contain embedded newline characters.
+
+It is similar to the `-print0` option of `find` and the `-0`
+option of `xargs`.
+
+E.g. to copy files listed by `find`:
+
+```console
+find /path -print0 | rclone copy --files-from0 - / remote:path
+```
+
 ### `--ignore-case` - make searches case insensitive

 By default, rclone filter patterns are case sensitive. The `--ignore-case`
@@ -846,7 +861,7 @@ This will stay constant across retries.

 - Safe to use with `rclone sync`; source and destination selections will match.
 - **Do not** use with `--delete-excluded`, as this could delete unselected files.
- Ignored if `--files-from` is used.
+- Ignored if `--files-from`, `--files-from-raw` or `--files-from0` is used.

 #### Examples

@@ -45,6 +45,11 @@ var OptionsInfo = fs.Options{{
 	Default: []string{},
 	Help:    "Read list of source-file names from file without any processing of lines (use - to read from stdin)",
 	Groups:  "Filter",
+}, {
+	Name:    "files_from0",
+	Default: []string{},
+	Help:    "Read list of source-file names from file using NUL as separator (use - to read from stdin)",
+	Groups:  "Filter",
 }, {
 	Name:    "min_age",
 	Default: fs.DurationOff,
@@ -145,6 +150,7 @@ type Options struct {
 	ExcludeFile    []string      `config:"exclude_if_present"`
 	FilesFrom      []string      `config:"files_from"`
 	FilesFromRaw   []string      `config:"files_from_raw"`
+	FilesFrom0     []string      `config:"files_from0"`
 	MetaRules      RulesOpt      `config:"metadata"`
 	MinAge         fs.Duration   `config:"min_age"`
 	MaxAge         fs.Duration   `config:"max_age"`
@@ -226,10 +232,10 @@ func NewFilter(opt *Options) (f *Filter, err error) {

 	for _, rule := range f.Opt.FilesFrom {
 		if !inActive {
-			return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw")
+			return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw or --files-from0")
 		}
 		f.initAddFile() // init to show --files-from set even if no files within
-		err := forEachLine(rule, false, func(line string) error {
+		err := forEachLine(rule, false, false, func(line string) error {
 			return f.AddFile(line)
 		})
 		if err != nil {
@@ -241,10 +247,23 @@ func NewFilter(opt *Options) (f *Filter, err error) {
 		// --files-from-raw can be used with --files-from, hence we do
 		// not need to get the value of f.InActive again
 		if !inActive {
-			return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from")
+			return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from or --files-from0")
 		}
 		f.initAddFile() // init to show --files-from set even if no files within
-		err := forEachLine(rule, true, func(line string) error {
+		err := forEachLine(rule, true, false, func(line string) error {
+			return f.AddFile(line)
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	for _, rule := range f.Opt.FilesFrom0 {
+		if !inActive {
+			return nil, fmt.Errorf("the usage of --files-from0 overrides all other filters, it should be used alone or with --files-from or --files-from-raw")
+		}
+		f.initAddFile() // init to show --files-from set even if no files within
+		err := forEachLine(rule, true, true, func(line string) error {
 			return f.AddFile(line)
 		})
 		if err != nil {
@@ -176,6 +176,57 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) {
 	}
 }

+func TestNewFilterForbiddenMixOfFilesFrom0AndFilterRule(t *testing.T) {
+	Opt := Opt
+
+	// Set up the input
+	Opt.FilterRule = []string{"- filter1", "- filter1b"}
+	Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\x00files2\x00")}
+
+	rm := func(p string) {
+		err := os.Remove(p)
+		if err != nil {
+			t.Logf("error removing %q: %v", p, err)
+		}
+	}
+	// Reset the input
+	defer func() {
+		rm(Opt.FilesFrom0[0])
+	}()
+
+	_, err := NewFilter(&Opt)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "the usage of --files-from0 overrides all other filters")
+}
+
+func TestNewFilterWithFilesFrom0(t *testing.T) {
+	Opt := Opt
+
+	// Set up the input: NUL-separated, with an embedded newline in one entry
+	Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\nmore\x00files2\x00")}
+
+	rm := func(p string) {
+		err := os.Remove(p)
+		if err != nil {
+			t.Logf("error removing %q: %v", p, err)
+		}
+	}
+	// Reset the input
+	defer func() {
+		rm(Opt.FilesFrom0[0])
+	}()
+
+	f, err := NewFilter(&Opt)
+	require.NoError(t, err)
+	assert.Len(t, f.files, 3)
+	for _, name := range []string{"#comment", "files1\nmore", "files2"} {
+		_, ok := f.files[name]
+		if !ok {
+			t.Errorf("Didn't find file %q in f.files", name)
+		}
+	}
+}
+
 func TestNewFilterFullExceptFilesFromOpt(t *testing.T) {
 	Opt := Opt

@@ -770,7 +821,7 @@ five
 		}()
 		fileName = "-"
 	}
-	err := forEachLine(fileName, raw, func(s string) error {
+	err := forEachLine(fileName, raw, false, func(s string) error {
 		lines = append(lines, s)
 		return nil
 	})
@@ -799,6 +850,71 @@ func TestFilterForEachLineStdinWithRaw(t *testing.T) {
 	testFilterForEachLine(t, true, true)
 }

+func testFilterForEachLineNul(t *testing.T, useStdin bool) {
+	file := testFile(t, "one\x00two\nthree\x00four\x00five\x00")
+	defer func() {
+		err := os.Remove(file)
+		require.NoError(t, err)
+	}()
+	lines := []string{}
+	fileName := file
+	if useStdin {
+		in, err := os.Open(file)
+		require.NoError(t, err)
+		oldStdin := os.Stdin
+		os.Stdin = in
+		defer func() {
+			os.Stdin = oldStdin
+			_ = in.Close()
+		}()
+		fileName = "-"
+	}
+	err := forEachLine(fileName, true, true, func(s string) error {
+		lines = append(lines, s)
+		return nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"one", "two\nthree", "four", "five"}, lines)
+}
+
+func TestFilterForEachLineNul(t *testing.T) {
+	testFilterForEachLineNul(t, false)
+}
+
+func TestFilterForEachLineNulStdin(t *testing.T) {
+	testFilterForEachLineNul(t, true)
+}
+
+func TestFilterForEachLineNulNoTrailing(t *testing.T) {
+	file := testFile(t, "one\x00two\x00three")
+	defer func() {
+		err := os.Remove(file)
+		require.NoError(t, err)
+	}()
+	var lines []string
+	err := forEachLine(file, true, true, func(s string) error {
+		lines = append(lines, s)
+		return nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"one", "two", "three"}, lines)
+}
+
+func TestFilterForEachLineNulConsecutive(t *testing.T) {
+	file := testFile(t, "one\x00\x00two\x00")
+	defer func() {
+		err := os.Remove(file)
+		require.NoError(t, err)
+	}()
+	var lines []string
+	err := forEachLine(file, true, true, func(s string) error {
+		lines = append(lines, s)
+		return nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, []string{"one", "", "two"}, lines)
+}
+
 func TestFilterMatchesFromDocs(t *testing.T) {
 	for _, test := range []struct {
 		glob       string
@@ -2,6 +2,7 @@ package filter

 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"os"
 	"regexp"
@@ -114,10 +115,26 @@ func (rs *rules) includeMany(remotes []string) bool {
 	return true
 }

+// scanNul is a split function for a Scanner that returns each NUL-terminated
+// sequence of bytes. It correctly handles the final segment even if it
+// lacks a trailing NUL.
+func scanNul(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+	if i := bytes.IndexByte(data, '\x00'); i >= 0 {
+		return i + 1, data[:i], nil
+	}
+	if atEOF {
+		return len(data), data, nil
+	}
+	return 0, nil, nil
+}
+
 // forEachLine calls fn on every line in the file pointed to by path
 //
 // It ignores empty lines and lines starting with '#' or ';' if raw is false
-func forEachLine(path string, raw bool, fn func(string) error) (err error) {
+func forEachLine(path string, raw bool, useNulDelimiter bool, fn func(string) error) (err error) {
 	var scanner *bufio.Scanner
 	if path == "-" {
 		scanner = bufio.NewScanner(os.Stdin)
@@ -129,6 +146,11 @@ func forEachLine(path string, raw bool, fn func(string) error) (err error) {
 		scanner = bufio.NewScanner(in)
 		defer fs.CheckClose(in, &err)
 	}
+
+	if useNulDelimiter {
+		scanner.Split(scanNul)
+	}
+
 	for scanner.Scan() {
 		line := scanner.Text()
 		if !raw {
@@ -199,7 +221,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {
 		addImplicitExclude = true
 	}
 	for _, rule := range opt.IncludeFrom {
-		err := forEachLine(rule, false, func(line string) error {
+		err := forEachLine(rule, false, false, func(line string) error {
 			return add(true, line)
 		})
 		if err != nil {
@@ -215,7 +237,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {
 		foundExcludeRule = true
 	}
 	for _, rule := range opt.ExcludeFrom {
-		err := forEachLine(rule, false, func(line string) error {
+		err := forEachLine(rule, false, false, func(line string) error {
 			return add(false, line)
 		})
 		if err != nil {
@@ -235,7 +257,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {
 		}
 	}
 	for _, rule := range opt.FilterFrom {
-		err := forEachLine(rule, false, func(rule string) error {
+		err := forEachLine(rule, false, false, func(rule string) error {
 			return addRule(rule, add, clear)
 		})
 		if err != nil {