dropbox: add ListR with ListP leaf casing and fail-closed reconstruction

Recursive list_folder is collected in full before any callback. Remotes are rebuilt from PathLower parents plus the last path component only. Empty path_lower, missing parents, or remotes that escape the root abort the listing so sync/bisync cannot treat a truncated result as deletes. Shared-folder modes leave ListR disabled.
2026-09-09 23:46:39 +02:00
parent d44d85dc05
commit 0964c7eb39
2 changed files with 297 additions and 3 deletions
@@ -16,9 +16,13 @@ casing. Changes to only the casing of paths won't be returned by
 list_folder/continue. This field will be null if the file or folder is
 not mounted. This field is optional.

-We solve this by not implementing the ListR interface.  The dropbox
-remote will recurse directory by directory only using the last element
-of path_display and all will be well.
+ListR is implemented with the same leaf-casing rule as List/ListP:
+only the last path component (Name / PathDisplay base) is trusted.
+Parent remotes are rebuilt from PathLower identity plus already-seen
+folder leaves. A full PathDisplay is never used as a remote. If an
+entry cannot be placed (empty PathLower, missing parent), ListR
+returns an error and emits nothing, so sync/bisync cannot treat a
+truncated listing as “delete the rest”.
 */

 import (
@@ -28,6 +32,7 @@ import (
 	"io"
 	"path"
 	"regexp"
+	"sort"
 	"strings"
 	"time"
 	"unicode/utf8"
@@ -661,6 +666,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}

 	f.features.Fill(ctx, f)
+	if f.opt.SharedFiles || f.opt.SharedFolders {
+		// Received/shared listing is a flat namespace; recursive ListR would be wrong.
+		f.features.ListR = nil
+	}

 	if f.opt.RootNsid != "" {
 		f.ns = f.opt.RootNsid
@@ -1204,6 +1213,251 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
 	return list.Flush()
 }

+// listRitem is one recursive list_folder entry before remote reconstruction.
+type listRitem struct {
+	pathLower string
+	leaf      string
+	folder    *files.FolderMetadata
+	file      *files.FileMetadata
+}
+
+func listRLeaf(md *files.Metadata) string {
+	if md.PathDisplay != "" {
+		return path.Base(md.PathDisplay)
+	}
+	return md.Name
+}
+
+func dropboxListRootLower(encRoot string) string {
+	if encRoot == "" || encRoot == "/" {
+		return "/"
+	}
+	lower := strings.ToLower(encRoot)
+	if !strings.HasPrefix(lower, "/") {
+		lower = "/" + lower
+	}
+	return lower
+}
+
+// reconstructCasedRemotes maps PathLower → rclone remote using List/ListP
+// leaf casing. It never trusts parent components of PathDisplay.
+// Missing parents or empty PathLower return an error (no partial map).
+func reconstructCasedRemotes(dir, listRootLower string, enc encoder.MultiEncoder, items []listRitem) (map[string]string, error) {
+	folders := make([]listRitem, 0, len(items))
+	for _, it := range items {
+		if it.pathLower == "" {
+			return nil, fmt.Errorf("dropbox ListR: refusing incomplete listing: empty path_lower for %q", it.leaf)
+		}
+		if it.pathLower == listRootLower {
+			continue
+		}
+		if it.folder != nil {
+			folders = append(folders, it)
+		}
+	}
+	sort.SliceStable(folders, func(i, j int) bool {
+		ci, cj := strings.Count(folders[i].pathLower, "/"), strings.Count(folders[j].pathLower, "/")
+		if ci != cj {
+			return ci < cj
+		}
+		return folders[i].pathLower < folders[j].pathLower
+	})
+	remotes := make(map[string]string, len(items))
+	for _, it := range folders {
+		parent := path.Dir(it.pathLower)
+		var parentRemote string
+		if parent == listRootLower {
+			parentRemote = dir
+		} else {
+			var ok bool
+			parentRemote, ok = remotes[parent]
+			if !ok {
+				return nil, fmt.Errorf("dropbox ListR: refusing incomplete listing: missing parent %q for folder %q", parent, it.pathLower)
+			}
+		}
+		remotes[it.pathLower] = path.Join(parentRemote, enc.ToStandardName(it.leaf))
+	}
+	for _, it := range items {
+		if it.folder != nil || it.pathLower == "" || it.pathLower == listRootLower {
+			continue
+		}
+		parent := path.Dir(it.pathLower)
+		var parentRemote string
+		if parent == listRootLower {
+			parentRemote = dir
+		} else {
+			var ok bool
+			parentRemote, ok = remotes[parent]
+			if !ok {
+				return nil, fmt.Errorf("dropbox ListR: refusing incomplete listing: missing parent %q for %q", parent, it.pathLower)
+			}
+		}
+		remotes[it.pathLower] = path.Join(parentRemote, enc.ToStandardName(it.leaf))
+	}
+	return remotes, nil
+}
+
+func underPathLower(pathLower, prefix string) bool {
+	return pathLower == prefix || strings.HasPrefix(pathLower, prefix+"/")
+}
+
+// ListR lists the objects and directories of the Fs starting from
+// dir recursively into out.
+//
+// Entries are not emitted until the recursive listing has completed
+// and every path has been reconstructed. A failure at any point
+// returns without a successful empty listing.
+func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
+	if f.opt.SharedFiles || f.opt.SharedFolders {
+		return errors.New("dropbox: ListR is not supported with shared_files or shared_folders")
+	}
+
+	root := f.slashRoot
+	if dir != "" {
+		root += "/" + dir
+	}
+	listRootLower := dropboxListRootLower(f.opt.Enc.FromStandardPath(root))
+
+	var items []listRitem
+	started := false
+	var res *files.ListFolderResult
+	for {
+		if !started {
+			arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(root))
+			arg.Recursive = true
+			arg.Limit = 1000
+			if root == "/" {
+				arg.Path = ""
+			}
+			err = f.pacer.Call(func() (bool, error) {
+				res, err = f.srv.ListFolderContext(ctx, arg)
+				return shouldRetry(ctx, err)
+			})
+			if err != nil {
+				switch e := err.(type) {
+				case files.ListFolderAPIError:
+					if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
+						err = fs.ErrorDirNotFound
+					}
+				}
+				return err
+			}
+			started = true
+		} else {
+			arg := files.ListFolderContinueArg{Cursor: res.Cursor}
+			err = f.pacer.Call(func() (bool, error) {
+				res, err = f.srv.ListFolderContinueContext(ctx, &arg)
+				return shouldRetry(ctx, err)
+			})
+			if err != nil {
+				return fmt.Errorf("list continue: %w", err)
+			}
+		}
+		for _, entry := range res.Entries {
+			switch info := entry.(type) {
+			case *files.FolderMetadata:
+				if info.PathLower == "" {
+					return fmt.Errorf("dropbox ListR: refusing incomplete listing: folder %q has empty path_lower", info.Name)
+				}
+				items = append(items, listRitem{
+					pathLower: info.PathLower,
+					leaf:      listRLeaf(&info.Metadata),
+					folder:    info,
+				})
+			case *files.FileMetadata:
+				if info.PathLower == "" {
+					return fmt.Errorf("dropbox ListR: refusing incomplete listing: file %q has empty path_lower", info.Name)
+				}
+				items = append(items, listRitem{
+					pathLower: info.PathLower,
+					leaf:      listRLeaf(&info.Metadata),
+					file:      info,
+				})
+			default:
+				fs.Errorf(f, "Unknown type %T", entry)
+			}
+		}
+		if !res.HasMore {
+			break
+		}
+	}
+
+	skipPrefixes := make([]string, 0)
+	for _, it := range items {
+		if it.folder == nil || it.folder.SharingInfo == nil || it.folder.SharingInfo.SharedFolderId == "" {
+			continue
+		}
+		remoteHint := it.leaf
+		if f.opt.SkipSharedFolders {
+			fs.Debugf(remoteHint, "Skipping shared folder")
+			skipPrefixes = append(skipPrefixes, it.pathLower)
+			continue
+		}
+		if f.opt.SkipUnownedFolders {
+			var sfMeta *sharing.SharedFolderMetadata
+			err = f.pacer.Call(func() (bool, error) {
+				var apiErr error
+				sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(it.folder.SharingInfo.SharedFolderId))
+				return shouldRetry(ctx, apiErr)
+			})
+			if err != nil {
+				fs.Errorf(remoteHint, "Failed to get shared folder metadata (defaulting to include): %v", err)
+			} else if sfMeta != nil && sfMeta.AccessType != nil && sfMeta.AccessType.Tag != sharing.AccessLevelOwner {
+				fs.Debugf(remoteHint, "Skipping unowned shared folder")
+				skipPrefixes = append(skipPrefixes, it.pathLower)
+			}
+		}
+	}
+
+	kept := items[:0]
+	for _, it := range items {
+		skip := false
+		for _, p := range skipPrefixes {
+			if underPathLower(it.pathLower, p) {
+				skip = true
+				break
+			}
+		}
+		if !skip {
+			kept = append(kept, it)
+		}
+	}
+	items = kept
+
+	remotes, err := reconstructCasedRemotes(dir, listRootLower, f.opt.Enc, items)
+	if err != nil {
+		return err
+	}
+
+	lh := list.NewHelper(callback)
+	for _, it := range items {
+		remote, ok := remotes[it.pathLower]
+		if !ok {
+			continue
+		}
+		if list.RemoteEscapesRoot(remote) {
+			return fmt.Errorf("dropbox ListR: refusing listing: remote %q escapes the root", remote)
+		}
+		if it.folder != nil {
+			d := fs.NewDir(remote, time.Time{}).SetID(it.folder.Id)
+			if err = lh.Add(d); err != nil {
+				return err
+			}
+			continue
+		}
+		o, err := f.newObjectWithInfo(ctx, remote, it.file)
+		if err != nil {
+			return err
+		}
+		if o.(*Object).exportType.listable() {
+			if err = lh.Add(o); err != nil {
+				return err
+			}
+		}
+	}
+	return lh.Flush()
+}
+
 // Put the object
 //
 // Copy the reader in to the new object which is returned.
@@ -2413,6 +2667,7 @@ var (
 	_ fs.PublicLinker = (*Fs)(nil)
 	_ fs.DirMover     = (*Fs)(nil)
 	_ fs.ListPer      = (*Fs)(nil)
+	_ fs.ListRer      = (*Fs)(nil)
 	_ fs.Abouter      = (*Fs)(nil)
 	_ fs.Shutdowner   = &Fs{}
 	_ fs.Object       = (*Object)(nil)
@@ -568,3 +568,42 @@ func TestListingChangeFromEntry(t *testing.T) {
 	assert.Equal(t, "sub", ch.Remote)
 	assert.True(t, ch.Deleted)
 }
+
+func TestReconstructCasedRemotes(t *testing.T) {
+	enc := encoder.Standard
+	// Files listed before folders; PathDisplay parents wrongly cased.
+	items := []listRitem{
+		{pathLower: "/photos/sub/a.txt", leaf: "a.txt"},
+		{pathLower: "/photos/sub", leaf: "Sub", folder: &files.FolderMetadata{}},
+		{pathLower: "/photos/other", leaf: "Other", folder: &files.FolderMetadata{}},
+	}
+	got, err := reconstructCasedRemotes("", "/photos", enc, items)
+	require.NoError(t, err)
+	assert.Equal(t, "Sub/a.txt", got["/photos/sub/a.txt"])
+	assert.Equal(t, "Sub", got["/photos/sub"])
+	assert.Equal(t, "Other", got["/photos/other"])
+
+	_, err = reconstructCasedRemotes("", "/photos", enc, []listRitem{
+		{pathLower: "/photos/missing/a.txt", leaf: "a.txt"},
+	})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "missing parent")
+
+	_, err = reconstructCasedRemotes("", "/", enc, []listRitem{
+		{pathLower: "", leaf: "x"},
+	})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "empty path_lower")
+
+	got, err = reconstructCasedRemotes("photos", "/photos", enc, []listRitem{
+		{pathLower: "/photos/sub", leaf: "Sub", folder: &files.FolderMetadata{}},
+		{pathLower: "/photos/sub/a.txt", leaf: "a.txt"},
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "photos/Sub/a.txt", got["/photos/sub/a.txt"])
+
+	_, err = reconstructCasedRemotes("", "/", enc, []listRitem{
+		{pathLower: "/../etc/passwd", leaf: "passwd"},
+	})
+	require.Error(t, err)
+}