diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 04b777181..507cefff6 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -16,9 +16,13 @@ casing. Changes to only the casing of paths won't be returned by list_folder/continue. This field will be null if the file or folder is not mounted. This field is optional. -We solve this by not implementing the ListR interface. The dropbox -remote will recurse directory by directory only using the last element -of path_display and all will be well. +ListR is implemented with the same leaf-casing rule as List/ListP: +only the last path component (Name / PathDisplay base) is trusted. +Parent remotes are rebuilt from PathLower identity plus already-seen +folder leaves. A full PathDisplay is never used as a remote. If an +entry cannot be placed (empty PathLower, missing parent), ListR +returns an error and emits nothing, so sync/bisync cannot treat a +truncated listing as “delete the rest”. */ import ( @@ -28,6 +32,7 @@ import ( "io" "path" "regexp" + "sort" "strings" "time" "unicode/utf8" @@ -661,6 +666,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } f.features.Fill(ctx, f) + if f.opt.SharedFiles || f.opt.SharedFolders { + // Received/shared listing is a flat namespace; recursive ListR would be wrong. + f.features.ListR = nil + } if f.opt.RootNsid != "" { f.ns = f.opt.RootNsid @@ -1204,6 +1213,251 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) ( return list.Flush() } +// listRitem is one recursive list_folder entry before remote reconstruction. +type listRitem struct { + pathLower string + leaf string + folder *files.FolderMetadata + file *files.FileMetadata +} + +func listRLeaf(md *files.Metadata) string { + if md.PathDisplay != "" { + return path.Base(md.PathDisplay) + } + return md.Name +} + +func dropboxListRootLower(encRoot string) string { + if encRoot == "" || encRoot == "/" { + return "/" + } + lower := strings.ToLower(encRoot) + if !strings.HasPrefix(lower, "/") { + lower = "/" + lower + } + return lower +} + +// reconstructCasedRemotes maps PathLower → rclone remote using List/ListP +// leaf casing. It never trusts parent components of PathDisplay. +// Missing parents or empty PathLower return an error (no partial map). +func reconstructCasedRemotes(dir, listRootLower string, enc encoder.MultiEncoder, items []listRitem) (map[string]string, error) { + folders := make([]listRitem, 0, len(items)) + for _, it := range items { + if it.pathLower == "" { + return nil, fmt.Errorf("dropbox ListR: refusing incomplete listing: empty path_lower for %q", it.leaf) + } + if it.pathLower == listRootLower { + continue + } + if it.folder != nil { + folders = append(folders, it) + } + } + sort.SliceStable(folders, func(i, j int) bool { + ci, cj := strings.Count(folders[i].pathLower, "/"), strings.Count(folders[j].pathLower, "/") + if ci != cj { + return ci < cj + } + return folders[i].pathLower < folders[j].pathLower + }) + remotes := make(map[string]string, len(items)) + for _, it := range folders { + parent := path.Dir(it.pathLower) + var parentRemote string + if parent == listRootLower { + parentRemote = dir + } else { + var ok bool + parentRemote, ok = remotes[parent] + if !ok { + return nil, fmt.Errorf("dropbox ListR: refusing incomplete listing: missing parent %q for folder %q", parent, it.pathLower) + } + } + remotes[it.pathLower] = path.Join(parentRemote, enc.ToStandardName(it.leaf)) + } + for _, it := range items { + if it.folder != nil || it.pathLower == "" || it.pathLower == listRootLower { + continue + } + parent := path.Dir(it.pathLower) + var parentRemote string + if parent == listRootLower { + parentRemote = dir + } else { + var ok bool + parentRemote, ok = remotes[parent] + if !ok { + return nil, fmt.Errorf("dropbox ListR: refusing incomplete listing: missing parent %q for %q", parent, it.pathLower) + } + } + remotes[it.pathLower] = path.Join(parentRemote, enc.ToStandardName(it.leaf)) + } + return remotes, nil +} + +func underPathLower(pathLower, prefix string) bool { + return pathLower == prefix || strings.HasPrefix(pathLower, prefix+"/") +} + +// ListR lists the objects and directories of the Fs starting from +// dir recursively into out. +// +// Entries are not emitted until the recursive listing has completed +// and every path has been reconstructed. A failure at any point +// returns without a successful empty listing. +func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { + if f.opt.SharedFiles || f.opt.SharedFolders { + return errors.New("dropbox: ListR is not supported with shared_files or shared_folders") + } + + root := f.slashRoot + if dir != "" { + root += "/" + dir + } + listRootLower := dropboxListRootLower(f.opt.Enc.FromStandardPath(root)) + + var items []listRitem + started := false + var res *files.ListFolderResult + for { + if !started { + arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(root)) + arg.Recursive = true + arg.Limit = 1000 + if root == "/" { + arg.Path = "" + } + err = f.pacer.Call(func() (bool, error) { + res, err = f.srv.ListFolderContext(ctx, arg) + return shouldRetry(ctx, err) + }) + if err != nil { + switch e := err.(type) { + case files.ListFolderAPIError: + if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound { + err = fs.ErrorDirNotFound + } + } + return err + } + started = true + } else { + arg := files.ListFolderContinueArg{Cursor: res.Cursor} + err = f.pacer.Call(func() (bool, error) { + res, err = f.srv.ListFolderContinueContext(ctx, &arg) + return shouldRetry(ctx, err) + }) + if err != nil { + return fmt.Errorf("list continue: %w", err) + } + } + for _, entry := range res.Entries { + switch info := entry.(type) { + case *files.FolderMetadata: + if info.PathLower == "" { + return fmt.Errorf("dropbox ListR: refusing incomplete listing: folder %q has empty path_lower", info.Name) + } + items = append(items, listRitem{ + pathLower: info.PathLower, + leaf: listRLeaf(&info.Metadata), + folder: info, + }) + case *files.FileMetadata: + if info.PathLower == "" { + return fmt.Errorf("dropbox ListR: refusing incomplete listing: file %q has empty path_lower", info.Name) + } + items = append(items, listRitem{ + pathLower: info.PathLower, + leaf: listRLeaf(&info.Metadata), + file: info, + }) + default: + fs.Errorf(f, "Unknown type %T", entry) + } + } + if !res.HasMore { + break + } + } + + skipPrefixes := make([]string, 0) + for _, it := range items { + if it.folder == nil || it.folder.SharingInfo == nil || it.folder.SharingInfo.SharedFolderId == "" { + continue + } + remoteHint := it.leaf + if f.opt.SkipSharedFolders { + fs.Debugf(remoteHint, "Skipping shared folder") + skipPrefixes = append(skipPrefixes, it.pathLower) + continue + } + if f.opt.SkipUnownedFolders { + var sfMeta *sharing.SharedFolderMetadata + err = f.pacer.Call(func() (bool, error) { + var apiErr error + sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(it.folder.SharingInfo.SharedFolderId)) + return shouldRetry(ctx, apiErr) + }) + if err != nil { + fs.Errorf(remoteHint, "Failed to get shared folder metadata (defaulting to include): %v", err) + } else if sfMeta != nil && sfMeta.AccessType != nil && sfMeta.AccessType.Tag != sharing.AccessLevelOwner { + fs.Debugf(remoteHint, "Skipping unowned shared folder") + skipPrefixes = append(skipPrefixes, it.pathLower) + } + } + } + + kept := items[:0] + for _, it := range items { + skip := false + for _, p := range skipPrefixes { + if underPathLower(it.pathLower, p) { + skip = true + break + } + } + if !skip { + kept = append(kept, it) + } + } + items = kept + + remotes, err := reconstructCasedRemotes(dir, listRootLower, f.opt.Enc, items) + if err != nil { + return err + } + + lh := list.NewHelper(callback) + for _, it := range items { + remote, ok := remotes[it.pathLower] + if !ok { + continue + } + if list.RemoteEscapesRoot(remote) { + return fmt.Errorf("dropbox ListR: refusing listing: remote %q escapes the root", remote) + } + if it.folder != nil { + d := fs.NewDir(remote, time.Time{}).SetID(it.folder.Id) + if err = lh.Add(d); err != nil { + return err + } + continue + } + o, err := f.newObjectWithInfo(ctx, remote, it.file) + if err != nil { + return err + } + if o.(*Object).exportType.listable() { + if err = lh.Add(o); err != nil { + return err + } + } + } + return lh.Flush() +} + // Put the object // // Copy the reader in to the new object which is returned. @@ -2413,6 +2667,7 @@ var ( _ fs.PublicLinker = (*Fs)(nil) _ fs.DirMover = (*Fs)(nil) _ fs.ListPer = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) _ fs.Abouter = (*Fs)(nil) _ fs.Shutdowner = &Fs{} _ fs.Object = (*Object)(nil) diff --git a/backend/dropbox/dropbox_internal_test.go b/backend/dropbox/dropbox_internal_test.go index fb30352da..b44789e6e 100644 --- a/backend/dropbox/dropbox_internal_test.go +++ b/backend/dropbox/dropbox_internal_test.go @@ -568,3 +568,42 @@ func TestListingChangeFromEntry(t *testing.T) { assert.Equal(t, "sub", ch.Remote) assert.True(t, ch.Deleted) } + +func TestReconstructCasedRemotes(t *testing.T) { + enc := encoder.Standard + // Files listed before folders; PathDisplay parents wrongly cased. + items := []listRitem{ + {pathLower: "/photos/sub/a.txt", leaf: "a.txt"}, + {pathLower: "/photos/sub", leaf: "Sub", folder: &files.FolderMetadata{}}, + {pathLower: "/photos/other", leaf: "Other", folder: &files.FolderMetadata{}}, + } + got, err := reconstructCasedRemotes("", "/photos", enc, items) + require.NoError(t, err) + assert.Equal(t, "Sub/a.txt", got["/photos/sub/a.txt"]) + assert.Equal(t, "Sub", got["/photos/sub"]) + assert.Equal(t, "Other", got["/photos/other"]) + + _, err = reconstructCasedRemotes("", "/photos", enc, []listRitem{ + {pathLower: "/photos/missing/a.txt", leaf: "a.txt"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing parent") + + _, err = reconstructCasedRemotes("", "/", enc, []listRitem{ + {pathLower: "", leaf: "x"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "empty path_lower") + + got, err = reconstructCasedRemotes("photos", "/photos", enc, []listRitem{ + {pathLower: "/photos/sub", leaf: "Sub", folder: &files.FolderMetadata{}}, + {pathLower: "/photos/sub/a.txt", leaf: "a.txt"}, + }) + require.NoError(t, err) + assert.Equal(t, "photos/Sub/a.txt", got["/photos/sub/a.txt"]) + + _, err = reconstructCasedRemotes("", "/", enc, []listRitem{ + {pathLower: "/../etc/passwd", leaf: "passwd"}, + }) + require.Error(t, err) +}