bisync: add Dropbox list_folder cursor listings via --delta-list

Skip the per-directory Dropbox walk on successive bisync runs by applying
list_folder/continue deltas to the stored listing. Cursor reset, --max-delete,
and atomic listing+cursor writes abort without a half-applied snapshot. The
local side still walks; there is no NTFS USN dependency.
This commit is contained in:
2026-09-09 23:36:25 +02:00
parent ef6968730f
commit d44d85dc05
8 changed files with 608 additions and 0 deletions
+111
View File
@@ -2292,6 +2292,117 @@ func (o *Object) Remove(ctx context.Context) (err error) {
return err
}
// ListingChange is one list_folder/continue entry for bisync --delta-list.
type ListingChange struct {
Remote string
Deleted bool
IsDir bool
Size int64
ModTime time.Time
Hash string
}
func isListFolderReset(err error) bool {
var cont files.ListFolderContinueAPIError
if errors.As(err, &cont) && cont.EndpointError != nil && cont.EndpointError.Tag == files.ListFolderContinueErrorReset {
return true
}
return false
}
// ListFolderDeltas returns Dropbox list_folder changes since cursor.
// An empty cursor performs a recursive snapshot (all entries as non-deletes).
// reset is true when Dropbox invalidated the cursor; deltas will be empty.
func (f *Fs) ListFolderDeltas(ctx context.Context, cursor string) (newCursor string, reset bool, deltas []ListingChange, err error) {
if f.opt.SharedFiles || f.opt.SharedFolders {
return "", false, nil, errors.New("dropbox: ListFolderDeltas is not supported with shared_files or shared_folders")
}
started := false
var res *files.ListFolderResult
for {
if !started {
if cursor == "" {
arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(f.slashRoot))
arg.Recursive = true
arg.Limit = 1000
if arg.Path == "/" {
arg.Path = ""
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContext(ctx, arg)
return shouldRetry(ctx, err)
})
} else {
arg := files.ListFolderContinueArg{Cursor: cursor}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
return shouldRetry(ctx, err)
})
}
if err != nil {
if isListFolderReset(err) {
return "", true, nil, nil
}
return "", false, nil, err
}
started = true
} else {
arg := files.ListFolderContinueArg{Cursor: res.Cursor}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
return shouldRetry(ctx, err)
})
if err != nil {
if isListFolderReset(err) {
return "", true, nil, nil
}
return "", false, nil, fmt.Errorf("list continue: %w", err)
}
}
for _, entry := range res.Entries {
ch, ok := f.listingChangeFromEntry(entry)
if !ok {
continue
}
deltas = append(deltas, ch)
}
newCursor = res.Cursor
if !res.HasMore {
break
}
}
return newCursor, false, deltas, nil
}
func (f *Fs) listingChangeFromEntry(entry files.IsMetadata) (ListingChange, bool) {
var (
ch ListingChange
entryPath string
)
switch info := entry.(type) {
case *files.FolderMetadata:
ch.IsDir = true
entryPath = info.PathDisplay
case *files.FileMetadata:
entryPath = info.PathDisplay
ch.Size = int64(info.Size)
ch.ModTime = time.Time(info.ClientModified)
ch.Hash = info.ContentHash
case *files.DeletedMetadata:
ch.Deleted = true
ch.IsDir = true
entryPath = info.PathDisplay
default:
return ListingChange{}, false
}
entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
if entryPath == "" {
return ListingChange{}, false
}
ch.Remote = f.opt.Enc.ToStandardPath(entryPath)
return ch, true
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
+33
View File
@@ -535,3 +535,36 @@ func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}
func TestListingChangeFromEntry(t *testing.T) {
f := &Fs{
slashRootSlash: "/Dropbox/",
opt: Options{Enc: encoder.Standard},
}
file := &files.FileMetadata{
Metadata: files.Metadata{
Name: "a.txt",
PathDisplay: "/Dropbox/sub/a.txt",
},
Size: 42,
ContentHash: "abc",
ClientModified: dropbox.DBXTime(time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC)),
}
ch, ok := f.listingChangeFromEntry(file)
require.True(t, ok)
assert.Equal(t, "sub/a.txt", ch.Remote)
assert.Equal(t, int64(42), ch.Size)
assert.Equal(t, "abc", ch.Hash)
assert.False(t, ch.Deleted)
del := &files.DeletedMetadata{
Metadata: files.Metadata{
Name: "sub",
PathDisplay: "/Dropbox/sub",
},
}
ch, ok = f.listingChangeFromEntry(del)
require.True(t, ok)
assert.Equal(t, "sub", ch.Remote)
assert.True(t, ch.Deleted)
}