bisync: add Dropbox list_folder cursor listings via --delta-list
Skip the per-directory Dropbox walk on successive bisync runs by applying list_folder/continue deltas to the stored listing. Cursor reset, --max-delete, and atomic listing+cursor writes abort without a half-applied snapshot. The local side still walks; there is no NTFS USN dependency.
This commit is contained in:
@@ -2292,6 +2292,117 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
// ListingChange is one list_folder/continue entry for bisync --delta-list.
|
||||
type ListingChange struct {
|
||||
Remote string
|
||||
Deleted bool
|
||||
IsDir bool
|
||||
Size int64
|
||||
ModTime time.Time
|
||||
Hash string
|
||||
}
|
||||
|
||||
func isListFolderReset(err error) bool {
|
||||
var cont files.ListFolderContinueAPIError
|
||||
if errors.As(err, &cont) && cont.EndpointError != nil && cont.EndpointError.Tag == files.ListFolderContinueErrorReset {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ListFolderDeltas returns Dropbox list_folder changes since cursor.
|
||||
// An empty cursor performs a recursive snapshot (all entries as non-deletes).
|
||||
// reset is true when Dropbox invalidated the cursor; deltas will be empty.
|
||||
func (f *Fs) ListFolderDeltas(ctx context.Context, cursor string) (newCursor string, reset bool, deltas []ListingChange, err error) {
|
||||
if f.opt.SharedFiles || f.opt.SharedFolders {
|
||||
return "", false, nil, errors.New("dropbox: ListFolderDeltas is not supported with shared_files or shared_folders")
|
||||
}
|
||||
started := false
|
||||
var res *files.ListFolderResult
|
||||
for {
|
||||
if !started {
|
||||
if cursor == "" {
|
||||
arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(f.slashRoot))
|
||||
arg.Recursive = true
|
||||
arg.Limit = 1000
|
||||
if arg.Path == "/" {
|
||||
arg.Path = ""
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContext(ctx, arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
} else {
|
||||
arg := files.ListFolderContinueArg{Cursor: cursor}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
if isListFolderReset(err) {
|
||||
return "", true, nil, nil
|
||||
}
|
||||
return "", false, nil, err
|
||||
}
|
||||
started = true
|
||||
} else {
|
||||
arg := files.ListFolderContinueArg{Cursor: res.Cursor}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
if isListFolderReset(err) {
|
||||
return "", true, nil, nil
|
||||
}
|
||||
return "", false, nil, fmt.Errorf("list continue: %w", err)
|
||||
}
|
||||
}
|
||||
for _, entry := range res.Entries {
|
||||
ch, ok := f.listingChangeFromEntry(entry)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
deltas = append(deltas, ch)
|
||||
}
|
||||
newCursor = res.Cursor
|
||||
if !res.HasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
return newCursor, false, deltas, nil
|
||||
}
|
||||
|
||||
func (f *Fs) listingChangeFromEntry(entry files.IsMetadata) (ListingChange, bool) {
|
||||
var (
|
||||
ch ListingChange
|
||||
entryPath string
|
||||
)
|
||||
switch info := entry.(type) {
|
||||
case *files.FolderMetadata:
|
||||
ch.IsDir = true
|
||||
entryPath = info.PathDisplay
|
||||
case *files.FileMetadata:
|
||||
entryPath = info.PathDisplay
|
||||
ch.Size = int64(info.Size)
|
||||
ch.ModTime = time.Time(info.ClientModified)
|
||||
ch.Hash = info.ContentHash
|
||||
case *files.DeletedMetadata:
|
||||
ch.Deleted = true
|
||||
ch.IsDir = true
|
||||
entryPath = info.PathDisplay
|
||||
default:
|
||||
return ListingChange{}, false
|
||||
}
|
||||
entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
|
||||
if entryPath == "" {
|
||||
return ListingChange{}, false
|
||||
}
|
||||
ch.Remote = f.opt.Enc.ToStandardPath(entryPath)
|
||||
return ch, true
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
|
||||
@@ -535,3 +535,36 @@ func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
|
||||
_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
|
||||
func TestListingChangeFromEntry(t *testing.T) {
|
||||
f := &Fs{
|
||||
slashRootSlash: "/Dropbox/",
|
||||
opt: Options{Enc: encoder.Standard},
|
||||
}
|
||||
file := &files.FileMetadata{
|
||||
Metadata: files.Metadata{
|
||||
Name: "a.txt",
|
||||
PathDisplay: "/Dropbox/sub/a.txt",
|
||||
},
|
||||
Size: 42,
|
||||
ContentHash: "abc",
|
||||
ClientModified: dropbox.DBXTime(time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC)),
|
||||
}
|
||||
ch, ok := f.listingChangeFromEntry(file)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "sub/a.txt", ch.Remote)
|
||||
assert.Equal(t, int64(42), ch.Size)
|
||||
assert.Equal(t, "abc", ch.Hash)
|
||||
assert.False(t, ch.Deleted)
|
||||
|
||||
del := &files.DeletedMetadata{
|
||||
Metadata: files.Metadata{
|
||||
Name: "sub",
|
||||
PathDisplay: "/Dropbox/sub",
|
||||
},
|
||||
}
|
||||
ch, ok = f.listingChangeFromEntry(del)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "sub", ch.Remote)
|
||||
assert.True(t, ch.Deleted)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user