package bisync import ( "context" "errors" "fmt" "github.com/rclone/rclone/backend/dropbox" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/terminal" ) func unwrapDropbox(f fs.Fs) *dropbox.Fs { for f != nil { if d, ok := f.(*dropbox.Fs); ok { return d } uw := f.Features().UnWrap if uw == nil { return nil } f = uw() } return nil } func (b *bisyncRun) makeDeltaListing(ctx context.Context) (*fileList, *fileList, error) { b.march.marchCtx = ctx b.setupListing() dbx1 := unwrapDropbox(b.fs1) dbx2 := unwrapDropbox(b.fs2) if dbx1 == nil && dbx2 == nil { return nil, nil, errors.New("--delta-list requires a Dropbox remote on Path1 and/or Path2") } var err error if dbx1 != nil { fs.Infof(nil, "Path1 Dropbox delta listing") b.march.ls1, err = b.refreshDropboxListing(ctx, dbx1, b.fs1, b.listing1, b.march.ls1) } else { fs.Infof(nil, "Path1 full listing (no USN/journal; local walk)") err = b.listOneFs(ctx, true) if err == nil { if prior, lerr := b.loadListing(b.listing1); lerr == nil { err = guardListingShrink(prior, b.march.ls1, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force}) } } } if err != nil { b.handleErr("delta-list", "error listing Path1", err, true, true) b.abort = true return b.march.ls1, b.march.ls2, err } if dbx2 != nil { fs.Infof(nil, "Path2 Dropbox delta listing") b.march.ls2, err = b.refreshDropboxListing(ctx, dbx2, b.fs2, b.listing2, b.march.ls2) } else { fs.Infof(nil, "Path2 full listing (no USN/journal; local walk)") err = b.listOneFs(ctx, false) if err == nil { if prior, lerr := b.loadListing(b.listing2); lerr == nil { err = guardListingShrink(prior, b.march.ls2, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force}) } } } if err != nil { b.handleErr("delta-list", "error listing Path2", err, true, true) b.abort = true return b.march.ls1, b.march.ls2, err } if err = b.march.ls1.save(b.newListing1); err != nil { return b.march.ls1, b.march.ls2, err } if err = b.march.ls2.save(b.newListing2); err != nil { return b.march.ls1, b.march.ls2, err } return b.march.ls1, b.march.ls2, nil } func (b *bisyncRun) listOneFs(ctx context.Context, isPath1 bool) error { f := b.fs1 if !isPath1 { f = b.fs2 } return walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error { for _, e := range entries { b.parse(e, isPath1) } return nil }) } func (b *bisyncRun) refreshDropboxListing(ctx context.Context, dbx *dropbox.Fs, f fs.Fs, listing string, ls *fileList) (*fileList, error) { ls.hash = b.hashTypeForFs(f) cursor, root, err := loadCursor(listing) if err != nil { return nil, err } wantRoot := bilibFsRoot(f) if cursor != "" && root != "" && root != wantRoot { fs.Infof(nil, "Dropbox cursor root %q != %q; full relist", root, wantRoot) cursor = "" } var prior *fileList if cursor != "" { loaded, err := b.loadListing(listing) if err != nil { fs.Infof(nil, "No usable prior listing (%v); full Dropbox relist", err) cursor = "" } else { ls = loaded ls.hash = b.hashTypeForFs(f) prior = loaded } } else if loaded, err := b.loadListing(listing); err == nil { prior = loaded } newCursor, reset, changes, err := dbx.ListFolderDeltas(ctx, cursor) if err != nil { return nil, err } if reset { fs.Log(nil, Color(terminal.YellowFg, "Dropbox listing cursor reset; performing full recursive relist")) newCursor, reset, changes, err = dbx.ListFolderDeltas(ctx, "") if err != nil { return nil, err } if reset { return nil, errDeltaCursorReset } ls = newFileList() ls.hash = b.hashTypeForFs(f) } else if cursor == "" { ls = newFileList() ls.hash = b.hashTypeForFs(f) } deltas := make([]listingDelta, 0, len(changes)) for _, ch := range changes { flags := "-" if ch.IsDir { flags = "d" } size := ch.Size if ch.IsDir { size = -1 } hashVal := ch.Hash if ls.hash == hash.None { hashVal = "" } deltas = append(deltas, listingDelta{ Remote: ch.Remote, Deleted: ch.Deleted, Size: size, ModTime: ch.ModTime.In(TZ), Hash: hashVal, Flags: flags, }) } applyOpts := deltaApplyOpts{ MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force, } priorCount := 0 hadPrior := prior != nil if hadPrior { priorCount = countListingFiles(prior) } applyErr := applyListingDeltas(ls, deltas, applyOpts) if applyErr != nil { return nil, applyErr } if hadPrior { if err := guardFileCount(priorCount, countListingFiles(ls), applyOpts); err != nil { return nil, err } } if b.opt.DryRun { fs.Infof(nil, "dry-run: not saving Dropbox listing cursor") return ls, nil } if err := saveListingAndCursor(listing, newCursor, wantRoot, ls); err != nil { return nil, err } return ls, nil } func (b *bisyncRun) hashTypeForFs(f fs.Fs) hash.Type { if f == b.fs1 { return b.opt.Compare.HashType1 } return b.opt.Compare.HashType2 } func bilibFsRoot(f fs.Fs) string { return fmt.Sprintf("%s:%s", f.Name(), f.Root()) }