rclone/cmd/bisync/delta_list_run.go

package bisync

import (
	"context"
	"errors"
	"fmt"

	"github.com/rclone/rclone/backend/dropbox"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/hash"
	"github.com/rclone/rclone/fs/walk"
	"github.com/rclone/rclone/lib/terminal"
)

func unwrapDropbox(f fs.Fs) *dropbox.Fs {
	for f != nil {
		if d, ok := f.(*dropbox.Fs); ok {
			return d
		}
		uw := f.Features().UnWrap
		if uw == nil {
			return nil
		}
		f = uw()
	}
	return nil
}

func (b *bisyncRun) makeDeltaListing(ctx context.Context) (*fileList, *fileList, error) {
	b.march.marchCtx = ctx
	b.setupListing()

	dbx1 := unwrapDropbox(b.fs1)
	dbx2 := unwrapDropbox(b.fs2)
	if dbx1 == nil && dbx2 == nil {
		return nil, nil, errors.New("--delta-list requires a Dropbox remote on Path1 and/or Path2")
	}

	var err error
	if dbx1 != nil {
		fs.Infof(nil, "Path1 Dropbox delta listing")
		b.march.ls1, err = b.refreshDropboxListing(ctx, dbx1, b.fs1, b.listing1, b.march.ls1)
	} else {
		fs.Infof(nil, "Path1 full listing (no USN/journal; local walk)")
		err = b.listOneFs(ctx, true)
		if err == nil {
			if prior, lerr := b.loadListing(b.listing1); lerr == nil {
				err = guardListingShrink(prior, b.march.ls1, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force})
			}
		}
	}
	if err != nil {
		b.handleErr("delta-list", "error listing Path1", err, true, true)
		b.abort = true
		return b.march.ls1, b.march.ls2, err
	}

	if dbx2 != nil {
		fs.Infof(nil, "Path2 Dropbox delta listing")
		b.march.ls2, err = b.refreshDropboxListing(ctx, dbx2, b.fs2, b.listing2, b.march.ls2)
	} else {
		fs.Infof(nil, "Path2 full listing (no USN/journal; local walk)")
		err = b.listOneFs(ctx, false)
		if err == nil {
			if prior, lerr := b.loadListing(b.listing2); lerr == nil {
				err = guardListingShrink(prior, b.march.ls2, deltaApplyOpts{MaxDelete: b.opt.MaxDelete, MaxDeleteFiles: b.opt.MaxDeleteFiles, MaxDeleteSize: b.opt.MaxDeleteSize, Force: b.opt.Force})
			}
		}
	}
	if err != nil {
		b.handleErr("delta-list", "error listing Path2", err, true, true)
		b.abort = true
		return b.march.ls1, b.march.ls2, err
	}

	if err = b.march.ls1.save(b.newListing1); err != nil {
		return b.march.ls1, b.march.ls2, err
	}
	if err = b.march.ls2.save(b.newListing2); err != nil {
		return b.march.ls1, b.march.ls2, err
	}
	return b.march.ls1, b.march.ls2, nil
}

func (b *bisyncRun) listOneFs(ctx context.Context, isPath1 bool) error {
	f := b.fs1
	if !isPath1 {
		f = b.fs2
	}
	return walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error {
		for _, e := range entries {
			b.parse(e, isPath1)
		}
		return nil
	})
}

func (b *bisyncRun) refreshDropboxListing(ctx context.Context, dbx *dropbox.Fs, f fs.Fs, listing string, ls *fileList) (*fileList, error) {
	ls.hash = b.hashTypeForFs(f)
	cursor, root, err := loadCursor(listing)
	if err != nil {
		return nil, err
	}
	wantRoot := bilibFsRoot(f)
	if cursor != "" && root != "" && root != wantRoot {
		fs.Infof(nil, "Dropbox cursor root %q != %q; full relist", root, wantRoot)
		cursor = ""
	}

	var prior *fileList
	if cursor != "" {
		loaded, err := b.loadListing(listing)
		if err != nil {
			fs.Infof(nil, "No usable prior listing (%v); full Dropbox relist", err)
			cursor = ""
		} else {
			ls = loaded
			ls.hash = b.hashTypeForFs(f)
			prior = loaded
		}
	} else if loaded, err := b.loadListing(listing); err == nil {
		prior = loaded
	}

	newCursor, reset, changes, err := dbx.ListFolderDeltas(ctx, cursor)
	if err != nil {
		return nil, err
	}
	if reset {
		fs.Log(nil, Color(terminal.YellowFg, "Dropbox listing cursor reset; performing full recursive relist"))
		newCursor, reset, changes, err = dbx.ListFolderDeltas(ctx, "")
		if err != nil {
			return nil, err
		}
		if reset {
			return nil, errDeltaCursorReset
		}
		ls = newFileList()
		ls.hash = b.hashTypeForFs(f)
	} else if cursor == "" {
		ls = newFileList()
		ls.hash = b.hashTypeForFs(f)
	}

	deltas := make([]listingDelta, 0, len(changes))
	for _, ch := range changes {
		flags := "-"
		if ch.IsDir {
			flags = "d"
		}
		size := ch.Size
		if ch.IsDir {
			size = -1
		}
		hashVal := ch.Hash
		if ls.hash == hash.None {
			hashVal = ""
		}
		deltas = append(deltas, listingDelta{
			Remote:  ch.Remote,
			Deleted: ch.Deleted,
			Size:    size,
			ModTime: ch.ModTime.In(TZ),
			Hash:    hashVal,
			Flags:   flags,
		})
	}

	applyOpts := deltaApplyOpts{
		MaxDelete:      b.opt.MaxDelete,
		MaxDeleteFiles: b.opt.MaxDeleteFiles,
		MaxDeleteSize:  b.opt.MaxDeleteSize,
		Force:          b.opt.Force,
	}
	priorCount := 0
	hadPrior := prior != nil
	if hadPrior {
		priorCount = countListingFiles(prior)
	}
	applyErr := applyListingDeltas(ls, deltas, applyOpts)
	if applyErr != nil {
		return nil, applyErr
	}
	if hadPrior {
		if err := guardFileCount(priorCount, countListingFiles(ls), applyOpts); err != nil {
			return nil, err
		}
	}

	if b.opt.DryRun {
		fs.Infof(nil, "dry-run: not saving Dropbox listing cursor")
		return ls, nil
	}
	if err := saveListingAndCursor(listing, newCursor, wantRoot, ls); err != nil {
		return nil, err
	}
	return ls, nil
}

func (b *bisyncRun) hashTypeForFs(f fs.Fs) hash.Type {
	if f == b.fs1 {
		return b.opt.Compare.HashType1
	}
	return b.opt.Compare.HashType2
}

func bilibFsRoot(f fs.Fs) string {
	return fmt.Sprintf("%s:%s", f.Name(), f.Root())
}