rclone/cmd/bisync/delta_list.go

package bisync

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/rclone/rclone/fs"
)

// errDeltaTooManyDeletes is returned when applying a delta would exceed --max-delete.
var errDeltaTooManyDeletes = errors.New("delta list: too many deletes")

// errDeltaCursorReset is returned when the remote invalidated the listing cursor.
// The in-memory listing must be left unchanged so the caller can full-relist.
var errDeltaCursorReset = errors.New("delta list: cursor reset")

// errDeltaListingEmptied is returned when a prior non-empty listing would be replaced by zero files.
var errDeltaListingEmptied = errors.New("delta list: listing became empty while prior listing had files")

// errDeltaListingCollapsed is returned when file count dropped beyond --max-delete.
var errDeltaListingCollapsed = errors.New("delta list: listing shrank beyond --max-delete")

func countListingFiles(ls *fileList) int {
	if ls == nil {
		return 0
	}
	n := 0
	for _, name := range ls.list {
		if !ls.isDir(name) {
			n++
		}
	}
	return n
}

func guardFileCount(oldC, newC int, opt deltaApplyOpts) error {
	if opt.Force || oldC == 0 {
		return nil
	}
	if newC == 0 {
		return errDeltaListingEmptied
	}
	dropped := oldC - newC
	if dropped <= 0 {
		return nil
	}
	o := &Options{MaxDelete: opt.MaxDelete, MaxDeleteFiles: opt.MaxDeleteFiles, MaxDeleteSize: opt.MaxDeleteSize, Force: opt.Force}
	if hit, _ := deleteBudgetExceeded(o, oldC, dropped, 0); hit {
		return errDeltaListingCollapsed
	}
	return nil
}

func guardListingShrink(prior, next *fileList, opt deltaApplyOpts) error {
	if prior == nil || next == nil {
		return nil
	}
	return guardFileCount(countListingFiles(prior), countListingFiles(next), opt)
}

func caseDeltaRemote(ls *fileList, remote string) string {
	parent := path.Dir(remote)
	if parent == "." {
		parent = ""
	}
	if parent == "" {
		return remote
	}
	if ls.has(parent) {
		return path.Join(parent, path.Base(remote))
	}
	for _, name := range ls.list {
		if strings.EqualFold(name, parent) {
			return path.Join(name, path.Base(remote))
		}
	}
	return remote
}

type listingDelta struct {
	Remote  string
	Deleted bool
	Size    int64
	ModTime time.Time
	Hash    string
	ID      string
	Flags   string
}

type deltaApplyOpts struct {
	MaxDelete      int
	MaxDeleteFiles int
	MaxDeleteSize  fs.SizeSuffix
	Force          bool
	Reset          bool
}

func cursorPath(listing string) string {
	return listing + ".cursor"
}

// applyListingDeltas mutates ls in place. On error, ls is restored to its
// previous contents (strong safety: never persist a half-applied delta).
func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts) error {
	if opt.Reset {
		return errDeltaCursorReset
	}
	backupList := append([]string(nil), ls.list...)
	backupInfo := make(map[string]*fileInfo, len(ls.info))
	for k, v := range ls.info {
		cp := *v
		backupInfo[k] = &cp
	}
	restore := func() {
		ls.list = backupList
		ls.info = backupInfo
	}

	oldCount := 0
	for _, name := range ls.list {
		if !ls.isDir(name) {
			oldCount++
		}
	}
	deletedFiles := 0
	var deletedBytes int64

	for _, d := range deltas {
		if d.Deleted {
			if ls.has(d.Remote) {
				// keep
			} else {
				for _, name := range ls.list {
					if strings.EqualFold(name, d.Remote) {
						d.Remote = name
						break
					}
				}
			}
		} else {
			d.Remote = caseDeltaRemote(ls, d.Remote)
		}
		if d.Deleted {
			// Dropbox DeletedMetadata: remove the path and all children.
			prefix := d.Remote
			if prefix != "" && !strings.HasSuffix(prefix, "/") {
				prefix += "/"
			}
			var toRemove []string
			for _, name := range ls.list {
				if name == d.Remote || (prefix != "" && strings.HasPrefix(name, prefix)) {
					toRemove = append(toRemove, name)
				}
			}
			for _, name := range toRemove {
				if !ls.isDir(name) {
					deletedFiles++
					deletedBytes += ls.getSize(name)
				}
				ls.remove(name)
			}
			continue
		}
		flags := d.Flags
		if flags == "" {
			flags = "-"
		}
		ls.put(d.Remote, d.Size, d.ModTime, d.Hash, d.ID, flags)
	}

	if !opt.Force && oldCount > 0 {
		o := &Options{MaxDelete: opt.MaxDelete, MaxDeleteFiles: opt.MaxDeleteFiles, MaxDeleteSize: opt.MaxDeleteSize}
		if hit, _ := deleteBudgetExceeded(o, oldCount, deletedFiles, deletedBytes); hit {
			restore()
			return errDeltaTooManyDeletes
		}
	}
	return nil
}

func saveListingAndCursor(listing, cursor, root string, ls *fileList) error {
	if err := os.MkdirAll(filepath.Dir(listing), 0700); err != nil {
		return err
	}
	tmpListing := listing + ".tmp"
	tmpCursor := cursorPath(listing) + ".tmp"
	if err := ls.save(tmpListing); err != nil {
		return err
	}
	cur := fmt.Sprintf("# rclone-bisync-cursor v1\nroot=%s\ncursor=%s\n", root, cursor)
	if err := os.WriteFile(tmpCursor, []byte(cur), 0600); err != nil {
		_ = os.Remove(tmpListing)
		return err
	}
	// Cursor first: a crash here leaves old listing + new cursor (missed
	// updates, not a mass-delete). Listing-first would leave a new listing
	// with a stale cursor, which can look like deletes on the next run.
	if err := os.Rename(tmpCursor, cursorPath(listing)); err != nil {
		_ = os.Remove(tmpListing)
		_ = os.Remove(tmpCursor)
		return err
	}
	if err := os.Rename(tmpListing, listing); err != nil {
		_ = os.Remove(tmpListing)
		_ = os.Remove(cursorPath(listing))
		return fmt.Errorf("cursor saved but listing rename failed (cursor removed to force full-list): %w", err)
	}
	return nil
}

func loadCursor(listing string) (cursor, root string, err error) {
	b, err := os.ReadFile(cursorPath(listing))
	if err != nil {
		if os.IsNotExist(err) {
			return "", "", nil
		}
		return "", "", err
	}
	sc := bufio.NewScanner(strings.NewReader(string(b)))
	for sc.Scan() {
		line := sc.Text()
		if strings.HasPrefix(line, "root=") {
			root = strings.TrimPrefix(line, "root=")
		}
		if strings.HasPrefix(line, "cursor=") {
			cursor = strings.TrimPrefix(line, "cursor=")
		}
	}
	return cursor, root, sc.Err()
}

func loadListingFile(listing string) (*fileList, error) {
	b := &bisyncRun{}
	return b.loadListing(listing)
}