rclone/cmd/bisync/delta_list.go

package bisync

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"
)

// errDeltaTooManyDeletes is returned when applying a delta would exceed --max-delete.
var errDeltaTooManyDeletes = errors.New("delta list: too many deletes")

// errDeltaCursorReset is returned when the remote invalidated the listing cursor.
// The in-memory listing must be left unchanged so the caller can full-relist.
var errDeltaCursorReset = errors.New("delta list: cursor reset")

type listingDelta struct {
	Remote  string
	Deleted bool
	Size    int64
	ModTime time.Time
	Hash    string
	ID      string
	Flags   string
}

type deltaApplyOpts struct {
	MaxDelete int
	Force     bool
	Reset     bool
}

func cursorPath(listing string) string {
	return listing + ".cursor"
}

// applyListingDeltas mutates ls in place. On error, ls is restored to its
// previous contents (strong safety: never persist a half-applied delta).
func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts) error {
	if opt.Reset {
		return errDeltaCursorReset
	}
	backupList := append([]string(nil), ls.list...)
	backupInfo := make(map[string]*fileInfo, len(ls.info))
	for k, v := range ls.info {
		cp := *v
		backupInfo[k] = &cp
	}
	restore := func() {
		ls.list = backupList
		ls.info = backupInfo
	}

	oldCount := 0
	for _, name := range ls.list {
		if !ls.isDir(name) {
			oldCount++
		}
	}
	deletedFiles := 0

	for _, d := range deltas {
		if d.Deleted {
			// Dropbox DeletedMetadata: remove the path and all children.
			prefix := d.Remote
			if prefix != "" && !strings.HasSuffix(prefix, "/") {
				prefix += "/"
			}
			var toRemove []string
			for _, name := range ls.list {
				if name == d.Remote || (prefix != "" && strings.HasPrefix(name, prefix)) {
					toRemove = append(toRemove, name)
				}
			}
			for _, name := range toRemove {
				if !ls.isDir(name) {
					deletedFiles++
				}
				ls.remove(name)
			}
			continue
		}
		flags := d.Flags
		if flags == "" {
			flags = "-"
		}
		ls.put(d.Remote, d.Size, d.ModTime, d.Hash, d.ID, flags)
	}

	if !opt.Force && opt.MaxDelete >= 0 && oldCount > 0 {
		maxRatio := float64(opt.MaxDelete) / 100.0
		if float64(deletedFiles)/float64(oldCount) > maxRatio {
			restore()
			return errDeltaTooManyDeletes
		}
	}
	return nil
}

func saveListingAndCursor(listing, cursor, root string, ls *fileList) error {
	if err := os.MkdirAll(filepath.Dir(listing), 0700); err != nil {
		return err
	}
	tmpListing := listing + ".tmp"
	tmpCursor := cursorPath(listing) + ".tmp"
	if err := ls.save(tmpListing); err != nil {
		return err
	}
	cur := fmt.Sprintf("# rclone-bisync-cursor v1\nroot=%s\ncursor=%s\n", root, cursor)
	if err := os.WriteFile(tmpCursor, []byte(cur), 0600); err != nil {
		_ = os.Remove(tmpListing)
		return err
	}
	if err := os.Rename(tmpListing, listing); err != nil {
		_ = os.Remove(tmpListing)
		_ = os.Remove(tmpCursor)
		return err
	}
	if err := os.Rename(tmpCursor, cursorPath(listing)); err != nil {
		return fmt.Errorf("listing saved but cursor rename failed (next run will full-list): %w", err)
	}
	return nil
}

func loadCursor(listing string) (cursor, root string, err error) {
	b, err := os.ReadFile(cursorPath(listing))
	if err != nil {
		if os.IsNotExist(err) {
			return "", "", nil
		}
		return "", "", err
	}
	sc := bufio.NewScanner(strings.NewReader(string(b)))
	for sc.Scan() {
		line := sc.Text()
		if strings.HasPrefix(line, "root=") {
			root = strings.TrimPrefix(line, "root=")
		}
		if strings.HasPrefix(line, "cursor=") {
			cursor = strings.TrimPrefix(line, "cursor=")
		}
	}
	return cursor, root, sc.Err()
}

func loadListingFile(listing string) (*fileList, error) {
	b := &bisyncRun{}
	return b.loadListing(listing)
}