package bisync import ( "bufio" "errors" "fmt" "os" "path/filepath" "strings" "time" ) // errDeltaTooManyDeletes is returned when applying a delta would exceed --max-delete. var errDeltaTooManyDeletes = errors.New("delta list: too many deletes") // errDeltaCursorReset is returned when the remote invalidated the listing cursor. // The in-memory listing must be left unchanged so the caller can full-relist. var errDeltaCursorReset = errors.New("delta list: cursor reset") type listingDelta struct { Remote string Deleted bool Size int64 ModTime time.Time Hash string ID string Flags string } type deltaApplyOpts struct { MaxDelete int Force bool Reset bool } func cursorPath(listing string) string { return listing + ".cursor" } // applyListingDeltas mutates ls in place. On error, ls is restored to its // previous contents (strong safety: never persist a half-applied delta). func applyListingDeltas(ls *fileList, deltas []listingDelta, opt deltaApplyOpts) error { if opt.Reset { return errDeltaCursorReset } backupList := append([]string(nil), ls.list...) backupInfo := make(map[string]*fileInfo, len(ls.info)) for k, v := range ls.info { cp := *v backupInfo[k] = &cp } restore := func() { ls.list = backupList ls.info = backupInfo } oldCount := 0 for _, name := range ls.list { if !ls.isDir(name) { oldCount++ } } deletedFiles := 0 for _, d := range deltas { if d.Deleted { // Dropbox DeletedMetadata: remove the path and all children. prefix := d.Remote if prefix != "" && !strings.HasSuffix(prefix, "/") { prefix += "/" } var toRemove []string for _, name := range ls.list { if name == d.Remote || (prefix != "" && strings.HasPrefix(name, prefix)) { toRemove = append(toRemove, name) } } for _, name := range toRemove { if !ls.isDir(name) { deletedFiles++ } ls.remove(name) } continue } flags := d.Flags if flags == "" { flags = "-" } ls.put(d.Remote, d.Size, d.ModTime, d.Hash, d.ID, flags) } if !opt.Force && opt.MaxDelete >= 0 && oldCount > 0 { maxRatio := float64(opt.MaxDelete) / 100.0 if float64(deletedFiles)/float64(oldCount) > maxRatio { restore() return errDeltaTooManyDeletes } } return nil } func saveListingAndCursor(listing, cursor, root string, ls *fileList) error { if err := os.MkdirAll(filepath.Dir(listing), 0700); err != nil { return err } tmpListing := listing + ".tmp" tmpCursor := cursorPath(listing) + ".tmp" if err := ls.save(tmpListing); err != nil { return err } cur := fmt.Sprintf("# rclone-bisync-cursor v1\nroot=%s\ncursor=%s\n", root, cursor) if err := os.WriteFile(tmpCursor, []byte(cur), 0600); err != nil { _ = os.Remove(tmpListing) return err } if err := os.Rename(tmpListing, listing); err != nil { _ = os.Remove(tmpListing) _ = os.Remove(tmpCursor) return err } if err := os.Rename(tmpCursor, cursorPath(listing)); err != nil { return fmt.Errorf("listing saved but cursor rename failed (next run will full-list): %w", err) } return nil } func loadCursor(listing string) (cursor, root string, err error) { b, err := os.ReadFile(cursorPath(listing)) if err != nil { if os.IsNotExist(err) { return "", "", nil } return "", "", err } sc := bufio.NewScanner(strings.NewReader(string(b))) for sc.Scan() { line := sc.Text() if strings.HasPrefix(line, "root=") { root = strings.TrimPrefix(line, "root=") } if strings.HasPrefix(line, "cursor=") { cursor = strings.TrimPrefix(line, "cursor=") } } return cursor, root, sc.Err() } func loadListingFile(listing string) (*fileList, error) { b := &bisyncRun{} return b.loadListing(listing) }