daemon: flock, dry-run resync gate, shared session, looping interval

README.md

@@ -1,8 +1,12 @@
 rclone-dropbox-bisync
 ====================
 
-Windows: D: (DataStorage) \Dropbox, workdir D:\rclone-bisync
+Windows: D:\Dropbox, workdir D:\rclone-bisync (volume DataStorage)
-Linux:   ~/DataStorage already mounted; Dropbox + workdir under that.
+Linux:   ~/DataStorage already mounted; Dropbox + rclone-bisync under that
+Session: datastorage-dropbox (same listing/lock names on both OSes)
+
+Caps: --max-delete 1% AND --max-delete-files 10 AND --max-delete-size 1G
+No --force, no --resilient. --max-lock 2h. Workdir mkdir lock.
 
 Windows (paste in PowerShell):
 
@@ -12,9 +16,9 @@ Linux:
 
   curl -fsSL https://git.denkena-consulting.com/f-denkena/rclone-dropbox-bisync/raw/branch/master/linux/install.sh | bash
 
-Needs git + go. Clones public f-denkena/rclone (bisync-delta-list) and this repo,
+First run is --dry-run --resync until:
-builds, writes config, creates workdir, RCLONE_TEST on the local Dropbox folder,
-sync on logon/startup and every 15 minutes, Windows tray / Linux systemd --user.
 
-Remote dropbox: must already be in rclone.conf. Copy RCLONE_TEST to dropbox: once
+  rclone-bisyncd --accept-resync
-so --check-access passes.
+
+enabled=true only after RCLONE_TEST exists on dropbox:. Looping daemon sleeps
+15 min after each finished run (no overlapping 15-min tasks).

cmd/rclone-bisyncd/main.go

@@ -2,22 +2,27 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 )
 
+const sessionName = "datastorage-dropbox"
+
 type Config struct {
-	Enabled          bool   `json:"enabled"`
+	Enabled         bool   `json:"enabled"`
-	Path1            string `json:"path1"`
+	Path1           string `json:"path1"`
-	Path2            string `json:"path2"`
+	Path2           string `json:"path2"`
-	Workdir          string `json:"workdir"`
+	Workdir         string `json:"workdir"`
-	IntervalMinutes  int    `json:"interval_minutes"`
+	IntervalMinutes int    `json:"interval_minutes"`
-	Rclone           string `json:"rclone"`
+	Rclone          string `json:"rclone"`
-	CheckFilename    string `json:"check_filename"`
+	CheckFilename   string `json:"check_filename"`
+	SessionName     string `json:"session_name"`
 }
 
 func defaultConfig() Config {
@@ -26,6 +31,7 @@ func defaultConfig() Config {
 		IntervalMinutes: 15,
 		Rclone:          "rclone",
 		CheckFilename:   "RCLONE_TEST",
+		SessionName:     sessionName,
 	}
 }
 
@@ -59,6 +65,12 @@ func loadConfig() (Config, error) {
 	if cfg.Rclone == "" {
 		cfg.Rclone = "rclone"
 	}
+	if cfg.SessionName == "" {
+		cfg.SessionName = sessionName
+	}
+	if cfg.CheckFilename == "" {
+		cfg.CheckFilename = "RCLONE_TEST"
+	}
 	return cfg, nil
 }
 
@@ -67,60 +79,150 @@ func notify(title, body string) {
 	case "linux":
 		_ = exec.Command("notify-send", "-a", "rclone-bisync", title, body).Run()
 	case "windows":
-		ps := fmt.Sprintf(`Add-Type -AssemblyName System.Windows.Forms; $n=New-Object System.Windows.Forms.NotifyIcon; $n.Icon=[System.Drawing.SystemIcons]::Information; $n.Visible=$true; $n.ShowBalloonTip(4000,'%s','%s',[System.Windows.Forms.ToolTipIcon]::Info); Start-Sleep -Seconds 5; $n.Dispose()`,
+		ps := fmt.Sprintf(`Add-Type -AssemblyName System.Windows.Forms; $n=New-Object System.Windows.Forms.NotifyIcon; $n.Icon=[System.Drawing.SystemIcons]::Information; $n.Visible=$true; $n.ShowBalloonTip(8000,'%s','%s',[System.Windows.Forms.ToolTipIcon]::Info); Start-Sleep -Seconds 8; $n.Dispose()`,
 			escapePS(title), escapePS(body))
 		_ = exec.Command("powershell", "-NoProfile", "-Command", ps).Run()
 	}
 }
 
 func escapePS(s string) string {
-	out := make([]byte, 0, len(s))
+	return strings.ReplaceAll(s, "'", "''")
-	for i := 0; i < len(s); i++ {
-		if s[i] == '\'' {
-			out = append(out, '\'', '\'')
-		} else {
-			out = append(out, s[i])
-		}
-	}
-	return string(out)
 }
 
-func runOnce(cfg Config) error {
+func listingsExist(cfg Config) bool {
-	if !cfg.Enabled {
+	base := filepath.Join(cfg.Workdir, cfg.SessionName)
-		return fmt.Errorf("disabled in %s", configPath())
+	_, e1 := os.Stat(base + ".path1.lst")
+	_, e2 := os.Stat(base + ".path2.lst")
+	return e1 == nil && e2 == nil
+}
+
+func remoteHasCheckFile(cfg Config) bool {
+	cmd := exec.Command(cfg.Rclone, "lsf", cfg.Path2, "--include", cfg.CheckFilename)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return false
 	}
-	if cfg.Path1 == "" || cfg.Path2 == "" || cfg.Workdir == "" {
+	return strings.Contains(string(out), cfg.CheckFilename)
-		return fmt.Errorf("path1, path2, and workdir must be set in %s", configPath())
+}
+
+func localHasCheckFile(cfg Config) bool {
+	_, err := os.Stat(filepath.Join(cfg.Path1, cfg.CheckFilename))
+	return err == nil
+}
+
+func acceptPath(cfg Config) string {
+	return filepath.Join(cfg.Workdir, "resync-accepted")
+}
+
+func acquireLock(workdir string) (func(), error) {
+	if err := os.MkdirAll(workdir, 0700); err != nil {
+		return nil, err
 	}
+	lockdir := filepath.Join(workdir, ".bisyncd.lock")
+	if err := os.Mkdir(lockdir, 0700); err != nil {
+		info, statErr := os.Stat(lockdir)
+		if statErr == nil && time.Since(info.ModTime()) > 3*time.Hour {
+			_ = os.RemoveAll(lockdir)
+			if err2 := os.Mkdir(lockdir, 0700); err2 == nil {
+				return func() { _ = os.RemoveAll(lockdir) }, nil
+			}
+		}
+		return nil, errors.New("another rclone-bisyncd holds " + lockdir)
+	}
+	_ = os.WriteFile(filepath.Join(lockdir, "pid"), []byte(fmt.Sprintf("%d", os.Getpid())), 0600)
+	return func() { _ = os.RemoveAll(lockdir) }, nil
+}
+
+func bisyncArgs(cfg Config, extra ...string) []string {
 	args := []string{
 		"bisync", cfg.Path1, cfg.Path2,
 		"--workdir", cfg.Workdir,
+		"--session-name", cfg.SessionName,
 		"--delta-list",
 		"--check-access",
 		"--check-filename", cfg.CheckFilename,
 		"--compare", "size,modtime,checksum",
 		"--slow-hash-sync-only",
-		"--max-delete", "10",
+		"--max-delete", "1",
-		"--resilient",
+		"--max-delete-files", "10",
+		"--max-delete-size", "1G",
+		"--max-lock", "2h",
 		"--recover",
 		"--conflict-resolve", "none",
 	}
-	cmd := exec.Command(cfg.Rclone, args...)
+	return append(args, extra...)
+}
+
+func runRclone(cfg Config, extra ...string) error {
+	cmd := exec.Command(cfg.Rclone, bisyncArgs(cfg, extra...)...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	return cmd.Run()
 }
 
+func runOnce(cfg Config, acceptResync bool) error {
+	if cfg.Path1 == "" || cfg.Path2 == "" || cfg.Workdir == "" {
+		return fmt.Errorf("path1, path2, and workdir must be set in %s", configPath())
+	}
+	unlock, err := acquireLock(cfg.Workdir)
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+	defer unlock()
+
+	if !localHasCheckFile(cfg) {
+		return fmt.Errorf("missing local %s under %s", cfg.CheckFilename, cfg.Path1)
+	}
+	if !remoteHasCheckFile(cfg) {
+		notify("rclone-bisync blocked", "No "+cfg.CheckFilename+" on "+cfg.Path2+" — copy it, then set enabled=true")
+		return fmt.Errorf("missing %s on %s", cfg.CheckFilename, cfg.Path2)
+	}
+
+	if !listingsExist(cfg) {
+		if acceptResync {
+			_ = os.WriteFile(acceptPath(cfg), []byte("ok\n"), 0600)
+		}
+		if _, err := os.Stat(acceptPath(cfg)); err != nil && !acceptResync {
+			notify("rclone-bisync dry-run resync", "No listings yet. Inspect output, then: rclone-bisyncd --accept-resync")
+			if err := runRclone(cfg, "--dry-run", "--resync"); err != nil {
+				notify("rclone-bisync dry-run failed", err.Error())
+				return err
+			}
+			return nil
+		}
+		notify("rclone-bisync", "Live --resync starting (one-shot)")
+		if err := runRclone(cfg, "--resync"); err != nil {
+			notify("rclone-bisync resync failed", err.Error())
+			return err
+		}
+		notify("rclone-bisync", "Live resync finished")
+		return nil
+	}
+
+	if !cfg.Enabled {
+		return fmt.Errorf("disabled in %s (listings exist; set enabled=true)", configPath())
+	}
+	if err := runRclone(cfg); err != nil {
+		notify("rclone-bisync failed", err.Error())
+		return err
+	}
+	notify("rclone-bisync", "Run finished")
+	return nil
+}
+
 func main() {
 	once := false
-	loop := true
+	accept := false
 	for _, a := range os.Args[1:] {
 		switch a {
 		case "--once":
 			once = true
-			loop = false
+		case "--accept-resync":
+			accept = true
+			once = true
 		case "--help", "-h":
-			fmt.Fprintf(os.Stderr, "rclone-bisyncd [--once]\nconfig: %s\n", configPath())
+			fmt.Fprintf(os.Stderr, "rclone-bisyncd [--once] [--accept-resync]\nconfig: %s\n", configPath())
 			os.Exit(0)
 		}
 	}
@@ -133,21 +235,15 @@ func main() {
 			if once {
 				os.Exit(1)
 			}
-		} else if !cfg.Enabled {
+		} else if err := runOnce(cfg, accept); err != nil {
-			fmt.Println("disabled; sleeping")
-		} else if err := runOnce(cfg); err != nil {
 			fmt.Fprintf(os.Stderr, "bisync: %v\n", err)
-			notify("rclone-bisync failed", err.Error())
 			if once {
 				os.Exit(1)
 			}
-		} else {
+		} else if once {
-			notify("rclone-bisync", "Run finished")
+			return
-			if once {
-				return
-			}
 		}
-		if !loop {
+		if once {
 			return
 		}
 		mins := 15

config.example.json

@@ -1,9 +1,10 @@
 {
-  "enabled": true,
+  "enabled": false,
   "path1": "~/DataStorage/Dropbox",
   "path2": "dropbox:",
   "workdir": "~/DataStorage/rclone-bisync",
   "interval_minutes": 15,
   "rclone": "rclone",
-  "check_filename": "RCLONE_TEST"
+  "check_filename": "RCLONE_TEST",
+  "session_name": "datastorage-dropbox"
 }

linux/install.sh

@@ -58,36 +58,37 @@ echo "building rclone..."
 echo "building rclone-bisyncd..."
 (cd "$SRC/packaging/cmd/rclone-bisyncd" && go build -trimpath -o "$BIN/rclone-bisyncd" .)
 
+ENABLED=false
+if "$BIN/rclone" lsf dropbox: --include RCLONE_TEST 2>/dev/null | grep -q RCLONE_TEST; then
+	ENABLED=true
+fi
 cat >"$CFG_DIR/config.json" <<EOF
 {
-  "enabled": true,
+  "enabled": $ENABLED,
   "path1": "$DROPBOX",
   "path2": "dropbox:",
   "workdir": "$WORKDIR",
   "interval_minutes": 15,
   "rclone": "$BIN/rclone",
-  "check_filename": "RCLONE_TEST"
+  "check_filename": "RCLONE_TEST",
+  "session_name": "datastorage-dropbox"
 }
 EOF
 
 install -m 644 "$SRC/packaging/linux/systemd/rclone-bisync.service" "$UNIT_DIR/rclone-bisync.service"
-install -m 644 "$SRC/packaging/linux/systemd/rclone-bisync.timer" "$UNIT_DIR/rclone-bisync.timer"
+sed -i "s|^ExecStart=.*|ExecStart=$BIN/rclone-bisyncd|" "$UNIT_DIR/rclone-bisync.service"
-# Point units at user binaries
-sed -i "s|^ExecStart=.*|ExecStart=$BIN/rclone-bisyncd --once|" "$UNIT_DIR/rclone-bisync.service"
 
-# Startup: enable timer + one-shot after login
 cat >"$HOME/.config/autostart/rclone-bisync.desktop" <<EOF
 [Desktop Entry]
 Type=Application
 Name=rclone Dropbox bisync
-Exec=sh -c 'systemctl --user start rclone-bisync.service; systemctl --user enable --now rclone-bisync.timer'
+Exec=systemctl --user start rclone-bisync.service
 X-GNOME-Autostart-enabled=true
 Terminal=false
 EOF
 
 systemctl --user daemon-reload
-systemctl --user enable --now rclone-bisync.timer
+systemctl --user enable --now rclone-bisync.service || true
-systemctl --user start rclone-bisync.service || true
 
 case ":$PATH:" in
 *:$BIN:*) ;;
@@ -95,5 +96,6 @@ case ":$PATH:" in
 esac
 
 echo "Configured path1=$DROPBOX workdir=$WORKDIR (~/DataStorage)"
-echo "Startup: autostart desktop + systemd --user timer every 15 min"
+echo "Startup: looping systemd --user service (sleeps 15 min after each run)"
-echo "Put RCLONE_TEST on dropbox: as well. Remote dropbox: must exist in rclone.conf."
+echo "If listings are empty: dry-run --resync only. Then: rclone-bisyncd --accept-resync"
+echo "enabled=$ENABLED (true only if dropbox: already has RCLONE_TEST)"

linux/session-start.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 systemctl --user daemon-reload 2>/dev/null || true
-systemctl --user enable --now rclone-bisync.timer 2>/dev/null || true
+systemctl --user enable --now rclone-bisync.service 2>/dev/null || true
 if command -v notify-send >/dev/null 2>&1; then
-	notify-send -a rclone-bisync "rclone-bisync" "Background timer enabled. Edit ~/.config/rclone-dropbox-bisync/config.json (set enabled=true)."
+	notify-send -a rclone-bisync "rclone-bisync" "Looping bisync started. First run is dry-run --resync until rclone-bisyncd --accept-resync."
 fi

linux/systemd/rclone-bisync.service

@@ -1,11 +1,13 @@
 [Unit]
-Description=rclone Dropbox bisync (fail-closed)
+Description=rclone Dropbox bisync (fail-closed loop)
 After=network-online.target
 Wants=network-online.target
 
 [Service]
-Type=oneshot
+Type=simple
-ExecStart=/opt/rclone-dropbox-bisync/bin/rclone-bisyncd --once
+ExecStart=/opt/rclone-dropbox-bisync/bin/rclone-bisyncd
+Restart=on-failure
+RestartSec=60
 Nice=10
 
 [Install]

linux/systemd/rclone-bisync.timer

@@ -1,9 +1,8 @@
 [Unit]
-Description=rclone Dropbox bisync timer
+Description=unused — looping rclone-bisync.service replaced the timer
 
 [Timer]
-OnBootSec=3min
+OnUnitInactiveSec=15min
-OnUnitActiveSec=15min
 Persistent=true
 Unit=rclone-bisync.service
 

windows/Install.ps1

@@ -11,6 +11,8 @@ if (-not (Test-Path $cfg)) {
     Copy-Item (Join-Path $Root 'config.example.json') $cfg
     $j = Get-Content $cfg -Raw | ConvertFrom-Json
     $j.rclone = Join-Path $Dest 'bin\rclone.exe'
+    $j.enabled = $false
+    $j.session_name = 'datastorage-dropbox'
     $j | ConvertTo-Json | Set-Content -Encoding UTF8 $cfg
 }
 
@@ -23,13 +25,16 @@ if ($userPath -notlike "*$bin*") {
 $run = 'HKCU:\Software\Microsoft\Windows\CurrentVersion\Run'
 New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync' -PropertyType String -Force `
     -Value "powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$Dest\tray.ps1`"" | Out-Null
-
+$daemon = Join-Path $Dest 'bin\rclone-bisyncd.exe'
-schtasks /Create /F /TN "rclone-dropbox-bisync" /SC MINUTE /MO 15 /TR `
+New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync-daemon' -PropertyType String -Force `
-    "`"$Dest\bin\rclone-bisyncd.exe`" --once" /ST 00:00 | Out-Null
+    -Value "`"$daemon`"" | Out-Null
+schtasks /Delete /F /TN 'rclone-dropbox-bisync' 2>$null
+schtasks /Create /F /TN 'rclone-dropbox-bisync' /SC ONLOGON /RL LIMITED /TR "`"$daemon`"" | Out-Null
 
 Start-Process powershell.exe -WindowStyle Hidden -ArgumentList @(
     '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', (Join-Path $Dest 'tray.ps1')
 )
+Start-Process $daemon -WindowStyle Hidden
 
 Add-Type -AssemblyName PresentationFramework
-[System.Windows.MessageBox]::Show("Installed to $Dest`r`nEdit config.json (enabled, path1, path2, workdir).`r`nPut RCLONE_TEST on both sides. Tray + 15-min task are on.", 'rclone-dropbox-bisync')
+[System.Windows.MessageBox]::Show("Installed to $Dest`r`nCopy RCLONE_TEST to dropbox:, set enabled=true, then rclone-bisyncd --accept-resync.", 'rclone-dropbox-bisync')

windows/bootstrap.ps1

@@ -58,13 +58,14 @@ if (-not (Test-Path $testFile)) { New-Item -ItemType File -Path $testFile | Out-
 
 $cfgPath = Join-Path $Dest 'config.json'
 @{
-    enabled           = $true
+    enabled           = $false
     path1             = $Dropbox
     path2             = 'dropbox:'
     workdir           = $Workdir
     interval_minutes  = 15
     rclone            = (Join-Path $Dest 'bin\rclone.exe')
     check_filename    = 'RCLONE_TEST'
+    session_name      = 'datastorage-dropbox'
 } | ConvertTo-Json | Set-Content -Encoding UTF8 $cfgPath
 $env:RCLONE_BISYNC_CONFIG = $cfgPath
 
@@ -79,23 +80,24 @@ $tray = Join-Path $Dest 'tray.ps1'
 New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync-tray' -PropertyType String -Force `
     -Value "powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$tray`"" | Out-Null
 
-# Sync at logon and every 15 minutes
 $daemon = Join-Path $Dest 'bin\rclone-bisyncd.exe'
-schtasks /Create /F /TN 'rclone-dropbox-bisync-startup' /SC ONLOGON /RL LIMITED /TR "`"$daemon`" --once" | Out-Null
+schtasks /Delete /F /TN 'rclone-dropbox-bisync' 2>$null
-schtasks /Create /F /TN 'rclone-dropbox-bisync' /SC MINUTE /MO 15 /TR "`"$daemon`" --once" /ST 00:00 | Out-Null
+schtasks /Create /F /TN 'rclone-dropbox-bisync' /SC ONLOGON /RL LIMITED /TR "`"$daemon`"" | Out-Null
+New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync-daemon' -PropertyType String -Force `
+    -Value "`"$daemon`"" | Out-Null
 
 Start-Process powershell.exe -WindowStyle Hidden -ArgumentList @(
     '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', $tray
 )
-Start-Process $daemon -ArgumentList '--once' -WindowStyle Hidden
+Start-Process $daemon -WindowStyle Hidden
 
 Write-Host @"
 Configured:
   path1    $Dropbox  (D: DataStorage)
   workdir  $Workdir
-  path2    dropbox:
+  session  datastorage-dropbox (shared with Linux)
-  startup  ONLOGON + tray
+  daemon   looping; 15 min after each run; mkdir lock
-  interval 15 min
+Copy RCLONE_TEST to dropbox:, set enabled=true, then:
-Put RCLONE_TEST on the Dropbox remote as well (rclone copy $testFile dropbox:RCLONE_TEST).
+  rclone-bisyncd --accept-resync
-rclone remote 'dropbox:' must already exist in rclone.conf.
+  (first run is --dry-run --resync until then)
 "@

windows/config.example.json

@@ -1,9 +1,10 @@
 {
-  "enabled": true,
+  "enabled": false,
   "path1": "D:\\Dropbox",
   "path2": "dropbox:",
   "workdir": "D:\\rclone-bisync",
   "interval_minutes": 15,
   "rclone": "rclone.exe",
-  "check_filename": "RCLONE_TEST"
+  "check_filename": "RCLONE_TEST",
+  "session_name": "datastorage-dropbox"
 }