diff --git a/README.md b/README.md index c3da66e..8249371 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ rclone-dropbox-bisync ==================== -Windows: D: (DataStorage) \Dropbox, workdir D:\rclone-bisync -Linux: ~/DataStorage already mounted; Dropbox + workdir under that. +Windows: D:\Dropbox, workdir D:\rclone-bisync (volume DataStorage) +Linux: ~/DataStorage already mounted; Dropbox + rclone-bisync under that +Session: datastorage-dropbox (same listing/lock names on both OSes) + +Caps: --max-delete 1% AND --max-delete-files 10 AND --max-delete-size 1G +No --force, no --resilient. --max-lock 2h. Workdir mkdir lock. Windows (paste in PowerShell): @@ -12,9 +16,9 @@ Linux: curl -fsSL https://git.denkena-consulting.com/f-denkena/rclone-dropbox-bisync/raw/branch/master/linux/install.sh | bash -Needs git + go. Clones public f-denkena/rclone (bisync-delta-list) and this repo, -builds, writes config, creates workdir, RCLONE_TEST on the local Dropbox folder, -sync on logon/startup and every 15 minutes, Windows tray / Linux systemd --user. +First run is --dry-run --resync until: -Remote dropbox: must already be in rclone.conf. Copy RCLONE_TEST to dropbox: once -so --check-access passes. + rclone-bisyncd --accept-resync + +enabled=true only after RCLONE_TEST exists on dropbox:. Looping daemon sleeps +15 min after each finished run (no overlapping 15-min tasks). diff --git a/cmd/rclone-bisyncd/main.go b/cmd/rclone-bisyncd/main.go index 493a961..1ec0f61 100644 --- a/cmd/rclone-bisyncd/main.go +++ b/cmd/rclone-bisyncd/main.go @@ -2,22 +2,27 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "os/exec" "path/filepath" "runtime" + "strings" "time" ) +const sessionName = "datastorage-dropbox" + type Config struct { - Enabled bool `json:"enabled"` - Path1 string `json:"path1"` - Path2 string `json:"path2"` - Workdir string `json:"workdir"` - IntervalMinutes int `json:"interval_minutes"` - Rclone string `json:"rclone"` - CheckFilename string `json:"check_filename"` + Enabled bool `json:"enabled"` + Path1 string `json:"path1"` + Path2 string `json:"path2"` + Workdir string `json:"workdir"` + IntervalMinutes int `json:"interval_minutes"` + Rclone string `json:"rclone"` + CheckFilename string `json:"check_filename"` + SessionName string `json:"session_name"` } func defaultConfig() Config { @@ -26,6 +31,7 @@ func defaultConfig() Config { IntervalMinutes: 15, Rclone: "rclone", CheckFilename: "RCLONE_TEST", + SessionName: sessionName, } } @@ -59,6 +65,12 @@ func loadConfig() (Config, error) { if cfg.Rclone == "" { cfg.Rclone = "rclone" } + if cfg.SessionName == "" { + cfg.SessionName = sessionName + } + if cfg.CheckFilename == "" { + cfg.CheckFilename = "RCLONE_TEST" + } return cfg, nil } @@ -67,60 +79,150 @@ func notify(title, body string) { case "linux": _ = exec.Command("notify-send", "-a", "rclone-bisync", title, body).Run() case "windows": - ps := fmt.Sprintf(`Add-Type -AssemblyName System.Windows.Forms; $n=New-Object System.Windows.Forms.NotifyIcon; $n.Icon=[System.Drawing.SystemIcons]::Information; $n.Visible=$true; $n.ShowBalloonTip(4000,'%s','%s',[System.Windows.Forms.ToolTipIcon]::Info); Start-Sleep -Seconds 5; $n.Dispose()`, + ps := fmt.Sprintf(`Add-Type -AssemblyName System.Windows.Forms; $n=New-Object System.Windows.Forms.NotifyIcon; $n.Icon=[System.Drawing.SystemIcons]::Information; $n.Visible=$true; $n.ShowBalloonTip(8000,'%s','%s',[System.Windows.Forms.ToolTipIcon]::Info); Start-Sleep -Seconds 8; $n.Dispose()`, escapePS(title), escapePS(body)) _ = exec.Command("powershell", "-NoProfile", "-Command", ps).Run() } } func escapePS(s string) string { - out := make([]byte, 0, len(s)) - for i := 0; i < len(s); i++ { - if s[i] == '\'' { - out = append(out, '\'', '\'') - } else { - out = append(out, s[i]) - } - } - return string(out) + return strings.ReplaceAll(s, "'", "''") } -func runOnce(cfg Config) error { - if !cfg.Enabled { - return fmt.Errorf("disabled in %s", configPath()) +func listingsExist(cfg Config) bool { + base := filepath.Join(cfg.Workdir, cfg.SessionName) + _, e1 := os.Stat(base + ".path1.lst") + _, e2 := os.Stat(base + ".path2.lst") + return e1 == nil && e2 == nil +} + +func remoteHasCheckFile(cfg Config) bool { + cmd := exec.Command(cfg.Rclone, "lsf", cfg.Path2, "--include", cfg.CheckFilename) + out, err := cmd.CombinedOutput() + if err != nil { + return false } - if cfg.Path1 == "" || cfg.Path2 == "" || cfg.Workdir == "" { - return fmt.Errorf("path1, path2, and workdir must be set in %s", configPath()) + return strings.Contains(string(out), cfg.CheckFilename) +} + +func localHasCheckFile(cfg Config) bool { + _, err := os.Stat(filepath.Join(cfg.Path1, cfg.CheckFilename)) + return err == nil +} + +func acceptPath(cfg Config) string { + return filepath.Join(cfg.Workdir, "resync-accepted") +} + +func acquireLock(workdir string) (func(), error) { + if err := os.MkdirAll(workdir, 0700); err != nil { + return nil, err } + lockdir := filepath.Join(workdir, ".bisyncd.lock") + if err := os.Mkdir(lockdir, 0700); err != nil { + info, statErr := os.Stat(lockdir) + if statErr == nil && time.Since(info.ModTime()) > 3*time.Hour { + _ = os.RemoveAll(lockdir) + if err2 := os.Mkdir(lockdir, 0700); err2 == nil { + return func() { _ = os.RemoveAll(lockdir) }, nil + } + } + return nil, errors.New("another rclone-bisyncd holds " + lockdir) + } + _ = os.WriteFile(filepath.Join(lockdir, "pid"), []byte(fmt.Sprintf("%d", os.Getpid())), 0600) + return func() { _ = os.RemoveAll(lockdir) }, nil +} + +func bisyncArgs(cfg Config, extra ...string) []string { args := []string{ "bisync", cfg.Path1, cfg.Path2, "--workdir", cfg.Workdir, + "--session-name", cfg.SessionName, "--delta-list", "--check-access", "--check-filename", cfg.CheckFilename, "--compare", "size,modtime,checksum", "--slow-hash-sync-only", - "--max-delete", "10", - "--resilient", + "--max-delete", "1", + "--max-delete-files", "10", + "--max-delete-size", "1G", + "--max-lock", "2h", "--recover", "--conflict-resolve", "none", } - cmd := exec.Command(cfg.Rclone, args...) + return append(args, extra...) +} + +func runRclone(cfg Config, extra ...string) error { + cmd := exec.Command(cfg.Rclone, bisyncArgs(cfg, extra...)...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } +func runOnce(cfg Config, acceptResync bool) error { + if cfg.Path1 == "" || cfg.Path2 == "" || cfg.Workdir == "" { + return fmt.Errorf("path1, path2, and workdir must be set in %s", configPath()) + } + unlock, err := acquireLock(cfg.Workdir) + if err != nil { + fmt.Println(err) + return nil + } + defer unlock() + + if !localHasCheckFile(cfg) { + return fmt.Errorf("missing local %s under %s", cfg.CheckFilename, cfg.Path1) + } + if !remoteHasCheckFile(cfg) { + notify("rclone-bisync blocked", "No "+cfg.CheckFilename+" on "+cfg.Path2+" — copy it, then set enabled=true") + return fmt.Errorf("missing %s on %s", cfg.CheckFilename, cfg.Path2) + } + + if !listingsExist(cfg) { + if acceptResync { + _ = os.WriteFile(acceptPath(cfg), []byte("ok\n"), 0600) + } + if _, err := os.Stat(acceptPath(cfg)); err != nil && !acceptResync { + notify("rclone-bisync dry-run resync", "No listings yet. Inspect output, then: rclone-bisyncd --accept-resync") + if err := runRclone(cfg, "--dry-run", "--resync"); err != nil { + notify("rclone-bisync dry-run failed", err.Error()) + return err + } + return nil + } + notify("rclone-bisync", "Live --resync starting (one-shot)") + if err := runRclone(cfg, "--resync"); err != nil { + notify("rclone-bisync resync failed", err.Error()) + return err + } + notify("rclone-bisync", "Live resync finished") + return nil + } + + if !cfg.Enabled { + return fmt.Errorf("disabled in %s (listings exist; set enabled=true)", configPath()) + } + if err := runRclone(cfg); err != nil { + notify("rclone-bisync failed", err.Error()) + return err + } + notify("rclone-bisync", "Run finished") + return nil +} + func main() { once := false - loop := true + accept := false for _, a := range os.Args[1:] { switch a { case "--once": once = true - loop = false + case "--accept-resync": + accept = true + once = true case "--help", "-h": - fmt.Fprintf(os.Stderr, "rclone-bisyncd [--once]\nconfig: %s\n", configPath()) + fmt.Fprintf(os.Stderr, "rclone-bisyncd [--once] [--accept-resync]\nconfig: %s\n", configPath()) os.Exit(0) } } @@ -133,21 +235,15 @@ func main() { if once { os.Exit(1) } - } else if !cfg.Enabled { - fmt.Println("disabled; sleeping") - } else if err := runOnce(cfg); err != nil { + } else if err := runOnce(cfg, accept); err != nil { fmt.Fprintf(os.Stderr, "bisync: %v\n", err) - notify("rclone-bisync failed", err.Error()) if once { os.Exit(1) } - } else { - notify("rclone-bisync", "Run finished") - if once { - return - } + } else if once { + return } - if !loop { + if once { return } mins := 15 diff --git a/config.example.json b/config.example.json index 6017ae2..ae81b8b 100644 --- a/config.example.json +++ b/config.example.json @@ -1,9 +1,10 @@ { - "enabled": true, + "enabled": false, "path1": "~/DataStorage/Dropbox", "path2": "dropbox:", "workdir": "~/DataStorage/rclone-bisync", "interval_minutes": 15, "rclone": "rclone", - "check_filename": "RCLONE_TEST" + "check_filename": "RCLONE_TEST", + "session_name": "datastorage-dropbox" } diff --git a/linux/install.sh b/linux/install.sh index d1711a4..f508539 100755 --- a/linux/install.sh +++ b/linux/install.sh @@ -58,36 +58,37 @@ echo "building rclone..." echo "building rclone-bisyncd..." (cd "$SRC/packaging/cmd/rclone-bisyncd" && go build -trimpath -o "$BIN/rclone-bisyncd" .) +ENABLED=false +if "$BIN/rclone" lsf dropbox: --include RCLONE_TEST 2>/dev/null | grep -q RCLONE_TEST; then + ENABLED=true +fi cat >"$CFG_DIR/config.json" <"$HOME/.config/autostart/rclone-bisync.desktop" </dev/null || true -systemctl --user enable --now rclone-bisync.timer 2>/dev/null || true +systemctl --user enable --now rclone-bisync.service 2>/dev/null || true if command -v notify-send >/dev/null 2>&1; then - notify-send -a rclone-bisync "rclone-bisync" "Background timer enabled. Edit ~/.config/rclone-dropbox-bisync/config.json (set enabled=true)." + notify-send -a rclone-bisync "rclone-bisync" "Looping bisync started. First run is dry-run --resync until rclone-bisyncd --accept-resync." fi diff --git a/linux/systemd/rclone-bisync.service b/linux/systemd/rclone-bisync.service index 758d0db..34b1c97 100644 --- a/linux/systemd/rclone-bisync.service +++ b/linux/systemd/rclone-bisync.service @@ -1,11 +1,13 @@ [Unit] -Description=rclone Dropbox bisync (fail-closed) +Description=rclone Dropbox bisync (fail-closed loop) After=network-online.target Wants=network-online.target [Service] -Type=oneshot -ExecStart=/opt/rclone-dropbox-bisync/bin/rclone-bisyncd --once +Type=simple +ExecStart=/opt/rclone-dropbox-bisync/bin/rclone-bisyncd +Restart=on-failure +RestartSec=60 Nice=10 [Install] diff --git a/linux/systemd/rclone-bisync.timer b/linux/systemd/rclone-bisync.timer index dd4340a..9966942 100644 --- a/linux/systemd/rclone-bisync.timer +++ b/linux/systemd/rclone-bisync.timer @@ -1,9 +1,8 @@ [Unit] -Description=rclone Dropbox bisync timer +Description=unused — looping rclone-bisync.service replaced the timer [Timer] -OnBootSec=3min -OnUnitActiveSec=15min +OnUnitInactiveSec=15min Persistent=true Unit=rclone-bisync.service diff --git a/windows/Install.ps1 b/windows/Install.ps1 index 5448175..ba75688 100644 --- a/windows/Install.ps1 +++ b/windows/Install.ps1 @@ -11,6 +11,8 @@ if (-not (Test-Path $cfg)) { Copy-Item (Join-Path $Root 'config.example.json') $cfg $j = Get-Content $cfg -Raw | ConvertFrom-Json $j.rclone = Join-Path $Dest 'bin\rclone.exe' + $j.enabled = $false + $j.session_name = 'datastorage-dropbox' $j | ConvertTo-Json | Set-Content -Encoding UTF8 $cfg } @@ -23,13 +25,16 @@ if ($userPath -notlike "*$bin*") { $run = 'HKCU:\Software\Microsoft\Windows\CurrentVersion\Run' New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync' -PropertyType String -Force ` -Value "powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$Dest\tray.ps1`"" | Out-Null - -schtasks /Create /F /TN "rclone-dropbox-bisync" /SC MINUTE /MO 15 /TR ` - "`"$Dest\bin\rclone-bisyncd.exe`" --once" /ST 00:00 | Out-Null +$daemon = Join-Path $Dest 'bin\rclone-bisyncd.exe' +New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync-daemon' -PropertyType String -Force ` + -Value "`"$daemon`"" | Out-Null +schtasks /Delete /F /TN 'rclone-dropbox-bisync' 2>$null +schtasks /Create /F /TN 'rclone-dropbox-bisync' /SC ONLOGON /RL LIMITED /TR "`"$daemon`"" | Out-Null Start-Process powershell.exe -WindowStyle Hidden -ArgumentList @( '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', (Join-Path $Dest 'tray.ps1') ) +Start-Process $daemon -WindowStyle Hidden Add-Type -AssemblyName PresentationFramework -[System.Windows.MessageBox]::Show("Installed to $Dest`r`nEdit config.json (enabled, path1, path2, workdir).`r`nPut RCLONE_TEST on both sides. Tray + 15-min task are on.", 'rclone-dropbox-bisync') +[System.Windows.MessageBox]::Show("Installed to $Dest`r`nCopy RCLONE_TEST to dropbox:, set enabled=true, then rclone-bisyncd --accept-resync.", 'rclone-dropbox-bisync') diff --git a/windows/bootstrap.ps1 b/windows/bootstrap.ps1 index 95da83e..13b1267 100644 --- a/windows/bootstrap.ps1 +++ b/windows/bootstrap.ps1 @@ -58,13 +58,14 @@ if (-not (Test-Path $testFile)) { New-Item -ItemType File -Path $testFile | Out- $cfgPath = Join-Path $Dest 'config.json' @{ - enabled = $true + enabled = $false path1 = $Dropbox path2 = 'dropbox:' workdir = $Workdir interval_minutes = 15 rclone = (Join-Path $Dest 'bin\rclone.exe') check_filename = 'RCLONE_TEST' + session_name = 'datastorage-dropbox' } | ConvertTo-Json | Set-Content -Encoding UTF8 $cfgPath $env:RCLONE_BISYNC_CONFIG = $cfgPath @@ -79,23 +80,24 @@ $tray = Join-Path $Dest 'tray.ps1' New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync-tray' -PropertyType String -Force ` -Value "powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$tray`"" | Out-Null -# Sync at logon and every 15 minutes $daemon = Join-Path $Dest 'bin\rclone-bisyncd.exe' -schtasks /Create /F /TN 'rclone-dropbox-bisync-startup' /SC ONLOGON /RL LIMITED /TR "`"$daemon`" --once" | Out-Null -schtasks /Create /F /TN 'rclone-dropbox-bisync' /SC MINUTE /MO 15 /TR "`"$daemon`" --once" /ST 00:00 | Out-Null +schtasks /Delete /F /TN 'rclone-dropbox-bisync' 2>$null +schtasks /Create /F /TN 'rclone-dropbox-bisync' /SC ONLOGON /RL LIMITED /TR "`"$daemon`"" | Out-Null +New-ItemProperty -Path $run -Name 'rclone-dropbox-bisync-daemon' -PropertyType String -Force ` + -Value "`"$daemon`"" | Out-Null Start-Process powershell.exe -WindowStyle Hidden -ArgumentList @( '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', $tray ) -Start-Process $daemon -ArgumentList '--once' -WindowStyle Hidden +Start-Process $daemon -WindowStyle Hidden Write-Host @" Configured: path1 $Dropbox (D: DataStorage) workdir $Workdir - path2 dropbox: - startup ONLOGON + tray - interval 15 min -Put RCLONE_TEST on the Dropbox remote as well (rclone copy $testFile dropbox:RCLONE_TEST). -rclone remote 'dropbox:' must already exist in rclone.conf. + session datastorage-dropbox (shared with Linux) + daemon looping; 15 min after each run; mkdir lock +Copy RCLONE_TEST to dropbox:, set enabled=true, then: + rclone-bisyncd --accept-resync + (first run is --dry-run --resync until then) "@ diff --git a/windows/config.example.json b/windows/config.example.json index da18e3c..30f8d50 100644 --- a/windows/config.example.json +++ b/windows/config.example.json @@ -1,9 +1,10 @@ { - "enabled": true, + "enabled": false, "path1": "D:\\Dropbox", "path2": "dropbox:", "workdir": "D:\\rclone-bisync", "interval_minutes": 15, "rclone": "rclone.exe", - "check_filename": "RCLONE_TEST" + "check_filename": "RCLONE_TEST", + "session_name": "datastorage-dropbox" }