f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make dedupe remove identical copies without asking and add non interactive mode - fixes #338

Browse Source 
* Now removes identical copies without asking
  * Now obeys `--dry-run`
  * Implement `--dedupe-mode` for non interactive running
    * `--dedupe-mode interactive` - interactive the default.
    * `--dedupe-mode skip` - removes identical files then skips anything left.
    * `--dedupe-mode first` - removes identical files then keeps the first one.
    * `--dedupe-mode newest` - removes identical files then keeps the newest one.
    * `--dedupe-mode oldest` - removes identical files then keeps the oldest one.
    * `--dedupe-mode rename` - removes identical files then renames the rest to be different.
  * Add tests which will only run on Google Drive.
This commit is contained in:
Nick Craig-Wood
2016-03-06 18:15:01 +00:00
parent 2119fb4314
commit 8ef551bf9c
5 changed files with 405 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/config.go
+19
View File
@@ -83,6 +83,7 @@ var (
lowLevelRetries = pflag.IntP("low-level-retries", "", 10, "Number of low level retries to do.")
updateOlder = pflag.BoolP("update", "u", false, "Skip files that are newer on the destination.")
noGzip = pflag.BoolP("no-gzip-encoding", "", false, "Don't set Accept-Encoding: gzip.")
dedupeMode = pflag.StringP("dedupe-mode", "", "interactive", "Dedupe mode interactive|skip|first|newest|oldest|rename.")
bwLimit SizeSuffix
// Key to use for password en/decryption.
@@ -203,6 +204,7 @@ type ConfigInfo struct {
LowLevelRetries int
UpdateOlder bool // Skip files that are newer on the destination
NoGzip bool // Disable compression
DedupeMode DeduplicateMode
}
// Transport returns an http.RoundTripper with the correct timeouts
@@ -311,6 +313,23 @@ func LoadConfig() {
Config.DeleteDuring = *deleteDuring
Config.DeleteAfter = *deleteAfter
switch strings.ToLower(*dedupeMode) {
case "interactive":
Config.DedupeMode = DeduplicateInteractive
case "skip":
Config.DedupeMode = DeduplicateSkip
case "first":
Config.DedupeMode = DeduplicateFirst
case "newest":
Config.DedupeMode = DeduplicateNewest
case "oldest":
Config.DedupeMode = DeduplicateOldest
case "rename":
Config.DedupeMode = DeduplicateRename
default:
log.Fatalf(`Unknown mode for --dedupe-mode %q.`, *dedupeMode)
}
switch {
case *deleteBefore && (*deleteDuring || *deleteAfter),
*deleteDuring && *deleteAfter:
fs/operations.go
+165 -55
View File
@@ -5,8 +5,10 @@ package fs
import (
"fmt"
"io"
"log"
"mime"
"path"
"sort"
"strings"
"sync"
"sync/atomic"
@@ -405,6 +407,23 @@ func PairMover(in ObjectPairChan, fdst Fs, wg *sync.WaitGroup) {
}
}
// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors.
func DeleteFile(dst Object) {
if Config.DryRun {
Log(dst, "Not deleting as --dry-run")
} else {
Stats.Checking(dst)
err := dst.Remove()
Stats.DoneChecking(dst)
if err != nil {
Stats.Error()
ErrorLog(dst, "Couldn't delete: %s", err)
} else {
Debug(dst, "Deleted")
}
}
}
// DeleteFiles removes all the files passed in the channel
func DeleteFiles(toBeDeleted ObjectsChan) {
var wg sync.WaitGroup
@@ -413,19 +432,7 @@ func DeleteFiles(toBeDeleted ObjectsChan) {
go func() {
defer wg.Done()
for dst := range toBeDeleted {
if Config.DryRun {
Log(dst, "Not deleting as --dry-run")
} else {
Stats.Checking(dst)
err := dst.Remove()
Stats.DoneChecking(dst)
if err != nil {
Stats.Error()
ErrorLog(dst, "Couldn't delete: %s", err)
} else {
Debug(dst, "Deleted")
}
}
DeleteFile(dst)
}
}()
}
@@ -958,15 +965,132 @@ func Delete(f Fs) error {
return err
}
// dedupeRename renames the objs slice to different names
func dedupeRename(remote string, objs []Object) {
f := objs[0].Fs()
mover, ok := f.(Mover)
if !ok {
log.Fatalf("Fs %v doesn't support Move", f)
}
ext := path.Ext(remote)
base := remote[:len(remote)-len(ext)]
for i, o := range objs {
newName := fmt.Sprintf("%s-%d%s", base, i+1, ext)
if !Config.DryRun {
newObj, err := mover.Move(o, newName)
if err != nil {
Stats.Error()
ErrorLog(o, "Failed to rename: %v", err)
continue
}
Log(newObj, "renamed from: %v", o)
} else {
Log(remote, "Not renaming to %q as --dry-run", newName)
}
}
}
// dedupeDeleteAllButOne deletes all but the one in keep
func dedupeDeleteAllButOne(keep int, remote string, objs []Object) {
for i, o := range objs {
if i == keep {
continue
}
DeleteFile(o)
}
Log(remote, "Deleted %d extra copies", len(objs)-1)
}
// dedupeDeleteIdentical deletes all but one of identical (by hash) copies
func dedupeDeleteIdentical(remote string, objs []Object) []Object {
// See how many of these duplicates are identical
byHash := make(map[string][]Object, len(objs))
for _, o := range objs {
md5sum, err := o.Hash(HashMD5)
if err == nil {
byHash[md5sum] = append(byHash[md5sum], o)
}
}
// Delete identical duplicates, refilling obj with the ones remaining
objs = nil
for md5sum, hashObjs := range byHash {
if len(hashObjs) > 1 {
Log(remote, "Deleting %d/%d identical duplicates (md5sum %q)", len(hashObjs)-1, len(hashObjs), md5sum)
for _, o := range hashObjs[1:] {
DeleteFile(o)
}
}
objs = append(objs, hashObjs[0])
}
return objs
}
// dedupeInteractive interactively dedupes the slice of objects
func dedupeInteractive(remote string, objs []Object) {
fmt.Printf("%s: %d duplicates remain\n", remote, len(objs))
for i, o := range objs {
md5sum, err := o.Hash(HashMD5)
if err != nil {
md5sum = err.Error()
}
fmt.Printf(" %d: %12d bytes, %s, md5sum %32s\n", i+1, o.Size(), o.ModTime().Format("2006-01-02 15:04:05.000000000"), md5sum)
}
switch Command([]string{"sSkip and do nothing", "kKeep just one (choose which in next step)", "rRename all to be different (by changing file.jpg to file-1.jpg)"}) {
case 's':
case 'k':
keep := ChooseNumber("Enter the number of the file to keep", 1, len(objs))
dedupeDeleteAllButOne(keep-1, remote, objs)
case 'r':
dedupeRename(remote, objs)
}
}
type objectsSortedByModTime []Object
func (objs objectsSortedByModTime) Len() int { return len(objs) }
func (objs objectsSortedByModTime) Swap(i, j int) { objs[i], objs[j] = objs[j], objs[i] }
func (objs objectsSortedByModTime) Less(i, j int) bool {
return objs[i].ModTime().Before(objs[j].ModTime())
}
// DeduplicateMode is how the dedupe command chooses what to do
type DeduplicateMode int
// Deduplicate modes
const (
DeduplicateInteractive DeduplicateMode = iota // interactively ask the user
DeduplicateSkip // skip all conflicts
DeduplicateFirst // choose the first object
DeduplicateNewest // choose the newest object
DeduplicateOldest // choose the oldest object
DeduplicateRename // rename the objects
)
func (mode DeduplicateMode) String() string {
switch mode {
case DeduplicateInteractive:
return "interactive"
case DeduplicateSkip:
return "skip"
case DeduplicateFirst:
return "first"
case DeduplicateNewest:
return "newest"
case DeduplicateOldest:
return "oldest"
case DeduplicateRename:
return "rename"
}
return "unknown"
}
// Deduplicate interactively finds duplicate files and offers to
// delete all but one or rename them to be different. Only useful with
// Google Drive which can have duplicate file names.
func Deduplicate(f Fs) error {
mover, ok := f.(Mover)
if !ok {
return fmt.Errorf("%v can't Move files", f)
}
Log(f, "Looking for duplicates")
func Deduplicate(f Fs, mode DeduplicateMode) error {
Log(f, "Looking for duplicates using %v mode.", mode)
files := map[string][]Object{}
for o := range f.List() {
remote := o.Remote()
@@ -974,43 +1098,29 @@ func Deduplicate(f Fs) error {
}
for remote, objs := range files {
if len(objs) > 1 {
fmt.Printf("%s: Found %d duplicates\n", remote, len(objs))
for i, o := range objs {
md5sum, err := o.Hash(HashMD5)
if err != nil {
md5sum = err.Error()
}
fmt.Printf(" %d: %12d bytes, %s, md5sum %32s\n", i+1, o.Size(), o.ModTime().Format("2006-01-02 15:04:05.000000000"), md5sum)
Log(remote, "Found %d duplicates - deleting identical copies", len(objs))
objs = dedupeDeleteIdentical(remote, objs)
if len(objs) <= 1 {
Log(remote, "All duplicates removed")
continue
}
switch Command([]string{"sSkip and do nothing", "kKeep just one (choose which in next step)", "rRename all to be different (by changing file.jpg to file-1.jpg)"}) {
case 's':
case 'k':
keep := ChooseNumber("Enter the number of the file to keep", 1, len(objs))
deleted := 0
for i, o := range objs {
if i+1 == keep {
continue
}
err := o.Remove()
if err != nil {
ErrorLog(o, "Failed to delete: %v", err)
continue
}
deleted++
}
fmt.Printf("%s: Deleted %d extra copies\n", remote, deleted)
case 'r':
ext := path.Ext(remote)
base := remote[:len(remote)-len(ext)]
for i, o := range objs {
newName := fmt.Sprintf("%s-%d%s", base, i+1, ext)
newObj, err := mover.Move(o, newName)
if err != nil {
ErrorLog(o, "Failed to rename: %v", err)
continue
}
fmt.Printf("%v: renamed from: %v\n", newObj, o)
}
switch mode {
case DeduplicateInteractive:
dedupeInteractive(remote, objs)
case DeduplicateFirst:
dedupeDeleteAllButOne(0, remote, objs)
case DeduplicateNewest:
sort.Sort(objectsSortedByModTime(objs)) // sort oldest first
dedupeDeleteAllButOne(len(objs)-1, remote, objs)
case DeduplicateOldest:
sort.Sort(objectsSortedByModTime(objs)) // sort oldest first
dedupeDeleteAllButOne(0, remote, objs)
case DeduplicateRename:
dedupeRename(remote, objs)
case DeduplicateSkip:
// skip
default:
//skip
}
}
}
fs/operations_test.go
+157
View File
@@ -975,3 +975,160 @@ func TestCheck(t *testing.T) {
fstest.CheckItems(t, r.flocal, file1, file2, file3)
check(5, 0)
}
func (r *Run) checkWithDuplicates(t *testing.T, items ...fstest.Item) {
objects, size, err := fs.Count(r.fremote)
if err != nil {
t.Fatalf("Error listing: %v", err)
}
if objects != int64(len(items)) {
t.Fatalf("Error listing want %d objects, got %d", len(items), objects)
}
wantSize := int64(0)
for _, item := range items {
wantSize += item.Size
}
if wantSize != size {
t.Fatalf("Error listing want %d size, got %d", wantSize, size)
}
}
func TestDeduplicateInteractive(t *testing.T) {
if *RemoteName != "TestDrive:" {
t.Skip("Can only test deduplicate on google drive")
}
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("one", "This is one", t1)
file2 := r.WriteObject("one", "This is one", t1)
file3 := r.WriteObject("one", "This is one", t1)
r.checkWithDuplicates(t, file1, file2, file3)
err := fs.Deduplicate(r.fremote, fs.DeduplicateInteractive)
if err != nil {
t.Fatalf("fs.Deduplicate returned error: %v", err)
}
fstest.CheckItems(t, r.fremote, file1)
}
func TestDeduplicateSkip(t *testing.T) {
if *RemoteName != "TestDrive:" {
t.Skip("Can only test deduplicate on google drive")
}
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("one", "This is one", t1)
file2 := r.WriteObject("one", "This is one", t1)
file3 := r.WriteObject("one", "This is another one", t1)
r.checkWithDuplicates(t, file1, file2, file3)
err := fs.Deduplicate(r.fremote, fs.DeduplicateSkip)
if err != nil {
t.Fatalf("fs.Deduplicate returned error: %v", err)
}
r.checkWithDuplicates(t, file1, file3)
}
func TestDeduplicateFirst(t *testing.T) {
if *RemoteName != "TestDrive:" {
t.Skip("Can only test deduplicate on google drive")
}
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("one", "This is one", t1)
file2 := r.WriteObject("one", "This is one A", t1)
file3 := r.WriteObject("one", "This is one BB", t1)
r.checkWithDuplicates(t, file1, file2, file3)
err := fs.Deduplicate(r.fremote, fs.DeduplicateFirst)
if err != nil {
t.Fatalf("fs.Deduplicate returned error: %v", err)
}
objects, size, err := fs.Count(r.fremote)
if err != nil {
t.Fatalf("Error listing: %v", err)
}
if objects != 1 {
t.Errorf("Expecting 1 object got %v", objects)
}
if size != file1.Size && size != file2.Size && size != file3.Size {
t.Errorf("Size not one of the object sizes %d", size)
}
}
func TestDeduplicateNewest(t *testing.T) {
if *RemoteName != "TestDrive:" {
t.Skip("Can only test deduplicate on google drive")
}
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("one", "This is one", t1)
file2 := r.WriteObject("one", "This is one too", t2)
file3 := r.WriteObject("one", "This is another one", t3)
r.checkWithDuplicates(t, file1, file2, file3)
err := fs.Deduplicate(r.fremote, fs.DeduplicateNewest)
if err != nil {
t.Fatalf("fs.Deduplicate returned error: %v", err)
}
fstest.CheckItems(t, r.fremote, file3)
}
func TestDeduplicateOldest(t *testing.T) {
if *RemoteName != "TestDrive:" {
t.Skip("Can only test deduplicate on google drive")
}
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("one", "This is one", t1)
file2 := r.WriteObject("one", "This is one too", t2)
file3 := r.WriteObject("one", "This is another one", t3)
r.checkWithDuplicates(t, file1, file2, file3)
err := fs.Deduplicate(r.fremote, fs.DeduplicateOldest)
if err != nil {
t.Fatalf("fs.Deduplicate returned error: %v", err)
}
fstest.CheckItems(t, r.fremote, file1)
}
func TestDeduplicateRename(t *testing.T) {
if *RemoteName != "TestDrive:" {
t.Skip("Can only test deduplicate on google drive")
}
r := NewRun(t)
defer r.Finalise()
file1 := r.WriteObject("one.txt", "This is one", t1)
file2 := r.WriteObject("one.txt", "This is one too", t2)
file3 := r.WriteObject("one.txt", "This is another one", t3)
r.checkWithDuplicates(t, file1, file2, file3)
err := fs.Deduplicate(r.fremote, fs.DeduplicateRename)
if err != nil {
t.Fatalf("fs.Deduplicate returned error: %v", err)
}
for o := range r.fremote.List() {
remote := o.Remote()
if remote != "one-1.txt" &&
remote != "one-2.txt" &&
remote != "one-3.txt" {
t.Errorf("Bad file name after rename %q", remote)
}
size := o.Size()
if size != file1.Size && size != file2.Size && size != file3.Size {
t.Errorf("Size not one of the object sizes %d", size)
}
}
}