Use --fast-list flag for sync/copy/move - fixes #1277

Redo test framework to take a -fast-list flag and test remotes with that flag.

fs/test_all.go

@@ -20,22 +20,84 @@ import (
 	"github.com/ncw/rclone/fstest"
 )
 
+type remoteConfig struct {
+	Name     string
+	SubDir   bool
+	FastList bool
+}
+
 var (
-	remotes = []string{
-		"TestAmazonCloudDrive:",
-		"TestB2:",
-		"TestCryptDrive:",
-		"TestCryptSwift:",
-		"TestDrive:",
-		"TestDropbox:",
-		"TestGoogleCloudStorage:",
-		"TestHubic:",
-		"TestOneDrive:",
-		"TestS3:",
-		"TestSftp:",
-		"TestSwift:",
-		"TestYandex:",
-		"TestFTP:",
+	remotes = []remoteConfig{
+		{
+			Name:     "TestAmazonCloudDrive:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestB2:",
+			SubDir:   true,
+			FastList: true,
+		},
+		{
+			Name:     "TestCryptDrive:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestCryptSwift:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestDrive:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestDropbox:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestGoogleCloudStorage:",
+			SubDir:   true,
+			FastList: true,
+		},
+		{
+			Name:     "TestHubic:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestOneDrive:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestS3:",
+			SubDir:   true,
+			FastList: true,
+		},
+		{
+			Name:     "TestSftp:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestSwift:",
+			SubDir:   true,
+			FastList: true,
+		},
+		{
+			Name:     "TestYandex:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestFTP:",
+			SubDir:   false,
+			FastList: false,
+		},
 	}
 	binary = "fs.test"
 	// Flags
@@ -60,7 +122,7 @@ type test struct {
 }
 
 // newTest creates a new test
-func newTest(remote string, subdir bool) *test {
+func newTest(remote string, subdir bool, fastlist bool) *test {
 	t := &test{
 		remote:  remote,
 		subdir:  subdir,
@@ -77,6 +139,9 @@ func newTest(remote string, subdir bool) *test {
 	if subdir {
 		t.cmdLine = append(t.cmdLine, "-subdir")
 	}
+	if fastlist {
+		t.cmdLine = append(t.cmdLine, "-fast-list")
+	}
 	t.cmdString = strings.Join(t.cmdLine, " ")
 	return t
 }
@@ -223,9 +288,25 @@ func removeTestBinary() {
 func main() {
 	flag.Parse()
 	if *runTests != "" {
-		remotes = strings.Split(*runTests, ",")
+		newRemotes := []remoteConfig{}
+		for _, name := range strings.Split(*runTests, ",") {
+			for i := range remotes {
+				if remotes[i].Name == name {
+					newRemotes = append(newRemotes, remotes[i])
+					goto found
+				}
+			}
+			log.Printf("Remote %q not found - inserting with default flags", name)
+			newRemotes = append(newRemotes, remoteConfig{Name: name})
+		found:
+		}
+		remotes = newRemotes
 	}
-	log.Printf("Testing remotes: %s", strings.Join(remotes, ", "))
+	var names []string
+	for _, remote := range remotes {
+		names = append(names, remote.Name)
+	}
+	log.Printf("Testing remotes: %s", strings.Join(names, ", "))
 
 	start := time.Now()
 	if *clean {
@@ -239,9 +320,14 @@ func main() {
 	results := make(chan *test, 8)
 	awaiting := 0
 	for _, remote := range remotes {
-		awaiting += 2
-		go newTest(remote, false).run(results)
-		go newTest(remote, true).run(results)
+		for _, subdir := range []bool{false, true} {
+			for _, fastlist := range []bool{false, true} {
+				if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
+					go newTest(remote.Name, subdir, fastlist).run(results)
+					awaiting++
+				}
+			}
+		}
 	}
 
 	// Wait for the tests to finish
@@ -261,6 +347,7 @@ func main() {
 		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
 		for _, t := range failed {
 			log.Printf("  * %s", t.cmdString)
+			log.Printf("    * Failed tests: %v", t.failedTests)
 		}
 		os.Exit(1)
 	}