fstest: refactor to decouple package from implementation

fstest/runs/config.go

@@ -0,0 +1,195 @@
+// Config handling
+
+// Package runs provides the types used by test_all
+package runs
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"slices"
+
+	"github.com/rclone/rclone/fs"
+	yaml "gopkg.in/yaml.v3"
+)
+
+// Test describes an integration test to run with `go test`
+type Test struct {
+	Path       string // path to the source directory
+	FastList   bool   // if it is possible to add -fast-list to tests
+	Short      bool   // if it is possible to run the test with -short
+	AddBackend bool   // set if Path needs the current backend appending
+	NoRetries  bool   // set if no retries should be performed
+	NoBinary   bool   // set to not build a binary in advance
+	LocalOnly  bool   // if set only run with the local backend
+}
+
+// Backend describes a backend test
+//
+// FIXME make bucket-based remotes set sub-dir automatically???
+type Backend struct {
+	Backend     string   // name of the backend directory
+	Remote      string   // name of the test remote
+	FastList    bool     // set to test with -fast-list
+	Short       bool     // set to test with -short
+	OneOnly     bool     // set to run only one backend test at once
+	MaxFile     string   // file size limit
+	CleanUp     bool     // when running clean, run cleanup first
+	Ignore      []string // test names to ignore the failure of
+	Tests       []string // paths of tests to run, blank for all
+	IgnoreTests []string // paths of tests not to run, blank for none
+	ListRetries int      // -list-retries if > 0
+	ExtraTime   float64  // factor to multiply the timeout by
+	Env         []string // environment variables to set in form KEY=VALUE
+}
+
+// includeTest returns true if this backend should be included in this
+// test
+func (b *Backend) includeTest(t *Test) bool {
+	// Is this test ignored
+	if slices.Contains(b.IgnoreTests, t.Path) {
+		return false
+	}
+	// Empty b.Tests imples do all of them except the ignored
+	if len(b.Tests) == 0 {
+		return true
+	}
+	return slices.Contains(b.Tests, t.Path)
+}
+
+// MakeRuns creates Run objects the Backend and Test
+//
+// There can be several created, one for each combination of optional
+// flags (e.g. FastList)
+func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
+	if !b.includeTest(t) {
+		return runs
+	}
+	maxSize := fs.SizeSuffix(0)
+	if b.MaxFile != "" {
+		if err := maxSize.Set(b.MaxFile); err != nil {
+			fs.Logf(nil, "Invalid maxfile value %q: %v", b.MaxFile, err)
+		}
+	}
+	fastlists := []bool{false}
+	if b.FastList && t.FastList {
+		fastlists = append(fastlists, true)
+	}
+	ignore := make(map[string]struct{}, len(b.Ignore))
+	for _, item := range b.Ignore {
+		ignore[item] = struct{}{}
+	}
+	for _, fastlist := range fastlists {
+		if t.LocalOnly && b.Backend != "local" {
+			continue
+		}
+		run := &Run{
+			Remote:      b.Remote,
+			Backend:     b.Backend,
+			Path:        t.Path,
+			FastList:    fastlist,
+			Short:       (b.Short && t.Short),
+			NoRetries:   t.NoRetries,
+			OneOnly:     b.OneOnly,
+			NoBinary:    t.NoBinary,
+			SizeLimit:   int64(maxSize),
+			Ignore:      ignore,
+			ListRetries: b.ListRetries,
+			ExtraTime:   b.ExtraTime,
+			Env:         b.Env,
+		}
+		if t.AddBackend {
+			run.Path = path.Join(run.Path, b.Backend)
+		}
+		runs = append(runs, run)
+	}
+	return runs
+}
+
+// Config describes the config for this program
+type Config struct {
+	Tests    []Test
+	Backends []Backend
+}
+
+// NewConfig reads the config file
+func NewConfig(configFile string) (*Config, error) {
+	d, err := os.ReadFile(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config file: %w", err)
+	}
+	config := &Config{}
+	err = yaml.Unmarshal(d, &config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config file: %w", err)
+	}
+	// d, err = yaml.Marshal(&config)
+	// if err != nil {
+	// 	log.Fatalf("error: %v", err)
+	// }
+	// fmt.Printf("--- m dump:\n%s\n\n", string(d))
+	return config, nil
+}
+
+// MakeRuns makes Run objects for each combination of Backend and Test
+// in the config
+func (c *Config) MakeRuns() (runs Runs) {
+	for _, backend := range c.Backends {
+		for _, test := range c.Tests {
+			runs = append(runs, backend.MakeRuns(&test)...)
+		}
+	}
+	return runs
+}
+
+// FilterBackendsByRemotes filters the Backends with the remotes passed in.
+//
+// If no backend is found with a remote is found then synthesize one
+func (c *Config) FilterBackendsByRemotes(remotes []string) {
+	var newBackends []Backend
+	for _, name := range remotes {
+		found := false
+		for i := range c.Backends {
+			if c.Backends[i].Remote == name {
+				newBackends = append(newBackends, c.Backends[i])
+				found = true
+			}
+		}
+		if !found {
+			fs.Logf(nil, "Remote %q not found - inserting with default flags", name)
+			// Lookup which backend
+			fsInfo, _, _, _, err := fs.ConfigFs(name)
+			if err != nil {
+				fs.Fatalf(nil, "couldn't find remote %q: %v", name, err)
+			}
+			newBackends = append(newBackends, Backend{Backend: fsInfo.FileName(), Remote: name})
+		}
+	}
+	c.Backends = newBackends
+}
+
+// FilterBackendsByBackends filters the Backends with the backendNames passed in
+func (c *Config) FilterBackendsByBackends(backendNames []string) {
+	var newBackends []Backend
+	for _, name := range backendNames {
+		for i := range c.Backends {
+			if c.Backends[i].Backend == name {
+				newBackends = append(newBackends, c.Backends[i])
+			}
+		}
+	}
+	c.Backends = newBackends
+}
+
+// FilterTests filters the incoming tests into the backends selected
+func (c *Config) FilterTests(paths []string) {
+	var newTests []Test
+	for _, path := range paths {
+		for i := range c.Tests {
+			if c.Tests[i].Path == path {
+				newTests = append(newTests, c.Tests[i])
+			}
+		}
+	}
+	c.Tests = newTests
+}

fstest/runs/report.go

@@ -0,0 +1,324 @@
+package runs
+
+import (
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"runtime"
+	"sort"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/file"
+	"github.com/skratchdot/open-golang/open"
+)
+
+const timeFormat = "2006-01-02-150405"
+
+// Report holds the info to make a report on a series of test runs
+type Report struct {
+	LogDir    string        // output directory for logs and report
+	StartTime time.Time     // time started
+	DateTime  string        // directory name for output
+	Duration  time.Duration // time the run took
+	Failed    Runs          // failed runs
+	Passed    Runs          // passed runs
+	Runs      []ReportRun   // runs to report
+	Version   string        // rclone version
+	Previous  string        // previous test name if known
+	IndexHTML string        // path to the index.html file
+	URL       string        // online version
+	Branch    string        // rclone branch
+	Commit    string        // rclone commit
+	GOOS      string        // Go OS
+	GOARCH    string        // Go Arch
+	GoVersion string        // Go Version
+}
+
+// ReportRun is used in the templates to report on a test run
+type ReportRun struct {
+	Name string
+	Runs Runs
+}
+
+// Parse version numbers
+// v1.49.0
+// v1.49.0-031-g2298834e-beta
+// v1.49.0-032-g20793a5f-sharefile-beta
+// match 1 is commit number
+// match 2 is branch name
+var parseVersion = regexp.MustCompile(`^v(?:[0-9.]+)-(?:\d+)-g([0-9a-f]+)(?:-(.*))?-beta$`)
+
+// FIXME take -issue or -pr parameter...
+
+// NewReport initialises and returns a Report
+func NewReport(Opt RunOpt) *Report {
+	r := &Report{
+		StartTime: time.Now(),
+		Version:   fs.Version,
+		GOOS:      runtime.GOOS,
+		GOARCH:    runtime.GOARCH,
+		GoVersion: runtime.Version(),
+	}
+	r.DateTime = r.StartTime.Format(timeFormat)
+
+	// Find previous log directory if possible
+	names, err := os.ReadDir(Opt.OutputDir)
+	if err == nil && len(names) > 0 {
+		r.Previous = names[len(names)-1].Name()
+	}
+
+	// Create output directory for logs and report
+	r.LogDir = path.Join(Opt.OutputDir, r.DateTime)
+	err = file.MkdirAll(r.LogDir, 0777)
+	if err != nil {
+		fs.Fatalf(nil, "Failed to make log directory: %v", err)
+	}
+
+	// Online version
+	r.URL = Opt.URLBase + r.DateTime + "/index.html"
+
+	// Get branch/commit out of version
+	parts := parseVersion.FindStringSubmatch(r.Version)
+	if len(parts) >= 3 {
+		r.Commit = parts[1]
+		r.Branch = parts[2]
+	}
+	if r.Branch == "" {
+		r.Branch = "master"
+	}
+
+	return r
+}
+
+// End should be called when the tests are complete
+func (r *Report) End() {
+	r.Duration = time.Since(r.StartTime)
+	sort.Sort(r.Failed)
+	sort.Sort(r.Passed)
+	r.Runs = []ReportRun{
+		{Name: "Failed", Runs: r.Failed},
+		{Name: "Passed", Runs: r.Passed},
+	}
+}
+
+// AllPassed returns true if there were no failed tests
+func (r *Report) AllPassed() bool {
+	return len(r.Failed) == 0
+}
+
+// RecordResult should be called with a Run when it has finished to be
+// recorded into the Report
+func (r *Report) RecordResult(t *Run) {
+	if !t.passed() {
+		r.Failed = append(r.Failed, t)
+	} else {
+		r.Passed = append(r.Passed, t)
+	}
+}
+
+// Title returns a human-readable summary title for the Report
+func (r *Report) Title() string {
+	if r.AllPassed() {
+		return fmt.Sprintf("PASS: All tests finished OK in %v", r.Duration)
+	}
+	return fmt.Sprintf("FAIL: %d tests failed in %v", len(r.Failed), r.Duration)
+}
+
+// LogSummary writes the summary to the log file
+func (r *Report) LogSummary() {
+	fs.Logf(nil, "Logs in %q", r.LogDir)
+
+	// Summarise results
+	fs.Logf(nil, "SUMMARY")
+	fs.Log(nil, r.Title())
+	if !r.AllPassed() {
+		for _, t := range r.Failed {
+			fs.Logf(nil, "  * %s", toShell(t.nextCmdLine()))
+			fs.Logf(nil, "    * Failed tests: %v", t.FailedTests)
+		}
+	}
+}
+
+// LogJSON writes the summary to index.json in LogDir
+func (r *Report) LogJSON() {
+	out, err := json.MarshalIndent(r, "", "\t")
+	if err != nil {
+		fs.Fatalf(nil, "Failed to marshal data for index.json: %v", err)
+	}
+	err = os.WriteFile(path.Join(r.LogDir, "index.json"), out, 0666)
+	if err != nil {
+		fs.Fatalf(nil, "Failed to write index.json: %v", err)
+	}
+}
+
+// LogHTML writes the summary to index.html in LogDir
+func (r *Report) LogHTML() {
+	r.IndexHTML = path.Join(r.LogDir, "index.html")
+	out, err := os.Create(r.IndexHTML)
+	if err != nil {
+		fs.Fatalf(nil, "Failed to open index.html: %v", err)
+	}
+	defer func() {
+		err := out.Close()
+		if err != nil {
+			fs.Fatalf(nil, "Failed to close index.html: %v", err)
+		}
+	}()
+	err = reportTemplate.Execute(out, r)
+	if err != nil {
+		fs.Fatalf(nil, "Failed to execute template: %v", err)
+	}
+	_ = open.Start("file://" + r.IndexHTML)
+}
+
+var reportHTML = `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>{{ .Title }}</title>
+<style>
+table {
+	border-collapse: collapse;
+	border-spacing: 0;
+	border: 1px solid #ddd;
+}
+table.tests {
+	width: 100%;
+}
+table, th, td {
+	border: 1px solid #264653;
+}
+.Failed {
+	color: #BE5B43;
+}
+.Passed {
+	color: #17564E;
+}
+.false {
+	font-weight: lighter;
+}
+.true {
+	font-weight: bold;
+}
+
+th, td {
+	text-align: left;
+	padding: 4px;
+}
+
+tr:nth-child(even) {
+    background-color: #f2f2f2;
+}
+
+a {
+	color: #5B1955;
+	text-decoration: none;
+}
+a:hover, a:focus {
+	color: #F4A261;
+	text-decoration:underline;
+}
+a:focus {
+	outline: thin dotted;
+	outline: 5px auto;
+}
+</style>
+</head>
+<body>
+<h1>{{ .Title }}</h1>
+
+<table>
+<tr><th>Version</th><td>{{ .Version }}</td></tr>
+<tr><th>Test</th><td><a href="{{ .URL }}">{{ .DateTime}}</a></td></tr>
+<tr><th>Branch</th><td><a href="https://github.com/rclone/rclone/tree/{{ .Branch }}">{{ .Branch }}</a></td></tr>
+{{ if .Commit}}<tr><th>Commit</th><td><a href="https://github.com/rclone/rclone/commit/{{ .Commit }}">{{ .Commit }}</a></td></tr>{{ end }}
+<tr><th>Go</th><td>{{ .GoVersion }} {{ .GOOS }}/{{ .GOARCH }}</td></tr>
+<tr><th>Duration</th><td>{{ .Duration }}</td></tr>
+{{ if .Previous}}<tr><th>Previous</th><td><a href="../{{ .Previous }}/index.html">{{ .Previous }}</a></td></tr>{{ end }}
+<tr><th>Up</th><td><a href="../">Older Tests</a></td></tr>
+</table>
+
+{{ range .Runs }}
+{{ if .Runs }}
+<h2 class="{{ .Name }}">{{ .Name }}: {{ len .Runs }}</h2>
+<table class="{{ .Name }} tests">
+<tr>
+<th>Backend</th>
+<th>Remote</th>
+<th>Test</th>
+<th>FastList</th>
+<th>Failed</th>
+<th>Logs</th>
+</tr>
+{{ $prevBackend := "" }}
+{{ $prevRemote := "" }}
+{{ range .Runs}}
+<tr>
+<td>{{ if ne $prevBackend .Backend }}{{ .Backend }}{{ end }}{{ $prevBackend = .Backend }}</td>
+<td>{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}</td>
+<td>{{ .Path }}</td>
+<td><span class="{{ .FastList }}">{{ .FastList }}</span></td>
+<td>{{ .FailedTestsCSV }}</td>
+<td>{{ range $i, $v := .Logs }}<a href="{{ $v }}">#{{ $i }}</a> {{ end }}</td>
+</tr>
+{{ end }}
+</table>
+{{ end }}
+{{ end }}
+</body>
+</html>
+`
+
+var reportTemplate = template.Must(template.New("Report").Parse(reportHTML))
+
+// EmailHTML sends the summary report to the email address supplied
+func (r *Report) EmailHTML(Opt RunOpt) {
+	if Opt.EmailReport == "" || r.IndexHTML == "" {
+		return
+	}
+	fs.Logf(nil, "Sending email summary to %q", Opt.EmailReport)
+	cmdLine := []string{"mail", "-a", "Content-Type: text/html", Opt.EmailReport, "-s", "rclone integration tests: " + r.Title()}
+	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
+	in, err := os.Open(r.IndexHTML)
+	if err != nil {
+		fs.Fatalf(nil, "Failed to open index.html: %v", err)
+	}
+	cmd.Stdin = in
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		fs.Fatalf(nil, "Failed to send email: %v", err)
+	}
+	_ = in.Close()
+}
+
+// uploadTo uploads a copy of the report online to the dir given
+func (r *Report) uploadTo(Opt RunOpt, uploadDir string) {
+	dst := path.Join(Opt.UploadPath, uploadDir)
+	fs.Logf(nil, "Uploading results to %q", dst)
+	cmdLine := []string{"rclone", "sync", "--stats-log-level", "NOTICE", r.LogDir, dst}
+	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		fs.Fatalf(nil, "Failed to upload results: %v", err)
+	}
+}
+
+// Upload uploads a copy of the report online
+func (r *Report) Upload(Opt RunOpt) {
+	if Opt.UploadPath == "" || r.IndexHTML == "" {
+		return
+	}
+	// Upload into dated directory
+	r.uploadTo(Opt, r.DateTime)
+	// And again into current
+	r.uploadTo(Opt, "current")
+}

fstest/runs/run.go

@@ -0,0 +1,481 @@
+// Run a test
+
+package runs
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"go/build"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/testserver"
+)
+
+// Control concurrency per backend if required
+var (
+	oneOnlyMu sync.Mutex
+	oneOnly   = map[string]*sync.Mutex{}
+)
+
+// RunOpt holds the options for the Run
+type RunOpt struct {
+	MaxTries     int           // Number of times to try each test
+	MaxN         int           // Maximum number of tests to run at once
+	TestRemotes  string        // Comma separated list of remotes to test, e.g. 'TestSwift:,TestS3'
+	TestBackends string        // Comma separated list of backends to test, e.g. 's3,googlecloudstorage
+	TestTests    string        // Comma separated list of tests to test, e.g. 'fs/sync,fs/operations'
+	Clean        bool          // Instead of testing, clean all left over test directories
+	RunOnly      string        // Run only those tests matching the regexp supplied
+	Timeout      time.Duration // Maximum time to run each test for before giving up
+	Race         bool          // If set run the tests under the race detector
+	ConfigFile   string        // Path to config file
+	OutputDir    string        // Place to store results
+	EmailReport  string        // Set to email the report to the address supplied
+	DryRun       bool          // Print commands which would be executed only
+	URLBase      string        // Base for the online version
+	UploadPath   string        // Set this to an rclone path to upload the results here
+	Verbose      bool          // Set to enable verbose logging in the tests
+	ListRetries  int           // Number or times to retry listing - set to override the default
+}
+
+// Run holds info about a running test
+//
+// A run just runs one command line, but it can be run multiple times
+// if retries are needed.
+type Run struct {
+	// Config
+	Remote      string // name of the test remote
+	Backend     string // name of the backend
+	Path        string // path to the source directory
+	FastList    bool   // add -fast-list to tests
+	Short       bool   // add -short
+	NoRetries   bool   // don't retry if set
+	OneOnly     bool   // only run test for this backend at once
+	NoBinary    bool   // set to not build a binary
+	SizeLimit   int64  // maximum test file size
+	Ignore      map[string]struct{}
+	ListRetries int      // -list-retries if > 0
+	ExtraTime   float64  // multiply the timeout by this
+	Env         []string // environment variables in form KEY=VALUE
+	// Internals
+	CmdLine     []string
+	CmdString   string
+	Try         int
+	err         error
+	output      []byte
+	FailedTests []string
+	RunFlag     string
+	LogDir      string   // directory to place the logs
+	TrialName   string   // name/log file name of current trial
+	TrialNames  []string // list of all the trials
+}
+
+// Runs records multiple Run objects
+type Runs []*Run
+
+// Sort interface
+func (rs Runs) Len() int      { return len(rs) }
+func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
+func (rs Runs) Less(i, j int) bool {
+	a, b := rs[i], rs[j]
+	if a.Backend < b.Backend {
+		return true
+	} else if a.Backend > b.Backend {
+		return false
+	}
+	if a.Remote < b.Remote {
+		return true
+	} else if a.Remote > b.Remote {
+		return false
+	}
+	if a.Path < b.Path {
+		return true
+	} else if a.Path > b.Path {
+		return false
+	}
+	if !a.FastList && b.FastList {
+		return true
+	} else if a.FastList && !b.FastList {
+		return false
+	}
+	return false
+}
+
+// dumpOutput prints the error output
+func (r *Run) dumpOutput() {
+	fs.Log(nil, "------------------------------------------------------------")
+	fs.Logf(nil, "---- %q ----", r.CmdString)
+	fs.Log(nil, string(r.output))
+	fs.Log(nil, "------------------------------------------------------------")
+}
+
+// trie for storing runs
+type trie map[string]trie
+
+// turn a trie into multiple regexp matches
+//
+// We can't ever have a / in a regexp as it doesn't work.
+func match(current trie) []string {
+	var names []string
+	var parts []string
+	for name, value := range current {
+		matchName := "^" + name + "$"
+		if len(value) == 0 {
+			names = append(names, name)
+		} else {
+			for _, part := range match(value) {
+				parts = append(parts, matchName+"/"+part)
+			}
+		}
+	}
+	sort.Strings(names)
+	if len(names) > 1 {
+		parts = append(parts, "^("+strings.Join(names, "|")+")$")
+	} else if len(names) == 1 {
+		parts = append(parts, "^"+names[0]+"$")
+	}
+	sort.Strings(parts)
+	return parts
+}
+
+// This converts a slice of test names into a regexp which matches
+// them.
+func testsToRegexp(tests []string) string {
+	split := trie{}
+	// Make a trie showing which parts are used at each level
+	for _, test := range tests {
+		parent := split
+		for _, name := range strings.Split(test, "/") {
+			current := parent[name]
+			if current == nil {
+				current = trie{}
+				parent[name] = current
+			}
+			parent = current
+		}
+	}
+	parts := match(split)
+	return strings.Join(parts, "|")
+}
+
+var failRe = regexp.MustCompile(`(?m)^\s*--- FAIL: (Test.*?) \(`)
+
+// findFailures looks for all the tests which failed
+func (r *Run) findFailures() {
+	oldFailedTests := r.FailedTests
+	r.FailedTests = nil
+	excludeParents := map[string]struct{}{}
+	ignored := 0
+	for _, matches := range failRe.FindAllSubmatch(r.output, -1) {
+		failedTest := string(matches[1])
+		// Skip any ignored failures
+		if _, found := r.Ignore[failedTest]; found {
+			ignored++
+		} else {
+			r.FailedTests = append(r.FailedTests, failedTest)
+		}
+		// Find all the parents of this test
+		parts := strings.Split(failedTest, "/")
+		for i := len(parts) - 1; i >= 1; i-- {
+			excludeParents[strings.Join(parts[:i], "/")] = struct{}{}
+		}
+	}
+	// Exclude the parents
+	newTests := r.FailedTests[:0]
+	for _, failedTest := range r.FailedTests {
+		if _, excluded := excludeParents[failedTest]; !excluded {
+			newTests = append(newTests, failedTest)
+		}
+	}
+	r.FailedTests = newTests
+	if len(r.FailedTests) == 0 && ignored > 0 {
+		fs.Logf(nil, "%q - Found %d ignored errors only - marking as good", r.CmdString, ignored)
+		r.err = nil
+		r.dumpOutput()
+		return
+	}
+	if len(r.FailedTests) != 0 {
+		r.RunFlag = testsToRegexp(r.FailedTests)
+	} else {
+		r.RunFlag = ""
+	}
+	if r.passed() && len(r.FailedTests) != 0 {
+		fs.Logf(nil, "%q - Expecting no errors but got: %v", r.CmdString, r.FailedTests)
+		r.dumpOutput()
+	} else if !r.passed() && len(r.FailedTests) == 0 {
+		fs.Logf(nil, "%q - Expecting errors but got none: %v", r.CmdString, r.FailedTests)
+		r.dumpOutput()
+		r.FailedTests = oldFailedTests
+	}
+}
+
+// nextCmdLine returns the next command line
+func (r *Run) nextCmdLine() []string {
+	CmdLine := r.CmdLine
+	if r.RunFlag != "" {
+		CmdLine = append(CmdLine, "-test.run", r.RunFlag)
+	}
+	return CmdLine
+}
+
+// trial runs a single test
+func (r *Run) trial(Opt RunOpt) {
+	CmdLine := r.nextCmdLine()
+	CmdString := toShell(CmdLine)
+	msg := fmt.Sprintf("%q - Starting (try %d/%d)", CmdString, r.Try, Opt.MaxTries)
+	fs.Log(nil, msg)
+	logName := path.Join(r.LogDir, r.TrialName)
+	out, err := os.Create(logName)
+	if err != nil {
+		fs.Fatalf(nil, "Couldn't create log file: %v", err)
+	}
+	defer func() {
+		err := out.Close()
+		if err != nil {
+			fs.Fatalf(nil, "Failed to close log file: %v", err)
+		}
+	}()
+	_, _ = fmt.Fprintln(out, msg)
+
+	// Early exit if --try-run
+	if Opt.DryRun {
+		fs.Logf(nil, "Not executing as --dry-run: %v", CmdLine)
+		_, _ = fmt.Fprintln(out, "--dry-run is set - not running")
+		return
+	}
+
+	// Start the test server if required
+	finish, err := testserver.Start(r.Remote)
+	if err != nil {
+		fs.Logf(nil, "%s: Failed to start test server: %v", r.Remote, err)
+		_, _ = fmt.Fprintf(out, "%s: Failed to start test server: %v\n", r.Remote, err)
+		r.err = err
+		return
+	}
+	defer finish()
+
+	// Internal buffer
+	var b bytes.Buffer
+	multiOut := io.MultiWriter(out, &b)
+
+	cmd := exec.Command(CmdLine[0], CmdLine[1:]...)
+	cmd.Stderr = multiOut
+	cmd.Stdout = multiOut
+	cmd.Dir = r.Path
+	cmd.Env = append(os.Environ(), r.Env...)
+	start := time.Now()
+	r.err = cmd.Run()
+	r.output = b.Bytes()
+	duration := time.Since(start)
+	r.findFailures()
+	if r.passed() {
+		msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", CmdString, duration, r.Try, Opt.MaxTries)
+	} else {
+		msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", CmdString, duration, r.Try, Opt.MaxTries, r.err, r.FailedTests)
+	}
+	fs.Log(nil, msg)
+	_, _ = fmt.Fprintln(out, msg)
+}
+
+// passed returns true if the test passed
+func (r *Run) passed() bool {
+	return r.err == nil
+}
+
+// GOPATH returns the current GOPATH
+func GOPATH() string {
+	gopath := os.Getenv("GOPATH")
+	if gopath == "" {
+		gopath = build.Default.GOPATH
+	}
+	return gopath
+}
+
+// BinaryName turns a package name into a binary name
+func (r *Run) BinaryName() string {
+	binary := path.Base(r.Path) + ".test"
+	if runtime.GOOS == "windows" {
+		binary += ".exe"
+	}
+	return binary
+}
+
+// BinaryPath turns a package name into a binary path
+func (r *Run) BinaryPath() string {
+	return path.Join(r.Path, r.BinaryName())
+}
+
+// PackagePath returns the path to the package
+func (r *Run) PackagePath() string {
+	return path.Join(GOPATH(), "src", r.Path)
+}
+
+// MakeTestBinary makes the binary we will run
+func (r *Run) MakeTestBinary(Opt RunOpt) {
+	binary := r.BinaryPath()
+	binaryName := r.BinaryName()
+	fs.Logf(nil, "%s: Making test binary %q", r.Path, binaryName)
+	CmdLine := []string{"go", "test", "-c"}
+	if Opt.Race {
+		CmdLine = append(CmdLine, "-race")
+	}
+	if Opt.DryRun {
+		fs.Logf(nil, "Not executing: %v", CmdLine)
+		return
+	}
+	cmd := exec.Command(CmdLine[0], CmdLine[1:]...)
+	cmd.Dir = r.Path
+	err := cmd.Run()
+	if err != nil {
+		fs.Fatalf(nil, "Failed to make test binary: %v", err)
+	}
+	if _, err := os.Stat(binary); err != nil {
+		fs.Fatalf(nil, "Couldn't find test binary %q", binary)
+	}
+}
+
+// RemoveTestBinary removes the binary made in makeTestBinary
+func (r *Run) RemoveTestBinary(Opt RunOpt) {
+	if Opt.DryRun {
+		return
+	}
+	binary := r.BinaryPath()
+	err := os.Remove(binary) // Delete the binary when finished
+	if err != nil {
+		fs.Logf(nil, "Error removing test binary %q: %v", binary, err)
+	}
+}
+
+// Name returns the run name as a file name friendly string
+func (r *Run) Name() string {
+	ns := []string{
+		r.Backend,
+		strings.ReplaceAll(r.Path, "/", "."),
+		r.Remote,
+	}
+	if r.FastList {
+		ns = append(ns, "fastlist")
+	}
+	ns = append(ns, fmt.Sprintf("%d", r.Try))
+	s := strings.Join(ns, "-")
+	s = strings.ReplaceAll(s, ":", "")
+	return s
+}
+
+// Init the Run
+func (r *Run) Init(Opt RunOpt) {
+	prefix := "-test."
+	if r.NoBinary {
+		prefix = "-"
+		r.CmdLine = []string{"go", "test"}
+	} else {
+		r.CmdLine = []string{"./" + r.BinaryName()}
+	}
+	testTimeout := Opt.Timeout
+	if r.ExtraTime > 0 {
+		testTimeout = time.Duration(float64(testTimeout) * r.ExtraTime)
+	}
+	r.CmdLine = append(r.CmdLine, prefix+"v", prefix+"timeout", testTimeout.String(), "-remote", r.Remote)
+	listRetries := Opt.ListRetries
+	if r.ListRetries > 0 {
+		listRetries = r.ListRetries
+	}
+	if listRetries > 0 {
+		r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(listRetries))
+	}
+	r.Try = 1
+	ci := fs.GetConfig(context.Background())
+	if Opt.Verbose {
+		r.CmdLine = append(r.CmdLine, "-verbose")
+		ci.LogLevel = fs.LogLevelDebug
+	}
+	if Opt.RunOnly != "" {
+		r.CmdLine = append(r.CmdLine, prefix+"run", Opt.RunOnly)
+	}
+	if r.FastList {
+		r.CmdLine = append(r.CmdLine, "-fast-list")
+	}
+	if r.Short {
+		r.CmdLine = append(r.CmdLine, "-short")
+	}
+	if r.SizeLimit > 0 {
+		r.CmdLine = append(r.CmdLine, "-size-limit", strconv.FormatInt(r.SizeLimit, 10))
+	}
+	r.CmdString = toShell(r.CmdLine)
+}
+
+// Logs returns all the log names
+func (r *Run) Logs() []string {
+	return r.TrialNames
+}
+
+// FailedTestsCSV returns the failed tests as a comma separated string, limiting the number
+func (r *Run) FailedTestsCSV() string {
+	const maxTests = 5
+	ts := r.FailedTests
+	if len(ts) > maxTests {
+		ts = ts[:maxTests:maxTests]
+		ts = append(ts, fmt.Sprintf("… (%d more)", len(r.FailedTests)-maxTests))
+	}
+	return strings.Join(ts, ", ")
+}
+
+// Run runs all the trials for this test
+func (r *Run) Run(Opt RunOpt, LogDir string, result chan<- *Run) {
+	if r.OneOnly {
+		oneOnlyMu.Lock()
+		mu := oneOnly[r.Backend]
+		if mu == nil {
+			mu = new(sync.Mutex)
+			oneOnly[r.Backend] = mu
+		}
+		oneOnlyMu.Unlock()
+		mu.Lock()
+		defer mu.Unlock()
+	}
+	r.Init(Opt)
+	r.LogDir = LogDir
+	for r.Try = 1; r.Try <= Opt.MaxTries; r.Try++ {
+		r.TrialName = r.Name() + ".txt"
+		r.TrialNames = append(r.TrialNames, r.TrialName)
+		fs.Logf(nil, "Starting run with log %q", r.TrialName)
+		r.trial(Opt)
+		if r.passed() || r.NoRetries {
+			break
+		}
+	}
+	if !r.passed() {
+		r.dumpOutput()
+	}
+	result <- r
+}
+
+// if matches then is definitely OK in the shell
+var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")
+
+// converts an argv style input into a shell command
+func toShell(args []string) (result string) {
+	for _, arg := range args {
+		if result != "" {
+			result += " "
+		}
+		if shellOK.MatchString(arg) {
+			result += arg
+		} else {
+			result += "'" + arg + "'"
+		}
+	}
+	return result
+}

fstest/runs/run_test.go

@@ -0,0 +1,179 @@
+package runs
+
+import (
+	"fmt"
+	"os/exec"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTestsToRegexp(t *testing.T) {
+	for _, test := range []struct {
+		in   []string
+		want string
+	}{
+		{
+			in:   []string{},
+			want: "",
+		},
+		{
+			in:   []string{"TestOne"},
+			want: "^TestOne$",
+		},
+		{
+			in:   []string{"TestOne", "TestTwo"},
+			want: "^(TestOne|TestTwo)$",
+		},
+		{
+			in:   []string{"TestOne", "TestTwo", "TestThree"},
+			want: "^(TestOne|TestThree|TestTwo)$",
+		},
+		{
+			in:   []string{"TestOne/Sub1"},
+			want: "^TestOne$/^Sub1$",
+		},
+		{
+			in: []string{
+				"TestOne/Sub1",
+				"TestTwo",
+			},
+			want: "^TestOne$/^Sub1$|^TestTwo$",
+		},
+		{
+			in: []string{
+				"TestOne/Sub1",
+				"TestOne/Sub2",
+				"TestTwo",
+			},
+			want: "^TestOne$/^(Sub1|Sub2)$|^TestTwo$",
+		},
+		{
+			in: []string{
+				"TestOne/Sub1",
+				"TestOne/Sub2/SubSub1",
+				"TestTwo",
+			},
+			want: "^TestOne$/^Sub1$|^TestOne$/^Sub2$/^SubSub1$|^TestTwo$",
+		},
+		{
+			in: []string{
+				"TestTests/A1",
+				"TestTests/B/B1",
+				"TestTests/C/C3/C31",
+			},
+			want: "^TestTests$/^A1$|^TestTests$/^B$/^B1$|^TestTests$/^C$/^C3$/^C31$",
+		},
+	} {
+		got := testsToRegexp(test.in)
+		assert.Equal(t, test.want, got, fmt.Sprintf("in=%v want=%q got=%q", test.in, test.want, got))
+	}
+}
+
+var runRe = regexp.MustCompile(`(?m)^\s*=== RUN\s*(Test.*?)\s*$`)
+
+// Test the regexp work with the -run flag in actually selecting the right tests
+func TestTestsToRegexpLive(t *testing.T) {
+	for _, test := range []struct {
+		in   []string
+		want []string
+	}{
+		{
+			in: []string{
+				"TestTests/A1",
+				"TestTests/C/C3",
+			},
+			want: []string{
+				"TestTests",
+				"TestTests/A1",
+				"TestTests/C",
+				"TestTests/C/C3",
+				"TestTests/C/C3/C31",
+				"TestTests/C/C3/C32",
+			},
+		},
+		{
+			in: []string{
+				"TestTests",
+				"TestTests/A1",
+				"TestTests/B",
+				"TestTests/B/B1",
+				"TestTests/C",
+			},
+			want: []string{
+				"TestTests",
+				"TestTests/A1",
+				"TestTests/B",
+				"TestTests/B/B1",
+				"TestTests/C",
+				"TestTests/C/C1",
+				"TestTests/C/C2",
+				"TestTests/C/C3",
+				"TestTests/C/C3/C31",
+				"TestTests/C/C3/C32",
+			},
+		},
+		{
+			in: []string{
+				"TestTests/A1",
+				"TestTests/B/B1",
+				"TestTests/C/C3/C31",
+			},
+			want: []string{
+				"TestTests",
+				"TestTests/A1",
+				"TestTests/B",
+				"TestTests/B/B1",
+				"TestTests/C",
+				"TestTests/C/C3",
+				"TestTests/C/C3/C31",
+			},
+		},
+		{
+			in: []string{
+				"TestTests/B/B1",
+				"TestTests/C/C3/C31",
+			},
+			want: []string{
+				"TestTests",
+				"TestTests/B",
+				"TestTests/B/B1",
+				"TestTests/C",
+				"TestTests/C/C3",
+				"TestTests/C/C3/C31",
+			},
+		},
+	} {
+		runRegexp := testsToRegexp(test.in)
+		cmd := exec.Command("go", "test", "-v", "-run", runRegexp)
+		out, err := cmd.CombinedOutput()
+		require.NoError(t, err)
+		var got []string
+		for _, match := range runRe.FindAllSubmatch(out, -1) {
+			got = append(got, string(match[1]))
+		}
+		assert.Equal(t, test.want, got, fmt.Sprintf("in=%v want=%v got=%v, runRegexp=%q", test.in, test.want, got, runRegexp))
+	}
+}
+
+var nilTest = func(t *testing.T) {}
+
+// Nested tests for TestTestsToRegexpLive to run
+func TestTests(t *testing.T) {
+	t.Run("A1", nilTest)
+	t.Run("A2", nilTest)
+	t.Run("B", func(t *testing.T) {
+		t.Run("B1", nilTest)
+		t.Run("B2", nilTest)
+	})
+	t.Run("C", func(t *testing.T) {
+		t.Run("C1", nilTest)
+		t.Run("C2", nilTest)
+		t.Run("C3", func(t *testing.T) {
+			t.Run("C31", nilTest)
+			t.Run("C32", nilTest)
+		})
+	})
+}