vfstests: move functional tests from mountlib and make them work with VFS

The tests are now run for the mount commands and for the plain VFS.

This makes the tests much easier to debug when running with a VFS than
through a mount.

vfs/vfstest/dir.go

@@ -0,0 +1,228 @@
+package vfstest
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestDirLs checks out listing
+func TestDirLs(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.checkDir(t, "")
+
+	run.mkdir(t, "a directory")
+	run.createFile(t, "a file", "hello")
+
+	run.checkDir(t, "a directory/|a file 5")
+
+	run.rmdir(t, "a directory")
+	run.rm(t, "a file")
+
+	run.checkDir(t, "")
+}
+
+// TestDirCreateAndRemoveDir tests creating and removing a directory
+func TestDirCreateAndRemoveDir(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.mkdir(t, "dir")
+	run.mkdir(t, "dir/subdir")
+	run.checkDir(t, "dir/|dir/subdir/")
+
+	// Check we can't delete a directory with stuff in
+	err := run.os.Remove(run.path("dir"))
+	assert.Error(t, err, "file exists")
+
+	// Now delete subdir then dir - should produce no errors
+	run.rmdir(t, "dir/subdir")
+	run.checkDir(t, "dir/")
+	run.rmdir(t, "dir")
+	run.checkDir(t, "")
+}
+
+// TestDirCreateAndRemoveFile tests creating and removing a file
+func TestDirCreateAndRemoveFile(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.mkdir(t, "dir")
+	run.createFile(t, "dir/file", "potato")
+	run.checkDir(t, "dir/|dir/file 6")
+
+	// Check we can't delete a directory with stuff in
+	err := run.os.Remove(run.path("dir"))
+	assert.Error(t, err, "file exists")
+
+	// Now delete file
+	run.rm(t, "dir/file")
+
+	run.checkDir(t, "dir/")
+	run.rmdir(t, "dir")
+	run.checkDir(t, "")
+}
+
+// TestDirRenameFile tests renaming a file
+func TestDirRenameFile(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.mkdir(t, "dir")
+	run.createFile(t, "file", "potato")
+	run.checkDir(t, "dir/|file 6")
+
+	err := run.os.Rename(run.path("file"), run.path("file2"))
+	require.NoError(t, err)
+	run.checkDir(t, "dir/|file2 6")
+
+	data := run.readFile(t, "file2")
+	assert.Equal(t, "potato", data)
+
+	err = run.os.Rename(run.path("file2"), run.path("dir/file3"))
+	require.NoError(t, err)
+	run.checkDir(t, "dir/|dir/file3 6")
+
+	data = run.readFile(t, "dir/file3")
+	require.NoError(t, err)
+	assert.Equal(t, "potato", data)
+
+	run.rm(t, "dir/file3")
+	run.rmdir(t, "dir")
+	run.checkDir(t, "")
+}
+
+// TestDirRenameEmptyDir tests renaming and empty directory
+func TestDirRenameEmptyDir(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.mkdir(t, "dir")
+	run.mkdir(t, "dir1")
+	run.checkDir(t, "dir/|dir1/")
+
+	err := run.os.Rename(run.path("dir1"), run.path("dir/dir2"))
+	require.NoError(t, err)
+	run.checkDir(t, "dir/|dir/dir2/")
+
+	err = run.os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
+	require.NoError(t, err)
+	run.checkDir(t, "dir/|dir/dir3/")
+
+	run.rmdir(t, "dir/dir3")
+	run.rmdir(t, "dir")
+	run.checkDir(t, "")
+}
+
+// TestDirRenameFullDir tests renaming a full directory
+func TestDirRenameFullDir(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.mkdir(t, "dir")
+	run.mkdir(t, "dir1")
+	run.createFile(t, "dir1/potato.txt", "maris piper")
+	run.checkDir(t, "dir/|dir1/|dir1/potato.txt 11")
+
+	err := run.os.Rename(run.path("dir1"), run.path("dir/dir2"))
+	require.NoError(t, err)
+	run.checkDir(t, "dir/|dir/dir2/|dir/dir2/potato.txt 11")
+
+	err = run.os.Rename(run.path("dir/dir2"), run.path("dir/dir3"))
+	require.NoError(t, err)
+	run.checkDir(t, "dir/|dir/dir3/|dir/dir3/potato.txt 11")
+
+	run.rm(t, "dir/dir3/potato.txt")
+	run.rmdir(t, "dir/dir3")
+	run.rmdir(t, "dir")
+	run.checkDir(t, "")
+}
+
+// TestDirModTime tests mod times
+func TestDirModTime(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.mkdir(t, "dir")
+	mtime := time.Date(2012, time.November, 18, 17, 32, 31, 0, time.UTC)
+	err := run.os.Chtimes(run.path("dir"), mtime, mtime)
+	require.NoError(t, err)
+
+	info, err := run.os.Stat(run.path("dir"))
+	require.NoError(t, err)
+
+	// avoid errors because of timezone differences
+	assert.Equal(t, info.ModTime().Unix(), mtime.Unix())
+
+	run.rmdir(t, "dir")
+}
+
+// TestDirCacheFlush tests fluching the dir cache
+func TestDirCacheFlush(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.checkDir(t, "")
+
+	run.mkdir(t, "dir")
+	run.mkdir(t, "otherdir")
+	run.createFile(t, "dir/file", "1")
+	run.createFile(t, "otherdir/file", "1")
+
+	dm := newDirMap("otherdir/|otherdir/file 1|dir/|dir/file 1")
+	localDm := make(dirMap)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	err := run.fremote.Mkdir(context.Background(), "dir/subdir")
+	require.NoError(t, err)
+
+	root, err := run.vfs.Root()
+	require.NoError(t, err)
+
+	// expect newly created "subdir" on remote to not show up
+	root.ForgetPath("otherdir", fs.EntryDirectory)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	root.ForgetPath("dir", fs.EntryDirectory)
+	dm = newDirMap("otherdir/|otherdir/file 1|dir/|dir/file 1|dir/subdir/")
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	run.rm(t, "otherdir/file")
+	run.rmdir(t, "otherdir")
+	run.rm(t, "dir/file")
+	run.rmdir(t, "dir/subdir")
+	run.rmdir(t, "dir")
+	run.checkDir(t, "")
+}
+
+// TestDirCacheFlushOnDirRename tests flushing the dir cache on rename
+func TestDirCacheFlushOnDirRename(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	run.mkdir(t, "dir")
+	run.createFile(t, "dir/file", "1")
+
+	dm := newDirMap("dir/|dir/file 1")
+	localDm := make(dirMap)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	// expect remotely created directory to not show up
+	err := run.fremote.Mkdir(context.Background(), "dir/subdir")
+	require.NoError(t, err)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	err = run.os.Rename(run.path("dir"), run.path("rid"))
+	require.NoError(t, err)
+
+	dm = newDirMap("rid/|rid/subdir/|rid/file 1")
+	localDm = make(dirMap)
+	run.readLocal(t, localDm, "")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+
+	run.rm(t, "rid/file")
+	run.rmdir(t, "rid/subdir")
+	run.rmdir(t, "rid")
+	run.checkDir(t, "")
+}

vfs/vfstest/edge_cases.go

@@ -0,0 +1,60 @@
+package vfstest
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestTouchAndDelete checks that writing a zero byte file and immediately
+// deleting it is not racy. See https://github.com/rclone/rclone/issues/1181
+func TestTouchAndDelete(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	run.checkDir(t, "")
+
+	run.createFile(t, "touched", "")
+	run.rm(t, "touched")
+
+	run.checkDir(t, "")
+}
+
+// TestRenameOpenHandle checks that a file with open writers is successfully
+// renamed after all writers close. See https://github.com/rclone/rclone/issues/2130
+func TestRenameOpenHandle(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on Windows")
+	}
+
+	run.checkDir(t, "")
+
+	// create file
+	example := []byte("Some Data")
+	path := run.path("rename")
+	file, err := osCreate(path)
+	require.NoError(t, err)
+
+	// write some data
+	_, err = file.Write(example)
+	require.NoError(t, err)
+	err = file.Sync()
+	require.NoError(t, err)
+
+	// attempt to rename open file
+	err = run.os.Rename(path, path+"bla")
+	require.NoError(t, err)
+
+	// close open writers to allow rename on remote to go through
+	err = file.Close()
+	require.NoError(t, err)
+
+	run.waitForWriters()
+
+	// verify file was renamed properly
+	run.checkDir(t, "renamebla 9")
+
+	// cleanup
+	run.rm(t, "renamebla")
+	run.checkDir(t, "")
+}

vfs/vfstest/file.go

@@ -0,0 +1,74 @@
+package vfstest
+
+import (
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/vfs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestFileModTime tests mod times on files
+func TestFileModTime(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.createFile(t, "file", "123")
+
+	mtime := time.Date(2012, time.November, 18, 17, 32, 31, 0, time.UTC)
+	err := run.os.Chtimes(run.path("file"), mtime, mtime)
+	require.NoError(t, err)
+
+	info, err := run.os.Stat(run.path("file"))
+	require.NoError(t, err)
+
+	// avoid errors because of timezone differences
+	assert.Equal(t, info.ModTime().Unix(), mtime.Unix())
+
+	run.rm(t, "file")
+}
+
+// run.os.Create without opening for write too
+func osCreate(name string) (vfs.OsFiler, error) {
+	return run.os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+}
+
+// run.os.Create with append
+func osAppend(name string) (vfs.OsFiler, error) {
+	return run.os.OpenFile(name, os.O_WRONLY|os.O_APPEND, 0666)
+}
+
+// TestFileModTimeWithOpenWriters tests mod time on open files
+func TestFileModTimeWithOpenWriters(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on Windows")
+	}
+
+	mtime := time.Date(2012, time.November, 18, 17, 32, 31, 0, time.UTC)
+	filepath := run.path("cp-archive-test")
+
+	f, err := osCreate(filepath)
+	require.NoError(t, err)
+
+	_, err = f.Write([]byte{104, 105})
+	require.NoError(t, err)
+
+	err = run.os.Chtimes(filepath, mtime, mtime)
+	require.NoError(t, err)
+
+	err = f.Close()
+	require.NoError(t, err)
+
+	run.waitForWriters()
+
+	info, err := run.os.Stat(filepath)
+	require.NoError(t, err)
+
+	// avoid errors because of timezone differences
+	assert.Equal(t, info.ModTime().Unix(), mtime.Unix())
+
+	run.rm(t, "cp-archive-test")
+}

vfs/vfstest/fs.go

@@ -0,0 +1,467 @@
+// Test suite for rclonefs
+
+package vfstest
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	_ "github.com/rclone/rclone/backend/all" // import all the backends
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/vfs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type (
+	// UnmountFn is called to unmount the file system
+	UnmountFn func() error
+	// MountFn is called to mount the file system
+	MountFn func(f fs.Fs, mountpoint string) (vfs *vfs.VFS, unmountResult <-chan error, unmount func() error, err error)
+)
+
+var (
+	mountFn MountFn
+)
+
+// RunTests runs all the tests against all the VFS cache modes
+//
+// If useVFS is set then it runs the tests against a VFS rather than amount
+func RunTests(t *testing.T, useVFS bool, fn MountFn) {
+	mountFn = fn
+	flag.Parse()
+	cacheModes := []vfs.CacheMode{
+		vfs.CacheModeOff,
+		vfs.CacheModeMinimal,
+		vfs.CacheModeWrites,
+		vfs.CacheModeFull,
+	}
+	run = newRun(useVFS)
+	for _, cacheMode := range cacheModes {
+		run.cacheMode(cacheMode)
+		log.Printf("Starting test run with cache mode %v", cacheMode)
+		ok := t.Run(fmt.Sprintf("CacheMode=%v", cacheMode), func(t *testing.T) {
+			t.Run("TestTouchAndDelete", TestTouchAndDelete)
+			t.Run("TestRenameOpenHandle", TestRenameOpenHandle)
+			t.Run("TestDirLs", TestDirLs)
+			t.Run("TestDirCreateAndRemoveDir", TestDirCreateAndRemoveDir)
+			t.Run("TestDirCreateAndRemoveFile", TestDirCreateAndRemoveFile)
+			t.Run("TestDirRenameFile", TestDirRenameFile)
+			t.Run("TestDirRenameEmptyDir", TestDirRenameEmptyDir)
+			t.Run("TestDirRenameFullDir", TestDirRenameFullDir)
+			t.Run("TestDirModTime", TestDirModTime)
+			t.Run("TestDirCacheFlush", TestDirCacheFlush)
+			t.Run("TestDirCacheFlushOnDirRename", TestDirCacheFlushOnDirRename)
+			t.Run("TestFileModTime", TestFileModTime)
+			t.Run("TestFileModTimeWithOpenWriters", TestFileModTimeWithOpenWriters)
+			t.Run("TestMount", TestMount)
+			t.Run("TestRoot", TestRoot)
+			t.Run("TestReadByByte", TestReadByByte)
+			t.Run("TestReadChecksum", TestReadChecksum)
+			t.Run("TestReadFileDoubleClose", TestReadFileDoubleClose)
+			t.Run("TestReadSeek", TestReadSeek)
+			t.Run("TestWriteFileNoWrite", TestWriteFileNoWrite)
+			t.Run("TestWriteFileWrite", TestWriteFileWrite)
+			t.Run("TestWriteFileOverwrite", TestWriteFileOverwrite)
+			t.Run("TestWriteFileDoubleClose", TestWriteFileDoubleClose)
+			t.Run("TestWriteFileFsync", TestWriteFileFsync)
+			t.Run("TestWriteFileDup", TestWriteFileDup)
+			t.Run("TestWriteFileAppend", TestWriteFileAppend)
+		})
+		log.Printf("Finished test run with cache mode %v (ok=%v)", cacheMode, ok)
+		if !ok {
+			break
+		}
+	}
+	run.Finalise()
+}
+
+// Run holds the remotes for a test run
+type Run struct {
+	os           Oser
+	vfs          *vfs.VFS
+	useVFS       bool // set if we are testing a VFS not a mount
+	mountPath    string
+	fremote      fs.Fs
+	fremoteName  string
+	cleanRemote  func()
+	umountResult <-chan error
+	umountFn     UnmountFn
+	skip         bool
+}
+
+// run holds the master Run data
+var run *Run
+
+// newRun initialise the remote mount for testing and returns a run
+// object.
+//
+// r.fremote is an empty remote Fs
+//
+// Finalise() will tidy them away when done.
+func newRun(useVFS bool) *Run {
+	r := &Run{
+		useVFS:       useVFS,
+		umountResult: make(chan error, 1),
+	}
+	fstest.Initialise()
+
+	var err error
+	r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote()
+	if err != nil {
+		log.Fatalf("Failed to open remote %q: %v", *fstest.RemoteName, err)
+	}
+
+	err = r.fremote.Mkdir(context.Background(), "")
+	if err != nil {
+		log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err)
+	}
+
+	if !r.useVFS {
+		r.mountPath = findMountPath()
+	}
+	// Mount it up
+	r.mount()
+
+	return r
+}
+
+func findMountPath() string {
+	if runtime.GOOS != "windows" {
+		mountPath, err := ioutil.TempDir("", "rclonefs-mount")
+		if err != nil {
+			log.Fatalf("Failed to create mount dir: %v", err)
+		}
+		return mountPath
+	}
+
+	// Find a free drive letter
+	drive := ""
+	for letter := 'E'; letter <= 'Z'; letter++ {
+		drive = string(letter) + ":"
+		_, err := os.Stat(drive + "\\")
+		if os.IsNotExist(err) {
+			goto found
+		}
+	}
+	log.Fatalf("Couldn't find free drive letter for test")
+found:
+	return drive
+}
+
+func (r *Run) mount() {
+	log.Printf("mount %q %q", r.fremote, r.mountPath)
+	var err error
+	r.vfs, r.umountResult, r.umountFn, err = mountFn(r.fremote, r.mountPath)
+	if err != nil {
+		log.Printf("mount FAILED: %v", err)
+		r.skip = true
+	} else {
+		log.Printf("mount OK")
+	}
+	if r.useVFS {
+		r.os = vfsOs{r.vfs}
+	} else {
+		r.os = realOs{}
+	}
+
+}
+
+func (r *Run) umount() {
+	if r.skip {
+		log.Printf("FUSE not found so skipping umount")
+		return
+	}
+	/*
+		log.Printf("Calling fusermount -u %q", r.mountPath)
+		err := exec.Command("fusermount", "-u", r.mountPath).Run()
+		if err != nil {
+			log.Printf("fusermount failed: %v", err)
+		}
+	*/
+	log.Printf("Unmounting %q", r.mountPath)
+	err := r.umountFn()
+	if err != nil {
+		log.Printf("signal to umount failed - retrying: %v", err)
+		time.Sleep(3 * time.Second)
+		err = r.umountFn()
+	}
+	if err != nil {
+		log.Fatalf("signal to umount failed: %v", err)
+	}
+	log.Printf("Waiting for umount")
+	err = <-r.umountResult
+	if err != nil {
+		log.Fatalf("umount failed: %v", err)
+	}
+
+	// Cleanup the VFS cache - umount has called Shutdown
+	err = r.vfs.CleanUp()
+	if err != nil {
+		log.Printf("Failed to cleanup the VFS cache: %v", err)
+	}
+}
+
+// cacheMode flushes the VFS and changes the CacheMode
+func (r *Run) cacheMode(cacheMode vfs.CacheMode) {
+	if r.skip {
+		log.Printf("FUSE not found so skipping cacheMode")
+		return
+	}
+	// Wait for writers to finish
+	r.vfs.WaitForWriters(30 * time.Second)
+	// Empty and remake the remote
+	r.cleanRemote()
+	err := r.fremote.Mkdir(context.Background(), "")
+	if err != nil {
+		log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err)
+	}
+	// Empty the cache
+	err = r.vfs.CleanUp()
+	if err != nil {
+		log.Printf("Failed to cleanup the VFS cache: %v", err)
+	}
+	// Reset the cache mode
+	r.vfs.SetCacheMode(cacheMode)
+	// Flush the directory cache
+	r.vfs.FlushDirCache()
+
+}
+
+func (r *Run) skipIfNoFUSE(t *testing.T) {
+	if r.skip {
+		t.Skip("FUSE not found so skipping test")
+	}
+}
+
+func (r *Run) skipIfVFS(t *testing.T) {
+	if r.useVFS {
+		t.Skip("Not running under VFS")
+	}
+}
+
+// Finalise cleans the remote and unmounts
+func (r *Run) Finalise() {
+	r.umount()
+	r.cleanRemote()
+	if r.useVFS {
+		// FIXME
+	} else {
+		err := os.RemoveAll(r.mountPath)
+		if err != nil {
+			log.Printf("Failed to clean mountPath %q: %v", r.mountPath, err)
+		}
+	}
+}
+
+// path returns an OS local path for filepath
+func (r *Run) path(filePath string) string {
+	if r.useVFS {
+		return filePath
+	}
+	// return windows drive letter root as E:\
+	if filePath == "" && runtime.GOOS == "windows" {
+		return run.mountPath + `\`
+	}
+	return filepath.Join(run.mountPath, filepath.FromSlash(filePath))
+}
+
+type dirMap map[string]struct{}
+
+// Create a dirMap from a string
+func newDirMap(dirString string) (dm dirMap) {
+	dm = make(dirMap)
+	for _, entry := range strings.Split(dirString, "|") {
+		if entry != "" {
+			dm[entry] = struct{}{}
+		}
+	}
+	return dm
+}
+
+// Returns a dirmap with only the files in
+func (dm dirMap) filesOnly() dirMap {
+	newDm := make(dirMap)
+	for name := range dm {
+		if !strings.HasSuffix(name, "/") {
+			newDm[name] = struct{}{}
+		}
+	}
+	return newDm
+}
+
+// reads the local tree into dir
+func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) {
+	realPath := r.path(filePath)
+	files, err := r.os.ReadDir(realPath)
+	require.NoError(t, err)
+	for _, fi := range files {
+		name := path.Join(filePath, fi.Name())
+		if fi.IsDir() {
+			dir[name+"/"] = struct{}{}
+			r.readLocal(t, dir, name)
+			assert.Equal(t, run.vfs.Opt.DirPerms&os.ModePerm, fi.Mode().Perm())
+		} else {
+			dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
+			assert.Equal(t, run.vfs.Opt.FilePerms&os.ModePerm, fi.Mode().Perm())
+		}
+	}
+}
+
+// reads the remote tree into dir
+func (r *Run) readRemote(t *testing.T, dir dirMap, filepath string) {
+	objs, dirs, err := walk.GetAll(context.Background(), r.fremote, filepath, true, 1)
+	if err == fs.ErrorDirNotFound {
+		return
+	}
+	require.NoError(t, err)
+	for _, obj := range objs {
+		dir[fmt.Sprintf("%s %d", obj.Remote(), obj.Size())] = struct{}{}
+	}
+	for _, d := range dirs {
+		name := d.Remote()
+		dir[name+"/"] = struct{}{}
+		r.readRemote(t, dir, name)
+	}
+}
+
+// checkDir checks the local and remote against the string passed in
+func (r *Run) checkDir(t *testing.T, dirString string) {
+	var retries = *fstest.ListRetries
+	sleep := time.Second / 5
+	var remoteOK, fuseOK bool
+	var dm, localDm, remoteDm dirMap
+	for i := 1; i <= retries; i++ {
+		dm = newDirMap(dirString)
+		localDm = make(dirMap)
+		r.readLocal(t, localDm, "")
+		remoteDm = make(dirMap)
+		r.readRemote(t, remoteDm, "")
+		// Ignore directories for remote compare
+		remoteOK = reflect.DeepEqual(dm.filesOnly(), remoteDm.filesOnly())
+		fuseOK = reflect.DeepEqual(dm, localDm)
+		if remoteOK && fuseOK {
+			return
+		}
+		sleep *= 2
+		t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
+		time.Sleep(sleep)
+	}
+	assert.Equal(t, dm.filesOnly(), remoteDm.filesOnly(), "expected vs remote")
+	assert.Equal(t, dm, localDm, "expected vs fuse mount")
+}
+
+// wait for any files being written to be released by fuse
+func (r *Run) waitForWriters() {
+	run.vfs.WaitForWriters(10 * time.Second)
+}
+
+// writeFile writes data to a file named by filename.
+// If the file does not exist, WriteFile creates it with permissions perm;
+// otherwise writeFile truncates it before writing.
+// If there is an error writing then writeFile
+// deletes it an existing file and tries again.
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := run.os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		err = run.os.Remove(filename)
+		if err != nil {
+			return err
+		}
+		f, err = run.os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, perm)
+		if err != nil {
+			return err
+		}
+	}
+	n, err := f.Write(data)
+	if err == nil && n < len(data) {
+		err = io.ErrShortWrite
+	}
+	if err1 := f.Close(); err == nil {
+		err = err1
+	}
+	return err
+}
+
+func (r *Run) createFile(t *testing.T, filepath string, contents string) {
+	filepath = r.path(filepath)
+	err := writeFile(filepath, []byte(contents), 0600)
+	require.NoError(t, err)
+	r.waitForWriters()
+}
+
+func (r *Run) readFile(t *testing.T, filepath string) string {
+	filepath = r.path(filepath)
+	result, err := run.os.ReadFile(filepath)
+	require.NoError(t, err)
+	time.Sleep(100 * time.Millisecond) // FIXME wait for Release
+	return string(result)
+}
+
+func (r *Run) mkdir(t *testing.T, filepath string) {
+	filepath = r.path(filepath)
+	err := run.os.Mkdir(filepath, 0700)
+	require.NoError(t, err)
+}
+
+func (r *Run) rm(t *testing.T, filepath string) {
+	filepath = r.path(filepath)
+	err := run.os.Remove(filepath)
+	require.NoError(t, err)
+
+	// Wait for file to disappear from listing
+	for i := 0; i < 100; i++ {
+		_, err := run.os.Stat(filepath)
+		if os.IsNotExist(err) {
+			return
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	assert.Fail(t, "failed to delete file", filepath)
+}
+
+func (r *Run) rmdir(t *testing.T, filepath string) {
+	filepath = r.path(filepath)
+	err := run.os.Remove(filepath)
+	require.NoError(t, err)
+}
+
+// TestMount checks that the Fs is mounted by seeing if the mountpoint
+// is in the mount output
+func TestMount(t *testing.T) {
+	run.skipIfVFS(t)
+	run.skipIfNoFUSE(t)
+	if runtime.GOOS == "windows" {
+		t.Skip("not running on windows")
+	}
+
+	out, err := exec.Command("mount").Output()
+	require.NoError(t, err)
+	assert.Contains(t, string(out), run.mountPath)
+}
+
+// TestRoot checks root directory is present and correct
+func TestRoot(t *testing.T) {
+	run.skipIfVFS(t)
+	run.skipIfNoFUSE(t)
+
+	fi, err := os.Lstat(run.mountPath)
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+	assert.Equal(t, run.vfs.Opt.DirPerms&os.ModePerm, fi.Mode().Perm())
+}

vfs/vfstest/os.go

@@ -0,0 +1,116 @@
+package vfstest
+
+import (
+	"io/ioutil"
+	"os"
+	"time"
+
+	"github.com/rclone/rclone/lib/file"
+	"github.com/rclone/rclone/vfs"
+)
+
+// Oser defines the things that the "os" package can do
+//
+// This covers what the VFS can do also
+type Oser interface {
+	Chtimes(name string, atime time.Time, mtime time.Time) error
+	Create(name string) (vfs.Handle, error)
+	Mkdir(name string, perm os.FileMode) error
+	Open(name string) (vfs.Handle, error)
+	OpenFile(name string, flags int, perm os.FileMode) (fd vfs.Handle, err error)
+	ReadDir(dirname string) ([]os.FileInfo, error)
+	ReadFile(filename string) (b []byte, err error)
+	Remove(name string) error
+	Rename(oldName, newName string) error
+	Stat(path string) (os.FileInfo, error)
+}
+
+// realOs is an implementation of Oser backed by the "os" package
+type realOs struct {
+}
+
+// realOsFile is an implementation of vfs.Handle
+type realOsFile struct {
+	*os.File
+}
+
+// Flush
+func (f realOsFile) Flush() error {
+	return nil
+}
+
+// Release
+func (f realOsFile) Release() error {
+	return f.File.Close()
+}
+
+// Node
+func (f realOsFile) Node() vfs.Node {
+	return nil
+}
+
+// Chtimes
+func (r realOs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return os.Chtimes(name, atime, mtime)
+}
+
+// Create
+func (r realOs) Create(name string) (vfs.Handle, error) {
+	fd, err := file.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return nil, err
+	}
+	return realOsFile{File: fd}, err
+}
+
+// Mkdir
+func (r realOs) Mkdir(name string, perm os.FileMode) error {
+	return os.Mkdir(name, perm)
+}
+
+// Open
+func (r realOs) Open(name string) (vfs.Handle, error) {
+	fd, err := os.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	return realOsFile{File: fd}, err
+}
+
+// OpenFile
+func (r realOs) OpenFile(name string, flags int, perm os.FileMode) (vfs.Handle, error) {
+	fd, err := file.OpenFile(name, flags, perm)
+	if err != nil {
+		return nil, err
+	}
+	return realOsFile{File: fd}, err
+}
+
+// ReadDir
+func (r realOs) ReadDir(dirname string) ([]os.FileInfo, error) {
+	return ioutil.ReadDir(dirname)
+}
+
+// ReadFile
+func (r realOs) ReadFile(filename string) (b []byte, err error) {
+	return ioutil.ReadFile(filename)
+}
+
+// Remove
+func (r realOs) Remove(name string) error {
+	return os.Remove(name)
+}
+
+// Rename
+func (r realOs) Rename(oldName, newName string) error {
+	return os.Rename(oldName, newName)
+}
+
+// Stat
+func (r realOs) Stat(path string) (os.FileInfo, error) {
+	return os.Stat(path)
+}
+
+// Check interfaces
+var _ Oser = &realOs{}
+var _ vfs.Handle = &realOsFile{}

vfs/vfstest/read.go

@@ -0,0 +1,126 @@
+package vfstest
+
+import (
+	"io"
+	"io/ioutil"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestReadByByte reads by byte including don't read any bytes
+func TestReadByByte(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	var data = []byte("hellohello")
+	run.createFile(t, "testfile", string(data))
+	run.checkDir(t, "testfile 10")
+
+	for i := 0; i < len(data); i++ {
+		fd, err := run.os.Open(run.path("testfile"))
+		assert.NoError(t, err)
+		for j := 0; j < i; j++ {
+			buf := make([]byte, 1)
+			n, err := io.ReadFull(fd, buf)
+			assert.NoError(t, err)
+			assert.Equal(t, 1, n)
+			assert.Equal(t, buf[0], data[j])
+		}
+		err = fd.Close()
+		assert.NoError(t, err)
+	}
+
+	time.Sleep(100 * time.Millisecond) // FIXME wait for Release
+	run.rm(t, "testfile")
+}
+
+// TestReadChecksum checks the checksum reading is working
+func TestReadChecksum(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	// create file big enough so we exceed any single FUSE read
+	// request
+	b := make([]rune, 3*128*1024)
+	for i := range b {
+		b[i] = 'r'
+	}
+	run.createFile(t, "bigfile", string(b))
+
+	// The hash comparison would fail in Flush, if we did not
+	// ensure we read the whole file
+	fd, err := run.os.Open(run.path("bigfile"))
+	assert.NoError(t, err)
+	buf := make([]byte, 10)
+	_, err = io.ReadFull(fd, buf)
+	assert.NoError(t, err)
+	err = fd.Close()
+	assert.NoError(t, err)
+
+	// The hash comparison would fail, because we only read parts
+	// of the file
+	fd, err = run.os.Open(run.path("bigfile"))
+	assert.NoError(t, err)
+	// read at start
+	_, err = io.ReadFull(fd, buf)
+	assert.NoError(t, err)
+	// read at end
+	_, err = fd.Seek(int64(len(b)-len(buf)), io.SeekStart)
+	assert.NoError(t, err)
+	_, err = io.ReadFull(fd, buf)
+	assert.NoError(t, err)
+	// ensure we don't compare hashes
+	err = fd.Close()
+	assert.NoError(t, err)
+
+	run.rm(t, "bigfile")
+}
+
+// TestReadSeek test seeking
+func TestReadSeek(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	var data = []byte("helloHELLO")
+	run.createFile(t, "testfile", string(data))
+	run.checkDir(t, "testfile 10")
+
+	fd, err := run.os.Open(run.path("testfile"))
+	assert.NoError(t, err)
+
+	// Seek to half way
+	_, err = fd.Seek(5, io.SeekStart)
+	assert.NoError(t, err)
+
+	buf, err := ioutil.ReadAll(fd)
+	assert.NoError(t, err)
+	assert.Equal(t, buf, []byte("HELLO"))
+
+	// Test seeking to the end
+	_, err = fd.Seek(10, io.SeekStart)
+	assert.NoError(t, err)
+
+	buf, err = ioutil.ReadAll(fd)
+	assert.NoError(t, err)
+	assert.Equal(t, buf, []byte(""))
+
+	// Test seeking beyond the end
+	_, err = fd.Seek(1000000, io.SeekStart)
+	assert.NoError(t, err)
+
+	buf, err = ioutil.ReadAll(fd)
+	assert.NoError(t, err)
+	assert.Equal(t, buf, []byte(""))
+
+	// Now back to the start
+	_, err = fd.Seek(0, io.SeekStart)
+	assert.NoError(t, err)
+
+	buf, err = ioutil.ReadAll(fd)
+	assert.NoError(t, err)
+	assert.Equal(t, buf, []byte("helloHELLO"))
+
+	err = fd.Close()
+	assert.NoError(t, err)
+
+	run.rm(t, "testfile")
+}

vfs/vfstest/read_non_unix.go

@@ -0,0 +1,13 @@
+// +build !linux,!darwin,!freebsd
+
+package vfstest
+
+import (
+	"runtime"
+	"testing"
+)
+
+// TestReadFileDoubleClose tests double close on read
+func TestReadFileDoubleClose(t *testing.T) {
+	t.Skip("not supported on " + runtime.GOOS)
+}

vfs/vfstest/read_unix.go

@@ -0,0 +1,53 @@
+// +build linux darwin freebsd
+
+package vfstest
+
+import (
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestReadFileDoubleClose tests double close on read
+func TestReadFileDoubleClose(t *testing.T) {
+	run.skipIfVFS(t)
+	run.skipIfNoFUSE(t)
+
+	run.createFile(t, "testdoubleclose", "hello")
+
+	in, err := run.os.Open(run.path("testdoubleclose"))
+	assert.NoError(t, err)
+	fd := in.Fd()
+
+	fd1, err := syscall.Dup(int(fd))
+	assert.NoError(t, err)
+
+	fd2, err := syscall.Dup(int(fd))
+	assert.NoError(t, err)
+
+	// close one of the dups - should produce no error
+	err = syscall.Close(fd1)
+	assert.NoError(t, err)
+
+	// read from the file
+	buf := make([]byte, 1)
+	_, err = in.Read(buf)
+	assert.NoError(t, err)
+
+	// close it
+	err = in.Close()
+	assert.NoError(t, err)
+
+	// read from the other dup - should produce no error as this
+	// file is now buffered
+	n, err := syscall.Read(fd2, buf)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, n)
+
+	// close the dup - should not produce an error
+	err = syscall.Close(fd2)
+	assert.NoError(t, err, "input/output error")
+
+	run.rm(t, "testdoubleclose")
+}

vfs/vfstest/vfs.go

@@ -0,0 +1,20 @@
+package vfstest
+
+import (
+	"os"
+
+	"github.com/rclone/rclone/vfs"
+)
+
+// vfsOs is an implementation of Oser backed by the "vfs" package
+type vfsOs struct {
+	*vfs.VFS
+}
+
+// Stat
+func (v vfsOs) Stat(path string) (os.FileInfo, error) {
+	return v.VFS.Stat(path)
+}
+
+// Check interfaces
+var _ Oser = vfsOs{}

vfs/vfstest/write.go

@@ -0,0 +1,179 @@
+package vfstest
+
+import (
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/rclone/rclone/vfs"
+)
+
+// TestWriteFileNoWrite tests writing a file with no write()'s to it
+func TestWriteFileNoWrite(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	fd, err := osCreate(run.path("testnowrite"))
+	assert.NoError(t, err)
+
+	err = fd.Close()
+	assert.NoError(t, err)
+
+	run.waitForWriters()
+
+	run.checkDir(t, "testnowrite 0")
+
+	run.rm(t, "testnowrite")
+}
+
+// FIXMETestWriteOpenFileInDirListing tests open file in directory listing
+func FIXMETestWriteOpenFileInDirListing(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	fd, err := osCreate(run.path("testnowrite"))
+	assert.NoError(t, err)
+
+	run.checkDir(t, "testnowrite 0")
+
+	err = fd.Close()
+	assert.NoError(t, err)
+
+	run.waitForWriters()
+
+	run.rm(t, "testnowrite")
+}
+
+// TestWriteFileWrite tests writing a file and reading it back
+func TestWriteFileWrite(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.createFile(t, "testwrite", "data")
+	run.checkDir(t, "testwrite 4")
+	contents := run.readFile(t, "testwrite")
+	assert.Equal(t, "data", contents)
+	run.rm(t, "testwrite")
+}
+
+// TestWriteFileOverwrite tests overwriting a file
+func TestWriteFileOverwrite(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	run.createFile(t, "testwrite", "data")
+	run.checkDir(t, "testwrite 4")
+	run.createFile(t, "testwrite", "potato")
+	contents := run.readFile(t, "testwrite")
+	assert.Equal(t, "potato", contents)
+	run.rm(t, "testwrite")
+}
+
+// TestWriteFileFsync tests Fsync
+//
+// NB the code for this is in file.go rather than write.go
+func TestWriteFileFsync(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	filepath := run.path("to be synced")
+	fd, err := osCreate(filepath)
+	require.NoError(t, err)
+	_, err = fd.Write([]byte("hello"))
+	require.NoError(t, err)
+	err = fd.Sync()
+	require.NoError(t, err)
+	err = fd.Close()
+	require.NoError(t, err)
+	run.waitForWriters()
+	run.rm(t, "to be synced")
+}
+
+// TestWriteFileDup tests behavior of mmap() in Python by using dup() on a file handle
+func TestWriteFileDup(t *testing.T) {
+	run.skipIfVFS(t)
+	run.skipIfNoFUSE(t)
+
+	if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
+		t.Skip("not supported on vfs-cache-mode < writes")
+		return
+	}
+
+	filepath := run.path("to be synced")
+	fh, err := osCreate(filepath)
+	require.NoError(t, err)
+
+	testData := []byte("0123456789")
+
+	err = fh.Truncate(int64(len(testData) + 2))
+	require.NoError(t, err)
+
+	err = fh.Sync()
+	require.NoError(t, err)
+
+	var dupFd uintptr
+	dupFd, err = writeTestDup(fh.Fd())
+	require.NoError(t, err)
+
+	dupFile := os.NewFile(dupFd, fh.Name())
+	_, err = dupFile.Write(testData)
+	require.NoError(t, err)
+
+	err = dupFile.Close()
+	require.NoError(t, err)
+
+	_, err = fh.Seek(int64(len(testData)), 0)
+	require.NoError(t, err)
+
+	_, err = fh.Write([]byte("10"))
+	require.NoError(t, err)
+
+	err = fh.Close()
+	require.NoError(t, err)
+
+	run.waitForWriters()
+	run.rm(t, "to be synced")
+}
+
+// TestWriteFileAppend tests that O_APPEND works on cache backends >= writes
+func TestWriteFileAppend(t *testing.T) {
+	run.skipIfNoFUSE(t)
+
+	if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
+		t.Skip("not supported on vfs-cache-mode < writes")
+		return
+	}
+
+	// TODO: Windows needs the v1.5 release of WinFsp to handle O_APPEND properly.
+	// Until it gets released, skip this test on Windows.
+	if runtime.GOOS == "windows" {
+		t.Skip("currently unsupported on Windows")
+	}
+
+	filepath := run.path("to be synced")
+	fh, err := osCreate(filepath)
+	require.NoError(t, err)
+
+	testData := []byte("0123456789")
+	appendData := []byte("10")
+
+	_, err = fh.Write(testData)
+	require.NoError(t, err)
+
+	err = fh.Close()
+	require.NoError(t, err)
+
+	fh, err = osAppend(filepath)
+	require.NoError(t, err)
+
+	_, err = fh.Write(appendData)
+	require.NoError(t, err)
+
+	err = fh.Close()
+	require.NoError(t, err)
+
+	info, err := run.os.Stat(filepath)
+	require.NoError(t, err)
+	require.EqualValues(t, len(testData)+len(appendData), info.Size())
+
+	run.waitForWriters()
+	run.rm(t, "to be synced")
+}

vfs/vfstest/write_non_unix.go

@@ -0,0 +1,25 @@
+// +build !linux,!darwin,!freebsd
+
+package vfstest
+
+import (
+	"runtime"
+	"testing"
+
+	"golang.org/x/sys/windows"
+)
+
+// TestWriteFileDoubleClose tests double close on write
+func TestWriteFileDoubleClose(t *testing.T) {
+	t.Skip("not supported on " + runtime.GOOS)
+}
+
+// writeTestDup performs the platform-specific implementation of the dup() syscall
+func writeTestDup(oldfd uintptr) (uintptr, error) {
+	p, err := windows.GetCurrentProcess()
+	if err != nil {
+		return 0, err
+	}
+	var h windows.Handle
+	return uintptr(h), windows.DuplicateHandle(p, windows.Handle(oldfd), p, &h, 0, true, windows.DUPLICATE_SAME_ACCESS)
+}

vfs/vfstest/write_unix.go

@@ -0,0 +1,69 @@
+// +build linux darwin freebsd
+
+package vfstest
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/sys/unix"
+
+	"github.com/rclone/rclone/vfs"
+)
+
+// TestWriteFileDoubleClose tests double close on write
+func TestWriteFileDoubleClose(t *testing.T) {
+	run.skipIfVFS(t)
+	run.skipIfNoFUSE(t)
+	if runtime.GOOS == "darwin" {
+		t.Skip("Skipping test on OSX")
+	}
+
+	out, err := osCreate(run.path("testdoubleclose"))
+	assert.NoError(t, err)
+	fd := out.Fd()
+
+	fd1, err := unix.Dup(int(fd))
+	assert.NoError(t, err)
+
+	fd2, err := unix.Dup(int(fd))
+	assert.NoError(t, err)
+
+	// close one of the dups - should produce no error
+	err = unix.Close(fd1)
+	assert.NoError(t, err)
+
+	// write to the file
+	buf := []byte("hello")
+	n, err := out.Write(buf)
+	assert.NoError(t, err)
+	assert.Equal(t, 5, n)
+
+	// close it
+	err = out.Close()
+	assert.NoError(t, err)
+
+	// write to the other dup
+	_, err = unix.Write(fd2, buf)
+	if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
+		// produces an error if cache mode < writes
+		assert.Error(t, err, "input/output error")
+	} else {
+		// otherwise does not produce an error
+		assert.NoError(t, err)
+	}
+
+	// close the dup - should not produce an error
+	err = unix.Close(fd2)
+	assert.NoError(t, err)
+
+	run.waitForWriters()
+	run.rm(t, "testdoubleclose")
+}
+
+// writeTestDup performs the platform-specific implementation of the dup() unix
+func writeTestDup(oldfd uintptr) (uintptr, error) {
+	newfd, err := unix.Dup(int(oldfd))
+	return uintptr(newfd), err
+}

vfs/vfstest_test.go

@@ -0,0 +1,30 @@
+// Run the more functional vfstest package on the vfs
+
+package vfs_test
+
+import (
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/all" // import all the backends
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/vfs"
+	"github.com/rclone/rclone/vfs/vfstest"
+)
+
+// TestExt runs more functional tests all the tests against all the
+// VFS cache modes
+func TestFunctional(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skip on non local")
+	}
+	vfstest.RunTests(t, true, func(f fs.Fs, mountpoint string) (VFS *vfs.VFS, unmountResult <-chan error, unmount func() error, err error) {
+		unmountResultChan := make(chan (error), 1)
+		unmount = func() error {
+			unmountResultChan <- nil
+			return nil
+		}
+		VFS = vfs.New(f, nil)
+		return VFS, unmountResultChan, unmount, nil
+	})
+}