Make tests for each Fs

Factor tests out of rclonetest
2014-07-24 23:13:32 +01:00
parent 02d50f8c6e
commit a5b0d88608
11 changed files with 851 additions and 185 deletions
@@ -0,0 +1,192 @@
+// Utilities for testing the fs
+
+package fstest
+
+// FIXME put name of test FS in Fs structure
+
+import (
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+)
+
+var Fatalf = log.Fatalf
+
+// Represents an item for checking
+type Item struct {
+	Path    string
+	Md5sum  string
+	ModTime time.Time
+	Size    int64
+}
+
+func (i *Item) Check(obj fs.Object) {
+	if obj == nil {
+		Fatalf("Object is nil")
+	}
+	// Check attributes
+	Md5sum, err := obj.Md5sum()
+	if err != nil {
+		Fatalf("Failed to read md5sum for %q: %v", obj.Remote(), err)
+	}
+	if i.Md5sum != Md5sum {
+		Fatalf("%s: Md5sum incorrect - expecting %q got %q", obj.Remote(), i.Md5sum, Md5sum)
+	}
+	if i.Size != obj.Size() {
+		Fatalf("%s: Size incorrect - expecting %d got %d", obj.Remote(), i.Size, obj.Size())
+	}
+	// check the mod time to the given precision
+	modTime := obj.ModTime()
+	dt := modTime.Sub(i.ModTime)
+	if dt >= fs.Config.ModifyWindow || dt <= -fs.Config.ModifyWindow {
+		Fatalf("%s: Modification time difference too big |%s| > %s (%s vs %s)", obj.Remote(), dt, fs.Config.ModifyWindow, modTime, i.ModTime)
+	}
+
+}
+
+// Represents all items for checking
+type Items struct {
+	byName map[string]*Item
+	items  []Item
+}
+
+// Make an Items
+func NewItems(items []Item) *Items {
+	is := &Items{
+		byName: make(map[string]*Item),
+		items:  items,
+	}
+	// Fill up byName
+	for i := range items {
+		is.byName[items[i].Path] = &items[i]
+	}
+	return is
+}
+
+// Check off an item
+func (is *Items) Find(obj fs.Object) {
+	i, ok := is.byName[obj.Remote()]
+	if !ok {
+		Fatalf("Unexpected file %q", obj.Remote())
+	}
+	delete(is.byName, obj.Remote())
+	i.Check(obj)
+}
+
+// Check all done
+func (is *Items) Done() {
+	if len(is.byName) != 0 {
+		for name := range is.byName {
+			log.Printf("Not found %q", name)
+		}
+		Fatalf("%d objects not found", len(is.byName))
+	}
+}
+
+// Checks the fs to see if it has the expected contents
+func CheckListing(f fs.Fs, items []Item) {
+	is := NewItems(items)
+	for obj := range f.List() {
+		is.Find(obj)
+	}
+	is.Done()
+}
+
+// Parse a time string or explode
+func Time(timeString string) time.Time {
+	t, err := time.Parse(time.RFC3339Nano, timeString)
+	if err != nil {
+		Fatalf("Failed to parse time %q: %v", timeString, err)
+	}
+	return t
+}
+
+// Create a random string
+func RandomString(n int) string {
+	source := "abcdefghijklmnopqrstuvwxyz0123456789"
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = source[rand.Intn(len(source))]
+	}
+	return string(out)
+}
+
+// Make a random bucket or subdirectory on the remote
+//
+// Call the finalise function returned to Purge the fs at the end (and
+// the parent if necessary)
+func RandomRemote(remoteName string, subdir bool) (fs.Fs, func()) {
+	// Make a directory if remote name is null
+	rmdir := ""
+	var err error
+	if remoteName == "" {
+		remoteName, err = ioutil.TempDir("", "rclone")
+		if err != nil {
+			Fatalf("Failed to create temp dir: %v", err)
+		}
+		rmdir = remoteName
+	}
+
+	if !strings.HasSuffix(remoteName, ":") {
+		remoteName += "/"
+	}
+	remoteName += RandomString(32)
+	var parentRemote fs.Fs
+	if subdir {
+		var err error
+		parentRemote, err = fs.NewFs(remoteName)
+		if err != nil {
+			log.Fatalf("Failed to make parent %q: %v", remoteName, err)
+		}
+		remoteName += "/" + RandomString(8)
+	}
+
+	remote, err := fs.NewFs(remoteName)
+	if err != nil {
+		log.Fatalf("Failed to make %q: %v", remoteName, err)
+	}
+
+	finalise := func() {
+		TestPurge(remote)
+		if parentRemote != nil {
+			TestPurge(parentRemote)
+		}
+		// Delete directory if we made one above
+		if rmdir != "" {
+			err := os.RemoveAll(rmdir)
+			if err != nil {
+				Fatalf("Failed to remove %q: %v", rmdir, err)
+			}
+		}
+	}
+
+	return remote, finalise
+}
+
+func TestMkdir(remote fs.Fs) {
+	err := fs.Mkdir(remote)
+	if err != nil {
+		Fatalf("Mkdir failed: %v", err)
+	}
+	CheckListing(remote, []Item{})
+}
+
+func TestPurge(remote fs.Fs) {
+	err := fs.Purge(remote)
+	if err != nil {
+		log.Fatalf("Purge failed: %v", err)
+	}
+	CheckListing(remote, []Item{})
+}
+
+func TestRmdir(remote fs.Fs) {
+	err := fs.Rmdir(remote)
+	if err != nil {
+		Fatalf("Rmdir failed: %v", err)
+	}
+}
@@ -0,0 +1,338 @@
+// Generic tests for testing the Fs and Object interfaces
+package fstests
+
+// FIXME need to check the limited file system
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fstest"
+)
+
+var (
+	remote         fs.Fs
+	RemoteName     = ""
+	remoteFinalise func()
+	NilObject      fs.Object
+	file1          = fstest.Item{
+		ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
+		Path:    "file name.txt",
+	}
+	file2 = fstest.Item{
+		ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"),
+		Path:    `hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`,
+	}
+)
+
+func TestInit(t *testing.T) {
+	fs.LoadConfig()
+	fs.Config.Verbose = false
+	fs.Config.Quiet = true
+	remote, remoteFinalise = fstest.RandomRemote(RemoteName, false)
+	// if err != nil {
+	// 	if strings.Contains(err.Error(), "Didn't find section in config file") {
+	// 		return
+	// 	}
+	// 	t.Fatalf("Couldn't start FS: %v", err)
+	// }
+	fstest.Fatalf = t.Fatalf
+	fstest.TestMkdir(remote)
+}
+
+func skipIfNotOk(t *testing.T) {
+	fstest.Fatalf = t.Fatalf
+	if remote == nil {
+		t.Skip("FS not configured")
+	}
+}
+
+// String returns a description of the FS
+
+func TestFsString(t *testing.T) {
+	skipIfNotOk(t)
+	str := remote.String()
+	if str == "" {
+		t.Fatal("Bad fs.String()")
+	}
+}
+
+type TestFile struct {
+	ModTime time.Time
+	Path    string
+	Size    int64
+	Md5sum  string
+}
+
+func TestFsRmdirEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestRmdir(remote)
+}
+
+func TestFsRmdirNotFound(t *testing.T) {
+	skipIfNotOk(t)
+	err := remote.Rmdir()
+	if err == nil {
+		t.Fatalf("Expecting error on Rmdir non existent")
+	}
+}
+
+func TestFsMkdir(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestMkdir(remote)
+	fstest.TestMkdir(remote)
+}
+
+func TestFsListEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(remote, []fstest.Item{})
+}
+
+func TestFsListDirEmpty(t *testing.T) {
+	skipIfNotOk(t)
+	for obj := range remote.ListDir() {
+		t.Error("Found unexpected item %q", obj.Name)
+	}
+}
+
+func TestFsNewFsObjectNotFound(t *testing.T) {
+	skipIfNotOk(t)
+	if remote.NewFsObject("potato") != nil {
+		t.Fatal("Didn't expect to find object")
+	}
+}
+
+func findObject(t *testing.T, Name string) fs.Object {
+	obj := remote.NewFsObject(Name)
+	if obj == nil {
+		t.Fatalf("nil object")
+	}
+	return obj
+}
+
+func testPut(t *testing.T, file *fstest.Item) {
+	buf := bytes.NewBufferString(fstest.RandomString(100))
+	hash := md5.New()
+	in := io.TeeReader(buf, hash)
+
+	file.Size = int64(buf.Len())
+	obj, err := remote.Put(in, file.Path, file.ModTime, file.Size)
+	if err != nil {
+		t.Fatal("Put error", err)
+	}
+	file.Md5sum = hex.EncodeToString(hash.Sum(nil))
+	file.Check(obj)
+	// Re-read the object and check again
+	obj = findObject(t, file.Path)
+	file.Check(obj)
+}
+
+func TestFsPutFile1(t *testing.T) {
+	skipIfNotOk(t)
+	testPut(t, &file1)
+}
+
+func TestFsPutFile2(t *testing.T) {
+	skipIfNotOk(t)
+	testPut(t, &file2)
+}
+
+func TestFsListDirFile2(t *testing.T) {
+	skipIfNotOk(t)
+	found := false
+	for obj := range remote.ListDir() {
+		if obj.Name != `hello? sausage` {
+			t.Errorf("Found unexpected item %q", obj.Name)
+		} else {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("Didn't find %q", `hello? sausage`)
+	}
+}
+
+func TestFsListFile1(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(remote, []fstest.Item{file1, file2})
+}
+
+func TestFsNewFsObject(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	file1.Check(obj)
+}
+
+func TestFsListFile1and2(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.CheckListing(remote, []fstest.Item{file1, file2})
+}
+
+func TestFsRmdirFull(t *testing.T) {
+	skipIfNotOk(t)
+	err := remote.Rmdir()
+	if err == nil {
+		t.Fatalf("Expecting error on RMdir on non empty remote")
+	}
+}
+
+func TestFsPrecision(t *testing.T) {
+	skipIfNotOk(t)
+	precision := remote.Precision()
+	if precision > time.Second || precision < 0 {
+		t.Fatalf("Precision out of range %v", precision)
+	}
+	// FIXME check expected precision
+}
+
+func TestObjectString(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	s := obj.String()
+	if s != file1.Path {
+		t.Errorf("String() wrong %v != %v", s, file1.Path)
+	}
+	obj = NilObject
+	s = obj.String()
+	if s != "<nil>" {
+		t.Errorf("String() wrong %v != %v", s, "<nil>")
+	}
+}
+
+func TestObjectFs(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Fs() != remote {
+		t.Errorf("Fs is wrong %v != %v", obj.Fs(), remote)
+	}
+}
+
+func TestObjectRemote(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Remote() != file1.Path {
+		t.Errorf("Remote is wrong %v != %v", obj.Remote(), file1.Path)
+	}
+}
+
+func TestObjectMd5sum(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	Md5sum, err := obj.Md5sum()
+	if err != nil {
+		t.Errorf("Error in Md5sum: %v", err)
+	}
+	if Md5sum != file1.Md5sum {
+		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
+	}
+}
+
+func TestObjectModTime(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if !obj.ModTime().Equal(file1.ModTime) {
+		t.Errorf("ModTime is wrong %v != %v", obj.ModTime(), file1.ModTime)
+	}
+}
+
+func TestObjectSetModTime(t *testing.T) {
+	skipIfNotOk(t)
+	newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z")
+	obj := findObject(t, file1.Path)
+	obj.SetModTime(newModTime)
+	// Check in this object
+	if !obj.ModTime().Equal(newModTime) {
+		t.Errorf("newModTime is wrong %v != %v", obj.ModTime(), newModTime)
+	}
+	file1.ModTime = newModTime
+	// And make a new object and read it from there too
+	TestObjectModTime(t)
+}
+
+func TestObjectSize(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if obj.Size() != file1.Size {
+		t.Errorf("Size is wrong %v != %v", obj.Size(), file1.Size)
+	}
+}
+
+func TestObjectOpen(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	in, err := obj.Open()
+	if err != nil {
+		t.Fatalf("Open() return error: %v", err)
+	}
+	hash := md5.New()
+	n, err := io.Copy(hash, in)
+	if err != nil {
+		t.Fatalf("io.Copy() return error: %v", err)
+	}
+	if n != file1.Size {
+		t.Fatalf("Read wrong number of bytes %d != %d", n, file1.Size)
+	}
+	err = in.Close()
+	if err != nil {
+		t.Fatalf("in.Close() return error: %v", err)
+	}
+	Md5sum := hex.EncodeToString(hash.Sum(nil))
+	if Md5sum != file1.Md5sum {
+		t.Errorf("Md5sum is wrong %v != %v", Md5sum, file1.Md5sum)
+	}
+}
+
+func TestObjectUpdate(t *testing.T) {
+	skipIfNotOk(t)
+	buf := bytes.NewBufferString(fstest.RandomString(200))
+	hash := md5.New()
+	in := io.TeeReader(buf, hash)
+
+	file1.Size = int64(buf.Len())
+	obj := findObject(t, file1.Path)
+	err := obj.Update(in, file1.ModTime, file1.Size)
+	if err != nil {
+		t.Fatal("Update error", err)
+	}
+	file1.Md5sum = hex.EncodeToString(hash.Sum(nil))
+	file1.Check(obj)
+	// Re-read the object and check again
+	obj = findObject(t, file1.Path)
+	file1.Check(obj)
+}
+
+func TestObjectStorable(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	if !obj.Storable() {
+		t.Fatalf("Expecting %v to be storable", obj)
+	}
+}
+
+func TestObjectRemove(t *testing.T) {
+	skipIfNotOk(t)
+	obj := findObject(t, file1.Path)
+	err := obj.Remove()
+	if err != nil {
+		t.Fatal("Remove error", err)
+	}
+	fstest.CheckListing(remote, []fstest.Item{file2})
+}
+
+func TestObjectPurge(t *testing.T) {
+	skipIfNotOk(t)
+	fstest.TestPurge(remote)
+	fstest.TestPurge(remote)
+}
+
+func TestFinalise(t *testing.T) {
+	skipIfNotOk(t)
+	if remoteFinalise != nil {
+		remoteFinalise()
+	}
+}