vfs: implement partial reads in --vfs-cache-mode full
This allows reads to only read part of the file and it keeps on disk a cache of what parts of each file have been loaded. File data itself is kept in sparse files.
This commit is contained in:
Nick Craig-Wood
@@ -0,0 +1,472 @@
|
||||
package vfscache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local" // import the local backend
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestMain drives the tests
|
||||
func TestMain(m *testing.M) {
|
||||
fstest.TestMain(m)
|
||||
}
|
||||
|
||||
// convert c.item to a string
|
||||
func itemAsString(c *Cache) []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var out []string
|
||||
for name, item := range c.item {
|
||||
out = append(out, fmt.Sprintf("name=%q opens=%d size=%d", filepath.ToSlash(name), item.opens, item.info.Size))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func TestCacheNew(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// FIXME need to be writing to the actual file here
|
||||
// need a item.WriteAt/item.ReadAt method I think
|
||||
|
||||
// Disable the cache cleaner as it interferes with these tests
|
||||
opt := vfscommon.DefaultOpt
|
||||
opt.CachePollInterval = 0
|
||||
c, err := New(ctx, r.Fremote, &opt)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Contains(t, c.root, "vfs")
|
||||
assert.Contains(t, c.fcache.Root(), filepath.Base(r.Fremote.Root()))
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
|
||||
// mkdir
|
||||
p, err := c.mkdir("potato")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "potato", filepath.Base(p))
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
|
||||
fi, err := os.Stat(filepath.Dir(p))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, fi.IsDir())
|
||||
|
||||
// get
|
||||
item, _ := c.get("potato")
|
||||
item2, _ := c.get("potato")
|
||||
assert.Equal(t, item, item2)
|
||||
assert.WithinDuration(t, time.Now(), item.info.ATime, time.Second)
|
||||
|
||||
// open
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
potato := c.Item("/potato")
|
||||
require.NoError(t, potato.Open(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
assert.WithinDuration(t, time.Now(), potato.info.ATime, time.Second)
|
||||
assert.Equal(t, 1, potato.opens)
|
||||
|
||||
// write the file
|
||||
require.NoError(t, potato.Truncate(5))
|
||||
atime := time.Now()
|
||||
potato.info.ATime = atime
|
||||
// err = ioutil.WriteFile(p, []byte("hello"), 0600)
|
||||
// require.NoError(t, err)
|
||||
|
||||
// read its atime
|
||||
|
||||
// updateAtimes
|
||||
//potato.ATime = time.Now().Add(-24 * time.Hour)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=1 size=5`,
|
||||
}, itemAsString(c))
|
||||
assert.True(t, atime.Equal(potato.info.ATime), fmt.Sprintf("%v != %v", atime, potato.info.ATime))
|
||||
|
||||
// try purging with file open
|
||||
c.purgeOld(10 * time.Second)
|
||||
// _, err = os.Stat(p)
|
||||
// assert.NoError(t, err)
|
||||
|
||||
// close
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=1 size=5`,
|
||||
}, itemAsString(c))
|
||||
require.NoError(t, potato.Truncate(6))
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=1 size=6`,
|
||||
}, itemAsString(c))
|
||||
require.NoError(t, potato.Close(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=0 size=6`,
|
||||
}, itemAsString(c))
|
||||
item, _ = c.get("potato")
|
||||
assert.WithinDuration(t, time.Now(), item.info.ATime, time.Second)
|
||||
assert.Equal(t, 0, item.opens)
|
||||
|
||||
// try purging with file closed
|
||||
c.purgeOld(10 * time.Second)
|
||||
// ...nothing should happend
|
||||
// _, err = os.Stat(p)
|
||||
// assert.NoError(t, err)
|
||||
|
||||
//.. purge again with -ve age
|
||||
c.purgeOld(-10 * time.Second)
|
||||
_, err = os.Stat(p)
|
||||
assert.True(t, os.IsNotExist(err))
|
||||
|
||||
// clean - have tested the internals already
|
||||
c.clean()
|
||||
|
||||
// cleanup
|
||||
err = c.CleanUp()
|
||||
require.NoError(t, err)
|
||||
_, err = os.Stat(c.root)
|
||||
assert.True(t, os.IsNotExist(err))
|
||||
}
|
||||
|
||||
func TestCacheOpens(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, c.CleanUp()) }()
|
||||
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
potato := c.Item("potato")
|
||||
require.NoError(t, potato.Open(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
require.NoError(t, potato.Open(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=2 size=0`,
|
||||
}, itemAsString(c))
|
||||
require.NoError(t, potato.Close(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
require.NoError(t, potato.Close(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="potato" opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
require.NoError(t, potato.Open(nil))
|
||||
a1 := c.Item("a//b/c/d/one")
|
||||
a2 := c.Item("a/b/c/d/e/two")
|
||||
a3 := c.Item("a/b/c/d/e/f/three")
|
||||
require.NoError(t, a1.Open(nil))
|
||||
require.NoError(t, a2.Open(nil))
|
||||
require.NoError(t, a3.Open(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="a/b/c/d/e/f/three" opens=1 size=0`,
|
||||
`name="a/b/c/d/e/two" opens=1 size=0`,
|
||||
`name="a/b/c/d/one" opens=1 size=0`,
|
||||
`name="potato" opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
require.NoError(t, potato.Close(nil))
|
||||
require.NoError(t, a1.Close(nil))
|
||||
require.NoError(t, a2.Close(nil))
|
||||
require.NoError(t, a3.Close(nil))
|
||||
assert.Equal(t, []string{
|
||||
`name="a/b/c/d/e/f/three" opens=0 size=0`,
|
||||
`name="a/b/c/d/e/two" opens=0 size=0`,
|
||||
`name="a/b/c/d/one" opens=0 size=0`,
|
||||
`name="potato" opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
}
|
||||
|
||||
// test the open, mkdir, purge, close, purge sequence
|
||||
func TestCacheOpenMkdir(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Disable the cache cleaner as it interferes with these tests
|
||||
opt := vfscommon.DefaultOpt
|
||||
opt.CachePollInterval = 0
|
||||
c, err := New(ctx, r.Fremote, &opt)
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, c.CleanUp()) }()
|
||||
|
||||
// open
|
||||
potato := c.Item("sub/potato")
|
||||
require.NoError(t, potato.Open(nil))
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/potato" opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// mkdir
|
||||
p, err := c.mkdir("sub/potato")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "potato", filepath.Base(p))
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/potato" opens=1 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// test directory exists
|
||||
fi, err := os.Stat(filepath.Dir(p))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, fi.IsDir())
|
||||
|
||||
// clean the cache
|
||||
c.purgeOld(-10 * time.Second)
|
||||
|
||||
// test directory still exists
|
||||
fi, err = os.Stat(filepath.Dir(p))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, fi.IsDir())
|
||||
|
||||
// close
|
||||
require.NoError(t, potato.Close(nil))
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/potato" opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// clean the cache
|
||||
c.purgeOld(-10 * time.Second)
|
||||
c.purgeEmptyDirs()
|
||||
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
|
||||
// FIXME test directory does not exist
|
||||
// _, err = os.Stat(filepath.Dir(p))
|
||||
// require.True(t, os.IsNotExist(err))
|
||||
}
|
||||
|
||||
func TestCachePurgeOld(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, c.CleanUp()) }()
|
||||
|
||||
// Test funcs
|
||||
var removed []string
|
||||
//removedDir := true
|
||||
removeFile := func(item *Item) {
|
||||
removed = append(removed, item.name)
|
||||
item._remove("TestCachePurgeOld")
|
||||
}
|
||||
// removeDir := func(name string) bool {
|
||||
// if removedDir {
|
||||
// removed = append(removed, filepath.ToSlash(name)+"/")
|
||||
// }
|
||||
// return removedDir
|
||||
// }
|
||||
|
||||
removed = nil
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
// FIXME c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
potato2 := c.Item("sub/dir2/potato2")
|
||||
require.NoError(t, potato2.Open(nil))
|
||||
potato := c.Item("sub/dir/potato")
|
||||
require.NoError(t, potato.Open(nil))
|
||||
require.NoError(t, potato2.Close(nil))
|
||||
require.NoError(t, potato.Open(nil))
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir/potato" opens=2 size=0`,
|
||||
`name="sub/dir2/potato2" opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
removed = nil
|
||||
// removedDir = true
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
// FIXME c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir2/potato2",
|
||||
}, removed)
|
||||
|
||||
require.NoError(t, potato.Close(nil))
|
||||
|
||||
removed = nil
|
||||
// removedDir = true
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
// FIXME c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
require.NoError(t, potato.Close(nil))
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir/potato" opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
removed = nil
|
||||
// removedDir = false
|
||||
c._purgeOld(10*time.Second, removeFile)
|
||||
// FIXME c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir/potato" opens=0 size=0`,
|
||||
}, itemAsString(c))
|
||||
|
||||
removed = nil
|
||||
// removedDir = true
|
||||
c._purgeOld(-10*time.Second, removeFile)
|
||||
// FIXME c._purgeEmptyDirs(removeDir)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir/potato",
|
||||
}, removed)
|
||||
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
}
|
||||
|
||||
func TestCachePurgeOverQuota(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Disable the cache cleaner as it interferes with these tests
|
||||
opt := vfscommon.DefaultOpt
|
||||
opt.CachePollInterval = 0
|
||||
c, err := New(ctx, r.Fremote, &opt)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Test funcs
|
||||
var removed []string
|
||||
remove := func(item *Item) {
|
||||
removed = append(removed, item.name)
|
||||
item._remove("TestCachePurgeOverQuota")
|
||||
}
|
||||
|
||||
removed = nil
|
||||
c._purgeOverQuota(-1, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
removed = nil
|
||||
c._purgeOverQuota(0, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
removed = nil
|
||||
c._purgeOverQuota(1, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
// Make some test files
|
||||
potato := c.Item("sub/dir/potato")
|
||||
require.NoError(t, potato.Open(nil))
|
||||
require.NoError(t, potato.Truncate(5))
|
||||
|
||||
potato2 := c.Item("sub/dir2/potato2")
|
||||
require.NoError(t, potato2.Open(nil))
|
||||
require.NoError(t, potato2.Truncate(6))
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir/potato" opens=1 size=5`,
|
||||
`name="sub/dir2/potato2" opens=1 size=6`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Check nothing removed
|
||||
removed = nil
|
||||
c._purgeOverQuota(1, remove)
|
||||
assert.Equal(t, []string(nil), removed)
|
||||
|
||||
// Close the files
|
||||
require.NoError(t, potato.Close(nil))
|
||||
require.NoError(t, potato2.Close(nil))
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir/potato" opens=0 size=5`,
|
||||
`name="sub/dir2/potato2" opens=0 size=6`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Update the stats to read the total size
|
||||
c.updateUsed()
|
||||
|
||||
// make potato2 definitely after potato
|
||||
t1 := time.Now().Add(10 * time.Second)
|
||||
require.NoError(t, potato2.Truncate(6))
|
||||
potato2.info.ATime = t1
|
||||
|
||||
// Check only potato removed to get below quota
|
||||
removed = nil
|
||||
c._purgeOverQuota(10, remove)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir/potato",
|
||||
}, removed)
|
||||
assert.Equal(t, int64(6), c.used)
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir2/potato2" opens=0 size=6`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Put potato back
|
||||
potato = c.Item("sub/dir/potato")
|
||||
require.NoError(t, potato.Open(nil))
|
||||
require.NoError(t, potato.Truncate(5))
|
||||
require.NoError(t, potato.Close(nil))
|
||||
|
||||
// Update the stats to read the total size
|
||||
c.updateUsed()
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir/potato" opens=0 size=5`,
|
||||
`name="sub/dir2/potato2" opens=0 size=6`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// make potato definitely after potato2
|
||||
t2 := t1.Add(20 * time.Second)
|
||||
require.NoError(t, potato.Truncate(5))
|
||||
potato.info.ATime = t2
|
||||
|
||||
// Check only potato2 removed to get below quota
|
||||
removed = nil
|
||||
c._purgeOverQuota(10, remove)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir2/potato2",
|
||||
}, removed)
|
||||
assert.Equal(t, int64(5), c.used)
|
||||
c.purgeEmptyDirs()
|
||||
|
||||
assert.Equal(t, []string{
|
||||
`name="sub/dir/potato" opens=0 size=5`,
|
||||
}, itemAsString(c))
|
||||
|
||||
// Now purge everything
|
||||
removed = nil
|
||||
c._purgeOverQuota(1, remove)
|
||||
assert.Equal(t, []string{
|
||||
"sub/dir/potato",
|
||||
}, removed)
|
||||
assert.Equal(t, int64(0), c.used)
|
||||
c.purgeEmptyDirs()
|
||||
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
|
||||
// Check nothing left behind
|
||||
c.clean()
|
||||
assert.Equal(t, int64(0), c.used)
|
||||
assert.Equal(t, []string(nil), itemAsString(c))
|
||||
}
|
||||
Reference in New Issue
Block a user