f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add context propagation to rclone

Browse Source 
- Change rclone/fs interfaces to accept context.Context
- Update interface implementations to use context.Context
- Change top level usage to propagate context to lover level functions

Context propagation is needed for stopping transfers and passing other
request-scoped values.
This commit is contained in:
Aleksandar Jankovic
2019-06-19 11:59:46 +01:00
committed by Nick Craig-Wood
parent a2c317b46e
commit f78cd1e043
156 changed files with 2570 additions and 2380 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/object/object.go
+22 -21
View File
@@ -3,6 +3,7 @@ package object
import (
"bytes"
"context"
"errors"
"io"
"io/ioutil"
@@ -43,13 +44,13 @@ type staticObjectInfo struct {
fs fs.Info
}
func (i *staticObjectInfo) Fs() fs.Info { return i.fs }
func (i *staticObjectInfo) Remote() string { return i.remote }
func (i *staticObjectInfo) String() string { return i.remote }
func (i *staticObjectInfo) ModTime() time.Time { return i.modTime }
func (i *staticObjectInfo) Size() int64 { return i.size }
func (i *staticObjectInfo) Storable() bool { return i.storable }
func (i *staticObjectInfo) Hash(h hash.Type) (string, error) {
func (i *staticObjectInfo) Fs() fs.Info { return i.fs }
func (i *staticObjectInfo) Remote() string { return i.remote }
func (i *staticObjectInfo) String() string { return i.remote }
func (i *staticObjectInfo) ModTime(ctx context.Context) time.Time { return i.modTime }
func (i *staticObjectInfo) Size() int64 { return i.size }
func (i *staticObjectInfo) Storable() bool { return i.storable }
func (i *staticObjectInfo) Hash(ctx context.Context, h hash.Type) (string, error) {
if len(i.hashes) == 0 {
return "", hash.ErrUnsupported
}
@@ -92,13 +93,13 @@ func (memoryFs) Features() *fs.Features { return &fs.Features{} }
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (memoryFs) List(dir string) (entries fs.DirEntries, err error) {
func (memoryFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
return nil, nil
}
// NewObject finds the Object at remote. If it can't be found
// it returns the error ErrorObjectNotFound.
func (memoryFs) NewObject(remote string) (fs.Object, error) {
func (memoryFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return nil, fs.ErrorObjectNotFound
}
@@ -107,22 +108,22 @@ func (memoryFs) NewObject(remote string) (fs.Object, error) {
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (memoryFs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
o := NewMemoryObject(src.Remote(), src.ModTime(), nil)
return o, o.Update(in, src, options...)
func (memoryFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
o := NewMemoryObject(src.Remote(), src.ModTime(ctx), nil)
return o, o.Update(ctx, in, src, options...)
}
// Mkdir makes the directory (container, bucket)
//
// Shouldn't return an error if it already exists
func (memoryFs) Mkdir(dir string) error {
func (memoryFs) Mkdir(ctx context.Context, dir string) error {
return errors.New("memoryFs: can't make directory")
}
// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (memoryFs) Rmdir(dir string) error {
func (memoryFs) Rmdir(ctx context.Context, dir string) error {
return fs.ErrorDirNotFound
}
@@ -165,7 +166,7 @@ func (o *MemoryObject) String() string {
}
// ModTime returns the modification date of the file
func (o *MemoryObject) ModTime() time.Time {
func (o *MemoryObject) ModTime(ctx context.Context) time.Time {
return o.modTime
}
@@ -180,7 +181,7 @@ func (o *MemoryObject) Storable() bool {
}
// Hash returns the requested hash of the contents
func (o *MemoryObject) Hash(h hash.Type) (string, error) {
func (o *MemoryObject) Hash(ctx context.Context, h hash.Type) (string, error) {
hash, err := hash.NewMultiHasherTypes(hash.Set(h))
if err != nil {
return "", err
@@ -193,13 +194,13 @@ func (o *MemoryObject) Hash(h hash.Type) (string, error) {
}
// SetModTime sets the metadata on the object to set the modification date
func (o *MemoryObject) SetModTime(modTime time.Time) error {
func (o *MemoryObject) SetModTime(ctx context.Context, modTime time.Time) error {
o.modTime = modTime
return nil
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *MemoryObject) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
func (o *MemoryObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
content := o.content
for _, option := range options {
switch x := option.(type) {
@@ -219,7 +220,7 @@ func (o *MemoryObject) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
// Update in to the object with the modTime given of the given size
//
// This re-uses the internal buffer if at all possible.
func (o *MemoryObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
func (o *MemoryObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
size := src.Size()
if size == 0 {
o.content = nil
@@ -229,11 +230,11 @@ func (o *MemoryObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.Ope
o.content = o.content[:size]
_, err = io.ReadFull(in, o.content)
}
o.modTime = src.ModTime()
o.modTime = src.ModTime(ctx)
return err
}
// Remove this object
func (o *MemoryObject) Remove() error {
func (o *MemoryObject) Remove(ctx context.Context) error {
return errors.New("memoryObject.Remove not supported")
}
fs/object/object_test.go
+26 -25
View File
@@ -2,6 +2,7 @@ package object_test
import (
"bytes"
"context"
"io"
"io/ioutil"
"testing"
@@ -23,26 +24,26 @@ func TestStaticObject(t *testing.T) {
assert.Equal(t, object.MemoryFs, o.Fs())
assert.Equal(t, remote, o.Remote())
assert.Equal(t, remote, o.String())
assert.Equal(t, now, o.ModTime())
assert.Equal(t, now, o.ModTime(context.Background()))
assert.Equal(t, size, o.Size())
assert.Equal(t, true, o.Storable())
Hash, err := o.Hash(hash.MD5)
Hash, err := o.Hash(context.Background(), hash.MD5)
assert.NoError(t, err)
assert.Equal(t, "", Hash)
o = object.NewStaticObjectInfo(remote, now, size, true, nil, nil)
_, err = o.Hash(hash.MD5)
_, err = o.Hash(context.Background(), hash.MD5)
assert.Equal(t, hash.ErrUnsupported, err)
hs := map[hash.Type]string{
hash.MD5: "potato",
}
o = object.NewStaticObjectInfo(remote, now, size, true, hs, nil)
Hash, err = o.Hash(hash.MD5)
Hash, err = o.Hash(context.Background(), hash.MD5)
assert.NoError(t, err)
assert.Equal(t, "potato", Hash)
_, err = o.Hash(hash.SHA1)
_, err = o.Hash(context.Background(), hash.SHA1)
assert.Equal(t, hash.ErrUnsupported, err)
}
@@ -55,27 +56,27 @@ func TestMemoryFs(t *testing.T) {
assert.Equal(t, hash.Supported, f.Hashes())
assert.Equal(t, &fs.Features{}, f.Features())
entries, err := f.List("")
entries, err := f.List(context.Background(), "")
assert.NoError(t, err)
assert.Nil(t, entries)
o, err := f.NewObject("obj")
o, err := f.NewObject(context.Background(), "obj")
assert.Equal(t, fs.ErrorObjectNotFound, err)
assert.Nil(t, o)
buf := bytes.NewBufferString("potato")
now := time.Now()
src := object.NewStaticObjectInfo("remote", now, int64(buf.Len()), true, nil, nil)
o, err = f.Put(buf, src)
o, err = f.Put(context.Background(), buf, src)
assert.NoError(t, err)
hash, err := o.Hash(hash.SHA1)
hash, err := o.Hash(context.Background(), hash.SHA1)
assert.NoError(t, err)
assert.Equal(t, "3e2e95f5ad970eadfa7e17eaf73da97024aa5359", hash)
err = f.Mkdir("dir")
err = f.Mkdir(context.Background(), "dir")
assert.Error(t, err)
err = f.Rmdir("dir")
err = f.Rmdir(context.Background(), "dir")
assert.Equal(t, fs.ErrorDirNotFound, err)
}
@@ -91,22 +92,22 @@ func TestMemoryObject(t *testing.T) {
assert.Equal(t, object.MemoryFs, o.Fs())
assert.Equal(t, remote, o.Remote())
assert.Equal(t, remote, o.String())
assert.Equal(t, now, o.ModTime())
assert.Equal(t, now, o.ModTime(context.Background()))
assert.Equal(t, int64(len(content)), o.Size())
assert.Equal(t, true, o.Storable())
Hash, err := o.Hash(hash.MD5)
Hash, err := o.Hash(context.Background(), hash.MD5)
assert.NoError(t, err)
assert.Equal(t, "8ee2027983915ec78acc45027d874316", Hash)
Hash, err = o.Hash(hash.SHA1)
Hash, err = o.Hash(context.Background(), hash.SHA1)
assert.NoError(t, err)
assert.Equal(t, "3e2e95f5ad970eadfa7e17eaf73da97024aa5359", Hash)
newNow := now.Add(time.Minute)
err = o.SetModTime(newNow)
err = o.SetModTime(context.Background(), newNow)
assert.NoError(t, err)
assert.Equal(t, newNow, o.ModTime())
assert.Equal(t, newNow, o.ModTime(context.Background()))
checkOpen := func(rc io.ReadCloser, expected string) {
actual, err := ioutil.ReadAll(rc)
@@ -117,18 +118,18 @@ func TestMemoryObject(t *testing.T) {
}
checkContent := func(o fs.Object, expected string) {
rc, err := o.Open()
rc, err := o.Open(context.Background())
assert.NoError(t, err)
checkOpen(rc, expected)
}
checkContent(o, string(content))
rc, err := o.Open(&fs.RangeOption{Start: 1, End: 3})
rc, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 3})
assert.NoError(t, err)
checkOpen(rc, "ot")
rc, err = o.Open(&fs.SeekOption{Offset: 3})
rc, err = o.Open(context.Background(), &fs.SeekOption{Offset: 3})
assert.NoError(t, err)
checkOpen(rc, "ato")
@@ -137,10 +138,10 @@ func TestMemoryObject(t *testing.T) {
newContent := bytes.NewBufferString("Rutabaga")
assert.True(t, newContent.Len() < cap(content)) // fits within cap(content)
src := object.NewStaticObjectInfo(remote, newNow, int64(newContent.Len()), true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, "Rutabaga")
assert.Equal(t, newNow, o.ModTime())
assert.Equal(t, newNow, o.ModTime(context.Background()))
assert.Equal(t, "Rutaba", string(content)) // check we re-used the buffer
// not within the buffer
@@ -149,7 +150,7 @@ func TestMemoryObject(t *testing.T) {
newContent = bytes.NewBufferString(newStr)
assert.True(t, newContent.Len() > cap(content)) // does not fit within cap(content)
src = object.NewStaticObjectInfo(remote, newNow, int64(newContent.Len()), true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, newStr)
assert.Equal(t, "Rutaba", string(content)) // check we didn't re-use the buffer
@@ -158,7 +159,7 @@ func TestMemoryObject(t *testing.T) {
newStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
newContent = bytes.NewBufferString(newStr)
src = object.NewStaticObjectInfo(remote, newNow, -1, true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, newStr)
@@ -166,10 +167,10 @@ func TestMemoryObject(t *testing.T) {
newStr = ""
newContent = bytes.NewBufferString(newStr)
src = object.NewStaticObjectInfo(remote, newNow, 0, true, nil, nil)
err = o.Update(newContent, src)
err = o.Update(context.Background(), newContent, src)
assert.NoError(t, err)
checkContent(o, newStr)
err = o.Remove()
err = o.Remove(context.Background())
assert.Error(t, err)
}