f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Break the fs package up into smaller parts.

Browse Source 
The purpose of this is to make it easier to maintain and eventually to
allow the rclone backends to be re-used in other projects without
having to use the rclone configuration system.

The new code layout is documented in CONTRIBUTING.
This commit is contained in:
Nick Craig-Wood
2018-01-15 17:51:14 +00:00
parent 92624bbbf1
commit 11da2a6c9b
183 changed files with 5749 additions and 5063 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/oauthutil/oauthutil.go
+36 -34
View File
@@ -12,6 +12,8 @@ import (
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config"
"github.com/ncw/rclone/fs/fshttp"
"github.com/pkg/errors"
"github.com/skratchdot/open-golang/open"
"golang.org/x/net/context"
@@ -54,7 +56,7 @@ type oldToken struct {
// GetToken returns the token saved in the config file under
// section name.
func GetToken(name string) (*oauth2.Token, error) {
tokenString := fs.ConfigFileGet(name, fs.ConfigToken)
tokenString := config.FileGet(name, config.ConfigToken)
if tokenString == "" {
return nil, errors.New("empty token found - please run rclone config again")
}
@@ -94,9 +96,9 @@ func PutToken(name string, token *oauth2.Token, newSection bool) error {
return err
}
tokenString := string(tokenBytes)
old := fs.ConfigFileGet(name, fs.ConfigToken)
old := config.FileGet(name, config.ConfigToken)
if tokenString != old {
err = fs.ConfigSetValueAndSave(name, fs.ConfigToken, tokenString)
err = config.SetValueAndSave(name, config.ConfigToken, tokenString)
if newSection && err != nil {
fs.Debugf(name, "Added new token to config, still needs to be saved")
} else if err != nil {
@@ -197,31 +199,31 @@ func Context(client *http.Client) context.Context {
// config file if they are not blank.
// If any value is overridden, true is returned.
// the origConfig is copied
func overrideCredentials(name string, origConfig *oauth2.Config) (config *oauth2.Config, changed bool) {
config = new(oauth2.Config)
*config = *origConfig
func overrideCredentials(name string, origConfig *oauth2.Config) (newConfig *oauth2.Config, changed bool) {
newConfig = new(oauth2.Config)
*newConfig = *origConfig
changed = false
ClientID := fs.ConfigFileGet(name, fs.ConfigClientID)
ClientID := config.FileGet(name, config.ConfigClientID)
if ClientID != "" {
config.ClientID = ClientID
newConfig.ClientID = ClientID
changed = true
}
ClientSecret := fs.ConfigFileGet(name, fs.ConfigClientSecret)
ClientSecret := config.FileGet(name, config.ConfigClientSecret)
if ClientSecret != "" {
config.ClientSecret = ClientSecret
newConfig.ClientSecret = ClientSecret
changed = true
}
AuthURL := fs.ConfigFileGet(name, fs.ConfigAuthURL)
AuthURL := config.FileGet(name, config.ConfigAuthURL)
if AuthURL != "" {
config.Endpoint.AuthURL = AuthURL
newConfig.Endpoint.AuthURL = AuthURL
changed = true
}
TokenURL := fs.ConfigFileGet(name, fs.ConfigTokenURL)
TokenURL := config.FileGet(name, config.ConfigTokenURL)
if TokenURL != "" {
config.Endpoint.TokenURL = TokenURL
newConfig.Endpoint.TokenURL = TokenURL
changed = true
}
return config, changed
return newConfig, changed
}
// NewClientWithBaseClient gets a token from the config file and
@@ -252,8 +254,8 @@ func NewClientWithBaseClient(name string, config *oauth2.Config, baseClient *htt
// NewClient gets a token from the config file and configures a Client
// with it. It returns the client and a TokenSource which Invalidate may need to be called on
func NewClient(name string, config *oauth2.Config) (*http.Client, *TokenSource, error) {
return NewClientWithBaseClient(name, config, fs.Config.Client())
func NewClient(name string, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) {
return NewClientWithBaseClient(name, oauthConfig, fshttp.NewClient(fs.Config))
}
// Config does the initial creation of the token
@@ -269,26 +271,26 @@ func ConfigNoOffline(id, name string, config *oauth2.Config, opts ...oauth2.Auth
return doConfig(id, name, config, false, opts)
}
func doConfig(id, name string, config *oauth2.Config, offline bool, opts []oauth2.AuthCodeOption) error {
config, changed := overrideCredentials(name, config)
automatic := fs.ConfigFileGet(name, fs.ConfigAutomatic) != ""
func doConfig(id, name string, oauthConfig *oauth2.Config, offline bool, opts []oauth2.AuthCodeOption) error {
oauthConfig, changed := overrideCredentials(name, oauthConfig)
automatic := config.FileGet(name, config.ConfigAutomatic) != ""
if changed {
fmt.Printf("Make sure your Redirect URL is set to %q in your custom config.\n", config.RedirectURL)
fmt.Printf("Make sure your Redirect URL is set to %q in your custom config.\n", RedirectURL)
}
// See if already have a token
tokenString := fs.ConfigFileGet(name, "token")
tokenString := config.FileGet(name, "token")
if tokenString != "" {
fmt.Printf("Already have a token - refresh?\n")
if !fs.Confirm() {
if !config.Confirm() {
return nil
}
}
// Detect whether we should use internal web server
useWebServer := false
switch config.RedirectURL {
switch RedirectURL {
case RedirectURL, RedirectPublicURL, RedirectLocalhostURL:
useWebServer = true
if automatic {
@@ -297,12 +299,12 @@ func doConfig(id, name string, config *oauth2.Config, offline bool, opts []oauth
fmt.Printf("Use auto config?\n")
fmt.Printf(" * Say Y if not sure\n")
fmt.Printf(" * Say N if you are working on a remote or headless machine\n")
auto := fs.Confirm()
auto := config.Confirm()
if !auto {
fmt.Printf("For this to work, you will need rclone available on a machine that has a web browser available.\n")
fmt.Printf("Execute the following on your machine:\n")
if changed {
fmt.Printf("\trclone authorize %q %q %q\n", id, config.ClientID, config.ClientSecret)
fmt.Printf("\trclone authorize %q %q %q\n", id, oauthConfig.ClientID, oauthConfig.ClientSecret)
} else {
fmt.Printf("\trclone authorize %q\n", id)
}
@@ -310,7 +312,7 @@ func doConfig(id, name string, config *oauth2.Config, offline bool, opts []oauth
code := ""
for code == "" {
fmt.Printf("result> ")
code = strings.TrimSpace(fs.ReadLine())
code = strings.TrimSpace(config.ReadLine())
}
token := &oauth2.Token{}
err := json.Unmarshal([]byte(code), token)
@@ -325,13 +327,13 @@ func doConfig(id, name string, config *oauth2.Config, offline bool, opts []oauth
fmt.Printf("Use auto config?\n")
fmt.Printf(" * Say Y if not sure\n")
fmt.Printf(" * Say N if you are working on a remote or headless machine or Y didn't work\n")
useWebServer = fs.Confirm()
useWebServer = config.Confirm()
}
if useWebServer {
// copy the config and set to use the internal webserver
configCopy := *config
config = &configCopy
config.RedirectURL = RedirectURL
configCopy := *oauthConfig
oauthConfig = &configCopy
oauthConfig.RedirectURL = RedirectURL
}
}
@@ -345,7 +347,7 @@ func doConfig(id, name string, config *oauth2.Config, offline bool, opts []oauth
if offline {
opts = append(opts, oauth2.AccessTypeOffline)
}
authURL := config.AuthCodeURL(state, opts...)
authURL := oauthConfig.AuthCodeURL(state, opts...)
// Prepare webserver
server := authServer{
@@ -378,9 +380,9 @@ func doConfig(id, name string, config *oauth2.Config, offline bool, opts []oauth
} else {
// Read the code, and exchange it for a token.
fmt.Printf("Enter verification code> ")
authCode = fs.ReadLine()
authCode = config.ReadLine()
}
token, err := config.Exchange(oauth2.NoContext, authCode)
token, err := oauthConfig.Exchange(oauth2.NoContext, authCode)
if err != nil {
return errors.Wrap(err, "failed to get token")
}
lib/pacer/pacer.go
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/fserrors"
)
// Pacer state
@@ -339,7 +340,7 @@ func (p *Pacer) call(fn Paced, retries int) (err error) {
fs.Debugf("pacer", "low level retry %d/%d (error %v)", i, retries, err)
}
if retry {
err = fs.RetryError(err)
err = fserrors.RetryError(err)
}
return err
}
lib/pacer/pacer_test.go
+4 -3
View File
@@ -6,6 +6,7 @@ import (
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/fserrors"
"github.com/pkg/errors"
)
@@ -401,7 +402,7 @@ func Test_callRetry(t *testing.T) {
if err == errFoo {
t.Errorf("err didn't want %v got %v", errFoo, err)
}
_, ok := err.(fs.Retrier)
_, ok := err.(fserrors.Retrier)
if !ok {
t.Errorf("didn't return a retry error")
}
@@ -415,7 +416,7 @@ func TestCall(t *testing.T) {
if dp.called != 20 {
t.Errorf("called want %d got %d", 20, dp.called)
}
_, ok := err.(fs.Retrier)
_, ok := err.(fserrors.Retrier)
if !ok {
t.Errorf("didn't return a retry error")
}
@@ -429,7 +430,7 @@ func TestCallNoRetry(t *testing.T) {
if dp.called != 1 {
t.Errorf("called want %d got %d", 1, dp.called)
}
_, ok := err.(fs.Retrier)
_, ok := err.(fserrors.Retrier)
if !ok {
t.Errorf("didn't return a retry error")
}
lib/readers/counting_reader.go
+28
View File
@@ -0,0 +1,28 @@
package readers
import "io"
// NewCountingReader returns a CountingReader, which will read from the given
// reader while keeping track of how many bytes were read.
func NewCountingReader(in io.Reader) *CountingReader {
return &CountingReader{in: in}
}
// CountingReader holds a reader and a read count of how many bytes were read
// so far.
type CountingReader struct {
in io.Reader
read uint64
}
// Read reads from the underlying reader.
func (cr *CountingReader) Read(b []byte) (int, error) {
n, err := cr.in.Read(b)
cr.read += uint64(n)
return n, err
}
// BytesRead returns how many bytes were read from the underlying reader so far.
func (cr *CountingReader) BytesRead() uint64 {
return cr.read
}
lib/readers/readfill.go
+18
View File
@@ -0,0 +1,18 @@
package readers
import "io"
// ReadFill reads as much data from r into buf as it can
//
// It reads until the buffer is full or r.Read returned an error.
//
// This is io.ReadFull but when you just want as much data as
// possible, not an exact size of block.
func ReadFill(r io.Reader, buf []byte) (n int, err error) {
var nn int
for n < len(buf) && err == nil {
nn, err = r.Read(buf[n:])
n += nn
}
return n, err
}
lib/readers/readfill_test.go
+42
View File
@@ -0,0 +1,42 @@
package readers
import (
"io"
"testing"
"github.com/stretchr/testify/assert"
)
type byteReader struct {
c byte
}
func (br *byteReader) Read(p []byte) (n int, err error) {
if br.c == 0 {
err = io.EOF
} else if len(p) >= 1 {
p[0] = br.c
n = 1
br.c--
}
return
}
func TestReadFill(t *testing.T) {
buf := []byte{9, 9, 9, 9, 9}
n, err := ReadFill(&byteReader{0}, buf)
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, n)
assert.Equal(t, []byte{9, 9, 9, 9, 9}, buf)
n, err = ReadFill(&byteReader{3}, buf)
assert.Equal(t, io.EOF, err)
assert.Equal(t, 3, n)
assert.Equal(t, []byte{3, 2, 1, 9, 9}, buf)
n, err = ReadFill(&byteReader{8}, buf)
assert.Equal(t, nil, err)
assert.Equal(t, 5, n)
assert.Equal(t, []byte{8, 7, 6, 5, 4}, buf)
}
lib/readers/repeatable.go
+96
View File
@@ -0,0 +1,96 @@
package readers
import (
"io"
"github.com/pkg/errors"
)
// A RepeatableReader implements the io.ReadSeeker it allow to seek cached data
// back and forth within the reader but will only read data from the internal Reader as necessary
// and will play nicely with the Account and io.LimitedReader to reflect current speed
type RepeatableReader struct {
in io.Reader // Input reader
i int64 // current reading index
b []byte // internal cache buffer
}
var _ io.ReadSeeker = (*RepeatableReader)(nil)
// Seek implements the io.Seeker interface.
// If seek position is passed the cache buffer length the function will return
// the maximum offset that can be used and "fs.RepeatableReader.Seek: offset is unavailable" Error
func (r *RepeatableReader) Seek(offset int64, whence int) (int64, error) {
var abs int64
cacheLen := int64(len(r.b))
switch whence {
case 0: //io.SeekStart
abs = offset
case 1: //io.SeekCurrent
abs = r.i + offset
case 2: //io.SeekEnd
abs = cacheLen + offset
default:
return 0, errors.New("fs.RepeatableReader.Seek: invalid whence")
}
if abs < 0 {
return 0, errors.New("fs.RepeatableReader.Seek: negative position")
}
if abs > cacheLen {
return offset - (abs - cacheLen), errors.New("fs.RepeatableReader.Seek: offset is unavailable")
}
r.i = abs
return abs, nil
}
// Read data from original Reader into bytes
// Data is either served from the underlying Reader or from cache if was already read
func (r *RepeatableReader) Read(b []byte) (n int, err error) {
cacheLen := int64(len(r.b))
if r.i == cacheLen {
n, err = r.in.Read(b)
if n > 0 {
r.b = append(r.b, b[:n]...)
}
} else {
n = copy(b, r.b[r.i:])
}
r.i += int64(n)
return n, err
}
// NewRepeatableReader create new repeatable reader from Reader r
func NewRepeatableReader(r io.Reader) *RepeatableReader {
return &RepeatableReader{in: r}
}
// NewRepeatableReaderSized create new repeatable reader from Reader r
// with an initial buffer of size.
func NewRepeatableReaderSized(r io.Reader, size int) *RepeatableReader {
return &RepeatableReader{
in: r,
b: make([]byte, 0, size),
}
}
// NewRepeatableLimitReader create new repeatable reader from Reader r
// with an initial buffer of size wrapped in a io.LimitReader to read
// only size.
func NewRepeatableLimitReader(r io.Reader, size int) *RepeatableReader {
return NewRepeatableReaderSized(io.LimitReader(r, int64(size)), size)
}
// NewRepeatableReaderBuffer create new repeatable reader from Reader r
// using the buffer passed in.
func NewRepeatableReaderBuffer(r io.Reader, buf []byte) *RepeatableReader {
return &RepeatableReader{
in: r,
b: buf[:0],
}
}
// NewRepeatableLimitReaderBuffer create new repeatable reader from
// Reader r and buf wrapped in a io.LimitReader to read only size.
func NewRepeatableLimitReaderBuffer(r io.Reader, buf []byte, size int64) *RepeatableReader {
return NewRepeatableReaderBuffer(io.LimitReader(r, int64(size)), buf)
}
lib/readers/repeatable_test.go
+100
View File
@@ -0,0 +1,100 @@
package readers
import (
"bytes"
"io"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRepeatableReader(t *testing.T) {
var dst []byte
var n int
var pos int64
var err error
b := []byte("Testbuffer")
buf := bytes.NewBuffer(b)
r := NewRepeatableReader(buf)
dst = make([]byte, 100)
n, err = r.Read(dst)
assert.Nil(t, err)
assert.Equal(t, 10, n)
require.Equal(t, b, dst[0:10])
// Test read EOF
n, err = r.Read(dst)
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, n)
// Test Seek Back to start
dst = make([]byte, 10)
pos, err = r.Seek(0, 0)
assert.Nil(t, err)
require.Equal(t, 0, int(pos))
n, err = r.Read(dst)
assert.Nil(t, err)
assert.Equal(t, 10, n)
require.Equal(t, b, dst)
// Test partial read
buf = bytes.NewBuffer(b)
r = NewRepeatableReader(buf)
dst = make([]byte, 5)
n, err = r.Read(dst)
assert.Nil(t, err)
assert.Equal(t, 5, n)
require.Equal(t, b[0:5], dst)
n, err = r.Read(dst)
assert.Nil(t, err)
assert.Equal(t, 5, n)
require.Equal(t, b[5:], dst)
// Test Seek
buf = bytes.NewBuffer(b)
r = NewRepeatableReader(buf)
// Should not allow seek past cache index
pos, err = r.Seek(5, 1)
assert.NotNil(t, err)
assert.Equal(t, "fs.RepeatableReader.Seek: offset is unavailable", err.Error())
assert.Equal(t, 0, int(pos))
// Should not allow seek to negative position start
pos, err = r.Seek(-1, 1)
assert.NotNil(t, err)
assert.Equal(t, "fs.RepeatableReader.Seek: negative position", err.Error())
assert.Equal(t, 0, int(pos))
// Should not allow seek with invalid whence
pos, err = r.Seek(0, 3)
assert.NotNil(t, err)
assert.Equal(t, "fs.RepeatableReader.Seek: invalid whence", err.Error())
assert.Equal(t, 0, int(pos))
// Should seek from index with io.SeekCurrent(1) whence
dst = make([]byte, 5)
_, _ = r.Read(dst)
pos, err = r.Seek(-3, 1)
assert.Nil(t, err)
require.Equal(t, 2, int(pos))
pos, err = r.Seek(1, 1)
assert.Nil(t, err)
require.Equal(t, 3, int(pos))
// Should seek from cache end with io.SeekEnd(2) whence
pos, err = r.Seek(-3, 2)
assert.Nil(t, err)
require.Equal(t, 2, int(pos))
// Should read from seek postion and past it
dst = make([]byte, 5)
n, err = io.ReadFull(r, dst)
assert.Nil(t, err)
assert.Equal(t, 5, n)
assert.Equal(t, b[2:7], dst)
}