f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Factor new vfs module out of cmd/mountlib

Browse Source 
This is an OS style file system abstraction with directory caching
used in mount, cmount, serve webdav and serve http.
This commit is contained in:
Nick Craig-Wood
2017-11-03 12:59:59 +00:00
parent 6da6b2556b
commit c1aaff220d
20 changed files with 197 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/cmount/fs.go
+44 -44
View File
@@ -12,8 +12,8 @@ import (
"time"
"github.com/billziss-gh/cgofuse/fuse"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
)
@@ -21,7 +21,7 @@ const fhUnset = ^uint64(0)
// FS represents the top level filing system
type FS struct {
FS *mountlib.FS
VFS *vfs.VFS
f fs.Fs
openDirs *openFiles
openFilesWr *openFiles
@@ -32,7 +32,7 @@ type FS struct {
// NewFS makes a new FS
func NewFS(f fs.Fs) *FS {
fsys := &FS{
FS: mountlib.NewFS(f),
VFS: vfs.New(f),
f: f,
openDirs: newOpenFiles(0x01),
openFilesWr: newOpenFiles(0x02),
@@ -45,7 +45,7 @@ func NewFS(f fs.Fs) *FS {
type openFiles struct {
mu sync.Mutex
mark uint8
nodes []mountlib.Noder
nodes []vfs.Noder
}
func newOpenFiles(mark uint8) *openFiles {
@@ -55,11 +55,11 @@ func newOpenFiles(mark uint8) *openFiles {
}
// Open a node returning a file handle
func (of *openFiles) Open(node mountlib.Noder) (fh uint64) {
func (of *openFiles) Open(node vfs.Noder) (fh uint64) {
of.mu.Lock()
defer of.mu.Unlock()
var i int
var oldNode mountlib.Noder
var oldNode vfs.Noder
for i, oldNode = range of.nodes {
if oldNode == nil {
of.nodes[i] = node
@@ -78,7 +78,7 @@ func (of *openFiles) InRange(fh uint64) bool {
}
// get the node for fh, call with the lock held
func (of *openFiles) get(fh uint64) (i int, node mountlib.Noder, errc int) {
func (of *openFiles) get(fh uint64) (i int, node vfs.Noder, errc int) {
receivedMark := uint8(fh)
if receivedMark != of.mark {
fs.Debugf(nil, "Bad file handle: bad mark 0x%X != 0x%X: 0x%X", receivedMark, of.mark, fh)
@@ -99,7 +99,7 @@ func (of *openFiles) get(fh uint64) (i int, node mountlib.Noder, errc int) {
}
// Get the node for the file handle
func (of *openFiles) Get(fh uint64) (node mountlib.Noder, errc int) {
func (of *openFiles) Get(fh uint64) (node vfs.Noder, errc int) {
of.mu.Lock()
_, node, errc = of.get(fh)
of.mu.Unlock()
@@ -118,18 +118,18 @@ func (of *openFiles) Close(fh uint64) (errc int) {
}
// lookup a Node given a path
func (fsys *FS) lookupNode(path string) (node mountlib.Node, errc int) {
node, err := fsys.FS.Lookup(path)
func (fsys *FS) lookupNode(path string) (node vfs.Node, errc int) {
node, err := fsys.VFS.Lookup(path)
return node, translateError(err)
}
// lookup a Dir given a path
func (fsys *FS) lookupDir(path string) (dir *mountlib.Dir, errc int) {
func (fsys *FS) lookupDir(path string) (dir *vfs.Dir, errc int) {
node, errc := fsys.lookupNode(path)
if errc != 0 {
return nil, errc
}
dir, ok := node.(*mountlib.Dir)
dir, ok := node.(*vfs.Dir)
if !ok {
return nil, -fuse.ENOTDIR
}
@@ -137,19 +137,19 @@ func (fsys *FS) lookupDir(path string) (dir *mountlib.Dir, errc int) {
}
// lookup a parent Dir given a path returning the dir and the leaf
func (fsys *FS) lookupParentDir(filePath string) (leaf string, dir *mountlib.Dir, errc int) {
func (fsys *FS) lookupParentDir(filePath string) (leaf string, dir *vfs.Dir, errc int) {
parentDir, leaf := path.Split(filePath)
dir, errc = fsys.lookupDir(parentDir)
return leaf, dir, errc
}
// lookup a File given a path
func (fsys *FS) lookupFile(path string) (file *mountlib.File, errc int) {
func (fsys *FS) lookupFile(path string) (file *vfs.File, errc int) {
node, errc := fsys.lookupNode(path)
if errc != 0 {
return nil, errc
}
file, ok := node.(*mountlib.File)
file, ok := node.(*vfs.File)
if !ok {
return nil, -fuse.EISDIR
}
@@ -170,7 +170,7 @@ func (fsys *FS) getOpenFilesFromFh(fh uint64) (of *openFiles, errc int) {
}
// Get the underlying handle from the file handle
func (fsys *FS) getHandleFromFh(fh uint64) (handle mountlib.Noder, errc int) {
func (fsys *FS) getHandleFromFh(fh uint64) (handle vfs.Noder, errc int) {
of, errc := fsys.getOpenFilesFromFh(fh)
if errc != 0 {
return nil, errc
@@ -179,11 +179,11 @@ func (fsys *FS) getHandleFromFh(fh uint64) (handle mountlib.Noder, errc int) {
}
// get a node from the path or from the fh if not fhUnset
func (fsys *FS) getNode(path string, fh uint64) (node mountlib.Node, errc int) {
func (fsys *FS) getNode(path string, fh uint64) (node vfs.Node, errc int) {
if fh == fhUnset {
node, errc = fsys.lookupNode(path)
} else {
var n mountlib.Noder
var n vfs.Noder
n, errc = fsys.getHandleFromFh(fh)
if errc == 0 {
node = n.Node()
@@ -193,27 +193,27 @@ func (fsys *FS) getNode(path string, fh uint64) (node mountlib.Node, errc int) {
}
// stat fills up the stat block for Node
func (fsys *FS) stat(node mountlib.Node, stat *fuse.Stat_t) (errc int) {
func (fsys *FS) stat(node vfs.Node, stat *fuse.Stat_t) (errc int) {
var Size uint64
var Blocks uint64
var modTime time.Time
var Mode os.FileMode
switch x := node.(type) {
case *mountlib.Dir:
case *vfs.Dir:
modTime = x.ModTime()
Mode = mountlib.DirPerms | fuse.S_IFDIR
case *mountlib.File:
Mode = vfs.DirPerms | fuse.S_IFDIR
case *vfs.File:
modTime = x.ModTime()
Size = uint64(x.Size())
Blocks = (Size + 511) / 512
Mode = mountlib.FilePerms | fuse.S_IFREG
Mode = vfs.FilePerms | fuse.S_IFREG
}
//stat.Dev = 1
stat.Ino = node.Inode() // FIXME do we need to set the inode number?
stat.Mode = uint32(Mode)
stat.Nlink = 1
stat.Uid = mountlib.UID
stat.Gid = mountlib.GID
stat.Uid = vfs.UID
stat.Gid = vfs.GID
//stat.Rdev
stat.Size = int64(Size)
t := fuse.NewTimespec(modTime)
@@ -275,7 +275,7 @@ func (fsys *FS) Readdir(dirPath string,
return errc
}
dir, ok := node.(*mountlib.Dir)
dir, ok := node.(*vfs.Dir)
if !ok {
return -fuse.ENOTDIR
}
@@ -342,7 +342,7 @@ func (fsys *FS) Open(path string, flags int) (errc int, fh uint64) {
}
rdwrMode := flags & fuse.O_ACCMODE
var err error
var handle mountlib.Noder
var handle vfs.Noder
switch {
case rdwrMode == fuse.O_RDONLY:
handle, err = file.OpenRead()
@@ -385,7 +385,7 @@ func (fsys *FS) Truncate(path string, size int64, fh uint64) (errc int) {
if errc != 0 {
return errc
}
file, ok := node.(*mountlib.File)
file, ok := node.(*vfs.File)
if !ok {
return -fuse.EIO
}
@@ -406,7 +406,7 @@ func (fsys *FS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) {
if errc != 0 {
return errc
}
rfh, ok := handle.(*mountlib.ReadFileHandle)
rfh, ok := handle.(*vfs.ReadFileHandle)
if !ok {
// Can only read from read file handle
return -fuse.EIO
@@ -425,7 +425,7 @@ func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) {
if errc != 0 {
return errc
}
wfh, ok := handle.(*mountlib.WriteFileHandle)
wfh, ok := handle.(*vfs.WriteFileHandle)
if !ok {
// Can only write to write file handle
return -fuse.EIO
@@ -446,9 +446,9 @@ func (fsys *FS) Flush(path string, fh uint64) (errc int) {
}
var err error
switch x := handle.(type) {
case *mountlib.ReadFileHandle:
case *vfs.ReadFileHandle:
err = x.Flush()
case *mountlib.WriteFileHandle:
case *vfs.WriteFileHandle:
err = x.Flush()
default:
return -fuse.EIO
@@ -470,9 +470,9 @@ func (fsys *FS) Release(path string, fh uint64) (errc int) {
_ = of.Close(fh)
var err error
switch x := handle.(type) {
case *mountlib.ReadFileHandle:
case *vfs.ReadFileHandle:
err = x.Release()
case *mountlib.WriteFileHandle:
case *vfs.WriteFileHandle:
err = x.Release()
default:
return -fuse.EIO
@@ -540,9 +540,9 @@ func (fsys *FS) Utimens(path string, tmsp []fuse.Timespec) (errc int) {
}
var err error
switch x := node.(type) {
case *mountlib.Dir:
case *vfs.Dir:
err = x.SetModTime(t)
case *mountlib.File:
case *vfs.File:
err = x.SetModTime(t)
}
return translateError(err)
@@ -633,21 +633,21 @@ func translateError(err error) (errc int) {
return 0
}
cause := errors.Cause(err)
if mErr, ok := cause.(mountlib.Error); ok {
if mErr, ok := cause.(vfs.Error); ok {
switch mErr {
case mountlib.OK:
case vfs.OK:
return 0
case mountlib.ENOENT:
case vfs.ENOENT:
return -fuse.ENOENT
case mountlib.ENOTEMPTY:
case vfs.ENOTEMPTY:
return -fuse.ENOTEMPTY
case mountlib.EEXIST:
case vfs.EEXIST:
return -fuse.EEXIST
case mountlib.ESPIPE:
case vfs.ESPIPE:
return -fuse.ESPIPE
case mountlib.EBADF:
case vfs.EBADF:
return -fuse.EBADF
case mountlib.EROFS:
case vfs.EROFS:
return -fuse.EROFS
}
}
cmd/cmount/mount.go
+4 -3
View File
@@ -19,6 +19,7 @@ import (
"github.com/billziss-gh/cgofuse/fuse"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
)
@@ -69,7 +70,7 @@ func mountOptions(device string, mountpoint string) (options []string) {
if mountlib.DefaultPermissions {
options = append(options, "-o", "default_permissions")
}
if mountlib.ReadOnly {
if vfs.ReadOnly {
options = append(options, "-o", "ro")
}
if mountlib.WritebackCache {
@@ -90,7 +91,7 @@ func mountOptions(device string, mountpoint string) (options []string) {
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error, error) {
func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
fs.Debugf(f, "Mounting on %q", mountpoint)
// Check the mountpoint - in Windows the mountpoint musn't exist before the mount
@@ -160,7 +161,7 @@ func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error
found:
}
return fsys.FS, errChan, unmount, nil
return fsys.VFS, errChan, unmount, nil
}
// Mount mounts the remote at mountpoint.
cmd/mount/dir.go
+8 -14
View File
@@ -9,8 +9,8 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
@@ -27,13 +27,7 @@ type DirEntry struct {
// Dir represents a directory entry
type Dir struct {
*mountlib.Dir
// f fs.Fs
// path string
// modTime time.Time
// mu sync.RWMutex // protects the following
// read time.Time // time directory entry last read
// items map[string]*DirEntry
*vfs.Dir
}
// Check interface satsified
@@ -42,9 +36,9 @@ var _ fusefs.Node = (*Dir)(nil)
// Attr updates the attributes of a directory
func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) {
defer fs.Trace(d, "")("attr=%+v, err=%v", a, &err)
a.Gid = mountlib.GID
a.Uid = mountlib.UID
a.Mode = os.ModeDir | mountlib.DirPerms
a.Gid = vfs.GID
a.Uid = vfs.UID
a.Mode = os.ModeDir | vfs.DirPerms
modTime := d.ModTime()
a.Atime = modTime
a.Mtime = modTime
@@ -61,7 +55,7 @@ var _ fusefs.NodeSetattrer = (*Dir)(nil)
// Setattr handles attribute changes from FUSE. Currently supports ModTime only.
func (d *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) {
defer fs.Trace(d, "stat=%+v", req)("err=%v", &err)
if mountlib.NoModTime {
if vfs.NoModTime {
return nil
}
@@ -90,9 +84,9 @@ func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.Lo
return nil, translateError(err)
}
switch x := mnode.(type) {
case *mountlib.File:
case *vfs.File:
return &File{x}, nil
case *mountlib.Dir:
case *vfs.Dir:
return &Dir{x}, nil
}
panic("bad type")
cmd/mount/file.go
+9 -15
View File
@@ -7,21 +7,15 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
// File represents a file
type File struct {
*mountlib.File
// size int64 // size of file - read and written with atomic int64 - must be 64 bit aligned
// d *Dir // parent directory - read only
// mu sync.RWMutex // protects the following
// o fs.Object // NB o may be nil if file is being written
// writers int // number of writers for this file
// pendingModTime time.Time // will be applied once o becomes available, i.e. after file was written
*vfs.File
}
// Check interface satisfied
@@ -33,9 +27,9 @@ func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) {
modTime := f.File.ModTime()
Size := uint64(f.File.Size())
Blocks := (Size + 511) / 512
a.Gid = mountlib.GID
a.Uid = mountlib.UID
a.Mode = mountlib.FilePerms
a.Gid = vfs.GID
a.Uid = vfs.UID
a.Mode = vfs.FilePerms
a.Size = Size
a.Atime = modTime
a.Mtime = modTime
@@ -51,7 +45,7 @@ var _ fusefs.NodeSetattrer = (*File)(nil)
// Setattr handles attribute changes from FUSE. Currently supports ModTime only.
func (f *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) (err error) {
defer fs.Trace(f, "a=%+v", req)("err=%v", &err)
if mountlib.NoModTime {
if vfs.NoModTime {
return nil
}
if req.Valid.MtimeNow() {
@@ -70,15 +64,15 @@ func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
defer fs.Trace(f, "flags=%v", req.Flags)("fh=%v, err=%v", &fh, &err)
switch {
case req.Flags.IsReadOnly():
if mountlib.NoSeek {
if vfs.NoSeek {
resp.Flags |= fuse.OpenNonSeekable
}
var rfh *mountlib.ReadFileHandle
var rfh *vfs.ReadFileHandle
rfh, err = f.File.OpenRead()
fh = &ReadFileHandle{rfh}
case req.Flags.IsWriteOnly() || (req.Flags.IsReadWrite() && (req.Flags&fuse.OpenTruncate) != 0):
resp.Flags |= fuse.OpenNonSeekable
var wfh *mountlib.WriteFileHandle
var wfh *vfs.WriteFileHandle
wfh, err = f.File.OpenWrite()
fh = &WriteFileHandle{wfh}
case req.Flags.IsReadWrite():
cmd/mount/fs.go
+13 -13
View File
@@ -9,15 +9,15 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
// FS represents the top level filing system
type FS struct {
*mountlib.FS
*vfs.VFS
f fs.Fs
}
@@ -27,8 +27,8 @@ var _ fusefs.FS = (*FS)(nil)
// NewFS makes a new FS
func NewFS(f fs.Fs) *FS {
fsys := &FS{
FS: mountlib.NewFS(f),
f: f,
VFS: vfs.New(f),
f: f,
}
return fsys
}
@@ -36,7 +36,7 @@ func NewFS(f fs.Fs) *FS {
// Root returns the root node
func (f *FS) Root() (node fusefs.Node, err error) {
defer fs.Trace("", "")("node=%+v, err=%v", &node, &err)
root, err := f.FS.Root()
root, err := f.VFS.Root()
if err != nil {
return nil, translateError(err)
}
@@ -69,21 +69,21 @@ func translateError(err error) error {
return nil
}
cause := errors.Cause(err)
if mErr, ok := cause.(mountlib.Error); ok {
if mErr, ok := cause.(vfs.Error); ok {
switch mErr {
case mountlib.OK:
case vfs.OK:
return nil
case mountlib.ENOENT:
case vfs.ENOENT:
return fuse.ENOENT
case mountlib.ENOTEMPTY:
case vfs.ENOTEMPTY:
return fuse.Errno(syscall.ENOTEMPTY)
case mountlib.EEXIST:
case vfs.EEXIST:
return fuse.EEXIST
case mountlib.ESPIPE:
case vfs.ESPIPE:
return fuse.Errno(syscall.ESPIPE)
case mountlib.EBADF:
case vfs.EBADF:
return fuse.Errno(syscall.EBADF)
case mountlib.EROFS:
case vfs.EROFS:
return fuse.Errno(syscall.EROFS)
}
}
cmd/mount/mount.go
+4 -3
View File
@@ -13,6 +13,7 @@ import (
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
)
@@ -48,7 +49,7 @@ func mountOptions(device string) (options []fuse.MountOption) {
if mountlib.DefaultPermissions {
options = append(options, fuse.DefaultPermissions())
}
if mountlib.ReadOnly {
if vfs.ReadOnly {
options = append(options, fuse.ReadOnly())
}
if mountlib.WritebackCache {
@@ -69,7 +70,7 @@ func mountOptions(device string) (options []fuse.MountOption) {
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error, error) {
func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
fs.Debugf(f, "Mounting on %q", mountpoint)
c, err := fuse.Mount(mountpoint, mountOptions(f.Name()+":"+f.Root())...)
if err != nil {
@@ -100,7 +101,7 @@ func mount(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error
return fuse.Unmount(mountpoint)
}
return filesys.FS, errChan, unmount, nil
return filesys.VFS, errChan, unmount, nil
}
// Mount mounts the remote at mountpoint.
cmd/mount/read.go
+2 -8
View File
@@ -5,20 +5,14 @@ package mount
import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"golang.org/x/net/context"
)
// ReadFileHandle is an open for read file handle on a File
type ReadFileHandle struct {
*mountlib.ReadFileHandle
// mu sync.Mutex
// closed bool // set if handle has been closed
// r *fs.Account
// o fs.Object
// readCalled bool // set if read has been called
// offset int64
*vfs.ReadFileHandle
}
// Check interface satisfied
cmd/mount/write.go
+2 -2
View File
@@ -7,8 +7,8 @@ import (
"bazil.org/fuse"
fusefs "bazil.org/fuse/fs"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"golang.org/x/net/context"
)
@@ -16,7 +16,7 @@ var errClosedFileHandle = errors.New("Attempt to use closed file handle")
// WriteFileHandle is an open for write handle on a File
type WriteFileHandle struct {
*mountlib.WriteFileHandle
*vfs.WriteFileHandle
}
// Check interface satisfied
cmd/mountlib/createinfo.go
-60
View File
@@ -1,60 +0,0 @@
package mountlib
import (
"time"
"github.com/ncw/rclone/fs"
)
// info to create a new object
type createInfo struct {
f fs.Fs
remote string
}
func newCreateInfo(f fs.Fs, remote string) *createInfo {
return &createInfo{
f: f,
remote: remote,
}
}
// Fs returns read only access to the Fs that this object is part of
func (ci *createInfo) Fs() fs.Info {
return ci.f
}
// String returns the remote path
func (ci *createInfo) String() string {
return ci.remote
}
// Remote returns the remote path
func (ci *createInfo) Remote() string {
return ci.remote
}
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
func (ci *createInfo) Hash(fs.HashType) (string, error) {
return "", fs.ErrHashUnsupported
}
// ModTime returns the modification date of the file
// It should return a best guess if one isn't available
func (ci *createInfo) ModTime() time.Time {
return time.Now()
}
// Size returns the size of the file
func (ci *createInfo) Size() int64 {
// FIXME this means this won't work with all remotes...
return 0
}
// Storable says whether this object can be stored
func (ci *createInfo) Storable() bool {
return true
}
var _ fs.ObjectInfo = (*createInfo)(nil)
cmd/mountlib/dir.go
-496
View File
@@ -1,496 +0,0 @@
package mountlib
import (
"os"
"path"
"sort"
"strings"
"sync"
"time"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
// Dir represents a directory entry
type Dir struct {
fsys *FS
inode uint64 // inode number
f fs.Fs
parent *Dir // parent, nil for root
path string
modTime time.Time
entry fs.Directory
mu sync.Mutex // protects the following
read time.Time // time directory entry last read
items map[string]Node // NB can be nil when directory not read yet
}
func newDir(fsys *FS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir {
return &Dir{
fsys: fsys,
f: f,
parent: parent,
entry: fsDir,
path: fsDir.Remote(),
modTime: fsDir.ModTime(),
inode: NewInode(),
}
}
// String converts it to printablee
func (d *Dir) String() string {
if d == nil {
return "<nil *Dir>"
}
return d.path + "/"
}
// IsFile returns false for Dir - satisfies Node interface
func (d *Dir) IsFile() bool {
return false
}
// IsDir returns true for Dir - satisfies Node interface
func (d *Dir) IsDir() bool {
return true
}
// Mode bits of the directory - satisfies Node interface
func (d *Dir) Mode() (mode os.FileMode) {
return os.ModeDir | 0777
}
// Name (base) of the directory - satisfies Node interface
func (d *Dir) Name() (name string) {
name = path.Base(d.path)
if name == "." {
name = "/"
}
return name
}
// Sys returns underlying data source (can be nil) - satisfies Node interface
func (d *Dir) Sys() interface{} {
return nil
}
// Inode returns the inode number - satisfies Node interface
func (d *Dir) Inode() uint64 {
return d.inode
}
// Node returns the Node assocuated with this - satisfies Noder interface
func (d *Dir) Node() Node {
return d
}
// ForgetAll ensures the directory and all its children are purged
// from the cache.
func (d *Dir) ForgetAll() {
d.ForgetPath("")
}
// ForgetPath clears the cache for itself and all subdirectories if
// they match the given path. The path is specified relative from the
// directory it is called from.
// It is not possible to traverse the directory tree upwards, i.e.
// you cannot clear the cache for the Dir's ancestors or siblings.
func (d *Dir) ForgetPath(relativePath string) {
absPath := path.Join(d.path, relativePath)
if absPath == "." {
absPath = ""
}
d.walk(absPath, func(dir *Dir) {
fs.Debugf(dir.path, "forgetting directory cache")
dir.read = time.Time{}
dir.items = nil
})
}
// walk runs a function on all directories whose path matches
// the given absolute one. It will be called on a directory's
// children first. It will not apply the function to parent
// nodes, regardless of the given path.
func (d *Dir) walk(absPath string, fun func(*Dir)) {
d.mu.Lock()
defer d.mu.Unlock()
if d.items != nil {
for _, node := range d.items {
if dir, ok := node.(*Dir); ok {
dir.walk(absPath, fun)
}
}
}
if d.path == absPath || absPath == "" || strings.HasPrefix(d.path, absPath+"/") {
fun(d)
}
}
// rename should be called after the directory is renamed
//
// Reset the directory to new state, discarding all the objects and
// reading everything again
func (d *Dir) rename(newParent *Dir, fsDir fs.Directory) {
d.ForgetAll()
d.parent = newParent
d.entry = fsDir
d.path = fsDir.Remote()
d.modTime = fsDir.ModTime()
d.read = time.Time{}
}
// addObject adds a new object or directory to the directory
//
// note that we add new objects rather than updating old ones
func (d *Dir) addObject(node Node) {
d.mu.Lock()
if d.items != nil {
d.items[node.Name()] = node
}
d.mu.Unlock()
}
// delObject removes an object from the directory
func (d *Dir) delObject(leaf string) {
d.mu.Lock()
if d.items != nil {
delete(d.items, leaf)
}
d.mu.Unlock()
}
// read the directory and sets d.items - must be called with the lock held
func (d *Dir) _readDir() error {
when := time.Now()
if d.read.IsZero() || d.items == nil {
// fs.Debugf(d.path, "Reading directory")
} else {
age := when.Sub(d.read)
if age < d.fsys.dirCacheTime {
return nil
}
fs.Debugf(d.path, "Re-reading directory (%v old)", age)
}
entries, err := fs.ListDirSorted(d.f, false, d.path)
if err == fs.ErrorDirNotFound {
// We treat directory not found as empty because we
// create directories on the fly
} else if err != nil {
return err
}
// NB when we re-read a directory after its cache has expired
// we drop the old files which should lead to correct
// behaviour but may not be very efficient.
// Keep a note of the previous contents of the directory
oldItems := d.items
// Cache the items by name
d.items = make(map[string]Node, len(entries))
for _, entry := range entries {
switch item := entry.(type) {
case fs.Object:
obj := item
name := path.Base(obj.Remote())
d.items[name] = newFile(d, obj, name)
case fs.Directory:
dir := item
name := path.Base(dir.Remote())
// Use old dir value if it exists
if oldItems != nil {
if oldNode, ok := oldItems[name]; ok {
if oldNode.IsDir() {
d.items[name] = oldNode
continue
}
}
}
d.items[name] = newDir(d.fsys, d.f, d, dir)
default:
err = errors.Errorf("unknown type %T", item)
fs.Errorf(d.path, "readDir error: %v", err)
return err
}
}
d.read = when
return nil
}
// lookup a single item in the directory
//
// returns ENOENT if not found.
func (d *Dir) lookup(leaf string) (Node, error) {
d.mu.Lock()
defer d.mu.Unlock()
err := d._readDir()
if err != nil {
return nil, err
}
item, ok := d.items[leaf]
if !ok {
return nil, ENOENT
}
return item, nil
}
// Check to see if a directory is empty
func (d *Dir) isEmpty() (bool, error) {
d.mu.Lock()
defer d.mu.Unlock()
err := d._readDir()
if err != nil {
return false, err
}
return len(d.items) == 0, nil
}
// ModTime returns the modification time of the directory
func (d *Dir) ModTime() time.Time {
// fs.Debugf(d.path, "Dir.ModTime %v", d.modTime)
return d.modTime
}
// Size of the directory
func (d *Dir) Size() int64 {
return 0
}
// SetModTime sets the modTime for this dir
func (d *Dir) SetModTime(modTime time.Time) error {
if d.fsys.readOnly {
return EROFS
}
d.mu.Lock()
defer d.mu.Unlock()
d.modTime = modTime
return nil
}
// Lookup looks up a specific entry in the receiver.
//
// Lookup should return a Node corresponding to the entry. If the
// name does not exist in the directory, Lookup should return ENOENT.
//
// Lookup need not to handle the names "." and "..".
func (d *Dir) Lookup(name string) (node Node, err error) {
path := path.Join(d.path, name)
// fs.Debugf(path, "Dir.Lookup")
node, err = d.lookup(name)
if err != nil {
if err != ENOENT {
fs.Errorf(path, "Dir.Lookup error: %v", err)
}
return nil, err
}
// fs.Debugf(path, "Dir.Lookup OK")
return node, nil
}
// ReadDirAll reads the contents of the directory sorted
func (d *Dir) ReadDirAll() (items Nodes, err error) {
// fs.Debugf(d.path, "Dir.ReadDirAll")
d.mu.Lock()
defer d.mu.Unlock()
err = d._readDir()
if err != nil {
fs.Debugf(d.path, "Dir.ReadDirAll error: %v", err)
return nil, err
}
for _, item := range d.items {
items = append(items, item)
}
sort.Sort(items)
// fs.Debugf(d.path, "Dir.ReadDirAll OK with %d entries", len(items))
return items, nil
}
// Create makes a new file
func (d *Dir) Create(name string) (*File, *WriteFileHandle, error) {
if d.fsys.readOnly {
return nil, nil, EROFS
}
path := path.Join(d.path, name)
// fs.Debugf(path, "Dir.Create")
src := newCreateInfo(d.f, path)
// This gets added to the directory when the file is written
file := newFile(d, nil, name)
fh, err := newWriteFileHandle(d, file, src)
if err != nil {
fs.Errorf(path, "Dir.Create error: %v", err)
return nil, nil, err
}
// fs.Debugf(path, "Dir.Create OK")
return file, fh, nil
}
// Mkdir creates a new directory
func (d *Dir) Mkdir(name string) (*Dir, error) {
if d.fsys.readOnly {
return nil, EROFS
}
path := path.Join(d.path, name)
// fs.Debugf(path, "Dir.Mkdir")
err := d.f.Mkdir(path)
if err != nil {
fs.Errorf(path, "Dir.Mkdir failed to create directory: %v", err)
return nil, err
}
fsDir := fs.NewDir(path, time.Now())
dir := newDir(d.fsys, d.f, d, fsDir)
d.addObject(dir)
// fs.Debugf(path, "Dir.Mkdir OK")
return dir, nil
}
// Remove the directory
func (d *Dir) Remove() error {
if d.fsys.readOnly {
return EROFS
}
// Check directory is empty first
empty, err := d.isEmpty()
if err != nil {
fs.Errorf(d.path, "Dir.Remove dir error: %v", err)
return err
}
if !empty {
fs.Errorf(d.path, "Dir.Remove not empty")
return ENOTEMPTY
}
// remove directory
err = d.f.Rmdir(d.path)
if err != nil {
fs.Errorf(d.path, "Dir.Remove failed to remove directory: %v", err)
return err
}
// Remove the item from the parent directory listing
if d.parent != nil {
d.parent.delObject(d.Name())
}
return nil
}
// RemoveAll removes the directory and any contents recursively
func (d *Dir) RemoveAll() error {
if d.fsys.readOnly {
return EROFS
}
// Remove contents of the directory
nodes, err := d.ReadDirAll()
if err != nil {
fs.Errorf(d.path, "Dir.RemoveAll failed to read directory: %v", err)
return err
}
for _, node := range nodes {
err = node.RemoveAll()
if err != nil {
fs.Errorf(node.DirEntry(), "Dir.RemoveAll failed to remove: %v", err)
return err
}
}
return d.Remove()
}
// DirEntry returns the underlying fs.DirEntry
func (d *Dir) DirEntry() (entry fs.DirEntry) {
return d.entry
}
// RemoveName removes the entry with the given name from the receiver,
// which must be a directory. The entry to be removed may correspond
// to a file (unlink) or to a directory (rmdir).
func (d *Dir) RemoveName(name string) error {
if d.fsys.readOnly {
return EROFS
}
path := path.Join(d.path, name)
// fs.Debugf(path, "Dir.Remove")
node, err := d.lookup(name)
if err != nil {
fs.Errorf(path, "Dir.Remove error: %v", err)
return err
}
return node.Remove()
}
// Rename the file
func (d *Dir) Rename(oldName, newName string, destDir *Dir) error {
if d.fsys.readOnly {
return EROFS
}
oldPath := path.Join(d.path, oldName)
newPath := path.Join(destDir.path, newName)
// fs.Debugf(oldPath, "Dir.Rename to %q", newPath)
oldNode, err := d.lookup(oldName)
if err != nil {
fs.Errorf(oldPath, "Dir.Rename error: %v", err)
return err
}
switch x := oldNode.DirEntry().(type) {
case fs.Object:
oldObject := x
// FIXME: could Copy then Delete if Move not available
// - though care needed if case insensitive...
doMove := d.f.Features().Move
if doMove == nil {
err := errors.Errorf("Fs %q can't rename files (no Move)", d.f)
fs.Errorf(oldPath, "Dir.Rename error: %v", err)
return err
}
newObject, err := doMove(oldObject, newPath)
if err != nil {
fs.Errorf(oldPath, "Dir.Rename error: %v", err)
return err
}
// Update the node with the new details
if oldNode != nil {
if oldFile, ok := oldNode.(*File); ok {
fs.Debugf(oldNode.DirEntry(), "Updating file with %v %p", newObject, oldFile)
oldFile.rename(destDir, newObject)
}
}
case fs.Directory:
doDirMove := d.f.Features().DirMove
if doDirMove == nil {
err := errors.Errorf("Fs %q can't rename directories (no DirMove)", d.f)
fs.Errorf(oldPath, "Dir.Rename error: %v", err)
return err
}
srcRemote := x.Remote()
dstRemote := newPath
err = doDirMove(d.f, srcRemote, dstRemote)
if err != nil {
fs.Errorf(oldPath, "Dir.Rename error: %v", err)
return err
}
newDir := fs.NewDirCopy(x).SetRemote(newPath)
// Update the node with the new details
if oldNode != nil {
if oldDir, ok := oldNode.(*Dir); ok {
fs.Debugf(oldNode.DirEntry(), "Updating dir with %v %p", newDir, oldDir)
oldDir.rename(destDir, newDir)
}
}
default:
err = errors.Errorf("unknown type %T", oldNode)
fs.Errorf(d.path, "Dir.ReadDirAll error: %v", err)
return err
}
// Show moved - delete from old dir and add to new
d.delObject(oldName)
destDir.addObject(oldNode)
// fs.Debugf(newPath, "Dir.Rename renamed from %q", oldPath)
return nil
}
// Fsync the directory
//
// Note that we don't do anything except return OK
func (d *Dir) Fsync() error {
return nil
}
cmd/mountlib/errors.go
-39
View File
@@ -1,39 +0,0 @@
// Cross platform errors
package mountlib
import "fmt"
// Error describes low level errors in a cross platform way
type Error byte
// NB if changing errors translateError in cmd/mount/fs.go, cmd/cmount/fs.go, cmd/serve/webdav/webdav.go
// Low level errors
const (
OK Error = iota
ENOENT
ENOTEMPTY
EEXIST
ESPIPE
EBADF
EROFS
)
var errorNames = []string{
OK: "Success",
ENOENT: "No such file or directory",
ENOTEMPTY: "Directory not empty",
EEXIST: "File exists",
ESPIPE: "Illegal seek",
EBADF: "Bad file descriptor",
EROFS: "Read only file system",
}
// Error renders the error as a string
func (e Error) Error() string {
if int(e) >= len(errorNames) {
return fmt.Sprintf("Low level error %d", e)
}
return errorNames[e]
}
cmd/mountlib/file.go
-278
View File
@@ -1,278 +0,0 @@
package mountlib
import (
"os"
"path"
"sync"
"sync/atomic"
"time"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
// File represents a file
type File struct {
inode uint64 // inode number
size int64 // size of file - read and written with atomic int64 - must be 64 bit aligned
d *Dir // parent directory - read only
mu sync.RWMutex // protects the following
o fs.Object // NB o may be nil if file is being written
leaf string // leaf name of the object
writers int // number of writers for this file
pendingModTime time.Time // will be applied once o becomes available, i.e. after file was written
}
// newFile creates a new File
func newFile(d *Dir, o fs.Object, leaf string) *File {
return &File{
d: d,
o: o,
leaf: leaf,
inode: NewInode(),
}
}
// String converts it to printable
func (f *File) String() string {
if f == nil {
return "<nil *File>"
}
return path.Join(f.d.path, f.leaf)
}
// IsFile returns true for File - satisfies Node interface
func (f *File) IsFile() bool {
return true
}
// IsDir returns false for File - satisfies Node interface
func (f *File) IsDir() bool {
return false
}
// Mode bits of the file or directory - satisfies Node interface
func (f *File) Mode() (mode os.FileMode) {
return 0666
}
// Name (base) of the directory - satisfies Node interface
func (f *File) Name() (name string) {
return path.Base(f.o.Remote())
}
// Sys returns underlying data source (can be nil) - satisfies Node interface
func (f *File) Sys() interface{} {
return nil
}
// Inode returns the inode number - satisfies Node interface
func (f *File) Inode() uint64 {
return f.inode
}
// Node returns the Node assocuated with this - satisfies Noder interface
func (f *File) Node() Node {
return f
}
// rename should be called to update f.o and f.d after a rename
func (f *File) rename(d *Dir, o fs.Object) {
f.mu.Lock()
f.o = o
f.d = d
f.mu.Unlock()
}
// addWriters increments or decrements the writers
func (f *File) addWriters(n int) {
f.mu.Lock()
f.writers += n
f.mu.Unlock()
}
// ModTime returns the modified time of the file
//
// if NoModTime is set then it returns the mod time of the directory
func (f *File) ModTime() (modTime time.Time) {
f.mu.Lock()
defer f.mu.Unlock()
if !f.d.fsys.noModTime {
// if o is nil it isn't valid yet or there are writers, so return the size so far
if f.o == nil || f.writers != 0 {
if !f.pendingModTime.IsZero() {
return f.pendingModTime
}
} else {
return f.o.ModTime()
}
}
return f.d.modTime
}
// Size of the file
func (f *File) Size() int64 {
f.mu.Lock()
defer f.mu.Unlock()
// if o is nil it isn't valid yet or there are writers, so return the size so far
if f.o == nil || f.writers != 0 {
return atomic.LoadInt64(&f.size)
}
return f.o.Size()
}
// SetModTime sets the modtime for the file
func (f *File) SetModTime(modTime time.Time) error {
if f.d.fsys.readOnly {
return EROFS
}
f.mu.Lock()
defer f.mu.Unlock()
f.pendingModTime = modTime
if f.o != nil {
return f.applyPendingModTime()
}
// queue up for later, hoping f.o becomes available
return nil
}
// call with the mutex held
func (f *File) applyPendingModTime() error {
defer func() { f.pendingModTime = time.Time{} }()
if f.pendingModTime.IsZero() {
return nil
}
if f.o == nil {
return errors.New("Cannot apply ModTime, file object is not available")
}
err := f.o.SetModTime(f.pendingModTime)
switch err {
case nil:
fs.Debugf(f.o, "File.applyPendingModTime OK")
case fs.ErrorCantSetModTime, fs.ErrorCantSetModTimeWithoutDelete:
// do nothing, in order to not break "touch somefile" if it exists already
default:
fs.Errorf(f.o, "File.applyPendingModTime error: %v", err)
return err
}
return nil
}
// Update the size while writing
func (f *File) setSize(n int64) {
atomic.StoreInt64(&f.size, n)
}
// Update the object when written
func (f *File) setObject(o fs.Object) {
f.mu.Lock()
defer f.mu.Unlock()
f.o = o
_ = f.applyPendingModTime()
f.d.addObject(f)
}
// Wait for f.o to become non nil for a short time returning it or an
// error
//
// Call without the mutex held
func (f *File) waitForValidObject() (o fs.Object, err error) {
for i := 0; i < 50; i++ {
f.mu.Lock()
o = f.o
writers := f.writers
f.mu.Unlock()
if o != nil {
return o, nil
}
if writers == 0 {
return nil, errors.New("can't open file - writer failed")
}
time.Sleep(100 * time.Millisecond)
}
return nil, ENOENT
}
// OpenRead open the file for read
func (f *File) OpenRead() (fh *ReadFileHandle, err error) {
// if o is nil it isn't valid yet
o, err := f.waitForValidObject()
if err != nil {
return nil, err
}
// fs.Debugf(o, "File.OpenRead")
fh, err = newReadFileHandle(f, o)
err = errors.Wrap(err, "open for read")
if err != nil {
fs.Errorf(o, "File.OpenRead failed: %v", err)
return nil, err
}
return fh, nil
}
// OpenWrite open the file for write
func (f *File) OpenWrite() (fh *WriteFileHandle, err error) {
if f.d.fsys.readOnly {
return nil, EROFS
}
// if o is nil it isn't valid yet
o, err := f.waitForValidObject()
if err != nil {
return nil, err
}
// fs.Debugf(o, "File.OpenWrite")
src := newCreateInfo(f.d.f, o.Remote())
fh, err = newWriteFileHandle(f.d, f, src)
err = errors.Wrap(err, "open for write")
if err != nil {
fs.Errorf(o, "File.OpenWrite failed: %v", err)
return nil, err
}
return fh, nil
}
// Fsync the file
//
// Note that we don't do anything except return OK
func (f *File) Fsync() error {
return nil
}
// Remove the file
func (f *File) Remove() error {
if f.d.fsys.readOnly {
return EROFS
}
err := f.o.Remove()
if err != nil {
fs.Errorf(f.o, "File.Remove file error: %v", err)
return err
}
// Remove the item from the directory listing
f.d.delObject(f.Name())
return nil
}
// RemoveAll the file - same as remove for files
func (f *File) RemoveAll() error {
return f.Remove()
}
// DirEntry returns the underlying fs.DirEntry
func (f *File) DirEntry() (entry fs.DirEntry) {
return f.o
}
cmd/mountlib/fs.go
-158
View File
@@ -1,158 +0,0 @@
package mountlib
import (
"fmt"
"os"
"strings"
"sync/atomic"
"time"
"github.com/ncw/rclone/fs"
)
// Node represents either a *Dir or a *File
type Node interface {
os.FileInfo
IsFile() bool
Inode() uint64
SetModTime(modTime time.Time) error
Fsync() error
Remove() error
RemoveAll() error
DirEntry() fs.DirEntry
}
var (
_ Node = (*File)(nil)
_ Node = (*Dir)(nil)
)
// Nodes is a slice of Node
type Nodes []Node
// Sort functions
func (ns Nodes) Len() int { return len(ns) }
func (ns Nodes) Swap(i, j int) { ns[i], ns[j] = ns[j], ns[i] }
func (ns Nodes) Less(i, j int) bool { return ns[i].DirEntry().Remote() < ns[j].DirEntry().Remote() }
// Noder represents something which can return a node
type Noder interface {
fmt.Stringer
Node() Node
}
var (
_ Noder = (*File)(nil)
_ Noder = (*Dir)(nil)
_ Noder = (*ReadFileHandle)(nil)
_ Noder = (*WriteFileHandle)(nil)
)
// FS represents the top level filing system
type FS struct {
f fs.Fs
root *Dir
noSeek bool // don't allow seeking if set
noChecksum bool // don't check checksums if set
readOnly bool // if set FS is read only
noModTime bool // don't read mod times for files
dirCacheTime time.Duration // how long to consider directory listing cache valid
}
// NewFS creates a new filing system and root directory
func NewFS(f fs.Fs) *FS {
fsDir := fs.NewDir("", time.Now())
fsys := &FS{
f: f,
}
if NoSeek {
fsys.noSeek = true
}
if NoChecksum {
fsys.noChecksum = true
}
if ReadOnly {
fsys.readOnly = true
}
if NoModTime {
fsys.noModTime = true
}
fsys.dirCacheTime = DirCacheTime
fsys.root = newDir(fsys, f, nil, fsDir)
if PollInterval > 0 {
fsys.PollChanges(PollInterval)
}
return fsys
}
// PollChanges will poll the remote every pollInterval for changes if the remote
// supports it. If a non-polling option is used, the given time interval can be
// ignored
func (fsys *FS) PollChanges(pollInterval time.Duration) *FS {
doDirChangeNotify := fsys.f.Features().DirChangeNotify
if doDirChangeNotify != nil {
doDirChangeNotify(fsys.root.ForgetPath, pollInterval)
}
return fsys
}
// Root returns the root node
func (fsys *FS) Root() (*Dir, error) {
// fs.Debugf(fsys.f, "Root()")
return fsys.root, nil
}
var inodeCount uint64
// NewInode creates a new unique inode number
func NewInode() (inode uint64) {
return atomic.AddUint64(&inodeCount, 1)
}
// Lookup finds the Node by path starting from the root
func (fsys *FS) Lookup(path string) (node Node, err error) {
node = fsys.root
for path != "" {
i := strings.IndexRune(path, '/')
var name string
if i < 0 {
name, path = path, ""
} else {
name, path = path[:i], path[i+1:]
}
if name == "" {
continue
}
dir, ok := node.(*Dir)
if !ok {
// We need to look in a directory, but found a file
return nil, ENOENT
}
node, err = dir.Lookup(name)
if err != nil {
return nil, err
}
}
return
}
// Statfs is called to obtain file system metadata.
// It should write that data to resp.
func (fsys *FS) Statfs() error {
/* FIXME
const blockSize = 4096
const fsBlocks = (1 << 50) / blockSize
resp.Blocks = fsBlocks // Total data blocks in file system.
resp.Bfree = fsBlocks // Free blocks in file system.
resp.Bavail = fsBlocks // Free blocks in file system if you're not root.
resp.Files = 1E9 // Total files in file system.
resp.Ffree = 1E9 // Free files in file system.
resp.Bsize = blockSize // Block size
resp.Namelen = 255 // Maximum file name length?
resp.Frsize = blockSize // Fragment size, smallest addressable data size in the file system.
*/
return nil
}
cmd/mountlib/mount.go
+5 -37
View File
@@ -1,42 +1,25 @@
package mountlib
// Globals
import (
"log"
"os"
"time"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Options set by command line flags
var (
NoModTime = false
NoChecksum = false
DebugFUSE = false
NoSeek = false
DirCacheTime = 5 * 60 * time.Second
PollInterval = time.Minute
// mount options
ReadOnly = false
DebugFUSE = false
AllowNonEmpty = false
AllowRoot = false
AllowOther = false
DefaultPermissions = false
WritebackCache = false
MaxReadAhead fs.SizeSuffix = 128 * 1024
Umask = 0
UID = ^uint32(0) // these values instruct WinFSP-FUSE to use the current user
GID = ^uint32(0) // overriden for non windows in mount_unix.go
// foreground = false
// default permissions for directories - modified by umask in Mount
DirPerms = os.FileMode(0777)
FilePerms = os.FileMode(0666)
ExtraOptions *[]string
ExtraFlags *[]string
ExtraOptions *[]string
ExtraFlags *[]string
)
// NewMountCommand makes a mount command with the given name and Mount function
@@ -149,10 +132,6 @@ like this:
cmd.CheckArgs(2, 2, command, args)
fdst := cmd.NewFsDst(args)
// Mask permissions
DirPerms = 0777 &^ os.FileMode(Umask)
FilePerms = 0666 &^ os.FileMode(Umask)
// Show stats if the user has specifically requested them
if cmd.ShowStats() {
stopStats := cmd.StartStats()
@@ -184,18 +163,7 @@ like this:
//flags.BoolVarP(&foreground, "foreground", "", foreground, "Do not detach.")
// Add in the generic flags
AddFlags(flags)
vfs.AddFlags(flags)
return commandDefintion
}
// AddFlags adds the non filing system specific flags to the command
func AddFlags(flags *pflag.FlagSet) {
flags.BoolVarP(&NoModTime, "no-modtime", "", NoModTime, "Don't read/write the modification time (can speed things up).")
flags.BoolVarP(&NoChecksum, "no-checksum", "", NoChecksum, "Don't compare checksums on up/download.")
flags.BoolVarP(&NoSeek, "no-seek", "", NoSeek, "Don't allow seeking in files.")
flags.DurationVarP(&DirCacheTime, "dir-cache-time", "", DirCacheTime, "Time to cache directory entries for.")
flags.DurationVarP(&PollInterval, "poll-interval", "", PollInterval, "Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable.")
flags.BoolVarP(&ReadOnly, "read-only", "", ReadOnly, "Mount read-only.")
platformFlags(flags)
}
cmd/mountlib/mount_non_unix.go
-11
View File
@@ -1,11 +0,0 @@
// +build !linux,!darwin,!freebsd
package mountlib
import (
"github.com/spf13/pflag"
)
// add any extra platform specific flags
func platformFlags(flags *pflag.FlagSet) {
}
cmd/mountlib/mount_unix.go
-19
View File
@@ -1,19 +0,0 @@
// +build linux darwin freebsd
package mountlib
import (
"github.com/spf13/pflag"
"golang.org/x/sys/unix"
)
// add any extra platform specific flags
func platformFlags(flags *pflag.FlagSet) {
flags.IntVarP(&Umask, "umask", "", Umask, "Override the permission bits set by the filesystem.")
Umask = unix.Umask(0) // read the umask
unix.Umask(Umask) // set it back to what it was
UID = uint32(unix.Geteuid())
GID = uint32(unix.Getegid())
flags.Uint32VarP(&UID, "uid", "", UID, "Override the uid field set by the filesystem.")
flags.Uint32VarP(&GID, "gid", "", GID, "Override the gid field set by the filesystem.")
}
cmd/mountlib/mounttest/dir.go
+1 -1
View File
@@ -176,7 +176,7 @@ func TestDirCacheFlush(t *testing.T) {
err := run.fremote.Mkdir("dir/subdir")
require.NoError(t, err)
root, err := run.filesys.Root()
root, err := run.vfs.Root()
require.NoError(t, err)
// expect newly created "subdir" on remote to not show up
cmd/mountlib/mounttest/fs.go
+7 -7
View File
@@ -15,10 +15,10 @@ import (
"testing"
"time"
"github.com/ncw/rclone/cmd/mountlib"
"github.com/ncw/rclone/fs"
_ "github.com/ncw/rclone/fs/all" // import all the file systems
"github.com/ncw/rclone/fstest"
"github.com/ncw/rclone/vfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -27,7 +27,7 @@ type (
// UnmountFn is called to unmount the file system
UnmountFn func() error
// MountFn is called to mount the file system
MountFn func(f fs.Fs, mountpoint string) (*mountlib.FS, <-chan error, func() error, error)
MountFn func(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error)
)
var (
@@ -46,7 +46,7 @@ func TestMain(m *testing.M, fn MountFn) {
// Run holds the remotes for a test run
type Run struct {
filesys *mountlib.FS
vfs *vfs.VFS
mountPath string
fremote fs.Fs
fremoteName string
@@ -112,7 +112,7 @@ func newRun() *Run {
func (r *Run) mount() {
log.Printf("mount %q %q", r.fremote, r.mountPath)
var err error
r.filesys, r.umountResult, r.umountFn, err = mountFn(r.fremote, r.mountPath)
r.vfs, r.umountResult, r.umountFn, err = mountFn(r.fremote, r.mountPath)
if err != nil {
log.Printf("mount failed: %v", err)
r.skip = true
@@ -207,10 +207,10 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filepath string) {
if fi.IsDir() {
dir[name+"/"] = struct{}{}
r.readLocal(t, dir, name)
assert.Equal(t, mountlib.DirPerms, fi.Mode().Perm())
assert.Equal(t, vfs.DirPerms, fi.Mode().Perm())
} else {
dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{}
assert.Equal(t, mountlib.FilePerms, fi.Mode().Perm())
assert.Equal(t, vfs.FilePerms, fi.Mode().Perm())
}
}
}
@@ -292,5 +292,5 @@ func TestRoot(t *testing.T) {
fi, err := os.Lstat(run.mountPath)
require.NoError(t, err)
assert.True(t, fi.IsDir())
assert.Equal(t, fi.Mode().Perm(), mountlib.DirPerms)
assert.Equal(t, fi.Mode().Perm(), vfs.DirPerms)
}
cmd/mountlib/read.go
-364
View File
@@ -1,364 +0,0 @@
package mountlib
import (
"io"
"sync"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
// ReadFileHandle is an open for read file handle on a File
type ReadFileHandle struct {
mu sync.Mutex
closed bool // set if handle has been closed
r *fs.Account
o fs.Object
readCalled bool // set if read has been called
offset int64 // offset of read of o
roffset int64 // offset of Read() calls
noSeek bool
file *File
hash *fs.MultiHasher
opened bool
}
// Check interfaces
var (
_ io.Reader = (*ReadFileHandle)(nil)
_ io.ReaderAt = (*ReadFileHandle)(nil)
_ io.Seeker = (*ReadFileHandle)(nil)
_ io.Closer = (*ReadFileHandle)(nil)
)
func newReadFileHandle(f *File, o fs.Object) (*ReadFileHandle, error) {
var hash *fs.MultiHasher
var err error
if !f.d.fsys.noChecksum {
hash, err = fs.NewMultiHasherTypes(o.Fs().Hashes())
if err != nil {
fs.Errorf(o.Fs(), "newReadFileHandle hash error: %v", err)
}
}
fh := &ReadFileHandle{
o: o,
noSeek: f.d.fsys.noSeek,
file: f,
hash: hash,
}
fs.Stats.Transferring(fh.o.Remote())
return fh, nil
}
// openPending opens the file if there is a pending open
// call with the lock held
func (fh *ReadFileHandle) openPending() (err error) {
if fh.opened {
return nil
}
r, err := fh.o.Open()
if err != nil {
return err
}
fh.r = fs.NewAccount(r, fh.o).WithBuffer() // account the transfer
fh.opened = true
return nil
}
// String converts it to printable
func (fh *ReadFileHandle) String() string {
if fh == nil {
return "<nil *ReadFileHandle>"
}
if fh.file == nil {
return "<nil *ReadFileHandle.file>"
}
return fh.file.String() + " (r)"
}
// Node returns the Node assocuated with this - satisfies Noder interface
func (fh *ReadFileHandle) Node() Node {
return fh.file
}
// seek to a new offset
//
// if reopen is true, then we won't attempt to use an io.Seeker interface
//
// Must be called with fh.mu held
func (fh *ReadFileHandle) seek(offset int64, reopen bool) (err error) {
if fh.noSeek {
return ESPIPE
}
fh.r.StopBuffering() // stop the background reading first
fh.hash = nil
oldReader := fh.r.GetReader()
r := oldReader
// Can we seek it directly?
if do, ok := oldReader.(io.Seeker); !reopen && ok {
fs.Debugf(fh.o, "ReadFileHandle.seek from %d to %d (io.Seeker)", fh.offset, offset)
_, err = do.Seek(offset, 0)
if err != nil {
fs.Debugf(fh.o, "ReadFileHandle.Read io.Seeker failed: %v", err)
return err
}
} else {
fs.Debugf(fh.o, "ReadFileHandle.seek from %d to %d", fh.offset, offset)
// close old one
err = oldReader.Close()
if err != nil {
fs.Debugf(fh.o, "ReadFileHandle.Read seek close old failed: %v", err)
}
// re-open with a seek
r, err = fh.o.Open(&fs.SeekOption{Offset: offset})
if err != nil {
fs.Debugf(fh.o, "ReadFileHandle.Read seek failed: %v", err)
return err
}
}
fh.r.UpdateReader(r)
fh.offset = offset
return nil
}
// Seek the file
func (fh *ReadFileHandle) Seek(offset int64, whence int) (n int64, err error) {
size := fh.o.Size()
switch whence {
case 0:
fh.roffset = 0
case 2:
fh.roffset = size
}
fh.roffset += offset
// we don't check the offset - the next Read will
return fh.roffset, nil
}
// ReadAt reads len(p) bytes into p starting at offset off in the
// underlying input source. It returns the number of bytes read (0 <=
// n <= len(p)) and any error encountered.
//
// When ReadAt returns n < len(p), it returns a non-nil error
// explaining why more bytes were not returned. In this respect,
// ReadAt is stricter than Read.
//
// Even if ReadAt returns n < len(p), it may use all of p as scratch
// space during the call. If some data is available but not len(p)
// bytes, ReadAt blocks until either all the data is available or an
// error occurs. In this respect ReadAt is different from Read.
//
// If the n = len(p) bytes returned by ReadAt are at the end of the
// input source, ReadAt may return either err == EOF or err == nil.
//
// If ReadAt is reading from an input source with a seek offset,
// ReadAt should not affect nor be affected by the underlying seek
// offset.
//
// Clients of ReadAt can execute parallel ReadAt calls on the same
// input source.
//
// Implementations must not retain p.
func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) {
fh.mu.Lock()
defer fh.mu.Unlock()
err = fh.openPending() // FIXME pending open could be more efficient in the presense of seek (and retried)
if err != nil {
return 0, err
}
// fs.Debugf(fh.o, "ReadFileHandle.Read size %d offset %d", reqSize, off)
if fh.closed {
fs.Errorf(fh.o, "ReadFileHandle.Read error: %v", EBADF)
return 0, EBADF
}
doSeek := off != fh.offset
var newOffset int64
retries := 0
reqSize := len(p)
doReopen := false
for {
if doSeek {
// Are we attempting to seek beyond the end of the
// file - if so just return EOF leaving the underlying
// file in an unchanged state.
if off >= fh.o.Size() {
fs.Debugf(fh.o, "ReadFileHandle.Read attempt to read beyond end of file: %d > %d", off, fh.o.Size())
return 0, nil
}
// Otherwise do the seek
err = fh.seek(off, doReopen)
} else {
err = nil
}
if err == nil {
if reqSize > 0 {
fh.readCalled = true
}
// One exception to the above is if we fail to fully populate a
// page cache page; a read into page cache is always page aligned.
// Make sure we never serve a partial read, to avoid that.
n, err = io.ReadFull(fh.r, p)
newOffset = fh.offset + int64(n)
// if err == nil && rand.Intn(10) == 0 {
// err = errors.New("random error")
// }
if err == nil {
break
} else if (err == io.ErrUnexpectedEOF || err == io.EOF) && newOffset == fh.o.Size() {
// Have read to end of file - reset error
err = nil
break
}
}
if retries >= fs.Config.LowLevelRetries {
break
}
retries++
fs.Errorf(fh.o, "ReadFileHandle.Read error: low level retry %d/%d: %v", retries, fs.Config.LowLevelRetries, err)
doSeek = true
doReopen = true
}
if err != nil {
fs.Errorf(fh.o, "ReadFileHandle.Read error: %v", err)
} else {
fh.offset = newOffset
// fs.Debugf(fh.o, "ReadFileHandle.Read OK")
if fh.hash != nil {
_, err = fh.hash.Write(p[:n])
if err != nil {
fs.Errorf(fh.o, "ReadFileHandle.Read HashError: %v", err)
return 0, err
}
}
}
return n, err
}
func (fh *ReadFileHandle) checkHash() error {
if fh.hash == nil || !fh.readCalled || fh.offset < fh.o.Size() {
return nil
}
for hashType, dstSum := range fh.hash.Sums() {
srcSum, err := fh.o.Hash(hashType)
if err != nil {
return err
}
if !fs.HashEquals(dstSum, srcSum) {
return errors.Errorf("corrupted on transfer: %v hash differ %q vs %q", hashType, dstSum, srcSum)
}
}
return nil
}
// Read reads up to len(p) bytes into p. It returns the number of bytes read (0
// <= n <= len(p)) and any error encountered. Even if Read returns n < len(p),
// it may use all of p as scratch space during the call. If some data is
// available but not len(p) bytes, Read conventionally returns what is
// available instead of waiting for more.
//
// When Read encounters an error or end-of-file condition after successfully
// reading n > 0 bytes, it returns the number of bytes read. It may return the
// (non-nil) error from the same call or return the error (and n == 0) from a
// subsequent call. An instance of this general case is that a Reader returning
// a non-zero number of bytes at the end of the input stream may return either
// err == EOF or err == nil. The next Read should return 0, EOF.
//
// Callers should always process the n > 0 bytes returned before considering
// the error err. Doing so correctly handles I/O errors that happen after
// reading some bytes and also both of the allowed EOF behaviors.
//
// Implementations of Read are discouraged from returning a zero byte count
// with a nil error, except when len(p) == 0. Callers should treat a return of
// 0 and nil as indicating that nothing happened; in particular it does not
// indicate EOF.
//
// Implementations must not retain p.
func (fh *ReadFileHandle) Read(p []byte) (n int, err error) {
if fh.roffset >= fh.o.Size() {
return 0, io.EOF
}
n, err = fh.ReadAt(p, fh.roffset)
fh.roffset += int64(n)
return n, err
}
// close the file handle returning EBADF if it has been
// closed already.
//
// Must be called with fh.mu held
func (fh *ReadFileHandle) close() error {
if fh.closed {
return EBADF
}
fh.closed = true
fs.Stats.DoneTransferring(fh.o.Remote(), true)
if err := fh.checkHash(); err != nil {
return err
}
return fh.r.Close()
}
// Flush is called each time the file or directory is closed.
// Because there can be multiple file descriptors referring to a
// single opened file, Flush can be called multiple times.
func (fh *ReadFileHandle) Flush() error {
fh.mu.Lock()
defer fh.mu.Unlock()
if !fh.opened {
return nil
}
// fs.Debugf(fh.o, "ReadFileHandle.Flush")
if err := fh.checkHash(); err != nil {
fs.Errorf(fh.o, "ReadFileHandle.Flush error: %v", err)
return err
}
// fs.Debugf(fh.o, "ReadFileHandle.Flush OK")
return nil
}
// Release is called when we are finished with the file handle
//
// It isn't called directly from userspace so the error is ignored by
// the kernel
func (fh *ReadFileHandle) Release() error {
fh.mu.Lock()
defer fh.mu.Unlock()
if !fh.opened {
return nil
}
if fh.closed {
fs.Debugf(fh.o, "ReadFileHandle.Release nothing to do")
return nil
}
fs.Debugf(fh.o, "ReadFileHandle.Release closing")
err := fh.close()
if err != nil {
fs.Errorf(fh.o, "ReadFileHandle.Release error: %v", err)
} else {
// fs.Debugf(fh.o, "ReadFileHandle.Release OK")
}
return err
}
// Size returns the size of the underlying file
func (fh *ReadFileHandle) Size() int64 {
return fh.o.Size()
}
// Close closes the file calling Flush then Release
func (fh *ReadFileHandle) Close() error {
err := fh.Flush()
err2 := fh.Release()
if err != nil {
return err
}
return err2
}
cmd/mountlib/write.go
-206
View File
@@ -1,206 +0,0 @@
package mountlib
import (
"io"
"sync"
"time"
"github.com/ncw/rclone/fs"
)
// WriteFileHandle is an open for write handle on a File
type WriteFileHandle struct {
mu sync.Mutex
closed bool // set if handle has been closed
remote string
pipeWriter *io.PipeWriter
o fs.Object
result chan error
file *File
writeCalled bool // set the first time Write() is called
offset int64
}
// Check interfaces
var (
_ io.Writer = (*WriteFileHandle)(nil)
_ io.WriterAt = (*WriteFileHandle)(nil)
_ io.Closer = (*WriteFileHandle)(nil)
)
func newWriteFileHandle(d *Dir, f *File, src fs.ObjectInfo) (*WriteFileHandle, error) {
fh := &WriteFileHandle{
remote: src.Remote(),
result: make(chan error, 1),
file: f,
}
var pipeReader *io.PipeReader
pipeReader, fh.pipeWriter = io.Pipe()
go func() {
o, err := fs.Rcat(d.f, src.Remote(), pipeReader, time.Now())
if err != nil {
fs.Errorf(fh.remote, "WriteFileHandle.New Rcat failed: %v", err)
}
// Close the pipeReader so the pipeWriter fails with ErrClosedPipe
_ = pipeReader.Close()
fh.o = o
fh.result <- err
}()
fh.file.addWriters(1)
fh.file.setSize(0)
return fh, nil
}
// String converts it to printable
func (fh *WriteFileHandle) String() string {
if fh == nil {
return "<nil *WriteFileHandle>"
}
if fh.file == nil {
return "<nil *WriteFileHandle.file>"
}
return fh.file.String() + " (w)"
}
// Node returns the Node assocuated with this - satisfies Noder interface
func (fh *WriteFileHandle) Node() Node {
return fh.file
}
// WriteAt writes len(p) bytes from p to the underlying data stream at offset
// off. It returns the number of bytes written from p (0 <= n <= len(p)) and
// any error encountered that caused the write to stop early. WriteAt must
// return a non-nil error if it returns n < len(p).
//
// If WriteAt is writing to a destination with a seek offset, WriteAt should
// not affect nor be affected by the underlying seek offset.
//
// Clients of WriteAt can execute parallel WriteAt calls on the same
// destination if the ranges do not overlap.
//
// Implementations must not retain p.
func (fh *WriteFileHandle) WriteAt(p []byte, off int64) (n int, err error) {
// fs.Debugf(fh.remote, "WriteFileHandle.Write len=%d", len(p))
fh.mu.Lock()
defer fh.mu.Unlock()
if fh.offset != off {
fs.Errorf(fh.remote, "WriteFileHandle.Write can't seek in file")
return 0, ESPIPE
}
if fh.closed {
fs.Errorf(fh.remote, "WriteFileHandle.Write error: %v", EBADF)
return 0, EBADF
}
fh.writeCalled = true
n, err = fh.pipeWriter.Write(p)
fh.offset += int64(n)
fh.file.setSize(fh.offset)
if err != nil {
fs.Errorf(fh.remote, "WriteFileHandle.Write error: %v", err)
return 0, err
}
// fs.Debugf(fh.remote, "WriteFileHandle.Write OK (%d bytes written)", n)
return n, nil
}
// Write writes len(p) bytes from p to the underlying data stream. It returns
// the number of bytes written from p (0 <= n <= len(p)) and any error
// encountered that caused the write to stop early. Write must return a non-nil
// error if it returns n < len(p). Write must not modify the slice data, even
// temporarily.
//
// Implementations must not retain p.
func (fh *WriteFileHandle) Write(p []byte) (n int, err error) {
// Since we can't seek, just call WriteAt with the current offset
return fh.WriteAt(p, fh.offset)
}
// Offset returns the offset of the file pointer
func (fh *WriteFileHandle) Offset() (offset int64) {
return fh.offset
}
// close the file handle returning EBADF if it has been
// closed already.
//
// Must be called with fh.mu held
func (fh *WriteFileHandle) close() error {
if fh.closed {
return EBADF
}
fh.closed = true
fh.file.addWriters(-1)
writeCloseErr := fh.pipeWriter.Close()
err := <-fh.result
if err == nil {
fh.file.setObject(fh.o)
err = writeCloseErr
}
return err
}
// Flush is called on each close() of a file descriptor. So if a
// filesystem wants to return write errors in close() and the file has
// cached dirty data, this is a good place to write back data and
// return any errors. Since many applications ignore close() errors
// this is not always useful.
//
// NOTE: The flush() method may be called more than once for each
// open(). This happens if more than one file descriptor refers to an
// opened file due to dup(), dup2() or fork() calls. It is not
// possible to determine if a flush is final, so each flush should be
// treated equally. Multiple write-flush sequences are relatively
// rare, so this shouldn't be a problem.
//
// Filesystems shouldn't assume that flush will always be called after
// some writes, or that if will be called at all.
func (fh *WriteFileHandle) Flush() error {
fh.mu.Lock()
defer fh.mu.Unlock()
// fs.Debugf(fh.remote, "WriteFileHandle.Flush")
// If Write hasn't been called then ignore the Flush - Release
// will pick it up
if !fh.writeCalled {
fs.Debugf(fh.remote, "WriteFileHandle.Flush ignoring flush on unwritten handle")
return nil
}
err := fh.close()
if err != nil {
fs.Errorf(fh.remote, "WriteFileHandle.Flush error: %v", err)
} else {
// fs.Debugf(fh.remote, "WriteFileHandle.Flush OK")
}
return err
}
// Release is called when we are finished with the file handle
//
// It isn't called directly from userspace so the error is ignored by
// the kernel
func (fh *WriteFileHandle) Release() error {
fh.mu.Lock()
defer fh.mu.Unlock()
if fh.closed {
fs.Debugf(fh.remote, "WriteFileHandle.Release nothing to do")
return nil
}
fs.Debugf(fh.remote, "WriteFileHandle.Release closing")
err := fh.close()
if err != nil {
fs.Errorf(fh.remote, "WriteFileHandle.Release error: %v", err)
} else {
// fs.Debugf(fh.remote, "WriteFileHandle.Release OK")
}
return err
}
// Close closes the file calling Flush then Release
func (fh *WriteFileHandle) Close() error {
err := fh.Flush()
err2 := fh.Release()
if err != nil {
return err
}
return err2
}