Nick Craig-Wood
0baac15c49
A folder shortcut pointing at one of its own ancestor folders (for example a shortcut to FolderX placed inside FolderX) made rclone recurse forever when dereferencing shortcuts, duplicating the folder contents until the disk was full. Rclone now detects when a folder shortcut targets an ancestor directory using the directory cache, leaves that shortcut out of the listing and logs an ERROR, so the rest of the drive can still be copied. Fixes #7118 Fixes #9565 Closes #9051
815 lines
25 KiB
Go
815 lines
25 KiB
Go
package drive
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
_ "github.com/rclone/rclone/backend/local"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/filter"
|
|
"github.com/rclone/rclone/fs/fserrors"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/rclone/rclone/fs/sync"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/fstest/fstests"
|
|
"github.com/rclone/rclone/lib/dircache"
|
|
"github.com/rclone/rclone/lib/random"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/api/drive/v3"
|
|
"google.golang.org/api/googleapi"
|
|
)
|
|
|
|
func TestDriveScopes(t *testing.T) {
|
|
for _, test := range []struct {
|
|
in string
|
|
want []string
|
|
wantFlag bool
|
|
}{
|
|
{"", []string{
|
|
"https://www.googleapis.com/auth/drive",
|
|
}, false},
|
|
{" drive.file , drive.readonly", []string{
|
|
"https://www.googleapis.com/auth/drive.file",
|
|
"https://www.googleapis.com/auth/drive.readonly",
|
|
}, false},
|
|
{" drive.file , drive.appfolder", []string{
|
|
"https://www.googleapis.com/auth/drive.file",
|
|
"https://www.googleapis.com/auth/drive.appfolder",
|
|
}, true},
|
|
} {
|
|
got := driveScopes(test.in)
|
|
assert.Equal(t, test.want, got, test.in)
|
|
gotFlag := driveScopesContainsAppFolder(got)
|
|
assert.Equal(t, test.wantFlag, gotFlag, test.in)
|
|
}
|
|
}
|
|
|
|
/*
|
|
var additionalMimeTypes = map[string]string{
|
|
"application/vnd.ms-excel.sheet.macroenabled.12": ".xlsm",
|
|
"application/vnd.ms-excel.template.macroenabled.12": ".xltm",
|
|
"application/vnd.ms-powerpoint.presentation.macroenabled.12": ".pptm",
|
|
"application/vnd.ms-powerpoint.slideshow.macroenabled.12": ".ppsm",
|
|
"application/vnd.ms-powerpoint.template.macroenabled.12": ".potm",
|
|
"application/vnd.ms-powerpoint": ".ppt",
|
|
"application/vnd.ms-word.document.macroenabled.12": ".docm",
|
|
"application/vnd.ms-word.template.macroenabled.12": ".dotm",
|
|
"application/vnd.openxmlformats-officedocument.presentationml.template": ".potx",
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.template": ".xltx",
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.template": ".dotx",
|
|
"application/vnd.sun.xml.writer": ".sxw",
|
|
"text/richtext": ".rtf",
|
|
}
|
|
*/
|
|
|
|
// Load the example export formats into exportFormats for testing
|
|
func TestInternalLoadExampleFormats(t *testing.T) {
|
|
fetchFormatsOnce.Do(func() {})
|
|
buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
|
|
var about struct {
|
|
ExportFormats map[string][]string `json:"exportFormats,omitempty"`
|
|
ImportFormats map[string][]string `json:"importFormats,omitempty"`
|
|
}
|
|
require.NoError(t, err)
|
|
require.NoError(t, json.Unmarshal(buf, &about))
|
|
_exportFormats = fixMimeTypeMap(about.ExportFormats)
|
|
_importFormats = fixMimeTypeMap(about.ImportFormats)
|
|
}
|
|
|
|
func TestInternalParseExtensions(t *testing.T) {
|
|
for _, test := range []struct {
|
|
in string
|
|
want []string
|
|
wantErr error
|
|
}{
|
|
{"doc", []string{".doc"}, nil},
|
|
{" docx ,XLSX, pptx,svg,md", []string{".docx", ".xlsx", ".pptx", ".svg", ".md"}, nil},
|
|
{"docx,svg,Docx", []string{".docx", ".svg"}, nil},
|
|
{"docx,potato,docx", []string{".docx"}, errors.New(`couldn't find MIME type for extension ".potato"`)},
|
|
} {
|
|
extensions, _, gotErr := parseExtensions(test.in)
|
|
if test.wantErr == nil {
|
|
assert.NoError(t, gotErr)
|
|
} else {
|
|
assert.EqualError(t, gotErr, test.wantErr.Error())
|
|
}
|
|
assert.Equal(t, test.want, extensions)
|
|
}
|
|
|
|
// Test it is appending
|
|
extensions, _, gotErr := parseExtensions("docx,svg", "docx,svg,xlsx")
|
|
assert.NoError(t, gotErr)
|
|
assert.Equal(t, []string{".docx", ".svg", ".xlsx"}, extensions)
|
|
}
|
|
|
|
func TestInternalFindExportFormat(t *testing.T) {
|
|
ctx := context.Background()
|
|
item := &drive.File{
|
|
Name: "file",
|
|
MimeType: "application/vnd.google-apps.document",
|
|
}
|
|
for _, test := range []struct {
|
|
extensions []string
|
|
wantExtension string
|
|
wantMimeType string
|
|
}{
|
|
{[]string{}, "", ""},
|
|
{[]string{".pdf"}, ".pdf", "application/pdf"},
|
|
{[]string{".pdf", ".rtf", ".xls"}, ".pdf", "application/pdf"},
|
|
{[]string{".xls", ".rtf", ".pdf"}, ".rtf", "application/rtf"},
|
|
{[]string{".xls", ".csv", ".svg"}, "", ""},
|
|
} {
|
|
f := new(Fs)
|
|
f.exportExtensions = test.extensions
|
|
gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(ctx, item)
|
|
assert.Equal(t, test.wantExtension, gotExtension)
|
|
if test.wantExtension != "" {
|
|
assert.Equal(t, item.Name+gotExtension, gotFilename)
|
|
} else {
|
|
assert.Equal(t, "", gotFilename)
|
|
}
|
|
assert.Equal(t, test.wantMimeType, gotMimeType)
|
|
assert.Equal(t, true, gotIsDocument)
|
|
}
|
|
}
|
|
|
|
func TestShortcutLoop(t *testing.T) {
|
|
f := &Fs{}
|
|
f.dirCache = dircache.New("", "root", f)
|
|
// root is FolderX with ID "X"
|
|
f.dirCache.Put("", "X")
|
|
// a real subfolder
|
|
f.dirCache.Put("sub", "sub-id")
|
|
// a nested subfolder reached via a shortcut, composite ID
|
|
f.dirCache.Put("sub/nested", joinID("nested-id", "shortcut-id"))
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
remote string
|
|
targetID string
|
|
want bool
|
|
}{
|
|
{"shortcut to the root itself", "loop", "X", true},
|
|
{"shortcut to the parent", "sub/loop", "sub-id", true},
|
|
{"shortcut to a grandparent", "sub/nested/loop", "X", true},
|
|
{"shortcut to a composite ancestor", "sub/nested/loop", "nested-id", true},
|
|
{"shortcut to an unrelated folder", "sub/loop", "other-id", false},
|
|
{"shortcut to a sibling", "sub/loop", "sub-id2", false},
|
|
} {
|
|
got := f.shortcutLoop(test.remote, test.targetID)
|
|
assert.Equal(t, test.want, got, test.name)
|
|
}
|
|
}
|
|
|
|
func TestMimeTypesToExtension(t *testing.T) {
|
|
for mimeType, extension := range _mimeTypeToExtension {
|
|
extensions, err := mime.ExtensionsByType(mimeType)
|
|
assert.NoError(t, err)
|
|
assert.Contains(t, extensions, extension)
|
|
}
|
|
}
|
|
|
|
func TestExtensionToMimeType(t *testing.T) {
|
|
for mimeType, extension := range _mimeTypeToExtension {
|
|
gotMimeType := mime.TypeByExtension(extension)
|
|
mediatype, _, err := mime.ParseMediaType(gotMimeType)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, mimeType, mediatype)
|
|
}
|
|
}
|
|
|
|
func TestExtensionsForExportFormats(t *testing.T) {
|
|
if _exportFormats == nil {
|
|
t.Error("exportFormats == nil")
|
|
}
|
|
for fromMT, toMTs := range _exportFormats {
|
|
for _, toMT := range toMTs {
|
|
if !isInternalMimeType(toMT) {
|
|
extensions, err := mime.ExtensionsByType(toMT)
|
|
assert.NoError(t, err, "invalid MIME type %q", toMT)
|
|
assert.NotEmpty(t, extensions, "No extension found for %q (from: %q)", fromMT, toMT)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestExtensionsForImportFormats(t *testing.T) {
|
|
t.Skip()
|
|
if _importFormats == nil {
|
|
t.Error("_importFormats == nil")
|
|
}
|
|
for fromMT := range _importFormats {
|
|
if !isInternalMimeType(fromMT) {
|
|
extensions, err := mime.ExtensionsByType(fromMT)
|
|
assert.NoError(t, err, "invalid MIME type %q", fromMT)
|
|
assert.NotEmpty(t, extensions, "No extension found for %q", fromMT)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (f *Fs) InternalTestShouldRetry(t *testing.T) {
|
|
ctx := context.Background()
|
|
gatewayTimeout := googleapi.Error{
|
|
Code: 503,
|
|
}
|
|
timeoutRetry, timeoutError := f.shouldRetry(ctx, &gatewayTimeout)
|
|
assert.True(t, timeoutRetry)
|
|
assert.Equal(t, &gatewayTimeout, timeoutError)
|
|
generic403 := googleapi.Error{
|
|
Code: 403,
|
|
}
|
|
rLEItem := googleapi.ErrorItem{
|
|
Reason: "rateLimitExceeded",
|
|
Message: "User rate limit exceeded.",
|
|
}
|
|
generic403.Errors = append(generic403.Errors, rLEItem)
|
|
oldStopUpload := f.opt.StopOnUploadLimit
|
|
oldStopDownload := f.opt.StopOnDownloadLimit
|
|
f.opt.StopOnUploadLimit = true
|
|
f.opt.StopOnDownloadLimit = true
|
|
defer func() {
|
|
f.opt.StopOnUploadLimit = oldStopUpload
|
|
f.opt.StopOnDownloadLimit = oldStopDownload
|
|
}()
|
|
expectedRLError := fserrors.FatalError(&generic403)
|
|
rateLimitRetry, rateLimitErr := f.shouldRetry(ctx, &generic403)
|
|
assert.False(t, rateLimitRetry)
|
|
assert.Equal(t, rateLimitErr, expectedRLError)
|
|
dQEItem := googleapi.ErrorItem{
|
|
Reason: "downloadQuotaExceeded",
|
|
}
|
|
generic403.Errors[0] = dQEItem
|
|
expectedDQError := fserrors.FatalError(&generic403)
|
|
downloadQuotaRetry, downloadQuotaError := f.shouldRetry(ctx, &generic403)
|
|
assert.False(t, downloadQuotaRetry)
|
|
assert.Equal(t, downloadQuotaError, expectedDQError)
|
|
tDFLEItem := googleapi.ErrorItem{
|
|
Reason: "teamDriveFileLimitExceeded",
|
|
}
|
|
generic403.Errors[0] = tDFLEItem
|
|
expectedTDFLError := fserrors.FatalError(&generic403)
|
|
teamDriveFileLimitRetry, teamDriveFileLimitError := f.shouldRetry(ctx, &generic403)
|
|
assert.False(t, teamDriveFileLimitRetry)
|
|
assert.Equal(t, teamDriveFileLimitError, expectedTDFLError)
|
|
qEItem := googleapi.ErrorItem{
|
|
Reason: "quotaExceeded",
|
|
}
|
|
generic403.Errors[0] = qEItem
|
|
expectedQuotaError := fserrors.FatalError(&generic403)
|
|
quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
|
|
assert.False(t, quotaExceededRetry)
|
|
assert.Equal(t, quotaExceededError, expectedQuotaError)
|
|
|
|
sqEItem := googleapi.ErrorItem{
|
|
Reason: "storageQuotaExceeded",
|
|
}
|
|
generic403.Errors[0] = sqEItem
|
|
expectedStorageQuotaError := fserrors.FatalError(&generic403)
|
|
storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
|
|
assert.False(t, storageQuotaExceededRetry)
|
|
assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
|
|
}
|
|
|
|
func (f *Fs) InternalTestDocumentImport(t *testing.T) {
|
|
oldAllow := f.opt.AllowImportNameChange
|
|
f.opt.AllowImportNameChange = true
|
|
defer func() {
|
|
f.opt.AllowImportNameChange = oldAllow
|
|
}()
|
|
|
|
testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
|
|
require.NoError(t, err)
|
|
|
|
testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
|
|
require.NoError(t, err)
|
|
|
|
_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
|
|
require.NoError(t, err)
|
|
|
|
err = operations.CopyFile(context.Background(), f, testFilesFs, "example2.doc", "example2.doc")
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func (f *Fs) InternalTestDocumentUpdate(t *testing.T) {
|
|
testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
|
|
require.NoError(t, err)
|
|
|
|
testFilesFs, err := fs.NewFs(context.Background(), testFilesPath)
|
|
require.NoError(t, err)
|
|
|
|
_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
|
|
require.NoError(t, err)
|
|
|
|
err = operations.CopyFile(context.Background(), f, testFilesFs, "example2.xlsx", "example1.ods")
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func (f *Fs) InternalTestDocumentExport(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
var err error
|
|
|
|
f.exportExtensions, _, err = parseExtensions("txt")
|
|
require.NoError(t, err)
|
|
|
|
obj, err := f.NewObject(context.Background(), "example2.txt")
|
|
require.NoError(t, err)
|
|
|
|
rc, err := obj.Open(context.Background())
|
|
require.NoError(t, err)
|
|
defer func() { require.NoError(t, rc.Close()) }()
|
|
|
|
_, err = io.Copy(&buf, rc)
|
|
require.NoError(t, err)
|
|
text := buf.String()
|
|
|
|
for _, excerpt := range []string{
|
|
"Lorem ipsum dolor sit amet, consectetur",
|
|
"porta at ultrices in, consectetur at augue.",
|
|
} {
|
|
require.Contains(t, text, excerpt)
|
|
}
|
|
}
|
|
|
|
func (f *Fs) InternalTestDocumentLink(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
var err error
|
|
|
|
f.exportExtensions, _, err = parseExtensions("link.html")
|
|
require.NoError(t, err)
|
|
|
|
obj, err := f.NewObject(context.Background(), "example2.link.html")
|
|
require.NoError(t, err)
|
|
|
|
rc, err := obj.Open(context.Background())
|
|
require.NoError(t, err)
|
|
defer func() { require.NoError(t, rc.Close()) }()
|
|
|
|
_, err = io.Copy(&buf, rc)
|
|
require.NoError(t, err)
|
|
text := buf.String()
|
|
|
|
require.True(t, strings.HasPrefix(text, "<html>"))
|
|
require.True(t, strings.HasSuffix(text, "</html>\n"))
|
|
for _, excerpt := range []string{
|
|
`<meta http-equiv="refresh"`,
|
|
`Loading <a href="`,
|
|
} {
|
|
require.Contains(t, text, excerpt)
|
|
}
|
|
}
|
|
|
|
const (
|
|
// from fstest/fstests/fstests.go
|
|
existingDir = "hello? sausage"
|
|
existingFile = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
|
|
existingSubDir = "êé"
|
|
)
|
|
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/Shortcuts
|
|
func (f *Fs) InternalTestShortcuts(t *testing.T) {
|
|
ctx := context.Background()
|
|
srcObj, err := f.NewObject(ctx, existingFile)
|
|
require.NoError(t, err)
|
|
srcHash, err := srcObj.Hash(ctx, hash.MD5)
|
|
require.NoError(t, err)
|
|
assert.NotEqual(t, "", srcHash)
|
|
t.Run("Errors", func(t *testing.T) {
|
|
_, err := f.makeShortcut(ctx, "", f, "")
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "can't be root")
|
|
|
|
_, err = f.makeShortcut(ctx, "notfound", f, "dst")
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "can't find source")
|
|
|
|
_, err = f.makeShortcut(ctx, existingFile, f, existingFile)
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "not overwriting")
|
|
assert.Contains(t, err.Error(), "existing file")
|
|
|
|
_, err = f.makeShortcut(ctx, existingFile, f, existingDir)
|
|
assert.Error(t, err)
|
|
assert.Contains(t, err.Error(), "not overwriting")
|
|
assert.Contains(t, err.Error(), "existing directory")
|
|
})
|
|
t.Run("File", func(t *testing.T) {
|
|
dstObj, err := f.makeShortcut(ctx, existingFile, f, "shortcut.txt")
|
|
require.NoError(t, err)
|
|
require.NotNil(t, dstObj)
|
|
assert.Equal(t, "shortcut.txt", dstObj.Remote())
|
|
dstHash, err := dstObj.Hash(ctx, hash.MD5)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, srcHash, dstHash)
|
|
require.NoError(t, dstObj.Remove(ctx))
|
|
})
|
|
t.Run("Dir", func(t *testing.T) {
|
|
dstObj, err := f.makeShortcut(ctx, existingDir, f, "shortcutdir")
|
|
require.NoError(t, err)
|
|
require.Nil(t, dstObj)
|
|
entries, err := f.List(ctx, "shortcutdir")
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, len(entries))
|
|
require.Equal(t, "shortcutdir/"+existingSubDir, entries[0].Remote())
|
|
require.NoError(t, f.Rmdir(ctx, "shortcutdir"))
|
|
})
|
|
t.Run("Command", func(t *testing.T) {
|
|
_, err := f.Command(ctx, "shortcut", []string{"one"}, nil)
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "need exactly 2 arguments")
|
|
|
|
_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
|
|
"target": "doesnotexistremote:",
|
|
})
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "couldn't find target")
|
|
|
|
_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
|
|
"target": ".",
|
|
})
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "target is not a drive backend")
|
|
|
|
dstObjI, err := f.Command(ctx, "shortcut", []string{existingFile, "shortcut2.txt"}, map[string]string{
|
|
"target": fs.ConfigString(f),
|
|
})
|
|
require.NoError(t, err)
|
|
dstObj := dstObjI.(*Object)
|
|
assert.Equal(t, "shortcut2.txt", dstObj.Remote())
|
|
dstHash, err := dstObj.Hash(ctx, hash.MD5)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, srcHash, dstHash)
|
|
require.NoError(t, dstObj.Remove(ctx))
|
|
|
|
dstObjI, err = f.Command(ctx, "shortcut", []string{existingFile, "shortcut3.txt"}, nil)
|
|
require.NoError(t, err)
|
|
dstObj = dstObjI.(*Object)
|
|
assert.Equal(t, "shortcut3.txt", dstObj.Remote())
|
|
dstHash, err = dstObj.Hash(ctx, hash.MD5)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, srcHash, dstHash)
|
|
require.NoError(t, dstObj.Remove(ctx))
|
|
})
|
|
}
|
|
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/UnTrash
|
|
func (f *Fs) InternalTestUnTrash(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
// Make some objects, one in a subdir
|
|
contents := random.String(100)
|
|
file1 := fstest.NewItem("trashDir/toBeTrashed", contents, time.Now())
|
|
obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
|
|
file2 := fstest.NewItem("trashDir/subdir/toBeTrashed", contents, time.Now())
|
|
_ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
|
|
|
|
// Check objects
|
|
checkObjects := func() {
|
|
fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{
|
|
file1,
|
|
file2,
|
|
}, []string{
|
|
"trashDir/subdir",
|
|
}, f.Precision())
|
|
}
|
|
checkObjects()
|
|
|
|
// Make sure we are using the trash
|
|
require.Equal(t, true, f.opt.UseTrash)
|
|
|
|
// Remove the object and the dir
|
|
require.NoError(t, obj1.Remove(ctx))
|
|
require.NoError(t, f.Purge(ctx, "trashDir/subdir"))
|
|
|
|
// Check objects gone
|
|
fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{}, []string{}, f.Precision())
|
|
|
|
// Restore the object and directory
|
|
r, err := f.unTrashDir(ctx, "trashDir", true)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, unTrashResult{Errors: 0, Untrashed: 2}, r)
|
|
|
|
// Check objects restored
|
|
checkObjects()
|
|
|
|
// Remove the test dir
|
|
require.NoError(t, f.Purge(ctx, "trashDir"))
|
|
}
|
|
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyOrMoveID
|
|
func (f *Fs) InternalTestCopyOrMoveID(t *testing.T) {
|
|
ctx := context.Background()
|
|
obj, err := f.NewObject(ctx, existingFile)
|
|
require.NoError(t, err)
|
|
o := obj.(*Object)
|
|
|
|
dir := t.TempDir()
|
|
|
|
checkFile := func(name string) {
|
|
filePath := filepath.Join(dir, name)
|
|
fi, err := os.Stat(filePath)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int64(100), fi.Size())
|
|
err = os.Remove(filePath)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
t.Run("BadID", func(t *testing.T) {
|
|
err = f.copyOrMoveID(ctx, "moveid", "ID-NOT-FOUND", dir+"/")
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "couldn't find id")
|
|
})
|
|
|
|
t.Run("Directory", func(t *testing.T) {
|
|
rootID, err := f.dirCache.RootID(ctx, false)
|
|
require.NoError(t, err)
|
|
err = f.copyOrMoveID(ctx, "moveid", rootID, dir+"/")
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "can't moveid directory")
|
|
})
|
|
|
|
t.Run("MoveWithoutDestName", func(t *testing.T) {
|
|
err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/")
|
|
require.NoError(t, err)
|
|
checkFile(path.Base(existingFile))
|
|
})
|
|
|
|
t.Run("CopyWithoutDestName", func(t *testing.T) {
|
|
err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/")
|
|
require.NoError(t, err)
|
|
checkFile(path.Base(existingFile))
|
|
})
|
|
|
|
t.Run("MoveWithDestName", func(t *testing.T) {
|
|
err = f.copyOrMoveID(ctx, "moveid", o.id, dir+"/potato.txt")
|
|
require.NoError(t, err)
|
|
checkFile("potato.txt")
|
|
})
|
|
|
|
t.Run("CopyWithDestName", func(t *testing.T) {
|
|
err = f.copyOrMoveID(ctx, "copyid", o.id, dir+"/potato.txt")
|
|
require.NoError(t, err)
|
|
checkFile("potato.txt")
|
|
})
|
|
}
|
|
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/Query
|
|
func (f *Fs) InternalTestQuery(t *testing.T) {
|
|
ctx := context.Background()
|
|
var err error
|
|
t.Run("BadQuery", func(t *testing.T) {
|
|
_, err = f.query(ctx, "this is a bad query")
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "failed to execute query")
|
|
})
|
|
|
|
t.Run("NoMatch", func(t *testing.T) {
|
|
results, err := f.query(ctx, fmt.Sprintf("name='%s' and name!='%s'", existingSubDir, existingSubDir))
|
|
require.NoError(t, err)
|
|
assert.Len(t, results, 0)
|
|
})
|
|
|
|
t.Run("GoodQuery", func(t *testing.T) {
|
|
pathSegments := strings.Split(existingFile, "/")
|
|
var parent string
|
|
for _, item := range pathSegments {
|
|
// the file name contains ' characters which must be escaped
|
|
escapedItem := f.opt.Enc.FromStandardName(item)
|
|
escapedItem = strings.ReplaceAll(escapedItem, `\`, `\\`)
|
|
escapedItem = strings.ReplaceAll(escapedItem, `'`, `\'`)
|
|
|
|
results, err := f.query(ctx, fmt.Sprintf("%strashed=false and name='%s'", parent, escapedItem))
|
|
require.NoError(t, err)
|
|
require.True(t, len(results) > 0)
|
|
for _, result := range results {
|
|
assert.True(t, len(result.Id) > 0)
|
|
assert.Equal(t, result.Name, item)
|
|
}
|
|
parent = fmt.Sprintf("'%s' in parents and ", results[0].Id)
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
|
|
func (f *Fs) InternalTestAgeQuery(t *testing.T) {
|
|
// Check set up for filtering
|
|
assert.True(t, f.Features().FilterAware)
|
|
|
|
opt := &filter.Options{}
|
|
err := opt.MaxAge.Set("1h")
|
|
assert.NoError(t, err)
|
|
flt, err := filter.NewFilter(opt)
|
|
assert.NoError(t, err)
|
|
|
|
defCtx := context.Background()
|
|
fltCtx := filter.ReplaceConfig(defCtx, flt)
|
|
|
|
testCtx1 := fltCtx
|
|
testCtx2 := filter.SetUseFilter(testCtx1, true)
|
|
testCtx3, testCancel := context.WithCancel(testCtx2)
|
|
testCtx4 := filter.SetUseFilter(testCtx3, false)
|
|
testCancel()
|
|
assert.False(t, filter.GetUseFilter(testCtx1))
|
|
assert.True(t, filter.GetUseFilter(testCtx2))
|
|
assert.True(t, filter.GetUseFilter(testCtx3))
|
|
assert.False(t, filter.GetUseFilter(testCtx4))
|
|
|
|
subRemote := fmt.Sprintf("%s:%s/%s", f.Name(), f.Root(), "agequery-testdir")
|
|
subFsResult, err := fs.NewFs(defCtx, subRemote)
|
|
require.NoError(t, err)
|
|
subFs, isDriveFs := subFsResult.(*Fs)
|
|
require.True(t, isDriveFs)
|
|
|
|
tempDir1 := t.TempDir()
|
|
tempFs1, err := fs.NewFs(defCtx, tempDir1)
|
|
require.NoError(t, err)
|
|
|
|
tempDir2 := t.TempDir()
|
|
tempFs2, err := fs.NewFs(defCtx, tempDir2)
|
|
require.NoError(t, err)
|
|
|
|
file1 := fstest.Item{ModTime: time.Now(), Path: "agequery.txt"}
|
|
_ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true)
|
|
|
|
// validate sync/copy
|
|
const timeQuery = "(modifiedTime >= '"
|
|
|
|
assert.NoError(t, sync.CopyDir(defCtx, subFs, tempFs1, false))
|
|
assert.NotContains(t, subFs.lastQuery, timeQuery)
|
|
|
|
assert.NoError(t, sync.CopyDir(fltCtx, subFs, tempFs1, false))
|
|
assert.Contains(t, subFs.lastQuery, timeQuery)
|
|
|
|
assert.NoError(t, sync.CopyDir(fltCtx, tempFs2, subFs, false))
|
|
assert.Contains(t, subFs.lastQuery, timeQuery)
|
|
|
|
assert.NoError(t, sync.CopyDir(defCtx, tempFs2, subFs, false))
|
|
assert.NotContains(t, subFs.lastQuery, timeQuery)
|
|
|
|
// validate list/walk
|
|
devNull, errOpen := os.OpenFile(os.DevNull, os.O_WRONLY, 0)
|
|
require.NoError(t, errOpen)
|
|
defer func() {
|
|
_ = devNull.Close()
|
|
}()
|
|
|
|
assert.NoError(t, operations.List(defCtx, subFs, devNull))
|
|
assert.NotContains(t, subFs.lastQuery, timeQuery)
|
|
|
|
assert.NoError(t, operations.List(fltCtx, subFs, devNull))
|
|
assert.Contains(t, subFs.lastQuery, timeQuery)
|
|
}
|
|
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/SingleQuoteFolder
|
|
func (f *Fs) InternalTestSingleQuoteFolder(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
// Test various folder names containing single quotes
|
|
for _, name := range []string{
|
|
"'",
|
|
"''",
|
|
"'a'",
|
|
"it's a test",
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
dir := "singleQuoteTest/" + name
|
|
err := f.Mkdir(ctx, dir)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
err := f.Rmdir(ctx, dir)
|
|
assert.NoError(t, err)
|
|
}()
|
|
|
|
entries, err := f.List(ctx, "singleQuoteTest")
|
|
require.NoError(t, err)
|
|
|
|
found := false
|
|
for _, entry := range entries {
|
|
if entry.Remote() == dir {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
assert.True(t, found, "directory %q not found in listing", name)
|
|
})
|
|
}
|
|
|
|
err := f.Rmdir(ctx, "singleQuoteTest")
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/MoveDuplicateParent
|
|
//
|
|
// Check that moving a file removes it from its real parent even when there
|
|
// are two directories with the same name and the directory cache resolves the
|
|
// path to the wrong duplicate. Otherwise the file ends up with two parents
|
|
// which fails on shared drives with a teamDrivesParentLimit error (#9472).
|
|
func (f *Fs) InternalTestMoveDuplicateParent(t *testing.T) {
|
|
ctx := context.Background()
|
|
if f.Features().Move == nil {
|
|
t.Skip("Move not supported")
|
|
}
|
|
|
|
rootID, err := f.dirCache.RootID(ctx, false)
|
|
require.NoError(t, err)
|
|
|
|
const dupName = "dup9472"
|
|
const dstName = "dst9472"
|
|
const fileName = "michael.txt"
|
|
|
|
// Create the first "dup9472" directory - this is the one the directory
|
|
// cache will resolve the path to.
|
|
dirA, err := f.createDir(ctx, rootID, dupName, nil)
|
|
require.NoError(t, err)
|
|
f.dirCache.Put(dupName, dirA.Id)
|
|
|
|
// Create a second directory with the SAME name and parent: a duplicate
|
|
// the directory cache does not know about.
|
|
dirB, err := f.createDir(ctx, rootID, dupName, nil)
|
|
require.NoError(t, err)
|
|
|
|
// Destination directory
|
|
dirDst, err := f.createDir(ctx, rootID, dstName, nil)
|
|
require.NoError(t, err)
|
|
|
|
defer func() {
|
|
_ = f.delete(ctx, dirA.Id, false)
|
|
_ = f.delete(ctx, dirB.Id, false)
|
|
_ = f.delete(ctx, dirDst.Id, false)
|
|
f.dirCache.Flush()
|
|
}()
|
|
|
|
// Upload a file directly into dirB (the duplicate the cache doesn't know).
|
|
createInfo := &drive.File{
|
|
Name: fileName,
|
|
Parents: []string{dirB.Id},
|
|
}
|
|
var fileInfo *drive.File
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
fileInfo, err = f.svc.Files.Create(createInfo).
|
|
Media(strings.NewReader("9472 duplicate parent test")).
|
|
Fields(f.getFileFields(ctx)).
|
|
SupportsAllDrives(true).
|
|
Context(ctx).Do()
|
|
return f.shouldRetry(ctx, err)
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, []string{dirB.Id}, fileInfo.Parents)
|
|
|
|
// Sanity check: the directory cache resolves the path to dirA, not the
|
|
// dirB the file actually lives in.
|
|
gotDirID, err := f.dirCache.FindDir(ctx, dupName, false)
|
|
require.NoError(t, err)
|
|
require.Equal(t, dirA.Id, gotDirID, "directory cache should resolve to the wrong duplicate")
|
|
|
|
// Build the source object pointing at the file in dirB and move it.
|
|
srcObj, err := f.newObjectWithInfo(ctx, dupName+"/"+fileName, fileInfo)
|
|
require.NoError(t, err)
|
|
|
|
dstObj, err := f.Move(ctx, srcObj, dstName+"/"+fileName)
|
|
require.NoError(t, err)
|
|
|
|
// The moved file must have exactly one parent: the destination.
|
|
movedInfo, err := f.getFile(ctx, dstObj.(fs.IDer).ID(), "id,parents")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{dirDst.Id}, movedInfo.Parents)
|
|
}
|
|
|
|
func (f *Fs) InternalTest(t *testing.T) {
|
|
// These tests all depend on each other so run them as nested tests
|
|
t.Run("DocumentImport", func(t *testing.T) {
|
|
f.InternalTestDocumentImport(t)
|
|
t.Run("DocumentUpdate", func(t *testing.T) {
|
|
f.InternalTestDocumentUpdate(t)
|
|
t.Run("DocumentExport", func(t *testing.T) {
|
|
f.InternalTestDocumentExport(t)
|
|
t.Run("DocumentLink", func(t *testing.T) {
|
|
f.InternalTestDocumentLink(t)
|
|
})
|
|
})
|
|
})
|
|
})
|
|
t.Run("Shortcuts", f.InternalTestShortcuts)
|
|
t.Run("UnTrash", f.InternalTestUnTrash)
|
|
t.Run("CopyOrMoveID", f.InternalTestCopyOrMoveID)
|
|
t.Run("Query", f.InternalTestQuery)
|
|
t.Run("AgeQuery", f.InternalTestAgeQuery)
|
|
t.Run("SingleQuoteFolder", f.InternalTestSingleQuoteFolder)
|
|
t.Run("MoveDuplicateParent", f.InternalTestMoveDuplicateParent)
|
|
t.Run("ShouldRetry", f.InternalTestShouldRetry)
|
|
}
|
|
|
|
var _ fstests.InternalTester = (*Fs)(nil)
|