rclone/backend/dropbox/dropbox_internal_test.go

package dropbox

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fstest/fstests"
	"github.com/rclone/rclone/lib/batcher"
	"github.com/rclone/rclone/lib/encoder"
	"github.com/rclone/rclone/lib/pacer"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

type paperMetadataClient struct {
	files.ContextClient
	info *files.FileMetadata
}

func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
	if arg.Path == "document" {
		return c.info, nil
	}
	return nil, files.GetMetadataAPIError{
		APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
		EndpointError: &files.GetMetadataError{
			Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
			Path: &files.LookupError{
				Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
			},
		},
	}
}

func TestInternalGetMetadataCancellation(t *testing.T) {
	requestStarted := make(chan struct{})
	releaseRequest := make(chan struct{})
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		close(requestStarted)
		<-releaseRequest
	}))
	defer server.Close()
	defer close(releaseRequest)

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	f := &Fs{
		srv: files.NewContext(dropbox.Config{
			Client: server.Client(),
			URLGenerator: func(hostType string, namespace string, route string) string {
				return server.URL
			},
		}),
		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
	}

	result := make(chan getMetadataResult, 1)
	go func() {
		result <- f.getMetadata(ctx, "/file")
	}()

	select {
	case <-requestStarted:
	case <-time.After(time.Second):
		t.Fatal("Dropbox request did not start")
	}
	cancel()

	select {
	case res := <-result:
		require.ErrorIs(t, res.err, context.Canceled)
	case <-time.After(time.Second):
		t.Fatal("Dropbox request did not observe cancellation")
	}
}

type changeNotifyClient struct {
	files.ContextClient
	entries []files.IsMetadata
}

func (c changeNotifyClient) ListFolderLongpollContext(context.Context, *files.ListFolderLongpollArg) (*files.ListFolderLongpollResult, error) {
	return files.NewListFolderLongpollResult(true), nil
}

func (c changeNotifyClient) ListFolderContinueContext(context.Context, *files.ListFolderContinueArg) (*files.ListFolderResult, error) {
	return files.NewListFolderResult(c.entries, "next", false), nil
}

func TestTrimPrefixFold(t *testing.T) {
	for _, test := range []struct {
		name   string
		path   string
		prefix string
		want   string
	}{
		{name: "exact casing", path: "/Docs/Sub", prefix: "/Docs/", want: "Sub"},
		{name: "different casing", path: "/docs/Sub", prefix: "/Docs/", want: "Sub"},
		{name: "different UTF-8 lengths", path: "/K/Sub", prefix: "/K/", want: "Sub"},
		{name: "exact root", path: "/docs", prefix: "/Docs/", want: ""},
		{name: "sibling boundary", path: "/Docs2/File", prefix: "/Docs/", want: "/Docs2/File"},
		{name: "root remote", path: "/File", prefix: "/", want: "File"},
	} {
		t.Run(test.name, func(t *testing.T) {
			assert.Equal(t, test.want, trimPrefixFold(test.path, test.prefix))
		})
	}
}

func TestChangeNotifyTrimsRootCaseInsensitively(t *testing.T) {
	ctx := context.Background()
	client := changeNotifyClient{entries: []files.IsMetadata{
		&files.FolderMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub"}},
		&files.FileMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub/File.TXT"}},
		&files.DeletedMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Gone.md"}},
	}}
	f := &Fs{
		ci:             fs.GetConfig(ctx),
		srv:            client,
		svc:            client,
		slashRootSlash: "/Docs/",
		pacer:          fs.NewPacer(ctx, pacer.NewDefault()),
	}

	type notification struct {
		path      string
		entryType fs.EntryType
	}
	var notifications []notification
	cursor, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
		notifications = append(notifications, notification{path: path, entryType: entryType})
	}, "start")
	require.NoError(t, err)
	assert.Equal(t, "next", cursor)
	assert.Equal(t, []notification{
		{path: "Sub", entryType: fs.EntryDirectory},
		{path: "Sub/File.TXT", entryType: fs.EntryObject},
		{path: "Gone.md", entryType: fs.EntryObject},
	}, notifications)
}

func TestInternalCheckPathLength(t *testing.T) {
	rep := func(n int, r rune) (out string) {
		rs := make([]rune, n)
		for i := range rs {
			rs[i] = r
		}
		return string(rs)
	}
	for _, test := range []struct {
		in string
		ok bool
	}{
		{in: "", ok: true},
		{in: rep(maxFileNameLength, 'a'), ok: true},
		{in: rep(maxFileNameLength+1, 'a'), ok: false},
		{in: rep(maxFileNameLength, '£'), ok: true},
		{in: rep(maxFileNameLength+1, '£'), ok: false},
		{in: rep(maxFileNameLength, '☺'), ok: true},
		{in: rep(maxFileNameLength+1, '☺'), ok: false},
		{in: rep(maxFileNameLength, '你'), ok: true},
		{in: rep(maxFileNameLength+1, '你'), ok: false},
		{in: "/ok/ok", ok: true},
		{in: "/ok/" + rep(maxFileNameLength, 'a') + "/ok", ok: true},
		{in: "/ok/" + rep(maxFileNameLength+1, 'a') + "/ok", ok: false},
		{in: "/ok/" + rep(maxFileNameLength, '£') + "/ok", ok: true},
		{in: "/ok/" + rep(maxFileNameLength+1, '£') + "/ok", ok: false},
		{in: "/ok/" + rep(maxFileNameLength, '☺') + "/ok", ok: true},
		{in: "/ok/" + rep(maxFileNameLength+1, '☺') + "/ok", ok: false},
		{in: "/ok/" + rep(maxFileNameLength, '你') + "/ok", ok: true},
		{in: "/ok/" + rep(maxFileNameLength+1, '你') + "/ok", ok: false},
	} {

		err := checkPathLength(test.in)
		assert.Equal(t, test.ok, err == nil, test.in)
	}
}

func TestInternalSharedFolderName(t *testing.T) {
	for _, test := range []struct {
		root string
		want string
	}{
		{root: "", want: ""},
		{root: "SharedFolder", want: "SharedFolder"},
		{root: "SharedFolder/subdir", want: "SharedFolder"},
		{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
		{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
	} {
		assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
	}
}

func TestPaperExportRemote(t *testing.T) {
	ctx := context.Background()
	info := &files.FileMetadata{
		ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
	}
	f := &Fs{
		exportExts: []exportExtension{"md"},
		pacer:      fs.NewPacer(ctx, pacer.NewDefault()),
		srv:        paperMetadataClient{info: info},
	}

	direct, err := f.NewObject(ctx, "document.md")
	require.NoError(t, err)
	assert.Equal(t, "document.md", direct.Remote())

	listed, err := f.newObjectWithInfo(ctx, "document.md", info)
	require.NoError(t, err)
	assert.Equal(t, "document.md.md", listed.Remote())

	legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
	require.NoError(t, err)
	assert.Equal(t, "document.md", legacy.Remote())
}

// uploadSessionClient is a mock files.ContextClient which records the
// chunked upload calls made to it
type uploadSessionClient struct {
	files.ContextClient
	appends      int    // number of UploadSessionAppendV2Context calls
	maxAppends   int    // fail the append after this many calls to stop runaway loops
	bytesWritten int64  // bytes received by UploadSessionAppendV2Context
	finishCalled bool   // set if UploadSessionFinishContext was called
	appended     func() // if set, called after each successful append
}

var errTooManyAppends = errors.New("too many appends - upload looping?")

func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
	return &files.UploadSessionStartResult{SessionId: "session"}, nil
}

func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
	// the real client fails the request if the context is cancelled
	if err := ctx.Err(); err != nil {
		return err
	}
	c.appends++
	if c.appends > c.maxAppends {
		return errTooManyAppends
	}
	n, err := io.Copy(io.Discard, content)
	if err != nil {
		return err
	}
	c.bytesWritten += n
	if c.appended != nil {
		c.appended()
	}
	return nil
}

func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
	c.finishCalled = true
	return &files.FileMetadata{}, nil
}

// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
	ctx := context.Background()
	f := &Fs{
		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
		srv:   srv,
	}
	f.opt.ChunkSize = chunkSize
	batcherOptions := defaultBatcherOptions
	batcherOptions.Mode = "off"
	var err error
	f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
	require.NoError(t, err)
	return f
}

// endlessReader supplies bytes forever
type endlessReader struct{}

func (endlessReader) Read(p []byte) (int, error) {
	for i := range p {
		p[i] = 'x'
	}
	return len(p), nil
}

func TestUploadChunkedEarlyEOF(t *testing.T) {
	ctx := context.Background()

	t.Run("MultiChunk", func(t *testing.T) {
		// The declared size spans 4 chunks but the source ends after 1.5
		client := &uploadSessionClient{maxAppends: 8}
		f := newUploadTestFs(t, client, 100)
		o := &Object{fs: f, remote: "test.bin"}
		_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
		require.Error(t, err)
		assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
		assert.False(t, client.finishCalled, "must not commit a truncated upload")
	})

	t.Run("SingleChunk", func(t *testing.T) {
		// The declared size fits in one chunk but the source ends early
		client := &uploadSessionClient{maxAppends: 8}
		f := newUploadTestFs(t, client, 500)
		o := &Object{fs: f, remote: "test.bin"}
		_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
		require.Error(t, err)
		assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
		assert.False(t, client.finishCalled, "must not commit a truncated upload")
	})

	t.Run("Complete", func(t *testing.T) {
		// A source which supplies exactly the declared size uploads OK
		client := &uploadSessionClient{maxAppends: 8}
		f := newUploadTestFs(t, client, 100)
		o := &Object{fs: f, remote: "test.bin"}
		entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
		require.NoError(t, err)
		require.NotNil(t, entry)
		assert.True(t, client.finishCalled)
		assert.Equal(t, int64(250), client.bytesWritten)
	})
}

func TestUploadChunkedCancel(t *testing.T) {
	// Cancelling the context must stop the upload even though every
	// append is succeeding
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	client := &uploadSessionClient{maxAppends: 8}
	client.appended = func() {
		if client.appends == 2 {
			cancel()
		}
	}
	f := newUploadTestFs(t, client, 100)
	o := &Object{fs: f, remote: "test.bin"}
	_, err := o.uploadChunked(ctx, endlessReader{}, files.NewCommitInfo("/test.bin"), -1)
	require.Error(t, err)
	assert.ErrorIs(t, err, context.Canceled)
	assert.False(t, client.finishCalled)
}

func (f *Fs) importPaperForTest(t *testing.T) {
	content := `# test doc

Lorem ipsum __dolor__ sit amet
[link](http://google.com)
`

	arg := files.PaperCreateArg{
		Path:         f.slashRootSlash + "export.paper",
		ImportFormat: &files.ImportFormat{Tagged: dropbox.Tagged{Tag: files.ImportFormatMarkdown}},
	}
	var err error
	err = f.pacer.Call(func() (bool, error) {
		reader := strings.NewReader(content)
		_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
		return shouldRetry(context.Background(), err)
	})
	require.NoError(t, err)
}

func (f *Fs) InternalTestPaperExport(t *testing.T) {
	ctx := context.Background()
	f.importPaperForTest(t)

	f.exportExts = []exportExtension{"html"}

	obj, err := f.NewObject(ctx, "export.html")
	require.NoError(t, err)

	rc, err := obj.Open(ctx)
	require.NoError(t, err)
	defer func() { require.NoError(t, rc.Close()) }()

	buf, err := io.ReadAll(rc)
	require.NoError(t, err)
	text := string(buf)

	for _, excerpt := range []string{
		"Lorem ipsum",
		"<b>dolor</b>",
		`href="http://google.com"`,
	} {
		require.Contains(t, text, excerpt)
	}
}
func (f *Fs) InternalTest(t *testing.T) {
	t.Run("PaperExport", f.InternalTestPaperExport)
}

var _ fstests.InternalTester = (*Fs)(nil)

// newSharingTestFs builds a minimal *Fs whose sharing client talks to a
// local mock server instead of the real Dropbox API.
func newSharingTestFs(t *testing.T, handler http.HandlerFunc) *Fs {
	t.Helper()
	ts := httptest.NewServer(handler)
	t.Cleanup(ts.Close)

	cfg := dropbox.Config{
		Client: &http.Client{Transport: http.DefaultTransport},
		URLGenerator: func(hostType, namespace, route string) string {
			return fmt.Sprintf("%s/2/%s/%s", ts.URL, namespace, route)
		},
	}

	return &Fs{
		opt: Options{
			Enc: encoder.Base |
				encoder.EncodeBackSlash |
				encoder.EncodeDel |
				encoder.EncodeRightSpace |
				encoder.EncodeInvalidUtf8,
		},
		sharing: sharing.NewContext(cfg),
		pacer:   fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond))),
	}
}

func mustParseDBXTime(t *testing.T, s string) *dropbox.DBXTime {
	t.Helper()
	tm, err := time.Parse(time.RFC3339, s)
	require.NoError(t, err)
	return (*dropbox.DBXTime)(&tm)
}

func receivedFilesHandler(t *testing.T, rawName string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.URL.Path, "list_received_files") {
			http.NotFound(w, r)
			return
		}
		resp := sharing.ListFilesResult{
			Entries: []*sharing.SharedFileMetadata{{
				Id:          "id:123",
				Name:        rawName,
				PreviewUrl:  "https://dropbox.com/preview/123",
				Policy:      &sharing.FolderPolicy{},
				TimeInvited: mustParseDBXTime(t, "2024-01-01T00:00:00Z"),
			}},
		}
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(resp)
	}
}

// TestListReceivedFilesDecodesName confirms received-file names are decoded
// via Enc.ToStandardName before being returned, matching listSharedFolders'
// handling of shared-folder names.
func TestListReceivedFilesDecodesName(t *testing.T) {
	const rawName = "report.txt␠"
	f := newSharingTestFs(t, receivedFilesHandler(t, rawName))

	wantName := f.opt.Enc.ToStandardName(rawName)
	require.NotEqual(t, rawName, wantName)

	var gotNames []string
	err := f.listReceivedFiles(context.Background(), func(entry fs.DirEntry) error {
		gotNames = append(gotNames, entry.Remote())
		return nil
	})
	require.NoError(t, err)
	require.Len(t, gotNames, 1)
	assert.Equal(t, wantName, gotNames[0])
}

// TestFindSharedFileResolvesDecodedName confirms findSharedFile can look up
// a received file by its standard, decoded rclone-visible name.
func TestFindSharedFileResolvesDecodedName(t *testing.T) {
	const rawName = "report.txt␠"
	f := newSharingTestFs(t, receivedFilesHandler(t, rawName))

	encodedName := f.opt.Enc.ToStandardName(rawName)
	require.NotEqual(t, rawName, encodedName)

	o, err := f.findSharedFile(context.Background(), encodedName)
	require.NoError(t, err)
	assert.Equal(t, encodedName, o.remote)
}

// sharedFoldersHandler serves a single shared folder from list_folders.
func sharedFoldersHandler(name, id string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.URL.Path, "list_folders") {
			http.NotFound(w, r)
			return
		}
		resp := sharing.ListFoldersResult{
			Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
		}
		w.Header().Set("Content-Type", "application/json")
		_ = json.NewEncoder(w).Encode(resp)
	}
}

// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
// lookup must match regardless of the case of the requested name.
func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
	f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))

	for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
		id, err := f.findSharedFolder(context.Background(), name)
		require.NoError(t, err, name)
		assert.Equal(t, "folder-id", id, name)
	}

	_, err := f.findSharedFolder(context.Background(), "no-such-folder")
	assert.ErrorIs(t, err, fs.ErrorDirNotFound)
}

// The Dropbox backend advertises CaseInsensitive: true, so a received file
// lookup must match regardless of the case of the requested name.
func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
	f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))

	for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
		o, err := f.findSharedFile(context.Background(), name)
		require.NoError(t, err, name)
		assert.Equal(t, "TestFile.txt", o.remote, name)
	}

	_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
	assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}

func TestListingChangeFromEntry(t *testing.T) {
	f := &Fs{
		slashRootSlash: "/Dropbox/",
		opt:            Options{Enc: encoder.Standard},
	}
	file := &files.FileMetadata{
		Metadata: files.Metadata{
			Name:        "a.txt",
			PathDisplay: "/Dropbox/sub/a.txt",
			PathLower:   "/dropbox/sub/a.txt",
		},
		Size:           42,
		ContentHash:    "abc",
		ClientModified: dropbox.DBXTime(time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC)),
	}
	ch, ok := f.listingChangeFromEntry(file)
	require.True(t, ok)
	assert.Equal(t, "sub/a.txt", ch.Remote)
	assert.Equal(t, int64(42), ch.Size)
	assert.Equal(t, "abc", ch.Hash)
	assert.False(t, ch.Deleted)

	del := &files.DeletedMetadata{
		Metadata: files.Metadata{
			Name:        "sub",
			PathDisplay: "/Dropbox/sub",
			PathLower:   "/dropbox/sub",
		},
	}
	ch, ok = f.listingChangeFromEntry(del)
	require.True(t, ok)
	assert.Equal(t, "sub", ch.Remote)
	assert.True(t, ch.Deleted)
}

func TestReconstructCasedRemotes(t *testing.T) {
	enc := encoder.Standard
	// Files listed before folders; PathDisplay parents wrongly cased.
	items := []listRitem{
		{pathLower: "/photos/sub/a.txt", leaf: "a.txt"},
		{pathLower: "/photos/sub", leaf: "Sub", folder: &files.FolderMetadata{}},
		{pathLower: "/photos/other", leaf: "Other", folder: &files.FolderMetadata{}},
	}
	got, err := reconstructCasedRemotes("", "/photos", enc, items)
	require.NoError(t, err)
	assert.Equal(t, "Sub/a.txt", got["/photos/sub/a.txt"])
	assert.Equal(t, "Sub", got["/photos/sub"])
	assert.Equal(t, "Other", got["/photos/other"])

	_, err = reconstructCasedRemotes("", "/photos", enc, []listRitem{
		{pathLower: "/photos/missing/a.txt", leaf: "a.txt"},
	})
	require.Error(t, err)
	assert.Contains(t, err.Error(), "missing parent")

	_, err = reconstructCasedRemotes("", "/", enc, []listRitem{
		{pathLower: "", leaf: "x"},
	})
	require.Error(t, err)
	assert.Contains(t, err.Error(), "empty path_lower")

	got, err = reconstructCasedRemotes("photos", "/photos", enc, []listRitem{
		{pathLower: "/photos/sub", leaf: "Sub", folder: &files.FolderMetadata{}},
		{pathLower: "/photos/sub/a.txt", leaf: "a.txt"},
	})
	require.NoError(t, err)
	assert.Equal(t, "photos/Sub/a.txt", got["/photos/sub/a.txt"])

	_, err = reconstructCasedRemotes("", "/", enc, []listRitem{
		{pathLower: "/../etc/passwd", leaf: "passwd"},
	})
	require.Error(t, err)
}