Nick Craig-Wood
bff17664ad
A source which returned EOF before supplying as many bytes as it declared would either commit a truncated file (if the shortfall was within the final chunk) or loop forever appending empty chunks to the upload session. Return an error wrapping io.ErrUnexpectedEOF instead. Note that all dropbox uploads use the chunked upload path with the default batch_mode of sync, so this affected uploads of every size.
288 lines
8.8 KiB
Go
288 lines
8.8 KiB
Go
package dropbox
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
|
|
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fstest/fstests"
|
|
"github.com/rclone/rclone/lib/batcher"
|
|
"github.com/rclone/rclone/lib/pacer"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
type paperMetadataClient struct {
|
|
files.ContextClient
|
|
info *files.FileMetadata
|
|
}
|
|
|
|
func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
|
|
if arg.Path == "document" {
|
|
return c.info, nil
|
|
}
|
|
return nil, files.GetMetadataAPIError{
|
|
APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
|
|
EndpointError: &files.GetMetadataError{
|
|
Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
|
|
Path: &files.LookupError{
|
|
Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestInternalGetMetadataCancellation(t *testing.T) {
|
|
requestStarted := make(chan struct{})
|
|
releaseRequest := make(chan struct{})
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
close(requestStarted)
|
|
<-releaseRequest
|
|
}))
|
|
defer server.Close()
|
|
defer close(releaseRequest)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
f := &Fs{
|
|
srv: files.NewContext(dropbox.Config{
|
|
Client: server.Client(),
|
|
URLGenerator: func(hostType string, namespace string, route string) string {
|
|
return server.URL
|
|
},
|
|
}),
|
|
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
|
|
}
|
|
|
|
result := make(chan getMetadataResult, 1)
|
|
go func() {
|
|
result <- f.getMetadata(ctx, "/file")
|
|
}()
|
|
|
|
select {
|
|
case <-requestStarted:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("Dropbox request did not start")
|
|
}
|
|
cancel()
|
|
|
|
select {
|
|
case res := <-result:
|
|
require.ErrorIs(t, res.err, context.Canceled)
|
|
case <-time.After(time.Second):
|
|
t.Fatal("Dropbox request did not observe cancellation")
|
|
}
|
|
}
|
|
|
|
func TestInternalCheckPathLength(t *testing.T) {
|
|
rep := func(n int, r rune) (out string) {
|
|
rs := make([]rune, n)
|
|
for i := range rs {
|
|
rs[i] = r
|
|
}
|
|
return string(rs)
|
|
}
|
|
for _, test := range []struct {
|
|
in string
|
|
ok bool
|
|
}{
|
|
{in: "", ok: true},
|
|
{in: rep(maxFileNameLength, 'a'), ok: true},
|
|
{in: rep(maxFileNameLength+1, 'a'), ok: false},
|
|
{in: rep(maxFileNameLength, '£'), ok: true},
|
|
{in: rep(maxFileNameLength+1, '£'), ok: false},
|
|
{in: rep(maxFileNameLength, '☺'), ok: true},
|
|
{in: rep(maxFileNameLength+1, '☺'), ok: false},
|
|
{in: rep(maxFileNameLength, '你'), ok: true},
|
|
{in: rep(maxFileNameLength+1, '你'), ok: false},
|
|
{in: "/ok/ok", ok: true},
|
|
{in: "/ok/" + rep(maxFileNameLength, 'a') + "/ok", ok: true},
|
|
{in: "/ok/" + rep(maxFileNameLength+1, 'a') + "/ok", ok: false},
|
|
{in: "/ok/" + rep(maxFileNameLength, '£') + "/ok", ok: true},
|
|
{in: "/ok/" + rep(maxFileNameLength+1, '£') + "/ok", ok: false},
|
|
{in: "/ok/" + rep(maxFileNameLength, '☺') + "/ok", ok: true},
|
|
{in: "/ok/" + rep(maxFileNameLength+1, '☺') + "/ok", ok: false},
|
|
{in: "/ok/" + rep(maxFileNameLength, '你') + "/ok", ok: true},
|
|
{in: "/ok/" + rep(maxFileNameLength+1, '你') + "/ok", ok: false},
|
|
} {
|
|
|
|
err := checkPathLength(test.in)
|
|
assert.Equal(t, test.ok, err == nil, test.in)
|
|
}
|
|
}
|
|
|
|
func TestPaperExportRemote(t *testing.T) {
|
|
ctx := context.Background()
|
|
info := &files.FileMetadata{
|
|
ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
|
|
}
|
|
f := &Fs{
|
|
exportExts: []exportExtension{"md"},
|
|
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
|
|
srv: paperMetadataClient{info: info},
|
|
}
|
|
|
|
direct, err := f.NewObject(ctx, "document.md")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "document.md", direct.Remote())
|
|
|
|
listed, err := f.newObjectWithInfo(ctx, "document.md", info)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "document.md.md", listed.Remote())
|
|
|
|
legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "document.md", legacy.Remote())
|
|
}
|
|
|
|
// uploadSessionClient is a mock files.ContextClient which records the
|
|
// chunked upload calls made to it
|
|
type uploadSessionClient struct {
|
|
files.ContextClient
|
|
appends int // number of UploadSessionAppendV2Context calls
|
|
maxAppends int // fail the append after this many calls to stop runaway loops
|
|
bytesWritten int64 // bytes received by UploadSessionAppendV2Context
|
|
finishCalled bool // set if UploadSessionFinishContext was called
|
|
}
|
|
|
|
var errTooManyAppends = errors.New("too many appends - upload looping?")
|
|
|
|
func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
|
|
return &files.UploadSessionStartResult{SessionId: "session"}, nil
|
|
}
|
|
|
|
func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
|
|
// the real client fails the request if the context is cancelled
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
c.appends++
|
|
if c.appends > c.maxAppends {
|
|
return errTooManyAppends
|
|
}
|
|
n, err := io.Copy(io.Discard, content)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
c.bytesWritten += n
|
|
return nil
|
|
}
|
|
|
|
func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
|
|
c.finishCalled = true
|
|
return &files.FileMetadata{}, nil
|
|
}
|
|
|
|
// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
|
|
func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
|
|
ctx := context.Background()
|
|
f := &Fs{
|
|
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
|
|
srv: srv,
|
|
}
|
|
f.opt.ChunkSize = chunkSize
|
|
batcherOptions := defaultBatcherOptions
|
|
batcherOptions.Mode = "off"
|
|
var err error
|
|
f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
|
|
require.NoError(t, err)
|
|
return f
|
|
}
|
|
|
|
func TestUploadChunkedEarlyEOF(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
t.Run("MultiChunk", func(t *testing.T) {
|
|
// The declared size spans 4 chunks but the source ends after 1.5
|
|
client := &uploadSessionClient{maxAppends: 8}
|
|
f := newUploadTestFs(t, client, 100)
|
|
o := &Object{fs: f, remote: "test.bin"}
|
|
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
|
require.Error(t, err)
|
|
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
|
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
|
})
|
|
|
|
t.Run("SingleChunk", func(t *testing.T) {
|
|
// The declared size fits in one chunk but the source ends early
|
|
client := &uploadSessionClient{maxAppends: 8}
|
|
f := newUploadTestFs(t, client, 500)
|
|
o := &Object{fs: f, remote: "test.bin"}
|
|
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
|
require.Error(t, err)
|
|
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
|
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
|
})
|
|
|
|
t.Run("Complete", func(t *testing.T) {
|
|
// A source which supplies exactly the declared size uploads OK
|
|
client := &uploadSessionClient{maxAppends: 8}
|
|
f := newUploadTestFs(t, client, 100)
|
|
o := &Object{fs: f, remote: "test.bin"}
|
|
entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, entry)
|
|
assert.True(t, client.finishCalled)
|
|
assert.Equal(t, int64(250), client.bytesWritten)
|
|
})
|
|
}
|
|
|
|
func (f *Fs) importPaperForTest(t *testing.T) {
|
|
content := `# test doc
|
|
|
|
Lorem ipsum __dolor__ sit amet
|
|
[link](http://google.com)
|
|
`
|
|
|
|
arg := files.PaperCreateArg{
|
|
Path: f.slashRootSlash + "export.paper",
|
|
ImportFormat: &files.ImportFormat{Tagged: dropbox.Tagged{Tag: files.ImportFormatMarkdown}},
|
|
}
|
|
var err error
|
|
err = f.pacer.Call(func() (bool, error) {
|
|
reader := strings.NewReader(content)
|
|
_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
|
|
return shouldRetry(context.Background(), err)
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func (f *Fs) InternalTestPaperExport(t *testing.T) {
|
|
ctx := context.Background()
|
|
f.importPaperForTest(t)
|
|
|
|
f.exportExts = []exportExtension{"html"}
|
|
|
|
obj, err := f.NewObject(ctx, "export.html")
|
|
require.NoError(t, err)
|
|
|
|
rc, err := obj.Open(ctx)
|
|
require.NoError(t, err)
|
|
defer func() { require.NoError(t, rc.Close()) }()
|
|
|
|
buf, err := io.ReadAll(rc)
|
|
require.NoError(t, err)
|
|
text := string(buf)
|
|
|
|
for _, excerpt := range []string{
|
|
"Lorem ipsum",
|
|
"<b>dolor</b>",
|
|
`href="http://google.com"`,
|
|
} {
|
|
require.Contains(t, text, excerpt)
|
|
}
|
|
}
|
|
func (f *Fs) InternalTest(t *testing.T) {
|
|
t.Run("PaperExport", f.InternalTestPaperExport)
|
|
}
|
|
|
|
var _ fstests.InternalTester = (*Fs)(nil)
|