diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go index 8e714efbf..0b2bd5a16 100644 --- a/fs/operations/lsjson.go +++ b/fs/operations/lsjson.go @@ -319,6 +319,38 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) // // Remove trailing / as rclone listings won't have them remote = strings.TrimRight(remote, "/") + + // For bucket-based backends with ListP, try listing the + // directory itself first using ListP. This is much cheaper than + // listing the parent when the parent has many entries. + // + // Bucket-based backends don't have directory IDs or other + // metadata that would be lost by returning a synthetic entry. + // + // If the listing returns empty we fall through to listing the + // parent to distinguish between an empty directory and a + // non-existent one. + features := fsrc.Features() + if features.BucketBased && features.ListP != nil { + errDirFound := errors.New("directory found") + err = features.ListP(ctx, remote, func(entries fs.DirEntries) error { + accounting.Stats(ctx).Listed(int64(len(entries))) + if len(entries) > 0 { + return errDirFound + } + return nil + }) + if err == errDirFound { + return lj.entry(ctx, fs.NewDir(remote, time.Now())) + } else if err == fs.ErrorDirNotFound { + return nil, nil + } + // Fall through to parent listing + // - on other errors (err != nil) + // - empty listing (err == nil) + } + + // List the parent to find the directory entry with proper metadata. parent := path.Dir(remote) if parent == "." || parent == "/" { parent = "" diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.go index 75d4728ba..e32aeefd4 100644 --- a/fs/operations/lsjson_test.go +++ b/fs/operations/lsjson_test.go @@ -2,11 +2,14 @@ package operations_test import ( "context" + "io" "sort" + "strings" "testing" "time" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" "github.com/stretchr/testify/assert" @@ -404,3 +407,58 @@ func TestStatJSON(t *testing.T) { assert.True(t, err != nil || f.Features().BucketBased, "Need an error for non bucket based backends") }) } + +// TestStatJSONMemory tests StatJSON against the memory backend. +// +// The memory backend is bucket based and implements ListP, so this +// exercises the fast path in StatJSON which lists the target directory +// itself rather than its (potentially huge) parent. It also checks that +// the entries listed on that path are accounted in the Listed stats. +func TestStatJSONMemory(t *testing.T) { + ctx := context.Background() + f, err := fs.NewFs(ctx, ":memory:") + require.NoError(t, err) + + // Check the memory backend triggers the fast path + require.True(t, f.Features().BucketBased, "memory backend should be bucket based") + require.NotNil(t, f.Features().ListP, "memory backend should implement ListP") + + // Put a file into a subdirectory + _, err = operations.Rcat(ctx, f, "sub/file1", io.NopCloser(strings.NewReader("hello")), t1, nil) + require.NoError(t, err) + + t.Run("Dir", func(t *testing.T) { + accounting.GlobalStats().ResetCounters() + got, err := operations.StatJSON(ctx, f, "sub", &operations.ListJSONOpt{}) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "sub", got.Path) + assert.Equal(t, "sub", got.Name) + assert.True(t, got.IsDir) + // The fast path lists the directory itself - check the entries + // it listed were accounted. + assert.Positive(t, accounting.GlobalStats().Listed(0), "expected the fast path listing to be accounted") + }) + + t.Run("DirWithTrailingSlash", func(t *testing.T) { + got, err := operations.StatJSON(ctx, f, "sub/", &operations.ListJSONOpt{}) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "sub", got.Path) + assert.True(t, got.IsDir) + }) + + t.Run("File", func(t *testing.T) { + got, err := operations.StatJSON(ctx, f, "sub/file1", &operations.ListJSONOpt{}) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, "sub/file1", got.Path) + assert.False(t, got.IsDir) + }) + + t.Run("NotFound", func(t *testing.T) { + got, err := operations.StatJSON(ctx, f, "notfound", &operations.ListJSONOpt{}) + require.NoError(t, err) + assert.Nil(t, got) + }) +}