operations: fix operations/stat for directories wth large parent dirs

When `operations/stat` / StatJSON is called on a directory path it
lists the parent directory to find the target entry. If the parent has
millions of entries this is very expensive.

This fixes the problem for bucket-based backends with ListP by listing
the target directory itself first. It will stop the listing
immediately if any files are found meaning it is safe to run on
directories with millions of files.

fs/operations/lsjson.go

@@ -319,6 +319,38 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
 	//
 	// Remove trailing / as rclone listings won't have them
 	remote = strings.TrimRight(remote, "/")
+
+	// For bucket-based backends with ListP, try listing the
+	// directory itself first using ListP. This is much cheaper than
+	// listing the parent when the parent has many entries.
+	//
+	// Bucket-based backends don't have directory IDs or other
+	// metadata that would be lost by returning a synthetic entry.
+	//
+	// If the listing returns empty we fall through to listing the
+	// parent to distinguish between an empty directory and a
+	// non-existent one.
+	features := fsrc.Features()
+	if features.BucketBased && features.ListP != nil {
+		errDirFound := errors.New("directory found")
+		err = features.ListP(ctx, remote, func(entries fs.DirEntries) error {
+			accounting.Stats(ctx).Listed(int64(len(entries)))
+			if len(entries) > 0 {
+				return errDirFound
+			}
+			return nil
+		})
+		if err == errDirFound {
+			return lj.entry(ctx, fs.NewDir(remote, time.Now()))
+		} else if err == fs.ErrorDirNotFound {
+			return nil, nil
+		}
+		// Fall through to parent listing
+		// - on other errors (err != nil)
+		// - empty listing (err == nil)
+	}
+
+	// List the parent to find the directory entry with proper metadata.
 	parent := path.Dir(remote)
 	if parent == "." || parent == "/" {
 		parent = ""

fs/operations/lsjson_test.go

@@ -2,11 +2,14 @@ package operations_test
 
 import (
 	"context"
+	"io"
 	"sort"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/stretchr/testify/assert"
@@ -404,3 +407,58 @@ func TestStatJSON(t *testing.T) {
 		assert.True(t, err != nil || f.Features().BucketBased, "Need an error for non bucket based backends")
 	})
 }
+
+// TestStatJSONMemory tests StatJSON against the memory backend.
+//
+// The memory backend is bucket based and implements ListP, so this
+// exercises the fast path in StatJSON which lists the target directory
+// itself rather than its (potentially huge) parent. It also checks that
+// the entries listed on that path are accounted in the Listed stats.
+func TestStatJSONMemory(t *testing.T) {
+	ctx := context.Background()
+	f, err := fs.NewFs(ctx, ":memory:")
+	require.NoError(t, err)
+
+	// Check the memory backend triggers the fast path
+	require.True(t, f.Features().BucketBased, "memory backend should be bucket based")
+	require.NotNil(t, f.Features().ListP, "memory backend should implement ListP")
+
+	// Put a file into a subdirectory
+	_, err = operations.Rcat(ctx, f, "sub/file1", io.NopCloser(strings.NewReader("hello")), t1, nil)
+	require.NoError(t, err)
+
+	t.Run("Dir", func(t *testing.T) {
+		accounting.GlobalStats().ResetCounters()
+		got, err := operations.StatJSON(ctx, f, "sub", &operations.ListJSONOpt{})
+		require.NoError(t, err)
+		require.NotNil(t, got)
+		assert.Equal(t, "sub", got.Path)
+		assert.Equal(t, "sub", got.Name)
+		assert.True(t, got.IsDir)
+		// The fast path lists the directory itself - check the entries
+		// it listed were accounted.
+		assert.Positive(t, accounting.GlobalStats().Listed(0), "expected the fast path listing to be accounted")
+	})
+
+	t.Run("DirWithTrailingSlash", func(t *testing.T) {
+		got, err := operations.StatJSON(ctx, f, "sub/", &operations.ListJSONOpt{})
+		require.NoError(t, err)
+		require.NotNil(t, got)
+		assert.Equal(t, "sub", got.Path)
+		assert.True(t, got.IsDir)
+	})
+
+	t.Run("File", func(t *testing.T) {
+		got, err := operations.StatJSON(ctx, f, "sub/file1", &operations.ListJSONOpt{})
+		require.NoError(t, err)
+		require.NotNil(t, got)
+		assert.Equal(t, "sub/file1", got.Path)
+		assert.False(t, got.IsDir)
+	})
+
+	t.Run("NotFound", func(t *testing.T) {
+		got, err := operations.StatJSON(ctx, f, "notfound", &operations.ListJSONOpt{})
+		require.NoError(t, err)
+		assert.Nil(t, got)
+	})
+}