walk: stop directory traversal when the context is cancelled - fixes #9788

The concurrent walker created by walk() only stopped when the callback
returned an error or the whole tree had been listed. Cancelling the
context (for example via the rc job/stop endpoint for an async
operations/size or recursive operations/list call) was therefore
ignored: the checkers kept pulling list jobs from the channel and kept
listing the entire tree, burning CPU and making job cancellation
useless for every backend without a native ListR implementation.

Make every checker select on ctx.Done() so a cancelled walk shuts down
promptly through the existing quit/drain path and reports the context
error. Also check the context between directory read chunks in the
local backend so a single huge directory does not block cancellation.

backend/local/local.go

@@ -688,6 +688,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}()
 
 	for {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, ctxErr
+		}
 		var fis []os.FileInfo
 		if useReadDir {
 			// Windows and Plan9 read the directory entries with the stat information in which

fs/walk/walk.go

@@ -394,6 +394,13 @@ func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel i
 		wg.Go(func() {
 			for {
 				select {
+				case <-ctx.Done():
+					closeQuit()
+					select {
+					case errs <- ctx.Err():
+					default:
+					}
+					return
 				case job, ok := <-in:
 					if !ok {
 						return

fs/walk/walk_context_cancel_test.go

@@ -0,0 +1,47 @@
+package walk
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/mockdir"
+	"github.com/stretchr/testify/assert"
+)
+
+// Test that walk stops listing when the context is cancelled.
+func TestWalkContextCancelled(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	started := make(chan struct{}, 1)
+	listDir := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
+		select {
+		case started <- struct{}{}:
+		default:
+		}
+		// Every directory contains one subdirectory so the walk never
+		// finishes on its own.
+		return fs.DirEntries{mockdir.New("sub")}, nil
+	}
+
+	done := make(chan error, 1)
+	go func() {
+		done <- walk(ctx, nil, "", false, -1, func(path string, entries fs.DirEntries, err error) error {
+			return nil
+		}, listDir)
+	}()
+
+	// Wait for the walk to start.
+	<-started
+
+	cancel()
+
+	select {
+	case err := <-done:
+		assert.ErrorIs(t, err, context.Canceled)
+	case <-time.After(10 * time.Second):
+		t.Fatal("walk did not stop after the context was cancelled")
+	}
+}