yandex: add app_folder option to support cloud_api:disk.app_folder OAuth scope - Fixes #9848

backend/yandex/yandex.go

@@ -100,6 +100,20 @@ normally enough to stop them, at the cost of slowing down uploads.
 Yandex support recommend a value of 1.5s - 3s.`,
 			Default:  fs.Duration(0),
 			Advanced: true,
+		}, {
+			Name: "app_folder",
+			Help: `Use the application folder as the root.
+
+If you registered your own OAuth application with Yandex with only
+the "Application Folder" permission (cloud_api:disk.app_folder)
+instead of full disk access, then rclone needs to address paths with
+the "app:/" prefix instead of the usual "disk:/" prefix, otherwise
+all requests fail with a 403 Forbidden error.
+
+Set this to true if your OAuth token only has access to the
+application folder.`,
+			Default:  false,
+			Advanced: true,
 		}}...),
 	})
 }
@@ -111,18 +125,20 @@ type Options struct {
 	Enc            encoder.MultiEncoder `config:"encoding"`
 	SpoofUserAgent bool                 `config:"spoof_ua"`
 	UploadWait     fs.Duration          `config:"upload_wait"`
+	AppFolder      bool                 `config:"app_folder"`
 }
 
 // Fs represents a remote yandex
 type Fs struct {
-	name     string
+	name      string
-	root     string         // root path
+	root      string         // root path
-	opt      Options        // parsed options
+	opt       Options        // parsed options
-	ci       *fs.ConfigInfo // global config
+	ci        *fs.ConfigInfo // global config
-	features *fs.Features   // optional features
+	features  *fs.Features   // optional features
-	srv      *rest.Client   // the connection to the yandex server
+	srv       *rest.Client   // the connection to the yandex server
-	pacer    *fs.Pacer      // pacer for API calls
+	pacer     *fs.Pacer      // pacer for API calls
-	diskRoot string         // root path with "disk:/" container name
+	diskRoot  string         // root path with the container prefix, e.g. "disk:/" or "app:/"
+	container string         // container prefix in use, e.g. "disk:" or "app:"
 }
 
 // Object describes a swift object
@@ -209,13 +225,21 @@ func errorHandler(resp *http.Response) error {
 func (f *Fs) setRoot(root string) {
 	//Set root path
 	f.root = strings.Trim(root, "/")
+	//Set the container prefix. This is "disk:" normally, or "app:" if the
+	//OAuth token only has access to the application folder (see the
+	//app_folder option).
+	if f.opt.AppFolder {
+		f.container = "app:"
+	} else {
+		f.container = "disk:"
+	}
 	//Set disk root path.
-	//Adding "disk:" to root path as all paths on disk start with it
+	//Adding the container prefix to root path as all paths on disk start with it
 	var diskRoot string
 	if f.root == "" {
-		diskRoot = "disk:/"
+		diskRoot = f.container + "/"
 	} else {
-		diskRoot = "disk:/" + f.root + "/"
+		diskRoot = f.container + "/" + f.root + "/"
 	}
 	f.diskRoot = diskRoot
 }
@@ -494,9 +518,13 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
 		NoResponse: true,
 	}
 
-	// If creating a directory with a : use (undocumented) disk: prefix
+	if f.opt.AppFolder {
-	if strings.ContainsRune(path, ':') {
+		// Bare relative paths are not resolved under the app folder root,
-		path = "disk:" + path
+		// unlike the disk root, so the "app:" prefix is always required.
+		path = f.container + path
+	} else if strings.ContainsRune(path, ':') {
+		// If creating a directory with a : use (undocumented) disk: prefix
+		path = f.container + path
 	}
 	opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(path))
 
@@ -518,8 +546,8 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
 func (f *Fs) mkDirs(ctx context.Context, path string) (err error) {
 	//trim filename from path
 	//dirString := strings.TrimSuffix(path, filepath.Base(path))
-	//trim "disk:" from path
+	//trim the container prefix, e.g. "disk:" or "app:", from path
-	dirString := strings.TrimPrefix(path, "disk:")
+	dirString := strings.TrimPrefix(path, f.container)
 	if dirString == "" {
 		return nil
 	}
@@ -808,7 +836,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	//fmt.Printf("Move src: %s (FullPath: %s), dst: %s (FullPath: %s)\n", srcRemote, srcPath, dstRemote, dstPath)
 
 	// Refuse to move to or from the root
-	if srcPath == "disk:/" || dstPath == "disk:/" {
+	if srcPath == srcFs.container+"/" || dstPath == f.container+"/" {
 		fs.Debugf(src, "DirMove error: Can't move root")
 		return errors.New("can't move root directory")
 	}

backend/yandex/yandex_internal_test.go

@@ -0,0 +1,88 @@
+package yandex
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestSetRoot checks that setRoot addresses paths with the "disk:/" prefix
+// by default, and with the "app:/" prefix when app_folder is set.
+func TestSetRoot(t *testing.T) {
+	for _, test := range []struct {
+		appFolder    bool
+		root         string
+		wantDiskRoot string
+		wantFilePath string
+	}{
+		{false, "", "disk:/", "disk:/file.txt"},
+		{false, "backups", "disk:/backups/", "disk:/backups/file.txt"},
+		{true, "", "app:/", "app:/file.txt"},
+		{true, "backups", "app:/backups/", "app:/backups/file.txt"},
+	} {
+		f := &Fs{opt: Options{AppFolder: test.appFolder}}
+		f.setRoot(test.root)
+		assert.Equal(t, test.wantDiskRoot, f.diskRoot, "diskRoot for root=%q app_folder=%v", test.root, test.appFolder)
+		assert.Equal(t, test.wantFilePath, f.filePath("file.txt"), "filePath for root=%q app_folder=%v", test.root, test.appFolder)
+	}
+}
+
+// TestCreateDir checks the path CreateDir sends to the API, both for the
+// default disk:/ root and for the app:/ root used when app_folder is set.
+// Yandex Disk does not resolve bare relative paths under the app folder
+// root, so those must be sent with an explicit "app:/" prefix, but the
+// default disk:/ behaviour (relying on the API's implicit disk root for
+// bare relative paths, only prefixing when the path itself contains a ':')
+// must stay unchanged for existing users.
+func TestCreateDir(t *testing.T) {
+	for _, test := range []struct {
+		appFolder bool
+		path      string
+		wantPath  string
+	}{
+		{false, "/backups/", "/backups/"},
+		{false, "/foo:bar/", "disk:/foo:bar/"},
+		{true, "/backups/", "app:/backups/"},
+		{true, "/foo:bar/", "app:/foo:bar/"},
+	} {
+		var gotPath string
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			gotPath = r.URL.Query().Get("path")
+			w.WriteHeader(http.StatusCreated)
+		}))
+
+		ctx := context.Background()
+		f := &Fs{
+			opt:   Options{AppFolder: test.appFolder},
+			srv:   rest.NewClient(server.Client()).SetRoot(server.URL),
+			pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+		}
+		f.setRoot("")
+
+		err := f.CreateDir(ctx, test.path)
+		require.NoError(t, err, "path=%q app_folder=%v", test.path, test.appFolder)
+		assert.Equal(t, test.wantPath, gotPath, "path=%q app_folder=%v", test.path, test.appFolder)
+		server.Close()
+	}
+}
+
+// TestDirMoveRoot checks that DirMove refuses to move a directory to the
+// container root, both for the default disk:/ root and for the app:/ root
+// used when app_folder is set.
+func TestDirMoveRoot(t *testing.T) {
+	for _, appFolder := range []bool{false, true} {
+		f := &Fs{opt: Options{AppFolder: appFolder}}
+		f.setRoot("")
+
+		err := f.DirMove(context.Background(), f, "somedir", "")
+		assert.EqualError(t, err, "can't move root directory", "app_folder=%v", appFolder)
+	}
+}

docs/content/yandex.md

@@ -125,6 +125,16 @@ are replaced.
 Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
 as they can't be used in JSON strings.
 
+### Application folder
+
+If you registered your own OAuth application with Yandex and only granted
+it the "Application Folder" permission (`cloud_api:disk.app_folder`)
+instead of full disk access, then you must set the `app_folder` option to
+`true`. Yandex Disk requires paths to be addressed with an `app:/` prefix
+instead of the usual `disk:/` prefix when using this restricted scope, and
+rclone will return a 403 Forbidden error on every request if this option
+isn't set correctly for the token in use.
+
 <!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/yandex/yandex.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->
 ### Standard options