operations: add operations/getfile remote control API endpoint

Add operations/cat endpoint to the Remote Control (RC) API to allow reading and streaming file contents in-process over librclone / FFI and HTTP RC. Supports range options (offset, count, head, tail), separator, optional maxSize buffer limit, and returns both string and base64 encoded results.
2026-09-08 10:43:48 +01:00
parent 4412cef804
commit a083861a46
3 changed files with 447 additions and 37 deletions
@@ -1313,50 +1313,16 @@ type readCloser struct {
 // if count >= 0 then only that many characters will be output
 func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []byte) error {
 	var mu sync.Mutex
-	ci := fs.GetConfig(ctx)
 	return ListFn(ctx, f, func(o fs.Object) {
-		var err error
-		tr := accounting.Stats(ctx).NewTransfer(o, nil)
-		defer func() {
-			tr.Done(ctx, err)
-		}()
-		opt := fs.RangeOption{Start: offset, End: -1}
-		size := o.Size()
-		if opt.Start < 0 {
-			opt.Start += size
-		}
-		if count >= 0 {
-			opt.End = opt.Start + count - 1
-		}
-		var options []fs.OpenOption
-		if opt.Start > 0 || opt.End >= 0 {
-			options = append(options, &opt)
-		}
-		for _, option := range ci.DownloadHeaders {
-			options = append(options, option)
-		}
-		var in io.ReadCloser
-		in, err = Open(ctx, o, options...)
-		if err != nil {
-			err = fs.CountError(ctx, err)
-			fs.Errorf(o, "Failed to open: %v", err)
-			return
-		}
-		if count >= 0 {
-			in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in}
-		}
-		in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
-		// take the lock just before we output stuff, so at the last possible moment
 		mu.Lock()
 		defer mu.Unlock()
-		_, err = io.Copy(w, in)
+		err := catObject(ctx, o, w, offset, count)
 		if err != nil {
-			err = fs.CountError(ctx, err)
 			fs.Errorf(o, "Failed to send to output: %v", err)
+			return
 		}
 		if len(sep) > 0 {
-			_, err = w.Write(sep)
-			if err != nil {
+			if _, err := w.Write(sep); err != nil {
 				err = fs.CountError(ctx, err)
 				fs.Errorf(o, "Failed to send separator to output: %v", err)
 			}
@@ -1364,6 +1330,46 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []b
 	})
 }

+// catObject sends an object or a range of it to the io.Writer
+func catObject(ctx context.Context, o fs.Object, w io.Writer, offset, count int64) (err error) {
+	tr := accounting.Stats(ctx).NewTransfer(o, nil)
+	defer func() {
+		tr.Done(ctx, err)
+	}()
+	opt := fs.RangeOption{Start: offset, End: -1}
+	size := o.Size()
+	if opt.Start < 0 && size >= 0 {
+		opt.Start += size
+	}
+	if count >= 0 {
+		opt.End = opt.Start + count - 1
+	}
+	var options []fs.OpenOption
+	if opt.Start > 0 || opt.End >= 0 {
+		options = append(options, &opt)
+	}
+	for _, option := range fs.GetConfig(ctx).DownloadHeaders {
+		options = append(options, option)
+	}
+	var in io.ReadCloser
+	in, err = Open(ctx, o, options...)
+	if err != nil {
+		err = fs.CountError(ctx, err)
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	if count >= 0 {
+		in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in}
+	}
+	in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
+	defer fs.CheckClose(in, &err)
+	_, err = io.Copy(w, in)
+	if err != nil {
+		err = fs.CountError(ctx, err)
+		return err
+	}
+	return nil
+}
+
 // Rcat reads data from the Reader until EOF and uploads it to a file on remote
 //
 // in is closed at the end of the transfer
@@ -1,7 +1,9 @@
 package operations

 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -11,11 +13,14 @@ import (
 	"path"
 	"strings"
 	"time"
+	"unicode/utf8"

 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/diskusage"
 )

@@ -1009,3 +1014,194 @@ func rcHashsumFile(ctx context.Context, in rc.Params) (out rc.Params, err error)
 	}
 	return out, err
 }
+
+// DefaultGetFileMaxSize is the built-in maximum size limit in bytes for operations/getfile (2 MiB)
+const DefaultGetFileMaxSize = 2 * 1024 * 1024
+
+// maxLimitWriter writes into a buffer up to limit bytes, returning an error if more data is written.
+type maxLimitWriter struct {
+	buf   bytes.Buffer
+	limit int64
+}
+
+func (w *maxLimitWriter) Write(p []byte) (n int, err error) {
+	rem := w.limit - int64(w.buf.Len())
+	if int64(len(p)) > rem {
+		if rem > 0 {
+			w.buf.Write(p[:rem])
+		}
+		return int(rem), fmt.Errorf("read exceeded maximum limit of %d bytes; use offset/count, maxSize or --rc-serve", w.limit)
+	}
+	return w.buf.Write(p)
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:  "operations/getfile",
+		Fn:    rcGetFile,
+		Title: "Get a single file into a JSON response.",
+		Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:path/to/file" or "drive:path/to/dir"
+- remote - (optional) a path within that remote e.g. "file.txt"
+- offset - (optional) start reading at offset N (or from end if negative) (default 0)
+- count - (optional) only read N bytes (default -1)
+- head - (optional) only read the first N bytes (default 0)
+- tail - (optional) only read the last N bytes (default 0)
+- maxSize - (optional) maximum size limit in bytes (can only lower the built-in maximum) (default 2097152 / 2MiB)
+- base64 - (optional) if true, returns the file contents base64-encoded (default false)
+
+Returns:
+
+- result - contents of the file as string or base64-encoded string
+
+For streaming file content over HTTP with Range support, use the --rc-serve flag instead.
+`,
+	})
+}
+
+// Get a single file into a JSON response
+func rcGetFile(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+	remote, err := in.GetString("remote")
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	fsString, err := in.GetString("fs")
+	if err != nil {
+		return nil, err
+	}
+	if remote != "" {
+		in = in.Copy()
+		in["fs"] = fspath.JoinRootPath(fsString, remote)
+	}
+
+	offset, err := in.GetInt64("offset")
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	count, err := in.GetInt64("count")
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	if rc.IsErrParamNotFound(err) {
+		count = -1
+	}
+	head, err := in.GetInt64("head")
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	tail, err := in.GetInt64("tail")
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+
+	usedOffset := offset != 0 || count >= 0
+	usedHead := head > 0
+	usedTail := tail > 0
+
+	if (usedHead && usedTail) || (usedHead && usedOffset) || (usedTail && usedOffset) {
+		return nil, errors.New("can only use one of head, tail or offset/count")
+	}
+	if head < 0 {
+		return nil, errors.New("head cannot be negative")
+	}
+	if tail < 0 {
+		return nil, errors.New("tail cannot be negative")
+	}
+	if head > 0 {
+		offset = 0
+		count = head
+	}
+	if tail > 0 {
+		offset = -tail
+		count = -1
+	}
+
+	limit := int64(DefaultGetFileMaxSize)
+	maxSize, err := in.GetInt64("maxSize")
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	if !rc.IsErrParamNotFound(err) {
+		if maxSize <= 0 {
+			return nil, errors.New("maxSize must be greater than 0")
+		}
+		if maxSize > DefaultGetFileMaxSize {
+			return nil, fmt.Errorf("maxSize %d cannot exceed built-in maximum of %d bytes", maxSize, DefaultGetFileMaxSize)
+		}
+		limit = maxSize
+	}
+
+	base64Encoded, err := in.GetBool("base64")
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+
+	ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs")
+	if err != nil {
+		return nil, err
+	}
+
+	ci := fs.GetConfig(ctx)
+	var (
+		fileCount int
+		w         = &maxLimitWriter{limit: limit}
+	)
+
+	err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+		for _, entry := range entries {
+			o, ok := entry.(fs.Object)
+			if !ok {
+				continue
+			}
+			fileCount++
+			if fileCount > 1 {
+				return errors.New("found more than one file; operations/getfile only supports a single file")
+			}
+
+			// Pre-check size if known
+			size := o.Size()
+			if size >= 0 {
+				start := offset
+				if start < 0 {
+					start += size
+				}
+				if start > size {
+					start = size
+				}
+				reqSize := size - start
+				if count >= 0 && count < reqSize {
+					reqSize = count
+				}
+				if reqSize > limit {
+					return fmt.Errorf("requested size (%d bytes) exceeds maximum limit of %d bytes; use offset/count, maxSize or --rc-serve", reqSize, limit)
+				}
+			}
+
+			err = catObject(ctx, o, w, offset, count)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	if fileCount == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	data := w.buf.Bytes()
+	if base64Encoded {
+		return rc.Params{
+			"result": base64.StdEncoding.EncodeToString(data),
+		}, nil
+	}
+	if !utf8.Valid(data) {
+		return nil, errors.New("file content is not valid UTF-8 (use base64=true for binary files)")
+	}
+	return rc.Params{
+		"result": string(data),
+	}, nil
+}
@@ -2,6 +2,7 @@ package operations_test

 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -893,3 +894,210 @@ func TestRcHashsumFile(t *testing.T) {
 	assert.Equal(t, "md5", out["hashType"])
 	assert.Equal(t, "0ef726ce9b1a7692357ff70dd321d595", out["hash"])
 }
+
+// operations/getfile: get a single file
+func TestRcGetFile(t *testing.T) {
+	ctx := context.Background()
+	r, call := rcNewRun(t, "operations/getfile")
+	r.Mkdir(ctx, r.Fremote)
+
+	file1Contents := "Hello Rclone GetFile Operation!"
+	file1 := r.WriteBoth(ctx, "getfile-file1.txt", file1Contents, t1)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file1)
+
+	// 1. Basic test with fs + remote
+	in := rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+	}
+	out, err := call.Fn(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, file1Contents, out["result"])
+
+	// 2. Combined fs parameter
+	in = rc.Params{
+		"fs": path.Join(r.FremoteName, file1.Path),
+	}
+	out, err = call.Fn(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, file1Contents, out["result"])
+
+	// 3. Range offset and count
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"offset": int64(6),
+		"count":  int64(6),
+	}
+	out, err = call.Fn(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, "Rclone", out["result"])
+
+	// 4. Head flag
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"head":   int64(5),
+	}
+	out, err = call.Fn(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, "Hello", out["result"])
+
+	// 5. Tail flag
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"tail":   int64(10),
+	}
+	out, err = call.Fn(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, "Operation!", out["result"])
+
+	// 6. Negative offset
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"offset": int64(-10),
+	}
+	out, err = call.Fn(ctx, in)
+	require.NoError(t, err)
+	assert.Equal(t, "Operation!", out["result"])
+
+	// 7. Base64 encoding for text file
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"base64": true,
+	}
+	out, err = call.Fn(ctx, in)
+	require.NoError(t, err)
+	decoded, err := base64.StdEncoding.DecodeString(out["result"].(string))
+	require.NoError(t, err)
+	assert.Equal(t, file1Contents, string(decoded))
+
+	// 8. Binary file: UTF-8 rejection vs base64 success
+	binaryData := "\x00\xff\xfe\xfd\x80\x81"
+	binFile := r.WriteBoth(ctx, "binary.dat", binaryData, t1)
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": binFile.Path,
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "not valid UTF-8")
+
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": binFile.Path,
+		"base64": true,
+	}
+	out, err = call.Fn(ctx, in)
+	require.NoError(t, err)
+	decodedBin, err := base64.StdEncoding.DecodeString(out["result"].(string))
+	require.NoError(t, err)
+	assert.Equal(t, []byte(binaryData), decodedBin)
+
+	// 9. Max size limit violation test
+	in = rc.Params{
+		"fs":      r.FremoteName,
+		"remote":  file1.Path,
+		"maxSize": int64(5),
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "exceeds maximum limit")
+
+	// 10. Max size exceeding built-in limit
+	in = rc.Params{
+		"fs":      r.FremoteName,
+		"remote":  file1.Path,
+		"maxSize": int64(operations.DefaultGetFileMaxSize + 1),
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cannot exceed built-in maximum")
+
+	// 11. Invalid maxSize values (<= 0)
+	in = rc.Params{
+		"fs":      r.FremoteName,
+		"remote":  file1.Path,
+		"maxSize": int64(0),
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "maxSize must be greater than 0")
+
+	in["maxSize"] = int64(-5)
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "maxSize must be greater than 0")
+
+	// 12. Invalid parameter types
+	for _, badParam := range []rc.Params{
+		{"fs": r.FremoteName, "remote": file1.Path, "offset": "not_an_int"},
+		{"fs": r.FremoteName, "remote": file1.Path, "count": "not_an_int"},
+		{"fs": r.FremoteName, "remote": file1.Path, "head": "not_an_int"},
+		{"fs": r.FremoteName, "remote": file1.Path, "tail": "not_an_int"},
+		{"fs": r.FremoteName, "remote": file1.Path, "maxSize": "not_an_int"},
+		{"fs": r.FremoteName, "remote": file1.Path, "base64": "not_a_bool"},
+		{"fs": r.FremoteName, "remote": 12345},
+	} {
+		_, err = call.Fn(ctx, badParam)
+		require.Error(t, err)
+	}
+
+	// 13. Conflicting and invalid bounds
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"head":   int64(5),
+		"tail":   int64(5),
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "can only use one of head, tail or offset/count")
+
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"head":   int64(-1),
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "head cannot be negative")
+
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": file1.Path,
+		"tail":   int64(-1),
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "tail cannot be negative")
+
+	// 14. Non-existent file / directory with 0 files
+	in = rc.Params{
+		"fs":     r.FremoteName,
+		"remote": "does_not_exist.txt",
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+
+	emptyDir := path.Join(r.LocalName, "emptydir")
+	require.NoError(t, os.Mkdir(emptyDir, 0755))
+	in = rc.Params{
+		"fs": emptyDir,
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
+
+	// 15. Directory containing multiple files
+	in = rc.Params{
+		"fs": r.FremoteName,
+	}
+	_, err = call.Fn(ctx, in)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "found more than one file")
+}