From a083861a46269d55e74e9df8833238eb0a0b1c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hakan=20=C4=B0SMA=C4=B0L?= <93117749+Hakanbaban53@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:43:48 +0300 Subject: [PATCH] operations: add operations/getfile remote control API endpoint Add operations/cat endpoint to the Remote Control (RC) API to allow reading and streaming file contents in-process over librclone / FFI and HTTP RC. Supports range options (offset, count, head, tail), separator, optional maxSize buffer limit, and returns both string and base64 encoded results. --- fs/operations/operations.go | 80 +++++++------- fs/operations/rc.go | 196 +++++++++++++++++++++++++++++++++ fs/operations/rc_test.go | 208 ++++++++++++++++++++++++++++++++++++ 3 files changed, 447 insertions(+), 37 deletions(-) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 3221c63a1..6b8248b6c 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1313,50 +1313,16 @@ type readCloser struct { // if count >= 0 then only that many characters will be output func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []byte) error { var mu sync.Mutex - ci := fs.GetConfig(ctx) return ListFn(ctx, f, func(o fs.Object) { - var err error - tr := accounting.Stats(ctx).NewTransfer(o, nil) - defer func() { - tr.Done(ctx, err) - }() - opt := fs.RangeOption{Start: offset, End: -1} - size := o.Size() - if opt.Start < 0 { - opt.Start += size - } - if count >= 0 { - opt.End = opt.Start + count - 1 - } - var options []fs.OpenOption - if opt.Start > 0 || opt.End >= 0 { - options = append(options, &opt) - } - for _, option := range ci.DownloadHeaders { - options = append(options, option) - } - var in io.ReadCloser - in, err = Open(ctx, o, options...) - if err != nil { - err = fs.CountError(ctx, err) - fs.Errorf(o, "Failed to open: %v", err) - return - } - if count >= 0 { - in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in} - } - in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer - // take the lock just before we output stuff, so at the last possible moment mu.Lock() defer mu.Unlock() - _, err = io.Copy(w, in) + err := catObject(ctx, o, w, offset, count) if err != nil { - err = fs.CountError(ctx, err) fs.Errorf(o, "Failed to send to output: %v", err) + return } if len(sep) > 0 { - _, err = w.Write(sep) - if err != nil { + if _, err := w.Write(sep); err != nil { err = fs.CountError(ctx, err) fs.Errorf(o, "Failed to send separator to output: %v", err) } @@ -1364,6 +1330,46 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []b }) } +// catObject sends an object or a range of it to the io.Writer +func catObject(ctx context.Context, o fs.Object, w io.Writer, offset, count int64) (err error) { + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + opt := fs.RangeOption{Start: offset, End: -1} + size := o.Size() + if opt.Start < 0 && size >= 0 { + opt.Start += size + } + if count >= 0 { + opt.End = opt.Start + count - 1 + } + var options []fs.OpenOption + if opt.Start > 0 || opt.End >= 0 { + options = append(options, &opt) + } + for _, option := range fs.GetConfig(ctx).DownloadHeaders { + options = append(options, option) + } + var in io.ReadCloser + in, err = Open(ctx, o, options...) + if err != nil { + err = fs.CountError(ctx, err) + return fmt.Errorf("failed to open file: %w", err) + } + if count >= 0 { + in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in} + } + in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer + defer fs.CheckClose(in, &err) + _, err = io.Copy(w, in) + if err != nil { + err = fs.CountError(ctx, err) + return err + } + return nil +} + // Rcat reads data from the Reader until EOF and uploads it to a file on remote // // in is closed at the end of the transfer diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 4c0828127..cdaa8f854 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -1,7 +1,9 @@ package operations import ( + "bytes" "context" + "encoding/base64" "errors" "fmt" "io" @@ -11,11 +13,14 @@ import ( "path" "strings" "time" + "unicode/utf8" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/diskusage" ) @@ -1009,3 +1014,194 @@ func rcHashsumFile(ctx context.Context, in rc.Params) (out rc.Params, err error) } return out, err } + +// DefaultGetFileMaxSize is the built-in maximum size limit in bytes for operations/getfile (2 MiB) +const DefaultGetFileMaxSize = 2 * 1024 * 1024 + +// maxLimitWriter writes into a buffer up to limit bytes, returning an error if more data is written. +type maxLimitWriter struct { + buf bytes.Buffer + limit int64 +} + +func (w *maxLimitWriter) Write(p []byte) (n int, err error) { + rem := w.limit - int64(w.buf.Len()) + if int64(len(p)) > rem { + if rem > 0 { + w.buf.Write(p[:rem]) + } + return int(rem), fmt.Errorf("read exceeded maximum limit of %d bytes; use offset/count, maxSize or --rc-serve", w.limit) + } + return w.buf.Write(p) +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/getfile", + Fn: rcGetFile, + Title: "Get a single file into a JSON response.", + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:path/to/file" or "drive:path/to/dir" +- remote - (optional) a path within that remote e.g. "file.txt" +- offset - (optional) start reading at offset N (or from end if negative) (default 0) +- count - (optional) only read N bytes (default -1) +- head - (optional) only read the first N bytes (default 0) +- tail - (optional) only read the last N bytes (default 0) +- maxSize - (optional) maximum size limit in bytes (can only lower the built-in maximum) (default 2097152 / 2MiB) +- base64 - (optional) if true, returns the file contents base64-encoded (default false) + +Returns: + +- result - contents of the file as string or base64-encoded string + +For streaming file content over HTTP with Range support, use the --rc-serve flag instead. +`, + }) +} + +// Get a single file into a JSON response +func rcGetFile(ctx context.Context, in rc.Params) (out rc.Params, err error) { + remote, err := in.GetString("remote") + if rc.NotErrParamNotFound(err) { + return nil, err + } + fsString, err := in.GetString("fs") + if err != nil { + return nil, err + } + if remote != "" { + in = in.Copy() + in["fs"] = fspath.JoinRootPath(fsString, remote) + } + + offset, err := in.GetInt64("offset") + if rc.NotErrParamNotFound(err) { + return nil, err + } + count, err := in.GetInt64("count") + if rc.NotErrParamNotFound(err) { + return nil, err + } + if rc.IsErrParamNotFound(err) { + count = -1 + } + head, err := in.GetInt64("head") + if rc.NotErrParamNotFound(err) { + return nil, err + } + tail, err := in.GetInt64("tail") + if rc.NotErrParamNotFound(err) { + return nil, err + } + + usedOffset := offset != 0 || count >= 0 + usedHead := head > 0 + usedTail := tail > 0 + + if (usedHead && usedTail) || (usedHead && usedOffset) || (usedTail && usedOffset) { + return nil, errors.New("can only use one of head, tail or offset/count") + } + if head < 0 { + return nil, errors.New("head cannot be negative") + } + if tail < 0 { + return nil, errors.New("tail cannot be negative") + } + if head > 0 { + offset = 0 + count = head + } + if tail > 0 { + offset = -tail + count = -1 + } + + limit := int64(DefaultGetFileMaxSize) + maxSize, err := in.GetInt64("maxSize") + if rc.NotErrParamNotFound(err) { + return nil, err + } + if !rc.IsErrParamNotFound(err) { + if maxSize <= 0 { + return nil, errors.New("maxSize must be greater than 0") + } + if maxSize > DefaultGetFileMaxSize { + return nil, fmt.Errorf("maxSize %d cannot exceed built-in maximum of %d bytes", maxSize, DefaultGetFileMaxSize) + } + limit = maxSize + } + + base64Encoded, err := in.GetBool("base64") + if rc.NotErrParamNotFound(err) { + return nil, err + } + + ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs") + if err != nil { + return nil, err + } + + ci := fs.GetConfig(ctx) + var ( + fileCount int + w = &maxLimitWriter{limit: limit} + ) + + err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error { + for _, entry := range entries { + o, ok := entry.(fs.Object) + if !ok { + continue + } + fileCount++ + if fileCount > 1 { + return errors.New("found more than one file; operations/getfile only supports a single file") + } + + // Pre-check size if known + size := o.Size() + if size >= 0 { + start := offset + if start < 0 { + start += size + } + if start > size { + start = size + } + reqSize := size - start + if count >= 0 && count < reqSize { + reqSize = count + } + if reqSize > limit { + return fmt.Errorf("requested size (%d bytes) exceeds maximum limit of %d bytes; use offset/count, maxSize or --rc-serve", reqSize, limit) + } + } + + err = catObject(ctx, o, w, offset, count) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + if fileCount == 0 { + return nil, fs.ErrorObjectNotFound + } + + data := w.buf.Bytes() + if base64Encoded { + return rc.Params{ + "result": base64.StdEncoding.EncodeToString(data), + }, nil + } + if !utf8.Valid(data) { + return nil, errors.New("file content is not valid UTF-8 (use base64=true for binary files)") + } + return rc.Params{ + "result": string(data), + }, nil +} diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go index 4877806dc..3cf9c558b 100644 --- a/fs/operations/rc_test.go +++ b/fs/operations/rc_test.go @@ -2,6 +2,7 @@ package operations_test import ( "context" + "encoding/base64" "fmt" "net/http" "net/http/httptest" @@ -893,3 +894,210 @@ func TestRcHashsumFile(t *testing.T) { assert.Equal(t, "md5", out["hashType"]) assert.Equal(t, "0ef726ce9b1a7692357ff70dd321d595", out["hash"]) } + +// operations/getfile: get a single file +func TestRcGetFile(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/getfile") + r.Mkdir(ctx, r.Fremote) + + file1Contents := "Hello Rclone GetFile Operation!" + file1 := r.WriteBoth(ctx, "getfile-file1.txt", file1Contents, t1) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + // 1. Basic test with fs + remote + in := rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + } + out, err := call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, file1Contents, out["result"]) + + // 2. Combined fs parameter + in = rc.Params{ + "fs": path.Join(r.FremoteName, file1.Path), + } + out, err = call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, file1Contents, out["result"]) + + // 3. Range offset and count + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "offset": int64(6), + "count": int64(6), + } + out, err = call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, "Rclone", out["result"]) + + // 4. Head flag + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "head": int64(5), + } + out, err = call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, "Hello", out["result"]) + + // 5. Tail flag + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "tail": int64(10), + } + out, err = call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, "Operation!", out["result"]) + + // 6. Negative offset + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "offset": int64(-10), + } + out, err = call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, "Operation!", out["result"]) + + // 7. Base64 encoding for text file + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "base64": true, + } + out, err = call.Fn(ctx, in) + require.NoError(t, err) + decoded, err := base64.StdEncoding.DecodeString(out["result"].(string)) + require.NoError(t, err) + assert.Equal(t, file1Contents, string(decoded)) + + // 8. Binary file: UTF-8 rejection vs base64 success + binaryData := "\x00\xff\xfe\xfd\x80\x81" + binFile := r.WriteBoth(ctx, "binary.dat", binaryData, t1) + in = rc.Params{ + "fs": r.FremoteName, + "remote": binFile.Path, + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "not valid UTF-8") + + in = rc.Params{ + "fs": r.FremoteName, + "remote": binFile.Path, + "base64": true, + } + out, err = call.Fn(ctx, in) + require.NoError(t, err) + decodedBin, err := base64.StdEncoding.DecodeString(out["result"].(string)) + require.NoError(t, err) + assert.Equal(t, []byte(binaryData), decodedBin) + + // 9. Max size limit violation test + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "maxSize": int64(5), + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "exceeds maximum limit") + + // 10. Max size exceeding built-in limit + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "maxSize": int64(operations.DefaultGetFileMaxSize + 1), + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot exceed built-in maximum") + + // 11. Invalid maxSize values (<= 0) + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "maxSize": int64(0), + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "maxSize must be greater than 0") + + in["maxSize"] = int64(-5) + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "maxSize must be greater than 0") + + // 12. Invalid parameter types + for _, badParam := range []rc.Params{ + {"fs": r.FremoteName, "remote": file1.Path, "offset": "not_an_int"}, + {"fs": r.FremoteName, "remote": file1.Path, "count": "not_an_int"}, + {"fs": r.FremoteName, "remote": file1.Path, "head": "not_an_int"}, + {"fs": r.FremoteName, "remote": file1.Path, "tail": "not_an_int"}, + {"fs": r.FremoteName, "remote": file1.Path, "maxSize": "not_an_int"}, + {"fs": r.FremoteName, "remote": file1.Path, "base64": "not_a_bool"}, + {"fs": r.FremoteName, "remote": 12345}, + } { + _, err = call.Fn(ctx, badParam) + require.Error(t, err) + } + + // 13. Conflicting and invalid bounds + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "head": int64(5), + "tail": int64(5), + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "can only use one of head, tail or offset/count") + + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "head": int64(-1), + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "head cannot be negative") + + in = rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "tail": int64(-1), + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "tail cannot be negative") + + // 14. Non-existent file / directory with 0 files + in = rc.Params{ + "fs": r.FremoteName, + "remote": "does_not_exist.txt", + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + + emptyDir := path.Join(r.LocalName, "emptydir") + require.NoError(t, os.Mkdir(emptyDir, 0755)) + in = rc.Params{ + "fs": emptyDir, + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Equal(t, fs.ErrorObjectNotFound, err) + + // 15. Directory containing multiple files + in = rc.Params{ + "fs": r.FremoteName, + } + _, err = call.Fn(ctx, in) + require.Error(t, err) + assert.Contains(t, err.Error(), "found more than one file") +}