f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rc: factor server code into rcserver and implement serving objects

Browse Source 
If a GET or HEAD request is receivied with a URL parameter of fs then
it will be served from that remote.
This commit is contained in:
Nick Craig-Wood
2018-11-03 11:32:00 +00:00
parent aa9b2c31f4
commit 0bfa9811f7
9 changed files with 253 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs/rc/cache.go
+12 -7
View File
@@ -14,16 +14,11 @@ var (
fsNewFs = fs.NewFs // for tests
)
// GetFsNamed gets a fs.Fs named fsName either from the cache or creates it afresh
func GetFsNamed(in Params, fsName string) (f fs.Fs, err error) {
// GetCachedFs gets a fs.Fs named fsString either from the cache or creates it afresh
func GetCachedFs(fsString string) (f fs.Fs, err error) {
fsCacheMu.Lock()
defer fsCacheMu.Unlock()
fsString, err := in.GetString(fsName)
if err != nil {
return nil, err
}
f = fsCache[fsString]
if f == nil {
f, err = fsNewFs(fsString)
@@ -34,6 +29,16 @@ func GetFsNamed(in Params, fsName string) (f fs.Fs, err error) {
return f, err
}
// GetFsNamed gets a fs.Fs named fsName either from the cache or creates it afresh
func GetFsNamed(in Params, fsName string) (f fs.Fs, err error) {
fsString, err := in.GetString(fsName)
if err != nil {
return nil, err
}
return GetCachedFs(fsString)
}
// GetFs gets a fs.Fs named "fs" either from the cache or creates it afresh
func GetFs(in Params) (f fs.Fs, err error) {
return GetFsNamed(in, "fs")
fs/rc/internal.go
+2 -3
View File
@@ -84,7 +84,7 @@ func rcError(in Params) (out Params, err error) {
// List the registered commands
func rcList(in Params) (out Params, err error) {
out = make(Params)
out["commands"] = registry.list()
out["commands"] = Calls.List()
return out, nil
}
@@ -125,7 +125,6 @@ func rcMemStats(in Params) (out Params, err error) {
// Do a garbage collection run
func rcGc(in Params) (out Params, err error) {
out = make(Params)
runtime.GC()
return out, nil
return nil, nil
}
fs/rc/rc.go
-171
View File
@@ -10,15 +10,9 @@ package rc
import (
"encoding/json"
"io"
"mime"
"net/http"
_ "net/http/pprof" // install the pprof http handlers
"strings"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
"github.com/skratchdot/open-golang/open"
)
// Options contains options for the remote control server
@@ -38,174 +32,9 @@ func init() {
DefaultOpt.HTTPOptions.ListenAddr = "localhost:5572"
}
// Start the remote control server if configured
func Start(opt *Options) {
if opt.Enabled {
s := newServer(opt)
go s.serve()
}
}
// server contains everything to run the server
type server struct {
srv *httplib.Server
files http.Handler
}
func newServer(opt *Options) *server {
// Serve on the DefaultServeMux so can have global registrations appear
mux := http.DefaultServeMux
s := &server{
srv: httplib.NewServer(mux, &opt.HTTPOptions),
}
mux.HandleFunc("/", s.handler)
// Add some more mime types which are often missing
mime.AddExtensionType(".wasm", "application/wasm")
mime.AddExtensionType(".js", "application/javascript")
// File handling
s.files = http.NewServeMux()
if opt.Files != "" {
fs.Logf(nil, "Serving files from %q", opt.Files)
s.files = http.FileServer(http.Dir(opt.Files))
}
return s
}
// serve runs the http server - doesn't return
func (s *server) serve() {
err := s.srv.Serve()
if err != nil {
fs.Errorf(nil, "Opening listener: %v", err)
}
fs.Logf(nil, "Serving remote control on %s", s.srv.URL())
// Open the files in the browser if set
if s.files != nil {
_ = open.Start(s.srv.URL())
}
s.srv.Wait()
}
// WriteJSON writes JSON in out to w
func WriteJSON(w io.Writer, out Params) error {
enc := json.NewEncoder(w)
enc.SetIndent("", "\t")
return enc.Encode(out)
}
// writeError writes a formatted error to the output
func writeError(path string, in Params, w http.ResponseWriter, err error, status int) {
fs.Errorf(nil, "rc: %q: error: %v", path, err)
// Adjust the error return for some well known errors
switch errors.Cause(err) {
case fs.ErrorDirNotFound, fs.ErrorObjectNotFound:
status = http.StatusNotFound
}
w.WriteHeader(status)
err = WriteJSON(w, Params{
"error": err.Error(),
"input": in,
})
if err != nil {
// can't return the error at this point
fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
}
}
// handler reads incoming requests and dispatches them
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
path := strings.Trim(r.URL.Path, "/")
w.Header().Add("Access-Control-Allow-Origin", "*")
// echo back access control headers client needs
reqAccessHeaders := r.Header.Get("Access-Control-Request-Headers")
w.Header().Add("Access-Control-Allow-Headers", reqAccessHeaders)
switch r.Method {
case "POST":
s.handlePost(w, r, path)
case "OPTIONS":
s.handleOptions(w, r, path)
case "GET":
s.handleGet(w, r, path)
default:
writeError(path, nil, w, errors.Errorf("method %q not allowed", r.Method), http.StatusMethodNotAllowed)
return
}
}
func (s *server) handlePost(w http.ResponseWriter, r *http.Request, path string) {
// Parse the POST and URL parameters into r.Form, for others r.Form will be empty value
err := r.ParseForm()
if err != nil {
writeError(path, nil, w, errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
return
}
// Read the POST and URL parameters into in
in := make(Params)
for k, vs := range r.Form {
if len(vs) > 0 {
in[k] = vs[len(vs)-1]
}
}
// Parse a JSON blob from the input
if r.Header.Get("Content-Type") == "application/json" {
err := json.NewDecoder(r.Body).Decode(&in)
if err != nil {
writeError(path, in, w, errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
return
}
}
// Find the call
call := registry.get(path)
if call == nil {
writeError(path, in, w, errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed)
return
}
// Check to see if it is async or not
isAsync, err := in.GetBool("_async")
if err != nil {
writeError(path, in, w, err, http.StatusBadRequest)
return
}
fs.Debugf(nil, "rc: %q: with parameters %+v", path, in)
var out Params
if isAsync {
out, err = StartJob(call.Fn, in)
} else {
out, err = call.Fn(in)
}
if err != nil {
writeError(path, in, w, err, http.StatusInternalServerError)
return
}
if out == nil {
out = make(Params)
}
fs.Debugf(nil, "rc: %q: reply %+v: %v", path, out, err)
err = WriteJSON(w, out)
if err != nil {
// can't return the error at this point
fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
}
}
func (s *server) handleOptions(w http.ResponseWriter, r *http.Request, path string) {
w.WriteHeader(http.StatusOK)
}
func (s *server) handleGet(w http.ResponseWriter, r *http.Request, path string) {
if s.files == nil {
w.WriteHeader(http.StatusNotFound)
return
}
s.files.ServeHTTP(w, r)
}
fs/rc/rcserver/rcserver.go
+199
View File
@@ -0,0 +1,199 @@
// Package rcserver implements the HTTP endpoint to serve the remote control
package rcserver
import (
"encoding/json"
"mime"
"net/http"
"strings"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/cmd/serve/httplib/serve"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/rc"
"github.com/pkg/errors"
"github.com/skratchdot/open-golang/open"
)
// Start the remote control server if configured
func Start(opt *rc.Options) {
if opt.Enabled {
s := newServer(opt)
go s.serve()
}
}
// server contains everything to run the server
type server struct {
srv *httplib.Server
files http.Handler
}
func newServer(opt *rc.Options) *server {
// Serve on the DefaultServeMux so can have global registrations appear
mux := http.DefaultServeMux
s := &server{
srv: httplib.NewServer(mux, &opt.HTTPOptions),
}
mux.HandleFunc("/", s.handler)
// Add some more mime types which are often missing
_ = mime.AddExtensionType(".wasm", "application/wasm")
_ = mime.AddExtensionType(".js", "application/javascript")
// File handling
if opt.Files != "" {
fs.Logf(nil, "Serving files from %q", opt.Files)
s.files = http.FileServer(http.Dir(opt.Files))
}
return s
}
// serve runs the http server - doesn't return
func (s *server) serve() {
err := s.srv.Serve()
if err != nil {
fs.Errorf(nil, "Opening listener: %v", err)
}
fs.Logf(nil, "Serving remote control on %s", s.srv.URL())
// Open the files in the browser if set
if s.files != nil {
_ = open.Start(s.srv.URL())
}
s.srv.Wait()
}
// writeError writes a formatted error to the output
func writeError(path string, in rc.Params, w http.ResponseWriter, err error, status int) {
fs.Errorf(nil, "rc: %q: error: %v", path, err)
// Adjust the error return for some well known errors
errOrig := errors.Cause(err)
switch {
case errOrig == fs.ErrorDirNotFound || errOrig == fs.ErrorObjectNotFound:
status = http.StatusNotFound
case rc.IsErrParamInvalid(err) || rc.IsErrParamNotFound(err):
status = http.StatusBadRequest
}
w.WriteHeader(status)
err = rc.WriteJSON(w, rc.Params{
"status": status,
"error": err.Error(),
"input": in,
"path": path,
})
if err != nil {
// can't return the error at this point
fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
}
}
// handler reads incoming requests and dispatches them
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
path := strings.Trim(r.URL.Path, "/")
w.Header().Add("Access-Control-Allow-Origin", "*")
// echo back access control headers client needs
reqAccessHeaders := r.Header.Get("Access-Control-Request-Headers")
w.Header().Add("Access-Control-Allow-Headers", reqAccessHeaders)
switch r.Method {
case "POST":
s.handlePost(w, r, path)
case "OPTIONS":
s.handleOptions(w, r, path)
case "GET":
s.handleGet(w, r, path)
default:
writeError(path, nil, w, errors.Errorf("method %q not allowed", r.Method), http.StatusMethodNotAllowed)
return
}
}
func (s *server) handlePost(w http.ResponseWriter, r *http.Request, path string) {
// Parse the POST and URL parameters into r.Form, for others r.Form will be empty value
err := r.ParseForm()
if err != nil {
writeError(path, nil, w, errors.Wrap(err, "failed to parse form/URL parameters"), http.StatusBadRequest)
return
}
// Read the POST and URL parameters into in
in := make(rc.Params)
for k, vs := range r.Form {
if len(vs) > 0 {
in[k] = vs[len(vs)-1]
}
}
// Parse a JSON blob from the input
if r.Header.Get("Content-Type") == "application/json" {
err := json.NewDecoder(r.Body).Decode(&in)
if err != nil {
writeError(path, in, w, errors.Wrap(err, "failed to read input JSON"), http.StatusBadRequest)
return
}
}
// Find the call
call := rc.Calls.Get(path)
if call == nil {
writeError(path, in, w, errors.Errorf("couldn't find method %q", path), http.StatusMethodNotAllowed)
return
}
// Check to see if it is async or not
isAsync, err := in.GetBool("_async")
if rc.NotErrParamNotFound(err) {
writeError(path, in, w, err, http.StatusBadRequest)
return
}
fs.Debugf(nil, "rc: %q: with parameters %+v", path, in)
var out rc.Params
if isAsync {
out, err = rc.StartJob(call.Fn, in)
} else {
out, err = call.Fn(in)
}
if err != nil {
writeError(path, in, w, err, http.StatusInternalServerError)
return
}
if out == nil {
out = make(rc.Params)
}
fs.Debugf(nil, "rc: %q: reply %+v: %v", path, out, err)
err = rc.WriteJSON(w, out)
if err != nil {
// can't return the error at this point
fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
}
}
func (s *server) handleOptions(w http.ResponseWriter, r *http.Request, path string) {
w.WriteHeader(http.StatusOK)
}
func (s *server) handleGet(w http.ResponseWriter, r *http.Request, path string) {
// if we have an &fs parameter we are serving from a different fs
fsName := r.URL.Query().Get("fs")
if fsName != "" {
f, err := rc.GetCachedFs(fsName)
if err != nil {
writeError(path, nil, w, errors.Wrap(err, "failed to make Fs"), http.StatusInternalServerError)
return
}
o, err := f.NewObject(path)
if err != nil {
writeError(path, nil, w, errors.Wrap(err, "failed to find object"), http.StatusInternalServerError)
return
}
serve.Object(w, r, o)
} else if s.files == nil {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
} else {
s.files.ServeHTTP(w, r)
}
}
fs/rc/registry.go
+8 -8
View File
@@ -36,7 +36,7 @@ func NewRegistry() *Registry {
}
// Add a call to the registry
func (r *Registry) add(call Call) {
func (r *Registry) Add(call Call) {
r.mu.Lock()
defer r.mu.Unlock()
call.Path = strings.Trim(call.Path, "/")
@@ -45,15 +45,15 @@ func (r *Registry) add(call Call) {
r.call[call.Path] = &call
}
// get a Call from a path or nil
func (r *Registry) get(path string) *Call {
// Get a Call from a path or nil
func (r *Registry) Get(path string) *Call {
r.mu.RLock()
defer r.mu.RUnlock()
return r.call[path]
}
// get a list of all calls in alphabetical order
func (r *Registry) list() (out []*Call) {
// List of all calls in alphabetical order
func (r *Registry) List() (out []*Call) {
r.mu.RLock()
defer r.mu.RUnlock()
var keys []string
@@ -67,10 +67,10 @@ func (r *Registry) list() (out []*Call) {
return out
}
// The global registry
var registry = NewRegistry()
// Calls is the global registry of Call objects
var Calls = NewRegistry()
// Add a function to the global registry
func Add(call Call) {
registry.add(call)
Calls.Add(call)
}