diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go
index 24a2e705b..417ca003c 100644
--- a/cmd/serve/dlna/dlna.go
+++ b/cmd/serve/dlna/dlna.go
@@ -244,6 +244,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 	}
 	listener, err := net.Listen(network, s.httpListenAddr)
 	if err != nil {
+		s.vfs.Shutdown()
 		return nil, err
 	}
 	s.HTTPConn = listener
@@ -402,6 +403,7 @@ func (s *server) Wait() {
 // Shutdown the DLNA server
 func (s *server) Shutdown() error {
 	err := s.HTTPConn.Close()
+	s.vfs.Shutdown()
 	close(s.waitChan)
 	if err != nil {
 		return fmt.Errorf("failed to shutdown DLNA server: %w", err)
diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go
index 83a917761..db71b90b1 100644
--- a/cmd/serve/ftp/ftp.go
+++ b/cmd/serve/ftp/ftp.go
@@ -354,7 +354,12 @@ func (d *driver) CheckPasswd(sctx *ftp.Context, user, pass string) (ok bool, err
 	return true, nil
 }
 
-// Get the VFS for this connection
+// getVFS returns the VFS for this connection.
+//
+// In proxy mode, getVFS calls proxy.Call on each FTP command which refreshes
+// the proxy cache timer (like http/webdav). Therefore, connection-level pinning
+// is not used; only individual transfers exceeding the cache expiry window
+// could be affected.
 func (d *driver) getVFS(sctx *ftp.Context) (VFS *vfs.VFS, err error) {
 	if !d.provider.IsProxy() {
 		// If no proxy always use the same VFS
diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.go
index 20bb6eddb..3a4b9a217 100644
--- a/cmd/serve/nfs/nfs.go
+++ b/cmd/serve/nfs/nfs.go
@@ -82,6 +82,17 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.AddFlagsFromOptions(flagSet, "", OptionsInfo)
 }
 
+type nfsHandler struct {
+	*Server
+	vfs *vfs.VFS
+}
+
+func (h *nfsHandler) Shutdown() error {
+	err := h.Server.Shutdown()
+	h.vfs.Shutdown()
+	return err
+}
+
 func init() {
 	vfsflags.AddFlags(Command.Flags())
 	AddFlags(Command.Flags())
@@ -98,10 +109,16 @@ func init() {
 		var opt = Opt // set default opts
 		err = rc.ParseOptions(in, "opt", &opt)
 		if err != nil {
+			VFS.Shutdown()
 			return nil, err
 		}
 		// Create server
-		return NewServer(ctx, VFS, &opt)
+		s, err := NewServer(ctx, VFS, &opt)
+		if err != nil {
+			VFS.Shutdown()
+			return nil, err
+		}
+		return &nfsHandler{Server: s, vfs: VFS}, nil
 	})
 }
 
@@ -111,7 +128,9 @@ func Run(command *cobra.Command, args []string) {
 	cmd.CheckArgs(1, 1, command, args)
 	f = cmd.NewFsSrc(args)
 	cmd.Run(false, true, command, func() error {
-		s, err := NewServer(context.Background(), vfs.New(context.Background(), f, &vfscommon.Opt), &Opt)
+		VFS := vfs.New(context.Background(), f, &vfscommon.Opt)
+		defer VFS.Shutdown()
+		s, err := NewServer(context.Background(), VFS, &Opt)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go
index 1f10839cc..d7935e41d 100644
--- a/cmd/serve/s3/server.go
+++ b/cmd/serve/s3/server.go
@@ -55,6 +55,9 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 	}
 	defer func() {
 		if err != nil {
+			if w.backend != nil {
+				w.backend.stopReaper()
+			}
 			w.provider.Shutdown()
 		}
 	}()
@@ -168,6 +171,7 @@ func (w *Server) Addr() net.Addr {
 
 // Shutdown the server
 func (w *Server) Shutdown() error {
+	w.backend.stopReaper()
 	err := w.server.Shutdown()
 	w.provider.Shutdown()
 	return err
diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.go
index 7640fe261..cc8d65676 100644
--- a/cmd/serve/sftp/server.go
+++ b/cmd/serve/sftp/server.go
@@ -99,6 +99,13 @@ func (s *server) acceptConnection(nConn net.Conn) {
 	// Discard all global out-of-band Requests
 	go ssh.DiscardRequests(reqs)
 
+	if sshConn.Permissions != nil && sshConn.Permissions.Extensions != nil {
+		if vfsKey := sshConn.Permissions.Extensions["_vfsKey"]; vfsKey != "" {
+			s.provider.Pin(vfsKey)
+			defer s.provider.Unpin(vfsKey)
+		}
+	}
+
 	c := &conn{
 		what: what,
 		vfs:  s.getVFS(what, sshConn),
@@ -111,7 +118,7 @@ func (s *server) acceptConnection(nConn net.Conn) {
 	c.handlers = newVFSHandler(c.vfs)
 
 	// Accept all channels
-	go c.handleChannels(chans)
+	c.handleChannels(chans)
 }
 
 // Accept connections and call them in a go routine