Nick Craig-Wood
52cf74dc59
An audit of the shell escaping alongside the PowerShell fix found the Windows Command Prompt path only rejected the double quote delimiter. A remote filename containing % or ! can trigger environment variable expansion even inside double quotes, and a newline or carriage return ends the command, so a crafted filename could alter the command run during server-side hashing. Reject paths containing these characters, matching the existing treatment of the double quote.
2854 lines
93 KiB
Go
2854 lines
93 KiB
Go
//go:build !plan9
|
||
|
||
// Package sftp provides a filesystem interface using github.com/pkg/sftp
|
||
package sftp
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/base64"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
iofs "io/fs"
|
||
"net"
|
||
"net/url"
|
||
"os"
|
||
"path"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"github.com/pkg/sftp"
|
||
"github.com/rclone/rclone/fs"
|
||
"github.com/rclone/rclone/fs/accounting"
|
||
"github.com/rclone/rclone/fs/config"
|
||
"github.com/rclone/rclone/fs/config/configmap"
|
||
"github.com/rclone/rclone/fs/config/configstruct"
|
||
"github.com/rclone/rclone/fs/config/obscure"
|
||
"github.com/rclone/rclone/fs/fserrors"
|
||
"github.com/rclone/rclone/fs/hash"
|
||
"github.com/rclone/rclone/lib/encoder"
|
||
"github.com/rclone/rclone/lib/env"
|
||
"github.com/rclone/rclone/lib/pacer"
|
||
"github.com/rclone/rclone/lib/readers"
|
||
sshagent "github.com/xanzy/ssh-agent"
|
||
"golang.org/x/crypto/ssh"
|
||
"golang.org/x/crypto/ssh/knownhosts"
|
||
)
|
||
|
||
const (
|
||
defaultShellType = "unix"
|
||
shellTypeNotSupported = "none"
|
||
hashCommandNotSupported = "none"
|
||
minSleep = 100 * time.Millisecond
|
||
maxSleep = 2 * time.Second
|
||
decayConstant = 2 // bigger for slower decay, exponential
|
||
keepAliveInterval = time.Minute // send keepalives every this long while running commands
|
||
maxHostKeys = 16 // caps the number of host_keys entries
|
||
)
|
||
|
||
var (
|
||
currentUser = env.CurrentUser()
|
||
posixWinAbsPathRegex = regexp.MustCompile(`^/[a-zA-Z]\:($|/)`) // E.g. "/C:" or anything starting with "/C:/"
|
||
unixShellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]")
|
||
)
|
||
|
||
func init() {
|
||
fsi := &fs.RegInfo{
|
||
Name: "sftp",
|
||
Description: "SSH/SFTP",
|
||
NewFs: NewFs,
|
||
Options: []fs.Option{{
|
||
Name: "host",
|
||
Help: "SSH host to connect to.\n\nE.g. \"example.com\".",
|
||
Required: true,
|
||
Sensitive: true,
|
||
}, {
|
||
Name: "user",
|
||
Help: "SSH username.",
|
||
Default: currentUser,
|
||
Sensitive: true,
|
||
}, {
|
||
Name: "port",
|
||
Help: "SSH port number.",
|
||
Default: 22,
|
||
}, {
|
||
Name: "pass",
|
||
Help: "SSH password, leave blank to use ssh-agent.",
|
||
IsPassword: true,
|
||
}, {
|
||
Name: "key_pem",
|
||
Help: `Raw PEM-encoded private key.
|
||
|
||
Note that this should be on a single line with line endings replaced with '\n', eg
|
||
|
||
key_pem = -----BEGIN RSA PRIVATE KEY-----\nMaMbaIXtE\n0gAMbMbaSsd\nMbaass\n-----END RSA PRIVATE KEY-----
|
||
|
||
This will generate the single line correctly:
|
||
|
||
awk '{printf "%s\\n", $0}' < ~/.ssh/id_rsa
|
||
|
||
If specified, it will override the key_file parameter.`,
|
||
Sensitive: true,
|
||
}, {
|
||
Name: "key_file",
|
||
Help: "Path to PEM-encoded private key file.\n\nLeave blank or set key-use-agent to use ssh-agent." + env.ShellExpandHelp,
|
||
}, {
|
||
Name: "key_file_pass",
|
||
Help: `The passphrase to decrypt the PEM-encoded private key file.
|
||
|
||
Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
|
||
in the new OpenSSH format can't be used.`,
|
||
IsPassword: true,
|
||
Sensitive: true,
|
||
}, {
|
||
Name: "pubkey",
|
||
Help: `SSH public certificate for public certificate based authentication.
|
||
Set this if you have a signed certificate you want to use for authentication.
|
||
If specified will override pubkey_file.`,
|
||
}, {
|
||
Name: "pubkey_file",
|
||
Help: `Optional path to public key file.
|
||
|
||
Set this if you have a signed certificate you want to use for authentication.` + env.ShellExpandHelp,
|
||
}, {
|
||
Name: "known_hosts_file",
|
||
Help: `Optional path to known_hosts file.
|
||
|
||
Set this value to enable server host key validation. Set to ` + "`none`" + ` to silence the "No host key validation" notice.` + env.ShellExpandHelp,
|
||
Advanced: true,
|
||
Examples: []fs.OptionExample{{
|
||
Value: "~/.ssh/known_hosts",
|
||
Help: "Use OpenSSH's known_hosts file.",
|
||
}},
|
||
}, {
|
||
Name: "pin_host_key",
|
||
Default: false,
|
||
Hide: fs.OptionHideConfigurator,
|
||
Help: `Pin the server host key on first connection (Trust On First Use).
|
||
|
||
Intended for one-time use as the ` + "`--sftp-pin-host-key`" + ` command-line
|
||
flag. Run rclone once with the flag and the server's host key will be
|
||
recorded into the host_keys config option. On subsequent runs (without
|
||
the flag) host_keys is consulted and any mismatch is refused.
|
||
|
||
Setting this option persistently in the config file is not
|
||
recommended. While it is set, rclone will also accept any new
|
||
host key algorithm the server later presents, which widens the trust
|
||
surface beyond the initial pin. To pin a new key after a legitimate
|
||
key change, re-run with the flag.
|
||
|
||
The first connection is unauthenticated, so ideally do it over a
|
||
trusted network or cross-check the fingerprint rclone logs against
|
||
one provided out of band.
|
||
|
||
If known_hosts_file is also set, that takes precedence and this option
|
||
is ignored.`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "host_keys",
|
||
Default: fs.CommaSepList{},
|
||
Help: `Pinned host keys for this remote, used to verify the server.
|
||
|
||
Comma-separated list of "algo base64-key" entries (the same format as
|
||
the second and third fields of an OpenSSH known_hosts line). Usually
|
||
populated automatically by running once with --sftp-pin-host-key, but
|
||
can be set by hand to pin a server's public key obtained out of band.
|
||
Note that each entry is the complete public key, not its SHA256
|
||
fingerprint.
|
||
|
||
When non-empty, the offered host key must match one of the entries or
|
||
the connection is refused. To re-pin after a legitimate key change,
|
||
clear this option and reconnect with --sftp-pin-host-key, or edit the
|
||
value directly.
|
||
|
||
At most ` + strconv.Itoa(maxHostKeys) + ` entries may be pinned.`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "key_use_agent",
|
||
Help: `When set forces the usage of the ssh-agent.
|
||
|
||
When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
|
||
requested from the ssh-agent. This allows to avoid ` + "`Too many authentication failures for *username*`" + ` errors
|
||
when the ssh-agent contains many keys.`,
|
||
Default: false,
|
||
}, {
|
||
Name: "use_insecure_cipher",
|
||
Help: `Enable the use of insecure ciphers and key exchange methods.
|
||
|
||
This enables the use of the following insecure ciphers and key exchange methods:
|
||
|
||
- aes128-cbc
|
||
- aes192-cbc
|
||
- aes256-cbc
|
||
- 3des-cbc
|
||
- diffie-hellman-group-exchange-sha256
|
||
- diffie-hellman-group-exchange-sha1
|
||
|
||
Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.
|
||
|
||
This must be false if you use either ciphers or key_exchange advanced options.
|
||
`,
|
||
Default: false,
|
||
Examples: []fs.OptionExample{
|
||
{
|
||
Value: "false",
|
||
Help: "Use default Cipher list.",
|
||
}, {
|
||
Value: "true",
|
||
Help: "Enables the use of the aes128-cbc cipher and diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1 key exchange.",
|
||
},
|
||
},
|
||
}, {
|
||
Name: "disable_hashcheck",
|
||
Default: false,
|
||
Help: "Disable the execution of SSH commands to determine if remote file hashing is available.\n\nLeave blank or set to false to enable hashing (recommended), set to true to disable hashing.",
|
||
}, {
|
||
Name: "ask_password",
|
||
Default: false,
|
||
Help: `Allow asking for SFTP password when needed.
|
||
|
||
If this is set and no password is supplied then rclone will:
|
||
- ask for a password
|
||
- not contact the ssh agent
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "path_override",
|
||
Default: "",
|
||
Help: `Override path used by SSH shell commands.
|
||
|
||
This allows checksum calculation when SFTP and SSH paths are
|
||
different. This issue affects among others Synology NAS boxes.
|
||
|
||
E.g. if shared folders can be found in directories representing volumes:
|
||
|
||
rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory
|
||
|
||
E.g. if home directory can be found in a shared folder called "home":
|
||
|
||
rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory
|
||
|
||
To specify only the path to the SFTP remote's root, and allow rclone to add any relative subpaths automatically (including unwrapping/decrypting remotes as necessary), add the '@' character to the beginning of the path.
|
||
|
||
E.g. the first example above could be rewritten as:
|
||
|
||
rclone sync /home/local/directory remote:/directory --sftp-path-override @/volume2
|
||
|
||
Note that when using this method with Synology "home" folders, the full "/homes/USER" path should be specified instead of "/home".
|
||
|
||
E.g. the second example above should be rewritten as:
|
||
|
||
rclone sync /home/local/directory remote:/homes/USER/directory --sftp-path-override @/volume1`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: config.ConfigEncoding,
|
||
Help: config.ConfigEncodingHelp,
|
||
Advanced: true,
|
||
Default: encoder.Display,
|
||
}, {
|
||
Name: "set_modtime",
|
||
Default: true,
|
||
Help: "Set the modified time on the remote if set.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "shell_type",
|
||
Default: "",
|
||
Help: "The type of SSH shell on remote server, if any.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
Examples: []fs.OptionExample{
|
||
{
|
||
Value: shellTypeNotSupported,
|
||
Help: "No shell access",
|
||
}, {
|
||
Value: "unix",
|
||
Help: "Unix shell",
|
||
}, {
|
||
Value: "powershell",
|
||
Help: "PowerShell",
|
||
}, {
|
||
Value: "cmd",
|
||
Help: "Windows Command Prompt",
|
||
},
|
||
},
|
||
}, {
|
||
Name: "hashes",
|
||
Help: `Comma separated list of supported checksum types.`,
|
||
Default: fs.CommaSepList{},
|
||
Advanced: true,
|
||
}, {
|
||
Name: "md5sum_command",
|
||
Default: "",
|
||
Help: "The command used to read MD5 hashes.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "sha1sum_command",
|
||
Default: "",
|
||
Help: "The command used to read SHA-1 hashes.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "crc32sum_command",
|
||
Default: "",
|
||
Help: "The command used to read CRC-32 hashes.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "sha256sum_command",
|
||
Default: "",
|
||
Help: "The command used to read SHA-256 hashes.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "blake3sum_command",
|
||
Default: "",
|
||
Help: "The command used to read BLAKE3 hashes.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "xxh3sum_command",
|
||
Default: "",
|
||
Help: "The command used to read XXH3 hashes.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "xxh128sum_command",
|
||
Default: "",
|
||
Help: "The command used to read XXH128 hashes.\n\nLeave blank for autodetect.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "skip_links",
|
||
Default: false,
|
||
Help: `Set to skip any symlinks and any other non regular files.
|
||
|
||
This only affects listing: symlinks and other non regular files are
|
||
omitted from directory listings. It is not a security control and does
|
||
not prevent writes from following symlinks on the server - confining an
|
||
SFTP account to a directory must be enforced server side (for example
|
||
with a chroot jail or restricted permissions).`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "subsystem",
|
||
Default: "sftp",
|
||
Help: "Specifies the SSH2 subsystem on the remote host.",
|
||
Advanced: true,
|
||
}, {
|
||
Name: "server_command",
|
||
Default: "",
|
||
Help: `Specifies the path or command to run a sftp server on the remote host.
|
||
|
||
The subsystem option is ignored when server_command is defined.
|
||
|
||
If adding server_command to the configuration file please note that
|
||
it should not be enclosed in quotes, since that will make rclone fail.
|
||
|
||
A working example is:
|
||
|
||
[remote_name]
|
||
type = sftp
|
||
server_command = sudo /usr/libexec/openssh/sftp-server`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "use_fstat",
|
||
Default: false,
|
||
Help: `If set use fstat instead of stat.
|
||
|
||
Some servers limit the amount of open files and calling Stat after opening
|
||
the file will throw an error from the server. Setting this flag will call
|
||
Fstat instead of Stat which is called on an already open file handle.
|
||
|
||
It has been found that this helps with IBM Sterling SFTP servers which have
|
||
"extractability" level set to 1 which means only 1 file can be opened at
|
||
any given time.
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "disable_concurrent_reads",
|
||
Default: false,
|
||
Help: `If set don't use concurrent reads.
|
||
|
||
Normally concurrent reads are safe to use and not using them will
|
||
degrade performance, so this option is disabled by default.
|
||
|
||
Some servers limit the amount number of times a file can be
|
||
downloaded. Using concurrent reads can trigger this limit, so if you
|
||
have a server which returns
|
||
|
||
Failed to copy: file does not exist
|
||
|
||
Then you may need to enable this flag.
|
||
|
||
If concurrent reads are disabled, the use_fstat option is ignored.
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "disable_concurrent_writes",
|
||
Default: false,
|
||
Help: `If set don't use concurrent writes.
|
||
|
||
Normally rclone uses concurrent writes to upload files. This improves
|
||
the performance greatly, especially for distant servers.
|
||
|
||
This option disables concurrent writes should that be necessary.
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "idle_timeout",
|
||
Default: fs.Duration(60 * time.Second),
|
||
Help: `Max time before closing idle connections.
|
||
|
||
If no connections have been returned to the connection pool in the time
|
||
given, rclone will empty the connection pool.
|
||
|
||
Set to 0 to keep connections indefinitely.
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "chunk_size",
|
||
Help: `Upload and download chunk size.
|
||
|
||
This controls the maximum size of payload in SFTP protocol packets.
|
||
The RFC limits this to 32768 bytes (32k), which is the default. However,
|
||
a lot of servers support larger sizes, typically limited to a maximum
|
||
total package size of 256k, and setting it larger will increase transfer
|
||
speed dramatically on high latency links. This includes OpenSSH, and,
|
||
for example, using the value of 255k works well, leaving plenty of room
|
||
for overhead while still being within a total packet size of 256k.
|
||
|
||
Make sure to test thoroughly before using a value higher than 32k,
|
||
and only use it if you always connect to the same server or after
|
||
sufficiently broad testing. If you get errors such as
|
||
"failed to send packet payload: EOF", lots of "connection lost",
|
||
or "corrupted on transfer", when copying a larger file, try lowering
|
||
the value. The server run by [rclone serve sftp](/commands/rclone_serve_sftp)
|
||
sends packets with standard 32k maximum payload so you must not
|
||
set a different chunk_size when downloading files, but it accepts
|
||
packets up to the 256k total size, so for uploads the chunk_size
|
||
can be set as for the OpenSSH example above.
|
||
`,
|
||
Default: 32 * fs.Kibi,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "concurrency",
|
||
Help: `The maximum number of outstanding requests for one file
|
||
|
||
This controls the maximum number of outstanding requests for one file.
|
||
Increasing it will increase throughput on high latency links at the
|
||
cost of using more memory.
|
||
`,
|
||
Default: 64,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "connections",
|
||
Help: strings.ReplaceAll(`Maximum number of SFTP simultaneous connections, 0 for unlimited.
|
||
|
||
Note that setting this is very likely to cause deadlocks so it should
|
||
be used with care.
|
||
|
||
If you are doing a sync or copy then make sure connections is one more
|
||
than the sum of |--transfers| and |--checkers|.
|
||
|
||
If you use |--check-first| then it just needs to be one more than the
|
||
maximum of |--checkers| and |--transfers|.
|
||
|
||
So for |connections 3| you'd use |--checkers 2 --transfers 2
|
||
--check-first| or |--checkers 1 --transfers 1|.
|
||
|
||
`, "|", "`"),
|
||
Default: 0,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "set_env",
|
||
Default: fs.SpaceSepList{},
|
||
Help: `Environment variables to pass to sftp and commands
|
||
|
||
Set environment variables in the form:
|
||
|
||
VAR=value
|
||
|
||
to be passed to the sftp client and to any commands run (eg md5sum).
|
||
|
||
Pass multiple variables space separated, eg
|
||
|
||
VAR1=value VAR2=value
|
||
|
||
and pass variables with spaces in quotes, eg
|
||
|
||
"VAR3=value with space" "VAR4=value with space" VAR5=nospacehere
|
||
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "ciphers",
|
||
Default: fs.SpaceSepList{},
|
||
Help: `Space separated list of ciphers to be used for session encryption, ordered by preference.
|
||
|
||
At least one must match with server configuration. This can be checked for example using ssh -Q cipher.
|
||
|
||
This must not be set if use_insecure_cipher is true.
|
||
|
||
Example:
|
||
|
||
aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "key_exchange",
|
||
Default: fs.SpaceSepList{},
|
||
Help: `Space separated list of key exchange algorithms, ordered by preference.
|
||
|
||
At least one must match with server configuration. This can be checked for example using ssh -Q kex.
|
||
|
||
This must not be set if use_insecure_cipher is true.
|
||
|
||
Example:
|
||
|
||
sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "macs",
|
||
Default: fs.SpaceSepList{},
|
||
Help: `Space separated list of MACs (message authentication code) algorithms, ordered by preference.
|
||
|
||
At least one must match with server configuration. This can be checked for example using ssh -Q mac.
|
||
|
||
Example:
|
||
|
||
umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "host_key_algorithms",
|
||
Default: fs.SpaceSepList{},
|
||
Help: `Space separated list of host key algorithms, ordered by preference.
|
||
|
||
At least one must match with server configuration. This can be checked for example using ssh -Q HostKeyAlgorithms.
|
||
|
||
Note: This can affect the outcome of key negotiation with the server even if server host key validation is not enabled.
|
||
|
||
Example:
|
||
|
||
ssh-ed25519 ssh-rsa ssh-dss
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "ssh",
|
||
Default: fs.SpaceSepList{},
|
||
Help: `Path and arguments to external ssh binary.
|
||
|
||
Normally rclone will use its internal ssh library to connect to the
|
||
SFTP server. However it does not implement all possible ssh options so
|
||
it may be desirable to use an external ssh binary.
|
||
|
||
Rclone ignores all the internal config if you use this option and
|
||
expects you to configure the ssh binary with the user/host/port and
|
||
any other options you need.
|
||
|
||
**Important** The ssh command must log in without asking for a
|
||
password so needs to be configured with keys or certificates.
|
||
|
||
Rclone will run the command supplied either with the additional
|
||
arguments "-s sftp" to access the SFTP subsystem or with commands such
|
||
as "md5sum /path/to/file" appended to read checksums.
|
||
|
||
Any arguments with spaces in should be surrounded by "double quotes".
|
||
|
||
An example setting might be:
|
||
|
||
ssh -o ServerAliveInterval=20 user@example.com
|
||
|
||
Note that when using an external ssh binary rclone makes a new ssh
|
||
connection for every hash it calculates.
|
||
`,
|
||
}, {
|
||
Name: "socks_proxy",
|
||
Default: "",
|
||
Help: `Socks 5 proxy host.
|
||
|
||
Supports the format user:pass@host:port, user@host:port, host:port.
|
||
|
||
Example:
|
||
|
||
myUser:myPass@localhost:9005
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "http_proxy",
|
||
Default: "",
|
||
Help: `URL for HTTP CONNECT proxy
|
||
|
||
Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
|
||
|
||
Supports the format http://user:pass@host:port, http://host:port, http://host.
|
||
|
||
Example:
|
||
|
||
http://myUser:myPass@proxyhostname.example.com:8000
|
||
`,
|
||
Advanced: true,
|
||
}, {
|
||
Name: "copy_is_hardlink",
|
||
Default: false,
|
||
Help: `Set to enable server side copies using hardlinks.
|
||
|
||
The SFTP protocol does not define a copy command so normally server
|
||
side copies are not allowed with the sftp backend.
|
||
|
||
However the SFTP protocol does support hardlinking, and if you enable
|
||
this flag then the sftp backend will support server side copies. These
|
||
will be implemented by doing a hardlink from the source to the
|
||
destination.
|
||
|
||
Not all sftp servers support this.
|
||
|
||
Note that hardlinking two files together will use no additional space
|
||
as the source and the destination will be the same file.
|
||
|
||
This feature may be useful backups made with --copy-dest.`,
|
||
Advanced: true,
|
||
}},
|
||
}
|
||
fs.Register(fsi)
|
||
}
|
||
|
||
// Options defines the configuration for this backend
|
||
type Options struct {
|
||
Host string `config:"host"`
|
||
User string `config:"user"`
|
||
Port string `config:"port"`
|
||
Pass string `config:"pass"`
|
||
KeyPem string `config:"key_pem"`
|
||
KeyFile string `config:"key_file"`
|
||
KeyFilePass string `config:"key_file_pass"`
|
||
PubKey string `config:"pubkey"`
|
||
PubKeyFile string `config:"pubkey_file"`
|
||
KnownHostsFile string `config:"known_hosts_file"`
|
||
PinHostKey bool `config:"pin_host_key"`
|
||
HostKeys fs.CommaSepList `config:"host_keys"`
|
||
KeyUseAgent bool `config:"key_use_agent"`
|
||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||
DisableHashCheck bool `config:"disable_hashcheck"`
|
||
AskPassword bool `config:"ask_password"`
|
||
PathOverride string `config:"path_override"`
|
||
Enc encoder.MultiEncoder `config:"encoding"`
|
||
SetModTime bool `config:"set_modtime"`
|
||
ShellType string `config:"shell_type"`
|
||
Hashes fs.CommaSepList `config:"hashes"`
|
||
Md5sumCommand string `config:"md5sum_command"`
|
||
Sha1sumCommand string `config:"sha1sum_command"`
|
||
Crc32sumCommand string `config:"crc32sum_command"`
|
||
Sha256sumCommand string `config:"sha256sum_command"`
|
||
Blake3sumCommand string `config:"blake3sum_command"`
|
||
Xxh3sumCommand string `config:"xxh3sum_command"`
|
||
Xxh128sumCommand string `config:"xxh128sum_command"`
|
||
SkipLinks bool `config:"skip_links"`
|
||
Subsystem string `config:"subsystem"`
|
||
ServerCommand string `config:"server_command"`
|
||
UseFstat bool `config:"use_fstat"`
|
||
DisableConcurrentReads bool `config:"disable_concurrent_reads"`
|
||
DisableConcurrentWrites bool `config:"disable_concurrent_writes"`
|
||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||
Concurrency int `config:"concurrency"`
|
||
Connections int `config:"connections"`
|
||
SetEnv fs.SpaceSepList `config:"set_env"`
|
||
Ciphers fs.SpaceSepList `config:"ciphers"`
|
||
KeyExchange fs.SpaceSepList `config:"key_exchange"`
|
||
MACs fs.SpaceSepList `config:"macs"`
|
||
HostKeyAlgorithms fs.SpaceSepList `config:"host_key_algorithms"`
|
||
SSH fs.SpaceSepList `config:"ssh"`
|
||
SocksProxy string `config:"socks_proxy"`
|
||
HTTPProxy string `config:"http_proxy"`
|
||
CopyIsHardlink bool `config:"copy_is_hardlink"`
|
||
}
|
||
|
||
// Fs stores the interface to the remote SFTP files
|
||
type Fs struct {
|
||
name string
|
||
root string
|
||
absRoot string
|
||
shellRoot string
|
||
shellType string
|
||
opt Options // parsed options
|
||
ci *fs.ConfigInfo // global config
|
||
m configmap.Mapper // config
|
||
features *fs.Features // optional features
|
||
config *ssh.ClientConfig
|
||
url string
|
||
mkdirLock *stringLock
|
||
cachedHashes *hash.Set
|
||
poolMu sync.Mutex
|
||
pool []*conn
|
||
drain *time.Timer // used to drain the pool when we stop using the connections
|
||
pacer *fs.Pacer // pacer for operations
|
||
savedpswd string
|
||
sessions atomic.Int32 // count in use sessions
|
||
tokens *pacer.TokenDispenser
|
||
proxyURL *url.URL // address of HTTP proxy read from environment
|
||
|
||
hostKeysMu sync.RWMutex
|
||
hostKeys map[string][][]byte // algo -> list of trusted marshalled key bytes
|
||
tofu bool // accept and pin unknown host keys on first use
|
||
}
|
||
|
||
// pendingKey records a host key accepted via --sftp-pin-host-key
|
||
// during the SSH handshake of a single connection but not yet
|
||
// persisted to the config file. We only commit it after that
|
||
// connection's authentication succeeds, so a failed login can't pin a
|
||
// key.
|
||
type pendingKey struct {
|
||
algo string
|
||
marshalled []byte
|
||
fingerprint string
|
||
hostname string
|
||
}
|
||
|
||
// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
|
||
type Object struct {
|
||
fs *Fs
|
||
remote string
|
||
size int64 // size of the object
|
||
modTime uint32 // modification time of the object as unix time
|
||
mode os.FileMode // mode bits from the file
|
||
md5sum *string // Cached MD5 checksum
|
||
sha1sum *string // Cached SHA-1 checksum
|
||
crc32sum *string // Cached CRC-32 checksum
|
||
sha256sum *string // Cached SHA-256 checksum
|
||
blake3sum *string // Cached BLAKE3 checksum
|
||
xxh3sum *string // Cached XXH3 checksum
|
||
xxh128sum *string // Cached XXH128 checksum
|
||
}
|
||
|
||
// conn encapsulates an ssh client and corresponding sftp client
|
||
type conn struct {
|
||
sshClient sshClient
|
||
sftpClient *sftp.Client
|
||
err chan error
|
||
}
|
||
|
||
// Wait for connection to close
|
||
func (c *conn) wait() {
|
||
c.err <- c.sshClient.Wait()
|
||
}
|
||
|
||
// Send keepalives every interval over the ssh connection until done is closed
|
||
func (c *conn) sendKeepAlives(interval time.Duration) (done chan struct{}) {
|
||
done = make(chan struct{})
|
||
go func() {
|
||
t := time.NewTicker(interval)
|
||
defer t.Stop()
|
||
for {
|
||
select {
|
||
case <-t.C:
|
||
c.sshClient.SendKeepAlive()
|
||
case <-done:
|
||
return
|
||
}
|
||
}
|
||
}()
|
||
return done
|
||
}
|
||
|
||
// Closes the connection
|
||
func (c *conn) close() error {
|
||
sftpErr := c.sftpClient.Close()
|
||
sshErr := c.sshClient.Close()
|
||
if sftpErr != nil {
|
||
return sftpErr
|
||
}
|
||
return sshErr
|
||
}
|
||
|
||
// Returns an error if closed
|
||
func (c *conn) closed() error {
|
||
select {
|
||
case err := <-c.err:
|
||
return err
|
||
default:
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Show that we are using an ssh session
|
||
//
|
||
// Call removeSession() when done
|
||
func (f *Fs) addSession() {
|
||
f.sessions.Add(1)
|
||
}
|
||
|
||
// Show the ssh session is no longer in use
|
||
func (f *Fs) removeSession() {
|
||
f.sessions.Add(-1)
|
||
}
|
||
|
||
// getSessions shows whether there are any sessions in use
|
||
func (f *Fs) getSessions() int32 {
|
||
return f.sessions.Load()
|
||
}
|
||
|
||
// Open a new connection to the SFTP server.
|
||
func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) {
|
||
// Rate limit rate of new connections
|
||
c = &conn{
|
||
err: make(chan error, 1),
|
||
}
|
||
if len(f.opt.SSH) == 0 {
|
||
sshConfig := f.config
|
||
var pending *pendingKey
|
||
if f.tofu {
|
||
// Give this dial its own host key callback and pending key
|
||
// slot so a key stashed here can only be committed by this
|
||
// connection, and only if its own authentication succeeds.
|
||
configCopy := *f.config
|
||
configCopy.HostKeyCallback = f.hostKeyCallback(&pending)
|
||
sshConfig = &configCopy
|
||
}
|
||
c.sshClient, err = f.newSSHClientInternal(ctx, "tcp", f.opt.Host+":"+f.opt.Port, sshConfig)
|
||
if err == nil && pending != nil {
|
||
// ssh.NewClientConn only returns success once the server is
|
||
// authenticated, so the stashed key is safe to persist.
|
||
f.commitHostKey(pending)
|
||
}
|
||
} else {
|
||
c.sshClient, err = f.newSSHClientExternal()
|
||
}
|
||
if err != nil {
|
||
return nil, fmt.Errorf("couldn't connect SSH: %w", err)
|
||
}
|
||
c.sftpClient, err = f.newSftpClient(c.sshClient)
|
||
if err != nil {
|
||
_ = c.sshClient.Close()
|
||
return nil, fmt.Errorf("couldn't initialise SFTP: %w", err)
|
||
}
|
||
go c.wait()
|
||
return c, nil
|
||
}
|
||
|
||
// Set any environment variables on the ssh.Session
|
||
func (f *Fs) setEnv(s sshSession) error {
|
||
for _, env := range f.opt.SetEnv {
|
||
equal := strings.IndexRune(env, '=')
|
||
if equal < 0 {
|
||
return fmt.Errorf("no = found in env var %q", env)
|
||
}
|
||
// fs.Debugf(f, "Setting env %q = %q", env[:equal], env[equal+1:])
|
||
err := s.Setenv(env[:equal], env[equal+1:])
|
||
if err != nil {
|
||
return fmt.Errorf("failed to set env var %q: %w", env[:equal], err)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Creates a new SFTP client on conn, using the specified subsystem
|
||
// or sftp server, and zero or more option functions
|
||
func (f *Fs) newSftpClient(client sshClient, opts ...sftp.ClientOption) (*sftp.Client, error) {
|
||
s, err := client.NewSession()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
err = f.setEnv(s)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
pw, err := s.StdinPipe()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
pr, err := s.StdoutPipe()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
if f.opt.ServerCommand != "" {
|
||
if err := s.Start(f.opt.ServerCommand); err != nil {
|
||
return nil, err
|
||
}
|
||
} else {
|
||
if err := s.RequestSubsystem(f.opt.Subsystem); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
|
||
opts = append(opts,
|
||
sftp.UseFstat(f.opt.UseFstat),
|
||
sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
|
||
sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites),
|
||
sftp.MaxPacketUnchecked(int(f.opt.ChunkSize)),
|
||
sftp.MaxConcurrentRequestsPerFile(f.opt.Concurrency),
|
||
)
|
||
return sftp.NewClientPipe(pr, pw, opts...)
|
||
}
|
||
|
||
// Get an SFTP connection from the pool, or open a new one
|
||
func (f *Fs) getSftpConnection(ctx context.Context) (c *conn, err error) {
|
||
accounting.LimitTPS(ctx)
|
||
if f.opt.Connections > 0 {
|
||
f.tokens.Get()
|
||
}
|
||
f.poolMu.Lock()
|
||
for len(f.pool) > 0 {
|
||
c = f.pool[0]
|
||
f.pool = f.pool[1:]
|
||
err := c.closed()
|
||
if err == nil {
|
||
break
|
||
}
|
||
fs.Errorf(f, "Discarding closed SSH connection: %v", err)
|
||
c = nil
|
||
}
|
||
f.poolMu.Unlock()
|
||
if c != nil {
|
||
return c, nil
|
||
}
|
||
err = f.pacer.Call(func() (bool, error) {
|
||
c, err = f.sftpConnection(ctx)
|
||
if err != nil {
|
||
return fserrors.ShouldRetry(err), err
|
||
}
|
||
return false, nil
|
||
})
|
||
if f.opt.Connections > 0 && c == nil {
|
||
f.tokens.Put()
|
||
}
|
||
return c, err
|
||
}
|
||
|
||
// Return an SFTP connection to the pool
|
||
//
|
||
// It nils the pointed to connection out so it can't be reused
|
||
//
|
||
// if err is not nil then it checks the connection is alive using a
|
||
// Getwd request
|
||
func (f *Fs) putSftpConnection(pc **conn, err error) {
|
||
if f.opt.Connections > 0 {
|
||
defer f.tokens.Put()
|
||
}
|
||
c := *pc
|
||
if !c.sshClient.CanReuse() {
|
||
return
|
||
}
|
||
*pc = nil
|
||
if err != nil {
|
||
// work out if this is an expected error
|
||
isRegularError := false
|
||
var statusErr *sftp.StatusError
|
||
var pathErr *os.PathError
|
||
switch {
|
||
case errors.Is(err, os.ErrNotExist):
|
||
isRegularError = true
|
||
case errors.As(err, &statusErr):
|
||
isRegularError = true
|
||
case errors.As(err, &pathErr):
|
||
isRegularError = true
|
||
}
|
||
// If not a regular SFTP error code then check the connection
|
||
if !isRegularError {
|
||
_, nopErr := c.sftpClient.Getwd()
|
||
if nopErr != nil {
|
||
fs.Debugf(f, "Connection failed, closing: %v", nopErr)
|
||
_ = c.close()
|
||
return
|
||
}
|
||
fs.Debugf(f, "Connection OK after error: %v", err)
|
||
}
|
||
}
|
||
f.poolMu.Lock()
|
||
f.pool = append(f.pool, c)
|
||
if f.opt.IdleTimeout > 0 {
|
||
f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
|
||
}
|
||
f.poolMu.Unlock()
|
||
}
|
||
|
||
// Drain the pool of any connections
|
||
func (f *Fs) drainPool(ctx context.Context) (err error) {
|
||
f.poolMu.Lock()
|
||
defer f.poolMu.Unlock()
|
||
if sessions := f.getSessions(); sessions != 0 {
|
||
fs.Debugf(f, "Not closing %d unused connections as %d sessions active", len(f.pool), sessions)
|
||
if f.opt.IdleTimeout > 0 {
|
||
f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer
|
||
}
|
||
return nil
|
||
}
|
||
if f.opt.IdleTimeout > 0 {
|
||
f.drain.Stop()
|
||
}
|
||
if len(f.pool) != 0 {
|
||
fs.Debugf(f, "Closing %d unused connections", len(f.pool))
|
||
}
|
||
for i, c := range f.pool {
|
||
if cErr := c.closed(); cErr == nil {
|
||
cErr = c.close()
|
||
if cErr != nil {
|
||
fs.Debugf(f, "Ignoring error closing connection: %v", cErr)
|
||
}
|
||
}
|
||
f.pool[i] = nil
|
||
}
|
||
f.pool = nil
|
||
return nil
|
||
}
|
||
|
||
// parseHostKeysField parses the host_keys config option, a list of
|
||
// "algo base64-key" entries (the second and third fields of an OpenSSH
|
||
// known_hosts line). Returns a map from algorithm name to the list of
|
||
// marshalled public-key bytes pinned for that algorithm. Empty input
|
||
// yields an empty (non-nil) map.
|
||
//
|
||
// Each entry must be a valid SSH public key (not a certificate) whose
|
||
// key type matches the stated algorithm. Duplicate entries are
|
||
// dropped, and more than maxHostKeys distinct entries is an error.
|
||
func parseHostKeysField(entries fs.CommaSepList) (map[string][][]byte, error) {
|
||
out := make(map[string][][]byte)
|
||
total := 0
|
||
entries:
|
||
for i, entry := range entries {
|
||
entry = strings.TrimSpace(entry)
|
||
if entry == "" {
|
||
continue
|
||
}
|
||
fields := strings.Fields(entry)
|
||
if len(fields) != 2 {
|
||
return nil, fmt.Errorf("host_keys entry %d: expected \"<algo> <base64>\", got %q", i+1, entry)
|
||
}
|
||
algo := fields[0]
|
||
marshalled, err := base64.StdEncoding.DecodeString(fields[1])
|
||
if err != nil {
|
||
return nil, fmt.Errorf("host_keys entry %d (%s): invalid base64: %w", i+1, algo, err)
|
||
}
|
||
pk, err := ssh.ParsePublicKey(marshalled)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("host_keys entry %d (%s): not a valid SSH public key: %w", i+1, algo, err)
|
||
}
|
||
if _, isCert := pk.(*ssh.Certificate); isCert {
|
||
return nil, fmt.Errorf("host_keys entry %d (%s): SSH certificates cannot be pinned; use known_hosts_file with an @cert-authority entry instead", i+1, algo)
|
||
}
|
||
if pk.Type() != algo {
|
||
hint := ""
|
||
if pk.Type() == ssh.KeyAlgoRSA && (algo == ssh.KeyAlgoRSASHA256 || algo == ssh.KeyAlgoRSASHA512) {
|
||
// A common slip: rsa-sha2-* name signature algorithms,
|
||
// but an RSA public key blob's format is ssh-rsa.
|
||
hint = fmt.Sprintf(" (%s is a signature algorithm, use %s)", algo, ssh.KeyAlgoRSA)
|
||
}
|
||
return nil, fmt.Errorf("host_keys entry %d: stated algorithm %q doesn't match the key's type %q%s", i+1, algo, pk.Type(), hint)
|
||
}
|
||
for _, existing := range out[algo] {
|
||
if bytes.Equal(existing, marshalled) {
|
||
continue entries
|
||
}
|
||
}
|
||
total++
|
||
if total > maxHostKeys {
|
||
return nil, fmt.Errorf("host_keys has more than %d entries; trim it or use known_hosts_file instead", maxHostKeys)
|
||
}
|
||
out[algo] = append(out[algo], marshalled)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// formatHostKeysField formats the in-memory pinned host keys back into a
|
||
// CommaSepList of "algo base64" entries in a deterministic order so
|
||
// config-file diffs stay clean.
|
||
func formatHostKeysField(keys map[string][][]byte) fs.CommaSepList {
|
||
algos := make([]string, 0, len(keys))
|
||
for algo := range keys {
|
||
algos = append(algos, algo)
|
||
}
|
||
sort.Strings(algos)
|
||
var parts fs.CommaSepList
|
||
for _, algo := range algos {
|
||
// sort the keys within an algorithm for deterministic output
|
||
list := append([][]byte(nil), keys[algo]...)
|
||
sort.Slice(list, func(i, j int) bool { return bytes.Compare(list[i], list[j]) < 0 })
|
||
for _, k := range list {
|
||
parts = append(parts, algo+" "+base64.StdEncoding.EncodeToString(k))
|
||
}
|
||
}
|
||
return parts
|
||
}
|
||
|
||
// pinnedHostKeyAlgorithms returns the host-key algorithms to negotiate
|
||
// for the pinned keys, in a deterministic order.
|
||
//
|
||
// RSA public keys report their type as "ssh-rsa" whichever signature
|
||
// algorithm signed the handshake, so a pinned "ssh-rsa" entry is
|
||
// expanded to the rsa-sha2 signature algorithms too. Offering only
|
||
// "ssh-rsa" would restrict the negotiation to SHA-1 signatures, which
|
||
// modern servers (OpenSSH >= 8.8) refuse by default.
|
||
func pinnedHostKeyAlgorithms(keys map[string][][]byte) []string {
|
||
algos := make([]string, 0, len(keys)+2)
|
||
for algo := range keys {
|
||
if algo == ssh.KeyAlgoRSA {
|
||
algos = append(algos, ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256)
|
||
}
|
||
algos = append(algos, algo)
|
||
}
|
||
sort.Strings(algos)
|
||
return algos
|
||
}
|
||
|
||
// formatPinnedFingerprints returns a comma-separated list of SHA256
|
||
// fingerprints for the supplied marshalled public keys, suitable for
|
||
// inclusion in user-facing error messages.
|
||
func formatPinnedFingerprints(algo string, keys [][]byte) string {
|
||
fps := make([]string, 0, len(keys))
|
||
for _, k := range keys {
|
||
pk, err := ssh.ParsePublicKey(k)
|
||
if err != nil {
|
||
fps = append(fps, "<unparseable>")
|
||
continue
|
||
}
|
||
fps = append(fps, ssh.FingerprintSHA256(pk))
|
||
}
|
||
return strings.Join(fps, ", ")
|
||
}
|
||
|
||
// hostKeyCallback returns an ssh.HostKeyCallback that validates the offered
|
||
// server key against f.hostKeys.
|
||
//
|
||
// When pin_host_key is set, pending must point at the dialing
|
||
// connection's own pending key slot: if no key is pinned yet for the
|
||
// offered algorithm the offered key is stashed there for the dialer to
|
||
// commit once its authentication succeeds. A nil pending disables
|
||
// accept-on-first-use, making the callback validate-only.
|
||
func (f *Fs) hostKeyCallback(pending **pendingKey) ssh.HostKeyCallback {
|
||
return func(hostname string, _ net.Addr, key ssh.PublicKey) error {
|
||
// SSH host certificates cannot be byte-pinned meaningfully: the
|
||
// certificate changes whenever it's re-signed even though the
|
||
// underlying CA is unchanged.
|
||
if _, isCert := key.(*ssh.Certificate); isCert {
|
||
return fserrors.NoLowLevelRetryError(fmt.Errorf("sftp: server presented an SSH certificate (%s) which pin_host_key cannot validate; use known_hosts_file with an @cert-authority entry instead", key.Type()))
|
||
}
|
||
algo := key.Type()
|
||
marshalled := key.Marshal()
|
||
|
||
f.hostKeysMu.RLock()
|
||
trusted, have := f.hostKeys[algo]
|
||
f.hostKeysMu.RUnlock()
|
||
|
||
if have {
|
||
for _, t := range trusted {
|
||
if bytes.Equal(t, marshalled) {
|
||
return nil
|
||
}
|
||
}
|
||
return fserrors.NoLowLevelRetryError(fmt.Errorf("sftp: host key mismatch for %s: server offered %s key with fingerprint %s; pinned fingerprints for this algorithm: %s -- if the server key has changed legitimately, clear host_keys and re-run with --sftp-pin-host-key",
|
||
hostname, algo, ssh.FingerprintSHA256(key), formatPinnedFingerprints(algo, trusted)))
|
||
}
|
||
|
||
// No pinned key for this algorithm. Either accept if --sftp-pin-host-key is set or reject.
|
||
if !f.opt.PinHostKey || pending == nil {
|
||
return fserrors.NoLowLevelRetryError(fmt.Errorf("sftp: server offered %s key with fingerprint %s but no key is pinned for this algorithm in host_keys; re-run with --sftp-pin-host-key to accept on first use",
|
||
algo, ssh.FingerprintSHA256(key)))
|
||
}
|
||
// Cap the host_keys growth
|
||
f.hostKeysMu.RLock()
|
||
total := 0
|
||
for _, list := range f.hostKeys {
|
||
total += len(list)
|
||
}
|
||
f.hostKeysMu.RUnlock()
|
||
if total >= maxHostKeys {
|
||
return fserrors.NoLowLevelRetryError(fmt.Errorf("sftp: host_keys already contains %d entries (cap %d); refusing to pin %s key with fingerprint %s -- clear or trim host_keys and re-run with --sftp-pin-host-key",
|
||
total, maxHostKeys, algo, ssh.FingerprintSHA256(key)))
|
||
}
|
||
// Stash the offered key for this connection's dialer to persist post-auth.
|
||
*pending = &pendingKey{
|
||
algo: algo,
|
||
marshalled: marshalled,
|
||
fingerprint: ssh.FingerprintSHA256(key),
|
||
hostname: hostname,
|
||
}
|
||
fs.Logf(f, "Accepted %s host key %s for %s on first use", algo, ssh.FingerprintSHA256(key), hostname)
|
||
return nil
|
||
}
|
||
}
|
||
|
||
// commitHostKey records a --sftp-pin-host-key accepted host key into
|
||
// both the durable config file and the in-memory f.hostKeys trust
|
||
// set. The in-memory update is what closes the trust loop after the
|
||
// callback deliberately left f.hostKeys untouched pre-auth.
|
||
func (f *Fs) commitHostKey(pk *pendingKey) {
|
||
// Hold the lock for the whole read-modify-write so concurrent
|
||
// commits from parallel dials can't lose each other's key.
|
||
f.hostKeysMu.Lock()
|
||
defer f.hostKeysMu.Unlock()
|
||
// Re-read the current stored value rather than reformatting our in-memory
|
||
// map. This narrows (but cannot close - the config layer has no locked
|
||
// read-modify-write) the window for clobbering an update made by a
|
||
// parallel rclone process.
|
||
current, _ := f.m.Get("host_keys")
|
||
var entries fs.CommaSepList
|
||
if err := entries.Set(current); err != nil {
|
||
fs.Errorf(f, "Not persisting %s host key %s for %s: existing host_keys value became unsplittable: %v -- fix or clear host_keys in the config and re-run with --sftp-pin-host-key",
|
||
pk.algo, pk.fingerprint, pk.hostname, err)
|
||
return
|
||
}
|
||
keys, err := parseHostKeysField(entries)
|
||
if err != nil {
|
||
// host_keys was parseable at NewFs time but isn't now...
|
||
snippet := current
|
||
const max = 120
|
||
if len(snippet) > max {
|
||
snippet = snippet[:max] + "..."
|
||
}
|
||
fs.Errorf(f, "Not persisting %s host key %s for %s: existing host_keys value became unparseable (%d bytes: %q): %v -- fix or clear host_keys in the config and re-run with --sftp-pin-host-key",
|
||
pk.algo, pk.fingerprint, pk.hostname, len(current), snippet, err)
|
||
return
|
||
}
|
||
// De-duplicate before appending. Even if already present durably, sync
|
||
// the in-memory view in case it lagged behind the file.
|
||
alreadyPresent := false
|
||
for _, existing := range keys[pk.algo] {
|
||
if bytes.Equal(existing, pk.marshalled) {
|
||
alreadyPresent = true
|
||
break
|
||
}
|
||
}
|
||
if !alreadyPresent {
|
||
// Re-check the cap against the stored value: it may have grown
|
||
// since the callback checked it.
|
||
total := 0
|
||
for _, list := range keys {
|
||
total += len(list)
|
||
}
|
||
if total >= maxHostKeys {
|
||
fs.Errorf(f, "Not persisting %s host key %s for %s: host_keys already contains %d entries (cap %d) -- clear or trim host_keys and re-run with --sftp-pin-host-key",
|
||
pk.algo, pk.fingerprint, pk.hostname, total, maxHostKeys)
|
||
return
|
||
}
|
||
keys[pk.algo] = append(keys[pk.algo], pk.marshalled)
|
||
}
|
||
|
||
f.hostKeys = keys
|
||
|
||
if alreadyPresent {
|
||
return
|
||
}
|
||
f.m.Set("host_keys", formatHostKeysField(keys).String())
|
||
if strings.HasPrefix(f.name, ":") {
|
||
fs.Logf(f, "%s host key %s captured but not persisted (on-the-fly remote); first-connect protection only", pk.algo, pk.fingerprint)
|
||
} else {
|
||
fs.Logf(f, "Pinned %s host key %s for %s in config", pk.algo, pk.fingerprint, pk.hostname)
|
||
}
|
||
}
|
||
|
||
// NewFs creates a new Fs object from the name and root. It connects to
|
||
// the host specified in the config file.
|
||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||
// This will hold the Fs object. We need to create it here
|
||
// so we can refer to it in the SSH callback, but it's populated
|
||
// in NewFsWithConnection
|
||
f := &Fs{
|
||
ci: fs.GetConfig(ctx),
|
||
}
|
||
// Parse config into Options struct
|
||
opt := new(Options)
|
||
err := configstruct.Set(m, opt)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if len(opt.SSH) != 0 && ((opt.User != currentUser && opt.User != "") || opt.Host != "" || (opt.Port != "22" && opt.Port != "")) {
|
||
fs.Logf(name, "--sftp-ssh is in use - ignoring user/host/port from config - set in the parameters to --sftp-ssh (remove them from the config to silence this warning)")
|
||
}
|
||
f.tokens = pacer.NewTokenDispenser(opt.Connections)
|
||
|
||
if opt.User == "" {
|
||
opt.User = currentUser
|
||
}
|
||
if opt.Port == "" {
|
||
opt.Port = "22"
|
||
}
|
||
|
||
// Set up sshConfig here from opt
|
||
// **NB** everything else should be setup in NewFsWithConnection
|
||
sshConfig := &ssh.ClientConfig{
|
||
User: opt.User,
|
||
Auth: []ssh.AuthMethod{},
|
||
Timeout: time.Duration(f.ci.ConnectTimeout),
|
||
ClientVersion: "SSH-2.0-" + f.ci.UserAgent,
|
||
}
|
||
|
||
// pin_host_key and host_keys only apply when rclone does the host
|
||
// key checking itself: known_hosts_file takes precedence and the
|
||
// external ssh program does its own validation, so in either case
|
||
// host_keys is ignored entirely, without being parsed.
|
||
usePinning := (opt.PinHostKey || len(opt.HostKeys) > 0) && opt.KnownHostsFile == "" && len(opt.SSH) == 0
|
||
var parsedHostKeys map[string][][]byte
|
||
if usePinning {
|
||
// Parse pinned host keys early so we can both validate the config
|
||
// and, when no host_key_algorithms is set, narrow the negotiated
|
||
// algorithm list to those we already trust.
|
||
parsed, err := parseHostKeysField(opt.HostKeys)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("couldn't parse host_keys: %w", err)
|
||
}
|
||
parsedHostKeys = parsed
|
||
f.hostKeys = parsed
|
||
}
|
||
|
||
if len(opt.HostKeyAlgorithms) != 0 {
|
||
sshConfig.HostKeyAlgorithms = []string(opt.HostKeyAlgorithms)
|
||
} else if len(parsedHostKeys) > 0 && !opt.PinHostKey {
|
||
// Restrict negotiated host-key algorithms to those we have a
|
||
// pinned key for, so a server offering an additional unpinned
|
||
// algorithm doesn't cause a spurious mismatch when we'd happily
|
||
// accept the same server's pinned algorithm.
|
||
//
|
||
// Not done with pin_host_key set: there any algorithm the
|
||
// server presents can be accepted and pinned on first use, so
|
||
// the negotiation must stay open to new algorithms.
|
||
sshConfig.HostKeyAlgorithms = pinnedHostKeyAlgorithms(parsedHostKeys)
|
||
}
|
||
|
||
// When the ssh option is set, the external ssh program makes the
|
||
// connection and does its own host key validation, so none of the
|
||
// host key checking configured below takes effect.
|
||
if len(opt.SSH) > 0 && (opt.PinHostKey || len(opt.HostKeys) > 0) {
|
||
fs.Logf(name, "pin_host_key and host_keys are ignored when the ssh option is set; the ssh program does its own host key validation")
|
||
}
|
||
|
||
switch {
|
||
case opt.KnownHostsFile == "none":
|
||
// Silence Host key validation warning if explicitly disabled
|
||
sshConfig.HostKeyCallback = ssh.InsecureIgnoreHostKey()
|
||
case opt.KnownHostsFile != "":
|
||
hostcallback, err := knownhosts.New(env.ShellExpand(opt.KnownHostsFile))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("couldn't parse known_hosts_file: %w", err)
|
||
}
|
||
sshConfig.HostKeyCallback = hostcallback
|
||
if opt.PinHostKey || len(opt.HostKeys) > 0 {
|
||
fs.Logf(name, "known_hosts_file is set; ignoring pin_host_key and host_keys")
|
||
}
|
||
case usePinning:
|
||
// Pinning mode: validate against host_keys. When pin_host_key
|
||
// is set, each dial swaps in its own accept-and-stash callback
|
||
// (see sftpConnection), so the shared config only ever
|
||
// validates.
|
||
sshConfig.HostKeyCallback = f.hostKeyCallback(nil)
|
||
f.tofu = opt.PinHostKey
|
||
if opt.PinHostKey && len(parsedHostKeys) == 0 {
|
||
fs.Logf(name, "pin_host_key is set with no pinned host_keys yet; accepting the next server key on first use")
|
||
}
|
||
if opt.PinHostKey && strings.HasPrefix(name, ":") {
|
||
fs.Logf(name, "pin_host_key on an on-the-fly remote cannot persist the pinned key; the server key will be re-accepted on every run")
|
||
}
|
||
default:
|
||
sshConfig.HostKeyCallback = ssh.InsecureIgnoreHostKey()
|
||
fs.Logf(name, "No host key validation is being performed. Set known_hosts_file (to \"none\" to silence this notice) or use --sftp-pin-host-key to enable it. See: https://rclone.org/sftp/#host-key-validation")
|
||
}
|
||
|
||
if opt.UseInsecureCipher && (opt.Ciphers != nil || opt.KeyExchange != nil) {
|
||
return nil, fmt.Errorf("use_insecure_cipher must be false if ciphers or key_exchange are set in advanced configuration")
|
||
}
|
||
|
||
sshConfig.Config.SetDefaults()
|
||
if opt.UseInsecureCipher {
|
||
sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc")
|
||
sshConfig.Config.KeyExchanges = append(sshConfig.Config.KeyExchanges, "diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256")
|
||
} else {
|
||
if opt.Ciphers != nil {
|
||
sshConfig.Config.Ciphers = opt.Ciphers
|
||
}
|
||
if opt.KeyExchange != nil {
|
||
sshConfig.Config.KeyExchanges = opt.KeyExchange
|
||
}
|
||
}
|
||
|
||
if opt.MACs != nil {
|
||
sshConfig.Config.MACs = opt.MACs
|
||
}
|
||
|
||
keyFile := env.ShellExpand(opt.KeyFile)
|
||
pubkeyFile := env.ShellExpand(opt.PubKeyFile)
|
||
//keyPem := env.ShellExpand(opt.KeyPem)
|
||
// Add ssh agent-auth if no password or file or key PEM specified
|
||
if (len(opt.SSH) == 0 && opt.Pass == "" && keyFile == "" && !opt.AskPassword && opt.KeyPem == "") || opt.KeyUseAgent {
|
||
sshAgentClient, _, err := sshagent.New()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("couldn't connect to ssh-agent: %w", err)
|
||
}
|
||
signers, err := sshAgentClient.Signers()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("couldn't read ssh agent signers: %w", err)
|
||
}
|
||
if keyFile != "" {
|
||
// If `opt.KeyUseAgent` is false, then it's expected that `opt.KeyFile` contains the private key
|
||
// and `${opt.KeyFile}.pub` contains the public key.
|
||
//
|
||
// If `opt.KeyUseAgent` is true, then it's expected that `opt.KeyFile` contains the public key.
|
||
// This is how it works with openssh; the `IdentityFile` in openssh config points to the public key.
|
||
// It's not necessary to specify the public key explicitly when using ssh-agent, since openssh and rclone
|
||
// will try all the keys they find in the ssh-agent until they find one that works. But just like
|
||
// `IdentityFile` is used in openssh config to limit the search to one specific key, so does
|
||
// `opt.KeyFile` in rclone config limit the search to that specific key.
|
||
//
|
||
// However, previous versions of rclone would always expect to find the public key in
|
||
// `${opt.KeyFile}.pub` even if `opt.KeyUseAgent` was true. So for the sake of backward compatibility
|
||
// we still first attempt to read the public key from `${opt.KeyFile}.pub`. But if it fails with
|
||
// an `fs.ErrNotExist` then we also try to read the public key from `opt.KeyFile`.
|
||
pubBytes, err := os.ReadFile(keyFile + ".pub")
|
||
if err != nil {
|
||
if errors.Is(err, iofs.ErrNotExist) && opt.KeyUseAgent {
|
||
pubBytes, err = os.ReadFile(keyFile)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to read public key file: %w", err)
|
||
}
|
||
} else {
|
||
return nil, fmt.Errorf("failed to read public key file: %w", err)
|
||
}
|
||
}
|
||
|
||
pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to parse public key file: %w", err)
|
||
}
|
||
pubM := pub.Marshal()
|
||
found := false
|
||
for _, s := range signers {
|
||
if bytes.Equal(pubM, s.PublicKey().Marshal()) {
|
||
sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(s))
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if !found {
|
||
return nil, errors.New("private key not found in the ssh-agent")
|
||
}
|
||
} else {
|
||
sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
|
||
}
|
||
}
|
||
|
||
// Load key file as a private key, if specified. This is only needed when not using an ssh agent.
|
||
if (keyFile != "" && !opt.KeyUseAgent) || opt.KeyPem != "" {
|
||
var key []byte
|
||
if opt.KeyPem == "" {
|
||
key, err = os.ReadFile(keyFile)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to read private key file: %w", err)
|
||
}
|
||
} else {
|
||
// wrap in quotes because the config is a coming as a literal without them.
|
||
opt.KeyPem, err = strconv.Unquote("\"" + opt.KeyPem + "\"")
|
||
if err != nil {
|
||
return nil, fmt.Errorf("pem key not formatted properly: %w", err)
|
||
}
|
||
key = []byte(opt.KeyPem)
|
||
}
|
||
clearpass := ""
|
||
if opt.KeyFilePass != "" {
|
||
clearpass, err = obscure.Reveal(opt.KeyFilePass)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
var signer ssh.Signer
|
||
if clearpass == "" {
|
||
signer, err = ssh.ParsePrivateKey(key)
|
||
} else {
|
||
signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(clearpass))
|
||
}
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to parse private key file: %w", err)
|
||
}
|
||
|
||
// If a public key has been specified then use that
|
||
if pubkeyFile != "" || opt.PubKey != "" {
|
||
pubKeyRaw := []byte(opt.PubKey)
|
||
// Use this error if public key is provided inline and is not a certificate
|
||
// if public key file is provided instead, use the err in the if block
|
||
notACertError := errors.New("public key provided is not a certificate: " + opt.PubKey)
|
||
if opt.PubKey == "" {
|
||
notACertError = errors.New("public key file is not a certificate file: " + pubkeyFile)
|
||
err := error(nil)
|
||
pubKeyRaw, err = os.ReadFile(pubkeyFile)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("unable to read cert file: %w", err)
|
||
}
|
||
}
|
||
|
||
pk, _, _, _, err := ssh.ParseAuthorizedKey(pubKeyRaw)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("unable to parse cert file: %w", err)
|
||
}
|
||
|
||
// And the signer for this, which includes the private key signer
|
||
// This is what we'll pass to the ssh client.
|
||
// Normally the ssh client will use the public key built
|
||
// into the private key, but we need to tell it to use the user
|
||
// specified public key cert. This signer is specific to the
|
||
// cert and will include the private key signer. Now ssh
|
||
// knows everything it needs.
|
||
cert, ok := pk.(*ssh.Certificate)
|
||
if !ok {
|
||
return nil, notACertError
|
||
}
|
||
pubsigner, err := ssh.NewCertSigner(cert, signer)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("error generating cert signer: %w", err)
|
||
}
|
||
sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(pubsigner))
|
||
} else {
|
||
sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer))
|
||
}
|
||
}
|
||
|
||
// Auth from password if specified
|
||
if opt.Pass != "" {
|
||
clearpass, err := obscure.Reveal(opt.Pass)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
sshConfig.Auth = append(sshConfig.Auth,
|
||
ssh.Password(clearpass),
|
||
ssh.KeyboardInteractive(func(user, instruction string, questions []string, echos []bool) ([]string, error) {
|
||
return f.keyboardInteractiveReponse(user, instruction, questions, echos, clearpass)
|
||
}),
|
||
)
|
||
}
|
||
|
||
// Config for password if none was defined and we're allowed to
|
||
// We don't ask now; we ask if the ssh connection succeeds
|
||
if opt.Pass == "" && opt.AskPassword {
|
||
sshConfig.Auth = append(sshConfig.Auth,
|
||
ssh.PasswordCallback(f.getPass),
|
||
ssh.KeyboardInteractive(func(user, instruction string, questions []string, echos []bool) ([]string, error) {
|
||
pass, _ := f.getPass()
|
||
return f.keyboardInteractiveReponse(user, instruction, questions, echos, pass)
|
||
}),
|
||
)
|
||
}
|
||
|
||
return NewFsWithConnection(ctx, f, name, root, m, opt, sshConfig)
|
||
}
|
||
|
||
// Do the keyboard interactive challenge
|
||
//
|
||
// Just send the password back for all questions
|
||
func (f *Fs) keyboardInteractiveReponse(user, instruction string, questions []string, echos []bool, pass string) ([]string, error) {
|
||
fs.Debugf(f, "Keyboard interactive auth requested")
|
||
answers := make([]string, len(questions))
|
||
for i := range answers {
|
||
answers[i] = pass
|
||
}
|
||
return answers, nil
|
||
}
|
||
|
||
// If we're in password mode and ssh connection succeeds then this
|
||
// callback is called. First time around we ask the user, and then
|
||
// save it so on reconnection we give back the previous string.
|
||
// This removes the ability to let the user correct a mistaken entry,
|
||
// but means that reconnects are transparent.
|
||
// We'll reuse config.Pass for this, 'cos we know it's not been
|
||
// specified.
|
||
func (f *Fs) getPass() (string, error) {
|
||
for f.savedpswd == "" {
|
||
_, _ = fmt.Fprint(os.Stderr, "Enter SFTP password: ")
|
||
f.savedpswd = config.ReadPassword()
|
||
}
|
||
return f.savedpswd, nil
|
||
}
|
||
|
||
// NewFsWithConnection creates a new Fs object from the name and root and an ssh.ClientConfig. It connects to
|
||
// the host specified in the ssh.ClientConfig
|
||
func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m configmap.Mapper, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
|
||
// Populate the Filesystem Object
|
||
f.name = name
|
||
f.opt = *opt
|
||
f.root = root
|
||
f.absRoot = f.opt.Enc.FromStandardPath(root)
|
||
f.shellRoot = f.absRoot
|
||
f.m = m
|
||
f.config = sshConfig
|
||
f.url = "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root
|
||
f.mkdirLock = newStringLock()
|
||
f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
|
||
f.savedpswd = ""
|
||
|
||
// set the pool drainer timer going
|
||
if f.opt.IdleTimeout > 0 {
|
||
f.drain = time.AfterFunc(time.Duration(f.opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
|
||
}
|
||
|
||
// get proxy URL if set
|
||
if opt.HTTPProxy != "" {
|
||
proxyURL, err := url.Parse(opt.HTTPProxy)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to parse HTTP Proxy URL: %w", err)
|
||
}
|
||
f.proxyURL = proxyURL
|
||
}
|
||
|
||
f.features = (&fs.Features{
|
||
CanHaveEmptyDirectories: true,
|
||
SlowHash: true,
|
||
PartialUploads: true,
|
||
DirModTimeUpdatesOnWrite: true, // indicate writing files to a directory updates its modtime
|
||
}).Fill(ctx, f)
|
||
if !opt.CopyIsHardlink {
|
||
// Disable server side copy unless --sftp-copy-is-hardlink is set
|
||
f.features.Copy = nil
|
||
}
|
||
// Make a connection and pool it to return errors early
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("NewFs: %w", err)
|
||
}
|
||
// Check remote shell type, try to auto-detect if not configured and save to config for later
|
||
if f.opt.ShellType != "" {
|
||
f.shellType = f.opt.ShellType
|
||
fs.Debugf(f, "Shell type %q from config", f.shellType)
|
||
} else {
|
||
session, err := c.sshClient.NewSession()
|
||
if err != nil {
|
||
f.shellType = shellTypeNotSupported
|
||
fs.Debugf(f, "Failed to get shell session for shell type detection command: %v", err)
|
||
} else {
|
||
var stdout, stderr bytes.Buffer
|
||
session.SetStdout(&stdout)
|
||
session.SetStderr(&stderr)
|
||
shellCmd := "echo ${ShellId}%ComSpec%"
|
||
fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd)
|
||
err = session.Run(shellCmd)
|
||
_ = session.Close()
|
||
f.shellType = defaultShellType
|
||
if err != nil {
|
||
fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes()))
|
||
} else {
|
||
outBytes := stdout.Bytes()
|
||
fs.Debugf(f, "Remote command result: %s", outBytes)
|
||
outString := string(bytes.TrimSpace(stdout.Bytes()))
|
||
if outString != "" {
|
||
if strings.HasPrefix(outString, "Microsoft.PowerShell") { // PowerShell: "Microsoft.PowerShell%ComSpec%"
|
||
f.shellType = "powershell"
|
||
} else if !strings.HasSuffix(outString, "%ComSpec%") { // Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
|
||
// Additional positive test, to avoid misdetection on unpredicted Unix shell variants
|
||
s := strings.ToLower(outString)
|
||
if strings.Contains(s, ".exe") || strings.Contains(s, ".com") {
|
||
f.shellType = "cmd"
|
||
}
|
||
} // POSIX-based Unix shell: "%ComSpec%"
|
||
} // fish Unix shell: ""
|
||
}
|
||
}
|
||
// Save permanently in config to avoid the extra work next time
|
||
fs.Debugf(f, "Shell type %q detected (set option shell_type to override)", f.shellType)
|
||
f.m.Set("shell_type", f.shellType)
|
||
}
|
||
// Ensure we have absolute path to root
|
||
// It appears that WS FTP doesn't like relative paths,
|
||
// and the openssh sftp tool also uses absolute paths.
|
||
if !path.IsAbs(f.absRoot) {
|
||
// Trying RealPath first, to perform proper server-side canonicalize.
|
||
// It may fail (SSH_FX_FAILURE reported on WS FTP) and will then resort
|
||
// to simple path join with current directory from Getwd (which can work
|
||
// on WS FTP, even though it is also based on RealPath).
|
||
absRoot, err := c.sftpClient.RealPath(f.absRoot)
|
||
if err != nil {
|
||
fs.Debugf(f, "Failed to resolve path using RealPath: %v", err)
|
||
cwd, err := c.sftpClient.Getwd()
|
||
if err != nil {
|
||
fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
|
||
} else {
|
||
f.absRoot = path.Join(cwd, f.absRoot)
|
||
fs.Debugf(f, "Relative path joined with current directory to get absolute path %q", f.absRoot)
|
||
}
|
||
} else {
|
||
f.absRoot = absRoot
|
||
fs.Debugf(f, "Relative path resolved to %q", f.absRoot)
|
||
}
|
||
}
|
||
f.putSftpConnection(&c, err)
|
||
if root != "" && !strings.HasSuffix(root, "/") {
|
||
// Check to see if the root is actually an existing file,
|
||
// and if so change the filesystem root to its parent directory.
|
||
oldAbsRoot := f.absRoot
|
||
remote := path.Base(root)
|
||
f.root = path.Dir(root)
|
||
f.absRoot = path.Dir(f.absRoot)
|
||
if f.root == "." {
|
||
f.root = ""
|
||
}
|
||
_, err = f.NewObject(ctx, remote)
|
||
if err != nil {
|
||
if err != fs.ErrorObjectNotFound && err != fs.ErrorIsDir {
|
||
return nil, err
|
||
}
|
||
// File doesn't exist so keep the old f
|
||
f.root = root
|
||
f.absRoot = oldAbsRoot
|
||
err = nil
|
||
} else {
|
||
// File exists so change fs to point to the parent and return it with an error
|
||
err = fs.ErrorIsFile
|
||
}
|
||
} else {
|
||
err = nil
|
||
}
|
||
fs.Debugf(f, "Using root directory %q", f.absRoot)
|
||
return f, err
|
||
}
|
||
|
||
// Name returns the configured name of the file system
|
||
func (f *Fs) Name() string {
|
||
return f.name
|
||
}
|
||
|
||
// Root returns the root for the filesystem
|
||
func (f *Fs) Root() string {
|
||
return f.root
|
||
}
|
||
|
||
// String returns the URL for the filesystem
|
||
func (f *Fs) String() string {
|
||
return f.url
|
||
}
|
||
|
||
// Features returns the optional features of this Fs
|
||
func (f *Fs) Features() *fs.Features {
|
||
return f.features
|
||
}
|
||
|
||
// Precision is the remote sftp file system's modtime precision, which we have no way of knowing. We estimate at 1s
|
||
func (f *Fs) Precision() time.Duration {
|
||
return time.Second
|
||
}
|
||
|
||
// NewObject creates a new remote sftp file object
|
||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||
o := &Object{
|
||
fs: f,
|
||
remote: remote,
|
||
}
|
||
err := o.stat(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return o, nil
|
||
}
|
||
|
||
// dirExists returns true,nil if the directory exists, false, nil if
|
||
// it doesn't or false, err
|
||
func (f *Fs) dirExists(ctx context.Context, dir string) (bool, error) {
|
||
if dir == "" {
|
||
dir = "."
|
||
}
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return false, fmt.Errorf("dirExists: %w", err)
|
||
}
|
||
info, err := c.sftpClient.Stat(dir)
|
||
f.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
return false, nil
|
||
}
|
||
return false, fmt.Errorf("dirExists stat failed: %w", err)
|
||
}
|
||
if !info.IsDir() {
|
||
return false, fs.ErrorIsFile
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// List the objects and directories in dir into entries. The
|
||
// entries can be returned in any order but should be for a
|
||
// complete directory.
|
||
//
|
||
// dir should be "" to list the root, and should not have
|
||
// trailing slashes.
|
||
//
|
||
// This should return ErrDirNotFound if the directory isn't
|
||
// found.
|
||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||
root := f.remotePath(dir)
|
||
sftpDir := root
|
||
if sftpDir == "" {
|
||
sftpDir = "."
|
||
}
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("List: %w", err)
|
||
}
|
||
infos, err := c.sftpClient.ReadDir(sftpDir)
|
||
f.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
if errors.Is(err, os.ErrNotExist) {
|
||
return nil, fs.ErrorDirNotFound
|
||
}
|
||
return nil, fmt.Errorf("error listing %q: %w", dir, err)
|
||
}
|
||
for _, info := range infos {
|
||
remote := path.Join(dir, f.opt.Enc.ToStandardName(info.Name()))
|
||
// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
|
||
// pick up the size and type of the destination, instead of the size and type of the symlink.
|
||
if !info.Mode().IsRegular() && !info.IsDir() {
|
||
if f.opt.SkipLinks {
|
||
// skip non regular file if SkipLinks is set
|
||
continue
|
||
}
|
||
oldInfo := info
|
||
info, err = f.stat(ctx, remote)
|
||
if err != nil {
|
||
if !os.IsNotExist(err) {
|
||
fs.Errorf(remote, "stat of non-regular file failed: %v", err)
|
||
}
|
||
info = oldInfo
|
||
}
|
||
}
|
||
if info.IsDir() {
|
||
d := fs.NewDir(remote, info.ModTime())
|
||
entries = append(entries, d)
|
||
} else {
|
||
o := &Object{
|
||
fs: f,
|
||
remote: remote,
|
||
}
|
||
o.setMetadata(info)
|
||
entries = append(entries, o)
|
||
}
|
||
}
|
||
return entries, nil
|
||
}
|
||
|
||
// Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime(ctx)>
|
||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||
err := f.mkParentDir(ctx, src.Remote())
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Put mkParentDir failed: %w", err)
|
||
}
|
||
// Temporary object under construction
|
||
o := &Object{
|
||
fs: f,
|
||
remote: src.Remote(),
|
||
}
|
||
err = o.Update(ctx, in, src, options...)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return o, nil
|
||
}
|
||
|
||
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
||
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||
return f.Put(ctx, in, src, options...)
|
||
}
|
||
|
||
// mkParentDir makes the parent of remote if necessary and any
|
||
// directories above that
|
||
func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
|
||
parent := path.Dir(remote)
|
||
return f.mkdir(ctx, f.remotePath(parent))
|
||
}
|
||
|
||
// mkdir makes the directory and parents using native paths
|
||
func (f *Fs) mkdir(ctx context.Context, dirPath string) error {
|
||
f.mkdirLock.Lock(dirPath)
|
||
defer f.mkdirLock.Unlock(dirPath)
|
||
if dirPath == "." || dirPath == "/" {
|
||
return nil
|
||
}
|
||
ok, err := f.dirExists(ctx, dirPath)
|
||
if err != nil {
|
||
return fmt.Errorf("mkdir dirExists failed: %w", err)
|
||
}
|
||
if ok {
|
||
return nil
|
||
}
|
||
parent := path.Dir(dirPath)
|
||
err = f.mkdir(ctx, parent)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("mkdir: %w", err)
|
||
}
|
||
err = c.sftpClient.Mkdir(dirPath)
|
||
f.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
if os.IsExist(err) {
|
||
fs.Debugf(f, "directory %q exists after Mkdir is attempted", dirPath)
|
||
return nil
|
||
}
|
||
return fmt.Errorf("mkdir %q failed: %w", dirPath, err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Mkdir makes the root directory of the Fs object
|
||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||
root := f.remotePath(dir)
|
||
return f.mkdir(ctx, root)
|
||
}
|
||
|
||
// DirSetModTime sets the directory modtime for dir
|
||
func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
|
||
o := Object{
|
||
fs: f,
|
||
remote: dir,
|
||
}
|
||
return o.SetModTime(ctx, modTime)
|
||
}
|
||
|
||
// Rmdir removes the root directory of the Fs object
|
||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||
// Check to see if directory is empty as some servers will
|
||
// delete recursively with RemoveDirectory
|
||
entries, err := f.List(ctx, dir)
|
||
if err != nil {
|
||
return fmt.Errorf("Rmdir: %w", err)
|
||
}
|
||
if len(entries) != 0 {
|
||
return fs.ErrorDirectoryNotEmpty
|
||
}
|
||
// Remove the directory
|
||
root := f.remotePath(dir)
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("Rmdir: %w", err)
|
||
}
|
||
err = c.sftpClient.RemoveDirectory(root)
|
||
f.putSftpConnection(&c, err)
|
||
return err
|
||
}
|
||
|
||
// Move renames a remote sftp file object
|
||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||
srcObj, ok := src.(*Object)
|
||
if !ok {
|
||
fs.Debugf(src, "Can't move - not same remote type")
|
||
return nil, fs.ErrorCantMove
|
||
}
|
||
err := f.mkParentDir(ctx, remote)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Move mkParentDir failed: %w", err)
|
||
}
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Move: %w", err)
|
||
}
|
||
srcPath, dstPath := srcObj.path(), f.remotePath(remote)
|
||
if _, ok := c.sftpClient.HasExtension("posix-rename@openssh.com"); ok {
|
||
err = c.sftpClient.PosixRename(srcPath, dstPath)
|
||
} else {
|
||
// If haven't got PosixRename then remove source first before renaming
|
||
err = c.sftpClient.Remove(dstPath)
|
||
if err != nil && !errors.Is(err, iofs.ErrNotExist) {
|
||
fs.Errorf(f, "Move: Failed to remove existing file %q: %v", dstPath, err)
|
||
}
|
||
err = c.sftpClient.Rename(srcPath, dstPath)
|
||
}
|
||
f.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Move Rename failed: %w", err)
|
||
}
|
||
dstObj, err := f.NewObject(ctx, remote)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Move NewObject failed: %w", err)
|
||
}
|
||
return dstObj, nil
|
||
}
|
||
|
||
// Copy server side copies a remote sftp file object using hardlinks
|
||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||
if !f.opt.CopyIsHardlink {
|
||
return nil, fs.ErrorCantCopy
|
||
}
|
||
srcObj, ok := src.(*Object)
|
||
if !ok {
|
||
fs.Debugf(src, "Can't copy - not same remote type")
|
||
return nil, fs.ErrorCantCopy
|
||
}
|
||
err := f.mkParentDir(ctx, remote)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Copy mkParentDir failed: %w", err)
|
||
}
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Copy: %w", err)
|
||
}
|
||
srcPath, dstPath := srcObj.path(), f.remotePath(remote)
|
||
err = c.sftpClient.Link(srcPath, dstPath)
|
||
f.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
if sftpErr, ok := err.(*sftp.StatusError); ok {
|
||
if sftpErr.FxCode() == sftp.ErrSSHFxOpUnsupported {
|
||
// Remote doesn't support Link
|
||
return nil, fs.ErrorCantCopy
|
||
}
|
||
}
|
||
return nil, fmt.Errorf("Copy failed: %w", err)
|
||
}
|
||
dstObj, err := f.NewObject(ctx, remote)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Copy NewObject failed: %w", err)
|
||
}
|
||
return dstObj, nil
|
||
}
|
||
|
||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||
// using server-side move operations.
|
||
//
|
||
// Will only be called if src.Fs().Name() == f.Name()
|
||
//
|
||
// If it isn't possible then return fs.ErrorCantDirMove
|
||
//
|
||
// If destination exists then return fs.ErrorDirExists
|
||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
||
srcFs, ok := src.(*Fs)
|
||
if !ok {
|
||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||
return fs.ErrorCantDirMove
|
||
}
|
||
srcPath := srcFs.remotePath(srcRemote)
|
||
dstPath := f.remotePath(dstRemote)
|
||
|
||
// Check if destination exists
|
||
ok, err := f.dirExists(ctx, dstPath)
|
||
if err != nil {
|
||
return fmt.Errorf("DirMove dirExists dst failed: %w", err)
|
||
}
|
||
if ok {
|
||
return fs.ErrorDirExists
|
||
}
|
||
|
||
// Make sure the parent directory exists
|
||
err = f.mkdir(ctx, path.Dir(dstPath))
|
||
if err != nil {
|
||
return fmt.Errorf("DirMove mkParentDir dst failed: %w", err)
|
||
}
|
||
|
||
// Do the move
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("DirMove: %w", err)
|
||
}
|
||
err = c.sftpClient.Rename(
|
||
srcPath,
|
||
dstPath,
|
||
)
|
||
f.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
return fmt.Errorf("DirMove Rename(%q,%q) failed: %w", srcPath, dstPath, err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// run runds cmd on the remote end returning standard output
|
||
func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
|
||
f.addSession() // Show session in use
|
||
defer f.removeSession()
|
||
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("run: get SFTP connection: %w", err)
|
||
}
|
||
defer f.putSftpConnection(&c, err)
|
||
|
||
// Send keepalives while the connection is open
|
||
defer close(c.sendKeepAlives(keepAliveInterval))
|
||
|
||
session, err := c.sshClient.NewSession()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("run: get SFTP session: %w", err)
|
||
}
|
||
err = f.setEnv(session)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer func() {
|
||
_ = session.Close()
|
||
}()
|
||
|
||
var stdout, stderr bytes.Buffer
|
||
session.SetStdout(&stdout)
|
||
session.SetStderr(&stderr)
|
||
|
||
fs.Debugf(f, "Running remote command: %s", cmd)
|
||
err = session.Run(cmd)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to run %q: %s: %w", cmd, bytes.TrimSpace(stderr.Bytes()), err)
|
||
}
|
||
fs.Debugf(f, "Remote command result: %s", bytes.TrimSpace(stdout.Bytes()))
|
||
|
||
return stdout.Bytes(), nil
|
||
}
|
||
|
||
// Hashes returns the supported hash types of the filesystem
|
||
func (f *Fs) Hashes() hash.Set {
|
||
ctx := context.TODO()
|
||
|
||
if f.cachedHashes != nil {
|
||
return *f.cachedHashes
|
||
}
|
||
|
||
hashTypesSupported := hash.NewHashSet()
|
||
f.cachedHashes = &hashTypesSupported
|
||
|
||
if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported {
|
||
return hashTypesSupported
|
||
}
|
||
|
||
hashTypes := hash.NewHashSet()
|
||
if len(f.opt.Hashes) > 0 {
|
||
for _, hashName := range f.opt.Hashes {
|
||
var hashType hash.Type
|
||
if err := hashType.Set(hashName); err != nil {
|
||
fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
|
||
}
|
||
hashTypes.Add(hashType)
|
||
}
|
||
} else {
|
||
hashTypes.Add(hash.MD5, hash.SHA1)
|
||
}
|
||
|
||
hashCommands := map[hash.Type]struct {
|
||
option *string
|
||
emptyHash string
|
||
hashCommands []struct{ hashFile, hashEmpty string }
|
||
}{
|
||
hash.MD5: {
|
||
&f.opt.Md5sumCommand,
|
||
"d41d8cd98f00b204e9800998ecf8427e",
|
||
[]struct{ hashFile, hashEmpty string }{
|
||
{"md5sum", "md5sum"},
|
||
{"md5 -r", "md5 -r"},
|
||
{"rclone md5sum", "rclone md5sum"},
|
||
},
|
||
},
|
||
hash.SHA1: {
|
||
&f.opt.Sha1sumCommand,
|
||
"da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
||
[]struct{ hashFile, hashEmpty string }{
|
||
{"sha1sum", "sha1sum"},
|
||
{"sha1 -r", "sha1 -r"},
|
||
{"rclone sha1sum", "rclone sha1sum"},
|
||
},
|
||
},
|
||
hash.CRC32: {
|
||
&f.opt.Sha1sumCommand,
|
||
"00000000",
|
||
[]struct{ hashFile, hashEmpty string }{
|
||
{"crc32", "crc32"},
|
||
{"rclone hashsum crc32", "rclone hashsum crc32"},
|
||
},
|
||
},
|
||
hash.SHA256: {
|
||
&f.opt.Sha256sumCommand,
|
||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||
[]struct{ hashFile, hashEmpty string }{
|
||
{"sha256sum", "sha1sum"},
|
||
{"sha256 -r", "sha1 -r"},
|
||
{"rclone hashsum sha256", "rclone hashsum sha256"},
|
||
},
|
||
},
|
||
hash.BLAKE3: {
|
||
&f.opt.Blake3sumCommand,
|
||
"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
|
||
[]struct{ hashFile, hashEmpty string }{
|
||
{"b3sum", "b3sum"},
|
||
{"rclone hashsum blake3", "rclone hashsum blake3"},
|
||
},
|
||
},
|
||
hash.XXH3: {
|
||
&f.opt.Xxh3sumCommand,
|
||
"2d06800538d394c2",
|
||
[]struct{ hashFile, hashEmpty string }{
|
||
// The xxhsum tool uses a non-standard prefix "XXH3_" preceding the hash output for the 64-bit variant
|
||
// of XXH3, to avoid confusion with the older 64-bit algorithm XXH64. This was introduced in version
|
||
// 0.8.3 released Dec 30, 2024. Older versions only supported the alternative BSD style output format,
|
||
// otherwise optional with argument --tag. We are currently not expecting these output formats and can
|
||
// therefore not use the "xxhsum -H3" command or its xxh3sum alias directly.
|
||
//{"xxh3sum", "xxh3sum"},
|
||
//{"xxhsum -H3", "xxhsum -H3"},
|
||
{"rclone hashsum xxh3", "rclone hashsum xxh3"},
|
||
},
|
||
},
|
||
hash.XXH128: {
|
||
&f.opt.Xxh128sumCommand,
|
||
"99aa06d3014798d86001c324468d497f",
|
||
[]struct{ hashFile, hashEmpty string }{
|
||
{"xxh128sum", "xxh128sum"},
|
||
{"xxhsum -H2", "xxhsum -H2"},
|
||
{"rclone hashsum xxh128", "rclone hashsum xxh128"},
|
||
},
|
||
},
|
||
}
|
||
if f.shellType == "powershell" {
|
||
for _, hashType := range []hash.Type{hash.MD5, hash.SHA1, hash.SHA256} {
|
||
if entry, ok := hashCommands[hashType]; ok {
|
||
entry.hashCommands = append(hashCommands[hashType].hashCommands, struct {
|
||
hashFile, hashEmpty string
|
||
}{
|
||
fmt.Sprintf("&{param($Path);Get-FileHash -Algorithm %v -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}", hashType),
|
||
fmt.Sprintf("Get-FileHash -Algorithm %v -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", hashType),
|
||
})
|
||
hashCommands[hashType] = entry
|
||
}
|
||
}
|
||
}
|
||
|
||
checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool {
|
||
if *hashCommand == hashCommandNotSupported {
|
||
return false
|
||
}
|
||
if *hashCommand != "" {
|
||
return true
|
||
}
|
||
fs.Debugf(f, "Checking default %v hash commands", hashType)
|
||
*changed = true
|
||
for _, command := range commands {
|
||
output, err := f.run(ctx, command.hashEmpty)
|
||
if err != nil {
|
||
fs.Debugf(f, "Hash command skipped: %v", err)
|
||
continue
|
||
}
|
||
output = bytes.TrimSpace(output)
|
||
if parseHash(output) == expected {
|
||
*hashCommand = command.hashFile
|
||
fs.Debugf(f, "Hash command accepted")
|
||
return true
|
||
}
|
||
fs.Debugf(f, "Hash command skipped: Wrong output")
|
||
}
|
||
*hashCommand = hashCommandNotSupported
|
||
return false
|
||
}
|
||
|
||
changed := false
|
||
for _, hashType := range hashTypes.Array() {
|
||
if entry, ok := hashCommands[hashType]; ok {
|
||
if works := checkHash(hashType, entry.hashCommands, entry.emptyHash, entry.option, &changed); works {
|
||
hashTypesSupported.Add(hashType)
|
||
}
|
||
}
|
||
}
|
||
|
||
if changed {
|
||
// Save permanently in config to avoid the extra work next time
|
||
for _, hashType := range hashTypes.Array() {
|
||
if entry, ok := hashCommands[hashType]; ok {
|
||
fs.Debugf(f, "Setting hash command for %v to %q (set %vsum_command to override)", hashType, *entry.option, hashType)
|
||
f.m.Set(fmt.Sprintf("%vsum_command", hashType), *entry.option)
|
||
}
|
||
}
|
||
}
|
||
|
||
return hashTypesSupported
|
||
}
|
||
|
||
// About gets usage stats
|
||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||
// If server implements the vendor-specific VFS statistics extension prefer that
|
||
// (OpenSSH implements it on using syscall.Statfs on Linux and API function GetDiskFreeSpace on Windows)
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var vfsStats *sftp.StatVFS
|
||
if _, found := c.sftpClient.HasExtension("statvfs@openssh.com"); found {
|
||
fs.Debugf(f, "Server has VFS statistics extension")
|
||
aboutPath := f.absRoot
|
||
if aboutPath == "" {
|
||
aboutPath = "/"
|
||
}
|
||
fs.Debugf(f, "About path %q", aboutPath)
|
||
vfsStats, err = c.sftpClient.StatVFS(aboutPath)
|
||
}
|
||
f.putSftpConnection(&c, err) // Return to pool asap, if running shell command below it will be reused
|
||
if vfsStats != nil {
|
||
total := vfsStats.TotalSpace()
|
||
free := vfsStats.FreeSpace()
|
||
used := total - free
|
||
return &fs.Usage{
|
||
Total: fs.NewUsageValue(total),
|
||
Used: fs.NewUsageValue(used),
|
||
Free: fs.NewUsageValue(free),
|
||
}, nil
|
||
} else if err != nil {
|
||
if errors.Is(err, os.ErrNotExist) {
|
||
return nil, err
|
||
}
|
||
fs.Debugf(f, "Failed to retrieve VFS statistics, trying shell command instead: %v", err)
|
||
} else {
|
||
fs.Debugf(f, "Server does not have the VFS statistics extension, trying shell command instead")
|
||
}
|
||
|
||
// Fall back to shell command method if possible
|
||
if f.shellType == shellTypeNotSupported || f.shellType == "cmd" {
|
||
fs.Debugf(f, "About shell command is not available for shell type %q (set option shell_type to override)", f.shellType)
|
||
return nil, fmt.Errorf("not supported with shell type %q", f.shellType)
|
||
}
|
||
aboutShellPath := f.remoteShellPath("")
|
||
if aboutShellPath == "" {
|
||
aboutShellPath = "/"
|
||
}
|
||
fs.Debugf(f, "About path %q", aboutShellPath)
|
||
aboutShellPathArg, err := f.quoteOrEscapeShellPath(aboutShellPath)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
// PowerShell
|
||
if f.shellType == "powershell" {
|
||
shellCmd := "Get-Item " + aboutShellPathArg + " -ErrorAction Stop|Select-Object -First 1 -ExpandProperty PSDrive|ForEach-Object{\"$($_.Used) $($_.Free)\"}"
|
||
fs.Debugf(f, "About using shell command for shell type %q", f.shellType)
|
||
stdout, err := f.run(ctx, shellCmd)
|
||
if err != nil {
|
||
fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err)
|
||
return nil, fmt.Errorf("powershell command failed: %w", err)
|
||
}
|
||
split := strings.Fields(string(stdout))
|
||
usage := &fs.Usage{}
|
||
if len(split) == 2 {
|
||
usedValue, usedErr := strconv.ParseInt(split[0], 10, 64)
|
||
if usedErr == nil {
|
||
usage.Used = fs.NewUsageValue(usedValue)
|
||
}
|
||
freeValue, freeErr := strconv.ParseInt(split[1], 10, 64)
|
||
if freeErr == nil {
|
||
usage.Free = fs.NewUsageValue(freeValue)
|
||
if usedErr == nil {
|
||
usage.Total = fs.NewUsageValue(usedValue + freeValue)
|
||
}
|
||
}
|
||
}
|
||
return usage, nil
|
||
}
|
||
// Unix/default shell
|
||
shellCmd := "df -k " + aboutShellPathArg
|
||
fs.Debugf(f, "About using shell command for shell type %q", f.shellType)
|
||
stdout, err := f.run(ctx, shellCmd)
|
||
if err != nil {
|
||
fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err)
|
||
return nil, fmt.Errorf("your remote may not have the required df utility: %w", err)
|
||
}
|
||
usageTotal, usageUsed, usageAvail := parseUsage(stdout)
|
||
usage := &fs.Usage{}
|
||
if usageTotal >= 0 {
|
||
usage.Total = fs.NewUsageValue(usageTotal)
|
||
}
|
||
if usageUsed >= 0 {
|
||
usage.Used = fs.NewUsageValue(usageUsed)
|
||
}
|
||
if usageAvail >= 0 {
|
||
usage.Free = fs.NewUsageValue(usageAvail)
|
||
}
|
||
return usage, nil
|
||
}
|
||
|
||
// Shutdown the backend, closing any background tasks and any
|
||
// cached connections.
|
||
func (f *Fs) Shutdown(ctx context.Context) error {
|
||
return f.drainPool(ctx)
|
||
}
|
||
|
||
// Fs is the filesystem this remote sftp file object is located within
|
||
func (o *Object) Fs() fs.Info {
|
||
return o.fs
|
||
}
|
||
|
||
// String returns the URL to the remote SFTP file
|
||
func (o *Object) String() string {
|
||
if o == nil {
|
||
return "<nil>"
|
||
}
|
||
return o.remote
|
||
}
|
||
|
||
// Remote the name of the remote SFTP file, relative to the fs root
|
||
func (o *Object) Remote() string {
|
||
return o.remote
|
||
}
|
||
|
||
// Hash returns the selected checksum of the file
|
||
// If no checksum is available it returns ""
|
||
func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||
if o.fs.opt.DisableHashCheck {
|
||
return "", nil
|
||
}
|
||
_ = o.fs.Hashes()
|
||
|
||
var hashCmd string
|
||
switch r {
|
||
case hash.MD5:
|
||
if o.md5sum != nil {
|
||
return *o.md5sum, nil
|
||
}
|
||
hashCmd = o.fs.opt.Md5sumCommand
|
||
case hash.SHA1:
|
||
if o.sha1sum != nil {
|
||
return *o.sha1sum, nil
|
||
}
|
||
hashCmd = o.fs.opt.Sha1sumCommand
|
||
case hash.CRC32:
|
||
if o.crc32sum != nil {
|
||
return *o.crc32sum, nil
|
||
}
|
||
hashCmd = o.fs.opt.Crc32sumCommand
|
||
case hash.SHA256:
|
||
if o.sha256sum != nil {
|
||
return *o.sha256sum, nil
|
||
}
|
||
hashCmd = o.fs.opt.Sha256sumCommand
|
||
case hash.BLAKE3:
|
||
if o.blake3sum != nil {
|
||
return *o.blake3sum, nil
|
||
}
|
||
hashCmd = o.fs.opt.Blake3sumCommand
|
||
case hash.XXH3:
|
||
if o.xxh3sum != nil {
|
||
return *o.xxh3sum, nil
|
||
}
|
||
hashCmd = o.fs.opt.Xxh3sumCommand
|
||
case hash.XXH128:
|
||
if o.xxh128sum != nil {
|
||
return *o.xxh128sum, nil
|
||
}
|
||
hashCmd = o.fs.opt.Xxh128sumCommand
|
||
default:
|
||
return "", hash.ErrUnsupported
|
||
}
|
||
if hashCmd == "" || hashCmd == hashCommandNotSupported {
|
||
return "", hash.ErrUnsupported
|
||
}
|
||
|
||
shellPathArg, err := o.fs.quoteOrEscapeShellPath(o.shellPath())
|
||
if err != nil {
|
||
return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
|
||
}
|
||
outBytes, err := o.fs.run(ctx, hashCmd+" "+shellPathArg)
|
||
if err != nil {
|
||
return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
|
||
}
|
||
hashString := parseHash(outBytes)
|
||
fs.Debugf(o, "Parsed hash: %s", hashString)
|
||
switch r {
|
||
case hash.MD5:
|
||
o.md5sum = &hashString
|
||
case hash.SHA1:
|
||
o.sha1sum = &hashString
|
||
case hash.CRC32:
|
||
o.crc32sum = &hashString
|
||
case hash.SHA256:
|
||
o.sha256sum = &hashString
|
||
case hash.BLAKE3:
|
||
o.blake3sum = &hashString
|
||
case hash.XXH3:
|
||
o.xxh3sum = &hashString
|
||
case hash.XXH128:
|
||
o.xxh128sum = &hashString
|
||
}
|
||
return hashString, nil
|
||
}
|
||
|
||
// powerShellQuoteEscaper doubles every character PowerShell accepts as a
|
||
// single-quote string delimiter. As well as the ASCII apostrophe, PowerShell
|
||
// treats the Unicode smart quotes U+2018, U+2019, U+201A and U+201B as single
|
||
// quotes, so a path wrapped in apostrophes must double all of them or an
|
||
// attacker controlled filename could close the literal and inject a statement.
|
||
// Doubling a delimiter is PowerShell's escape for a literal occurrence of it,
|
||
// and preserves the exact character.
|
||
var powerShellQuoteEscaper = strings.NewReplacer(
|
||
"'", "''",
|
||
"‘", "‘‘",
|
||
"’", "’’",
|
||
"‚", "‚‚",
|
||
"‛", "‛‛",
|
||
)
|
||
|
||
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
|
||
// and also ensures it cannot cause unintended behavior.
|
||
func quoteOrEscapeShellPath(shellType string, shellPath string) (string, error) {
|
||
// PowerShell
|
||
if shellType == "powershell" {
|
||
return "'" + powerShellQuoteEscaper.Replace(shellPath) + "'", nil
|
||
}
|
||
// Windows Command Prompt
|
||
//
|
||
// cmd has no reliable command-line escaping for the double quote used
|
||
// as the delimiter, while % expands environment variables and ! may
|
||
// expand them when delayed expansion is enabled, even inside double
|
||
// quotes. A newline or carriage return ends the command. None of these
|
||
// can be neutralised safely, so reject any path containing them rather
|
||
// than risk altering the command.
|
||
if shellType == "cmd" {
|
||
if strings.ContainsAny(shellPath, "\"%!\r\n") {
|
||
return "", fmt.Errorf("path is not valid in shell type %s: %q", shellType, shellPath)
|
||
}
|
||
return "\"" + shellPath + "\"", nil
|
||
}
|
||
// Unix shell
|
||
safe := unixShellEscapeRegex.ReplaceAllString(shellPath, `\$0`)
|
||
return strings.ReplaceAll(safe, "\n", "'\n'"), nil
|
||
}
|
||
|
||
// quoteOrEscapeShellPath makes path a valid string argument in configured shell
|
||
func (f *Fs) quoteOrEscapeShellPath(shellPath string) (string, error) {
|
||
return quoteOrEscapeShellPath(f.shellType, shellPath)
|
||
}
|
||
|
||
// remotePath returns the native SFTP path of the file or directory at the remote given
|
||
func (f *Fs) remotePath(remote string) string {
|
||
return path.Join(f.absRoot, f.opt.Enc.FromStandardPath(remote))
|
||
}
|
||
|
||
// remoteShellPath returns the SSH shell path of the file or directory at the remote given
|
||
func (f *Fs) remoteShellPath(remote string) string {
|
||
encodedRemote := f.opt.Enc.FromStandardPath(remote)
|
||
if f.opt.PathOverride != "" {
|
||
shellPath := path.Join(f.opt.PathOverride, encodedRemote)
|
||
if f.opt.PathOverride[0] == '@' {
|
||
shellPath = path.Join(strings.TrimPrefix(f.opt.PathOverride, "@"), f.absRoot, encodedRemote)
|
||
}
|
||
fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath)
|
||
return shellPath
|
||
}
|
||
shellPath := path.Join(f.absRoot, encodedRemote)
|
||
if f.shellType == "powershell" || f.shellType == "cmd" {
|
||
// If remote shell is powershell or cmd, then server is probably Windows.
|
||
// The sftp package converts everything to POSIX paths: Forward slashes, and
|
||
// absolute paths starts with a slash. An absolute path on a Windows server will
|
||
// then look like this "/C:/Windows/System32". We must remove the "/" prefix
|
||
// to make this a valid path for shell commands. In case of PowerShell there is a
|
||
// possibility that it is a Unix server, with PowerShell Core shell, but assuming
|
||
// root folders with names such as "C:" are rare, we just take this risk,
|
||
// and option path_override can always be used to work around corner cases.
|
||
if posixWinAbsPathRegex.MatchString(shellPath) {
|
||
shellPath = strings.TrimPrefix(shellPath, "/")
|
||
fs.Debugf(f, "Shell path adjusted to %q (set option path_override to override)", shellPath)
|
||
return shellPath
|
||
}
|
||
}
|
||
fs.Debugf(f, "Shell path %q", shellPath)
|
||
return shellPath
|
||
}
|
||
|
||
// Converts a byte array from the SSH session returned by
|
||
// an invocation of hash command to a hash string
|
||
// as expected by the rest of this application
|
||
func parseHash(bytes []byte) string {
|
||
// For strings with backslash *sum writes a leading \
|
||
// https://unix.stackexchange.com/q/313733/94054
|
||
return strings.ToLower(strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0]) // Split at hash / filename separator / all convert to lowercase
|
||
}
|
||
|
||
// Parses the byte array output from the SSH session
|
||
// returned by an invocation of df into
|
||
// the disk size, used space, and available space on the disk, in that order.
|
||
// Only works when `df` has output info on only one disk
|
||
func parseUsage(bytes []byte) (spaceTotal int64, spaceUsed int64, spaceAvail int64) {
|
||
spaceTotal, spaceUsed, spaceAvail = -1, -1, -1
|
||
lines := strings.Split(string(bytes), "\n")
|
||
if len(lines) < 2 {
|
||
return
|
||
}
|
||
split := strings.Fields(lines[1])
|
||
if len(split) < 6 {
|
||
return
|
||
}
|
||
spaceTotal, err := strconv.ParseInt(split[1], 10, 64)
|
||
if err != nil {
|
||
spaceTotal = -1
|
||
}
|
||
spaceUsed, err = strconv.ParseInt(split[2], 10, 64)
|
||
if err != nil {
|
||
spaceUsed = -1
|
||
}
|
||
spaceAvail, err = strconv.ParseInt(split[3], 10, 64)
|
||
if err != nil {
|
||
spaceAvail = -1
|
||
}
|
||
return spaceTotal * 1024, spaceUsed * 1024, spaceAvail * 1024
|
||
}
|
||
|
||
// Size returns the size in bytes of the remote sftp file
|
||
func (o *Object) Size() int64 {
|
||
return o.size
|
||
}
|
||
|
||
// ModTime returns the modification time of the remote sftp file
|
||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||
return time.Unix(int64(o.modTime), 0)
|
||
}
|
||
|
||
// path returns the native SFTP path of the object
|
||
func (o *Object) path() string {
|
||
return o.fs.remotePath(o.remote)
|
||
}
|
||
|
||
// shellPath returns the SSH shell path of the object
|
||
func (o *Object) shellPath() string {
|
||
return o.fs.remoteShellPath(o.remote)
|
||
}
|
||
|
||
// setMetadata updates the info in the object from the stat result passed in
|
||
func (o *Object) setMetadata(info os.FileInfo) {
|
||
o.modTime = info.Sys().(*sftp.FileStat).Mtime
|
||
o.size = info.Size()
|
||
o.mode = info.Mode()
|
||
}
|
||
|
||
// statRemote stats the file or directory at the remote given
|
||
func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) {
|
||
absPath := remote
|
||
if !strings.HasPrefix(remote, "/") {
|
||
absPath = f.remotePath(remote)
|
||
}
|
||
c, err := f.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("stat: %w", err)
|
||
}
|
||
info, err = c.sftpClient.Stat(absPath)
|
||
f.putSftpConnection(&c, err)
|
||
return info, err
|
||
}
|
||
|
||
// stat updates the info in the Object
|
||
func (o *Object) stat(ctx context.Context) error {
|
||
info, err := o.fs.stat(ctx, o.remote)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
return fs.ErrorObjectNotFound
|
||
}
|
||
return fmt.Errorf("stat failed: %w", err)
|
||
}
|
||
if info.IsDir() {
|
||
return fs.ErrorIsDir
|
||
}
|
||
o.setMetadata(info)
|
||
return nil
|
||
}
|
||
|
||
// SetModTime sets the modification and access time to the specified time
|
||
//
|
||
// it also updates the info field
|
||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||
if !o.fs.opt.SetModTime {
|
||
return nil
|
||
}
|
||
c, err := o.fs.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("SetModTime: %w", err)
|
||
}
|
||
err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
|
||
o.fs.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
return fmt.Errorf("SetModTime failed: %w", err)
|
||
}
|
||
err = o.stat(ctx)
|
||
if err != nil && err != fs.ErrorIsDir {
|
||
return fmt.Errorf("SetModTime stat failed: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Storable returns whether the remote sftp file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc.)
|
||
func (o *Object) Storable() bool {
|
||
return o.mode.IsRegular()
|
||
}
|
||
|
||
// objectReader represents a file open for reading on the SFTP server
|
||
type objectReader struct {
|
||
f *Fs
|
||
sftpFile *sftp.File
|
||
pipeReader *io.PipeReader
|
||
done chan struct{}
|
||
}
|
||
|
||
func (f *Fs) newObjectReader(sftpFile *sftp.File) *objectReader {
|
||
pipeReader, pipeWriter := io.Pipe()
|
||
file := &objectReader{
|
||
f: f,
|
||
sftpFile: sftpFile,
|
||
pipeReader: pipeReader,
|
||
done: make(chan struct{}),
|
||
}
|
||
// Show connection in use
|
||
f.addSession()
|
||
|
||
go func() {
|
||
// Use sftpFile.WriteTo to pump data so that it gets a
|
||
// chance to build the window up.
|
||
_, err := sftpFile.WriteTo(pipeWriter)
|
||
// Close the pipeWriter so the pipeReader fails with
|
||
// the same error or EOF if err == nil
|
||
_ = pipeWriter.CloseWithError(err)
|
||
// signal that we've finished
|
||
close(file.done)
|
||
}()
|
||
|
||
return file
|
||
}
|
||
|
||
// Read from a remote sftp file object reader
|
||
func (file *objectReader) Read(p []byte) (n int, err error) {
|
||
n, err = file.pipeReader.Read(p)
|
||
return n, err
|
||
}
|
||
|
||
// Close a reader of a remote sftp file
|
||
func (file *objectReader) Close() (err error) {
|
||
// Close the pipeReader so writes to the pipeWriter fail
|
||
_ = file.pipeReader.Close()
|
||
// Close the sftpFile - this will likely cause the WriteTo to error
|
||
err = file.sftpFile.Close()
|
||
// Wait for the background process to finish
|
||
<-file.done
|
||
// Show connection no longer in use
|
||
file.f.removeSession()
|
||
return err
|
||
}
|
||
|
||
// Open a remote sftp file object for reading. Seek is supported
|
||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||
var offset, limit int64 = 0, -1
|
||
for _, option := range options {
|
||
switch x := option.(type) {
|
||
case *fs.SeekOption:
|
||
offset = x.Offset
|
||
case *fs.RangeOption:
|
||
offset, limit = x.Decode(o.Size())
|
||
default:
|
||
if option.Mandatory() {
|
||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||
}
|
||
}
|
||
}
|
||
c, err := o.fs.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Open: %w", err)
|
||
}
|
||
sftpFile, err := c.sftpClient.Open(o.path())
|
||
o.fs.putSftpConnection(&c, err)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Open failed: %w", err)
|
||
}
|
||
if offset > 0 {
|
||
off, err := sftpFile.Seek(offset, io.SeekStart)
|
||
if err != nil || off != offset {
|
||
return nil, fmt.Errorf("Open Seek failed: %w", err)
|
||
}
|
||
}
|
||
in = readers.NewLimitedReadCloser(o.fs.newObjectReader(sftpFile), limit)
|
||
return in, nil
|
||
}
|
||
|
||
type sizeReader struct {
|
||
io.Reader
|
||
size int64
|
||
}
|
||
|
||
// Size returns the expected size of the stream
|
||
//
|
||
// It is used in sftpFile.ReadFrom as a hint to work out the
|
||
// concurrency needed
|
||
func (sr *sizeReader) Size() int64 {
|
||
return sr.size
|
||
}
|
||
|
||
// Update a remote sftp file using the data <in> and ModTime from <src>
|
||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||
o.fs.addSession() // Show session in use
|
||
defer o.fs.removeSession()
|
||
// Clear the hash cache since we are about to update the object
|
||
o.md5sum = nil
|
||
o.sha1sum = nil
|
||
o.crc32sum = nil
|
||
o.sha256sum = nil
|
||
o.blake3sum = nil
|
||
o.xxh3sum = nil
|
||
o.xxh128sum = nil
|
||
c, err := o.fs.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("Update: %w", err)
|
||
}
|
||
// Hang on to the connection for the whole upload so it doesn't get reused while we are uploading
|
||
file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
|
||
if err != nil {
|
||
o.fs.putSftpConnection(&c, err)
|
||
return fmt.Errorf("Update Create failed: %w", err)
|
||
}
|
||
// remove the file if upload failed
|
||
remove := func() {
|
||
c, removeErr := o.fs.getSftpConnection(ctx)
|
||
if removeErr != nil {
|
||
fs.Debugf(src, "Failed to open new SSH connection for delete: %v", removeErr)
|
||
return
|
||
}
|
||
removeErr = c.sftpClient.Remove(o.path())
|
||
o.fs.putSftpConnection(&c, removeErr)
|
||
if removeErr != nil {
|
||
fs.Debugf(src, "Failed to remove: %v", removeErr)
|
||
} else {
|
||
fs.Debugf(src, "Removed after failed upload: %v", err)
|
||
}
|
||
}
|
||
_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
|
||
if err != nil {
|
||
o.fs.putSftpConnection(&c, err)
|
||
remove()
|
||
return fmt.Errorf("Update ReadFrom failed: %w", err)
|
||
}
|
||
err = file.Close()
|
||
if err != nil {
|
||
o.fs.putSftpConnection(&c, err)
|
||
remove()
|
||
return fmt.Errorf("Update Close failed: %w", err)
|
||
}
|
||
// Release connection only when upload has finished so we don't upload multiple files on the same connection
|
||
o.fs.putSftpConnection(&c, err)
|
||
|
||
// Set the mod time - this stats the object if o.fs.opt.SetModTime == true
|
||
err = o.SetModTime(ctx, src.ModTime(ctx))
|
||
if err != nil {
|
||
return fmt.Errorf("Update SetModTime failed: %w", err)
|
||
}
|
||
|
||
// Stat the file after the upload to read its stats back if o.fs.opt.SetModTime == false
|
||
if !o.fs.opt.SetModTime {
|
||
err = o.stat(ctx)
|
||
if err == fs.ErrorObjectNotFound {
|
||
// In the specific case of o.fs.opt.SetModTime == false
|
||
// if the object wasn't found then don't return an error
|
||
fs.Debugf(o, "Not found after upload with set_modtime=false so returning best guess")
|
||
o.modTime = uint32(src.ModTime(ctx).Unix())
|
||
o.size = src.Size()
|
||
o.mode = os.FileMode(0666) // regular file
|
||
} else if err != nil {
|
||
return fmt.Errorf("Update stat failed: %w", err)
|
||
}
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// Remove a remote sftp file object
|
||
func (o *Object) Remove(ctx context.Context) error {
|
||
c, err := o.fs.getSftpConnection(ctx)
|
||
if err != nil {
|
||
return fmt.Errorf("Remove: %w", err)
|
||
}
|
||
err = c.sftpClient.Remove(o.path())
|
||
o.fs.putSftpConnection(&c, err)
|
||
return err
|
||
}
|
||
|
||
// Check the interfaces are satisfied
|
||
var (
|
||
_ fs.Fs = &Fs{}
|
||
_ fs.PutStreamer = &Fs{}
|
||
_ fs.Mover = &Fs{}
|
||
_ fs.Copier = &Fs{}
|
||
_ fs.DirMover = &Fs{}
|
||
_ fs.DirSetModTimer = &Fs{}
|
||
_ fs.Abouter = &Fs{}
|
||
_ fs.Shutdowner = &Fs{}
|
||
_ fs.Object = &Object{}
|
||
)
|