s3: add HCP provider and list_versions_oldest_first quirk
Hitachi Content Platform (HCP) returns object versions in ascending chronological order (oldest first), unlike the S3 standard which returns them newest first. This causes --s3-version-at to return the wrong version when used with HCP. Add a new list_versions_oldest_first quirk which reverses the Versions and DeleteMarkers lists before merging, so the existing versionAt filter works correctly regardless of backend sort order. Add HCP as a new provider with this quirk enabled by default. See: https://docs.hitachivantara.com/r/en-us/content-platform/9.6.x/mk-95hcph002/using-the-hitachi-api-for-amazon-s3/working-with-buckets/listing-bucket-contents-version-2
This commit is contained in:
Chris
Nick Craig-Wood
@@ -59,6 +59,7 @@ directories to and from different cloud storage providers.
|
||||
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
|
||||
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
|
||||
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
|
||||
- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp)
|
||||
- HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
- Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
|
||||
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
name: HCP
|
||||
description: Hitachi Content Platform (HCP)
|
||||
region: {}
|
||||
endpoint: {}
|
||||
location_constraint: {}
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
force_path_style: true
|
||||
list_url_encode: true
|
||||
# HCP returns a SHA256-based ETag for multipart uploads instead of the
|
||||
# standard MD5-based ETag, so ETag verification after multipart upload
|
||||
# must be disabled:
|
||||
use_multipart_etag: false
|
||||
# HCP returns object versions oldest first (ascending chronological order)
|
||||
# instead of the S3 standard newest first:
|
||||
# https://docs.hitachivantara.com/r/en-us/content-platform/9.6.x/mk-95hcph002/using-the-hitachi-api-for-amazon-s3/working-with-buckets/listing-bucket-contents-version-2
|
||||
list_versions_oldest_first: true
|
||||
@@ -30,6 +30,7 @@ type Quirks struct {
|
||||
MightGzip *bool `yaml:"might_gzip,omitempty"`
|
||||
UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
|
||||
UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
|
||||
ListVersionsOldestFirst *bool `yaml:"list_versions_oldest_first,omitempty"`
|
||||
UseXID *bool `yaml:"use_x_id,omitempty"`
|
||||
SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
|
||||
EtagIsNotMD5 *bool `yaml:"etag_is_not_md5,omitempty"`
|
||||
|
||||
@@ -742,6 +742,21 @@ knows about - please make a bug report if not.
|
||||
You can change this if you want to disable the use of multipart uploads.
|
||||
This shouldn't be necessary in normal operation.
|
||||
|
||||
This should be automatically set correctly for all providers rclone
|
||||
knows about - please make a bug report if not.
|
||||
`,
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "list_versions_oldest_first",
|
||||
Help: `Set if the backend returns object versions oldest first.
|
||||
|
||||
The S3 standard returns object versions newest first. Some backends
|
||||
(e.g. Hitachi HCP) return them oldest first instead.
|
||||
|
||||
Set this quirk if --s3-version-at or --s3-versions produce incorrect
|
||||
results with your backend.
|
||||
|
||||
This should be automatically set correctly for all providers rclone
|
||||
knows about - please make a bug report if not.
|
||||
`,
|
||||
@@ -1125,6 +1140,7 @@ type Options struct {
|
||||
IBMAPIKey string `config:"ibm_api_key"`
|
||||
IBMInstanceID string `config:"ibm_resource_instance_id"`
|
||||
IBMIAMEndpoint string `config:"ibm_iam_endpoint"`
|
||||
ListVersionsOldestFirst fs.Tristate `config:"list_versions_oldest_first"`
|
||||
UseXID fs.Tristate `config:"use_x_id"`
|
||||
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
|
||||
ObjectLockMode string `config:"object_lock_mode"`
|
||||
@@ -1741,6 +1757,7 @@ func setQuirks(opt *Options, provider *Provider) {
|
||||
set(&opt.UseXID, true, provider.Quirks.UseXID)
|
||||
set(&opt.SignAcceptEncoding, true, provider.Quirks.SignAcceptEncoding)
|
||||
set(&opt.ObjectLockSupported, true, provider.Quirks.ObjectLockSupported)
|
||||
set(&opt.ListVersionsOldestFirst, false, provider.Quirks.ListVersionsOldestFirst)
|
||||
}
|
||||
|
||||
// setRoot changes the root of the Fs
|
||||
@@ -2258,6 +2275,13 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
|
||||
//structs.SetFrom(resp, respVersions)
|
||||
setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(resp, respVersions)
|
||||
|
||||
// Some backends (e.g. Hitachi HCP) return versions oldest first instead
|
||||
// of newest first. Reverse both lists so mergeDeleteMarkers works correctly.
|
||||
if ls.f.opt.ListVersionsOldestFirst.Value {
|
||||
slices.Reverse(respVersions.Versions)
|
||||
slices.Reverse(respVersions.DeleteMarkers)
|
||||
}
|
||||
|
||||
// Merge in delete Markers as types.ObjectVersion if we need them
|
||||
if ls.hidden || ls.usingVersionAt {
|
||||
respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers)
|
||||
|
||||
@@ -149,6 +149,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="Hetzner Object Storage" home="https://www.hetzner.com/storage/object-storage/" config="/s3/#hetzner" >}}
|
||||
{{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}}
|
||||
{{< provider name="HiDrive" home="https://www.strato.de/cloud-speicher/" config="/hidrive/" >}}
|
||||
{{< provider name="Hitachi Content Platform (HCP)" home="https://www.hitachivantara.com/en-us/products/storage/content-platform.html" config="/s3/#hcp" >}}
|
||||
{{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
|
||||
{{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}}
|
||||
{{< provider name="iCloud Drive" home="https://icloud.com/" config="/iclouddrive/" >}}
|
||||
|
||||
@@ -27,6 +27,7 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="FileLu S5 (S3-Compatible Object Storage)" home="https://s5lu.com/" config="/s3/#filelu-s5" >}}
|
||||
{{< provider name="GCS" home="https://cloud.google.com/storage/docs" config="/s3/#google-cloud-storage" >}}
|
||||
{{< provider name="Hetzner" home="https://www.hetzner.com/storage/object-storage/" config="/s3/#hetzner" >}}
|
||||
{{< provider name="Hitachi Content Platform (HCP)" home="https://www.hitachivantara.com/en-us/products/storage/content-platform.html" config="/s3/#hcp" >}}
|
||||
{{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}}
|
||||
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
|
||||
{{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}}
|
||||
@@ -5585,6 +5586,132 @@ endpoint = hel1.your-objectstorage.com
|
||||
acl = private
|
||||
```
|
||||
|
||||
### Hitachi Content Platform (HCP) {#hcp}
|
||||
|
||||
Here is an example of making a [Hitachi Content Platform (HCP)](https://www.hitachivantara.com/en-us/products/storage/content-platform.html)
|
||||
configuration. First run:
|
||||
|
||||
```console
|
||||
rclone config
|
||||
```
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```text
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> my-hcp
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, GCS, HCP, Hetzner, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, Storj, Synology, TencentCOS, Wasabi, Qiniu and others
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
[snip]
|
||||
XX / Hitachi Content Platform (HCP)
|
||||
\ (HCP)
|
||||
[snip]
|
||||
provider> HCP
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth>
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> ACCESS_KEY
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> SECRET_KEY
|
||||
Option endpoint.
|
||||
Endpoint for HCP S3 compatible API.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
endpoint> https://hcp.example.com
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n>
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: HCP
|
||||
- access_key_id: ACCESS_KEY
|
||||
- secret_access_key: SECRET_KEY
|
||||
- endpoint: https://hcp.example.com
|
||||
Keep this "my-hcp" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d>
|
||||
Current remotes:
|
||||
|
||||
Name Type
|
||||
==== ====
|
||||
my-hcp s3
|
||||
|
||||
e) Edit existing remote
|
||||
n) New remote
|
||||
d) Delete remote
|
||||
r) Rename remote
|
||||
c) Copy remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
e/n/d/r/c/s/q>
|
||||
```
|
||||
|
||||
This will leave the config file looking like this.
|
||||
|
||||
```ini
|
||||
[my-hcp]
|
||||
type = s3
|
||||
provider = HCP
|
||||
access_key_id = ACCESS_KEY
|
||||
secret_access_key = SECRET_KEY
|
||||
endpoint = https://hcp.example.com
|
||||
```
|
||||
|
||||
#### Known limitations
|
||||
|
||||
These limitations have been observed with HCP version 9.7.x.
|
||||
|
||||
**Automatic directory objects**
|
||||
|
||||
HCP automatically creates zero-byte directory objects (keys ending in `/`) for
|
||||
every path component when an object is stored. These directory objects are
|
||||
managed by HCP internally: deleting one via the S3 API does not remove it but
|
||||
instead creates a delete marker with a special version ID (`-3`) that cannot
|
||||
itself be deleted. As a result, rclone cannot fully purge a bucket on HCP —
|
||||
leftover directory objects will remain after a `purge` or `delete` operation.
|
||||
|
||||
**Transparent gzip decompression**
|
||||
|
||||
HCP [transparently decompresses](https://docs.hitachivantara.com/r/en-us/content-platform/9.6.x/mk-95hcph002/using-the-hitachi-api-for-amazon-s3/working-with-objects/creating-an-object)
|
||||
objects uploaded with `Content-Encoding: gzip` before storing them. This means
|
||||
the stored object contains the raw (uncompressed) data, while the `Content-MD5`
|
||||
sent by rclone was computed over the compressed bytes. HCP will reject such
|
||||
uploads with a `BadDigest` error. To avoid this, do not set
|
||||
`content-encoding: gzip` in object metadata when using HCP.
|
||||
|
||||
### Huawei OBS {#huawei-obs}
|
||||
|
||||
Object Storage Service (OBS) provides stable, secure, efficient, and easy-to-use
|
||||
|
||||
Reference in New Issue
Block a user