diff --git a/README.md b/README.md index 4cf53a9fc..2aabfdffe 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ directories to and from different cloud storage providers. - Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner) - Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box) - HiDrive [:page_facing_up:](https://rclone.org/hidrive/) +- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp) - HTTP [:page_facing_up:](https://rclone.org/http/) - Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs) - iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/) diff --git a/backend/s3/provider/HCP.yaml b/backend/s3/provider/HCP.yaml new file mode 100644 index 000000000..66c094e75 --- /dev/null +++ b/backend/s3/provider/HCP.yaml @@ -0,0 +1,18 @@ +name: HCP +description: Hitachi Content Platform (HCP) +region: {} +endpoint: {} +location_constraint: {} +acl: {} +bucket_acl: true +quirks: + force_path_style: true + list_url_encode: true + # HCP returns a SHA256-based ETag for multipart uploads instead of the + # standard MD5-based ETag, so ETag verification after multipart upload + # must be disabled: + use_multipart_etag: false + # HCP returns object versions oldest first (ascending chronological order) + # instead of the S3 standard newest first: + # https://docs.hitachivantara.com/r/en-us/content-platform/9.6.x/mk-95hcph002/using-the-hitachi-api-for-amazon-s3/working-with-buckets/listing-bucket-contents-version-2 + list_versions_oldest_first: true diff --git a/backend/s3/providers.go b/backend/s3/providers.go index 898e6417e..7f1139705 100644 --- a/backend/s3/providers.go +++ b/backend/s3/providers.go @@ -30,6 +30,7 @@ type Quirks struct { MightGzip *bool `yaml:"might_gzip,omitempty"` UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"` UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"` + ListVersionsOldestFirst *bool `yaml:"list_versions_oldest_first,omitempty"` UseXID *bool `yaml:"use_x_id,omitempty"` SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"` EtagIsNotMD5 *bool `yaml:"etag_is_not_md5,omitempty"` diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 2543c01f9..eb4e4de57 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -742,6 +742,21 @@ knows about - please make a bug report if not. You can change this if you want to disable the use of multipart uploads. This shouldn't be necessary in normal operation. +This should be automatically set correctly for all providers rclone +knows about - please make a bug report if not. +`, + Default: fs.Tristate{}, + Advanced: true, + }, { + Name: "list_versions_oldest_first", + Help: `Set if the backend returns object versions oldest first. + +The S3 standard returns object versions newest first. Some backends +(e.g. Hitachi HCP) return them oldest first instead. + +Set this quirk if --s3-version-at or --s3-versions produce incorrect +results with your backend. + This should be automatically set correctly for all providers rclone knows about - please make a bug report if not. `, @@ -1125,6 +1140,7 @@ type Options struct { IBMAPIKey string `config:"ibm_api_key"` IBMInstanceID string `config:"ibm_resource_instance_id"` IBMIAMEndpoint string `config:"ibm_iam_endpoint"` + ListVersionsOldestFirst fs.Tristate `config:"list_versions_oldest_first"` UseXID fs.Tristate `config:"use_x_id"` SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"` ObjectLockMode string `config:"object_lock_mode"` @@ -1741,6 +1757,7 @@ func setQuirks(opt *Options, provider *Provider) { set(&opt.UseXID, true, provider.Quirks.UseXID) set(&opt.SignAcceptEncoding, true, provider.Quirks.SignAcceptEncoding) set(&opt.ObjectLockSupported, true, provider.Quirks.ObjectLockSupported) + set(&opt.ListVersionsOldestFirst, false, provider.Quirks.ListVersionsOldestFirst) } // setRoot changes the root of the Fs @@ -2258,6 +2275,13 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output, //structs.SetFrom(resp, respVersions) setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(resp, respVersions) + // Some backends (e.g. Hitachi HCP) return versions oldest first instead + // of newest first. Reverse both lists so mergeDeleteMarkers works correctly. + if ls.f.opt.ListVersionsOldestFirst.Value { + slices.Reverse(respVersions.Versions) + slices.Reverse(respVersions.DeleteMarkers) + } + // Merge in delete Markers as types.ObjectVersion if we need them if ls.hidden || ls.usingVersionAt { respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers) diff --git a/docs/content/_index.md b/docs/content/_index.md index 042b66874..7b6958577 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -149,6 +149,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Hetzner Object Storage" home="https://www.hetzner.com/storage/object-storage/" config="/s3/#hetzner" >}} {{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}} {{< provider name="HiDrive" home="https://www.strato.de/cloud-speicher/" config="/hidrive/" >}} +{{< provider name="Hitachi Content Platform (HCP)" home="https://www.hitachivantara.com/en-us/products/storage/content-platform.html" config="/s3/#hcp" >}} {{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}} {{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}} {{< provider name="iCloud Drive" home="https://icloud.com/" config="/iclouddrive/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index 369f8866f..a70f77962 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -27,6 +27,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="FileLu S5 (S3-Compatible Object Storage)" home="https://s5lu.com/" config="/s3/#filelu-s5" >}} {{< provider name="GCS" home="https://cloud.google.com/storage/docs" config="/s3/#google-cloud-storage" >}} {{< provider name="Hetzner" home="https://www.hetzner.com/storage/object-storage/" config="/s3/#hetzner" >}} +{{< provider name="Hitachi Content Platform (HCP)" home="https://www.hitachivantara.com/en-us/products/storage/content-platform.html" config="/s3/#hcp" >}} {{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} {{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}} @@ -5585,6 +5586,132 @@ endpoint = hel1.your-objectstorage.com acl = private ``` +### Hitachi Content Platform (HCP) {#hcp} + +Here is an example of making a [Hitachi Content Platform (HCP)](https://www.hitachivantara.com/en-us/products/storage/content-platform.html) +configuration. First run: + +```console +rclone config +``` + +This will guide you through an interactive setup process. + +```text +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n + +Enter name for new remote. +name> my-hcp +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, GCS, HCP, Hetzner, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, Storj, Synology, TencentCOS, Wasabi, Qiniu and others + \ (s3) +[snip] +Storage> s3 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] +XX / Hitachi Content Platform (HCP) + \ (HCP) +[snip] +provider> HCP +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> ACCESS_KEY +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> SECRET_KEY +Option endpoint. +Endpoint for HCP S3 compatible API. +Enter a value. Press Enter to leave empty. +endpoint> https://hcp.example.com +Edit advanced config? +y) Yes +n) No (default) +y/n> +Configuration complete. +Options: +- type: s3 +- provider: HCP +- access_key_id: ACCESS_KEY +- secret_access_key: SECRET_KEY +- endpoint: https://hcp.example.com +Keep this "my-hcp" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> +Current remotes: + +Name Type +==== ==== +my-hcp s3 + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> +``` + +This will leave the config file looking like this. + +```ini +[my-hcp] +type = s3 +provider = HCP +access_key_id = ACCESS_KEY +secret_access_key = SECRET_KEY +endpoint = https://hcp.example.com +``` + +#### Known limitations + +These limitations have been observed with HCP version 9.7.x. + +**Automatic directory objects** + +HCP automatically creates zero-byte directory objects (keys ending in `/`) for +every path component when an object is stored. These directory objects are +managed by HCP internally: deleting one via the S3 API does not remove it but +instead creates a delete marker with a special version ID (`-3`) that cannot +itself be deleted. As a result, rclone cannot fully purge a bucket on HCP — +leftover directory objects will remain after a `purge` or `delete` operation. + +**Transparent gzip decompression** + +HCP [transparently decompresses](https://docs.hitachivantara.com/r/en-us/content-platform/9.6.x/mk-95hcph002/using-the-hitachi-api-for-amazon-s3/working-with-objects/creating-an-object) +objects uploaded with `Content-Encoding: gzip` before storing them. This means +the stored object contains the raw (uncompressed) data, while the `Content-MD5` +sent by rclone was computed over the compressed bytes. HCP will reject such +uploads with a `BadDigest` error. To avoid this, do not set +`content-encoding: gzip` in object metadata when using HCP. + ### Huawei OBS {#huawei-obs} Object Storage Service (OBS) provides stable, secure, efficient, and easy-to-use