Nick Craig-Wood
19478503af
Add a new "use_arrow_list" advanced option which fetches directory listings using the ListBlobs Apache Arrow response format instead of XML. This can be substantially faster for very large containers. This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow" preview feature to be enabled on the storage account. It is not supported on accounts with a hierarchical namespace (ADLS Gen2). If the feature is not enabled the service returns XML and the listing transparently falls back to it. Connection string auth is not supported - it also falls back to the normal XML listing. The option is hidden from the configurator, command line help and docs until the feature is officially released.
630 lines
21 KiB
Go
630 lines
21 KiB
Go
//go:build !plan9 && !solaris && !js
|
|
|
|
package azureblob
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/md5"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/hash"
|
|
"github.com/rclone/rclone/fs/object"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/fstest/fstests"
|
|
"github.com/rclone/rclone/lib/random"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestBlockIDCreator(t *testing.T) {
|
|
// Check creation and random number
|
|
bic, err := newBlockIDCreator()
|
|
require.NoError(t, err)
|
|
bic2, err := newBlockIDCreator()
|
|
require.NoError(t, err)
|
|
assert.NotEqual(t, bic.random, bic2.random)
|
|
assert.NotEqual(t, bic.random, [8]byte{})
|
|
|
|
// Set random to known value for tests
|
|
bic.random = [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
|
|
chunkNumber := uint64(0xFEDCBA9876543210)
|
|
|
|
// Check creation of ID
|
|
want := base64.StdEncoding.EncodeToString([]byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, 1, 2, 3, 4, 5, 6, 7, 8})
|
|
assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", want)
|
|
got := bic.newBlockID(chunkNumber)
|
|
assert.Equal(t, want, got)
|
|
assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", got)
|
|
|
|
// Test checkID is working
|
|
assert.NoError(t, bic.checkID(chunkNumber, got))
|
|
assert.ErrorContains(t, bic.checkID(chunkNumber, "$"+got), "illegal base64")
|
|
assert.ErrorContains(t, bic.checkID(chunkNumber, "AAAA"+got), "bad block ID length")
|
|
assert.ErrorContains(t, bic.checkID(chunkNumber+1, got), "expecting decoded")
|
|
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
|
|
}
|
|
|
|
func (f *Fs) testFeatures(t *testing.T) {
|
|
// Check first feature flags are set on this remote
|
|
enabled := f.Features().SetTier
|
|
assert.True(t, enabled)
|
|
enabled = f.Features().GetTier
|
|
assert.True(t, enabled)
|
|
}
|
|
|
|
type ReadSeekCloser struct {
|
|
*strings.Reader
|
|
}
|
|
|
|
func (r *ReadSeekCloser) Close() error {
|
|
return nil
|
|
}
|
|
|
|
// Stage a block at remote but don't commit it
|
|
func (f *Fs) stageBlockWithoutCommit(ctx context.Context, t *testing.T, remote string) {
|
|
var (
|
|
containerName, blobPath = f.split(remote)
|
|
containerClient = f.cntSVC(containerName)
|
|
blobClient = containerClient.NewBlockBlobClient(blobPath)
|
|
data = "uncommitted data"
|
|
blockID = "1"
|
|
blockIDBase64 = base64.StdEncoding.EncodeToString([]byte(blockID))
|
|
)
|
|
r := &ReadSeekCloser{strings.NewReader(data)}
|
|
_, err := blobClient.StageBlock(ctx, blockIDBase64, r, nil)
|
|
require.NoError(t, err)
|
|
|
|
// Verify the block is staged but not committed
|
|
blockList, err := blobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
|
|
require.NoError(t, err)
|
|
found := false
|
|
for _, block := range blockList.UncommittedBlocks {
|
|
if *block.Name == blockIDBase64 {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
require.True(t, found, "Block ID not found in uncommitted blocks")
|
|
}
|
|
|
|
// This tests uploading a blob where it has uncommitted blocks with a different ID size.
|
|
//
|
|
// https://gauravmantri.com/2013/05/18/windows-azure-blob-storage-dealing-with-the-specified-blob-or-block-content-is-invalid-error/
|
|
//
|
|
// TestIntegration/FsMkdir/FsPutFiles/Internal/WriteUncommittedBlocks
|
|
func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
|
|
var (
|
|
ctx = context.Background()
|
|
remote = "testBlob"
|
|
)
|
|
|
|
// Multipart copy the blob please
|
|
oldUseCopyBlob, oldCopyCutoff := f.opt.UseCopyBlob, f.opt.CopyCutoff
|
|
f.opt.UseCopyBlob = false
|
|
f.opt.CopyCutoff = f.opt.ChunkSize
|
|
defer func() {
|
|
f.opt.UseCopyBlob, f.opt.CopyCutoff = oldUseCopyBlob, oldCopyCutoff
|
|
}()
|
|
|
|
// Create a blob with uncommitted blocks
|
|
f.stageBlockWithoutCommit(ctx, t, remote)
|
|
|
|
// Now attempt to overwrite the block with a different sized block ID to provoke this error
|
|
|
|
// Check the object does not exist
|
|
_, err := f.NewObject(ctx, remote)
|
|
require.Equal(t, fs.ErrorObjectNotFound, err)
|
|
|
|
// Upload a multipart file over the block with uncommitted chunks of a different ID size
|
|
size := 4*int(f.opt.ChunkSize) - 1
|
|
contents := random.String(size)
|
|
item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
|
|
o := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
|
|
|
// Check size
|
|
assert.Equal(t, int64(size), o.Size())
|
|
|
|
// Create a new blob with uncommitted blocks
|
|
newRemote := "testBlob2"
|
|
f.stageBlockWithoutCommit(ctx, t, newRemote)
|
|
|
|
// Copy over that block
|
|
dst, err := f.Copy(ctx, o, newRemote)
|
|
require.NoError(t, err)
|
|
|
|
// Check basics
|
|
assert.Equal(t, int64(size), dst.Size())
|
|
assert.Equal(t, newRemote, dst.Remote())
|
|
|
|
// Check contents
|
|
gotContents := fstests.ReadObject(ctx, t, dst, -1)
|
|
assert.Equal(t, contents, gotContents)
|
|
|
|
// Remove the object
|
|
require.NoError(t, dst.Remove(ctx))
|
|
}
|
|
|
|
func gz(t *testing.T, s string) string {
|
|
var buf bytes.Buffer
|
|
zw := gzip.NewWriter(&buf)
|
|
_, err := zw.Write([]byte(s))
|
|
require.NoError(t, err)
|
|
err = zw.Close()
|
|
require.NoError(t, err)
|
|
return buf.String()
|
|
}
|
|
|
|
func md5sum(t *testing.T, s string) string {
|
|
hash := md5.Sum([]byte(s))
|
|
return fmt.Sprintf("%x", hash)
|
|
}
|
|
|
|
func (f *Fs) testGzipEncoding(t *testing.T) {
|
|
ctx := context.Background()
|
|
original := random.String(1000)
|
|
contents := gz(t, original)
|
|
|
|
item := fstest.NewItem("test-gzip", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
metadata := fs.Metadata{
|
|
"content-encoding": "gzip",
|
|
"content-type": "text/plain",
|
|
}
|
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", metadata)
|
|
defer func() {
|
|
assert.NoError(t, obj.Remove(ctx))
|
|
}()
|
|
o := obj.(*Object)
|
|
|
|
// Test that the gzipped file we uploaded can be
|
|
// downloaded with and without decompression
|
|
checkDownload := func(wantContents string, wantSize int64, wantHash string) {
|
|
gotContents := fstests.ReadObject(ctx, t, o, -1)
|
|
assert.Equal(t, wantContents, gotContents)
|
|
assert.Equal(t, wantSize, o.Size())
|
|
gotHash, err := o.Hash(ctx, hash.MD5)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, wantHash, gotHash)
|
|
}
|
|
|
|
t.Run("NoDecompress", func(t *testing.T) {
|
|
checkDownload(contents, int64(len(contents)), md5sum(t, contents))
|
|
})
|
|
t.Run("Decompress", func(t *testing.T) {
|
|
f.opt.Decompress = true
|
|
defer func() {
|
|
f.opt.Decompress = false
|
|
}()
|
|
checkDownload(original, -1, "")
|
|
})
|
|
}
|
|
|
|
func (f *Fs) InternalTest(t *testing.T) {
|
|
t.Run("Features", f.testFeatures)
|
|
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
|
t.Run("Metadata", f.testMetadataPaths)
|
|
t.Run("GzipEncoding", f.testGzipEncoding)
|
|
t.Run("ArrowList", f.testArrowList)
|
|
}
|
|
|
|
// helper to read blob properties for an object
|
|
func getProps(ctx context.Context, t *testing.T, o fs.Object) *blob.GetPropertiesResponse {
|
|
ao := o.(*Object)
|
|
props, err := ao.readMetaDataAlways(ctx)
|
|
require.NoError(t, err)
|
|
return props
|
|
}
|
|
|
|
// helper to assert select headers and user metadata
|
|
func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, want map[string]string, wantUserMeta map[string]string) {
|
|
// Headers
|
|
get := func(p *string) string {
|
|
if p == nil {
|
|
return ""
|
|
}
|
|
return *p
|
|
}
|
|
if v, ok := want["content-type"]; ok {
|
|
assert.Equal(t, v, get(props.ContentType), "content-type")
|
|
}
|
|
if v, ok := want["cache-control"]; ok {
|
|
assert.Equal(t, v, get(props.CacheControl), "cache-control")
|
|
}
|
|
if v, ok := want["content-disposition"]; ok {
|
|
assert.Equal(t, v, get(props.ContentDisposition), "content-disposition")
|
|
}
|
|
if v, ok := want["content-encoding"]; ok {
|
|
assert.Equal(t, v, get(props.ContentEncoding), "content-encoding")
|
|
}
|
|
if v, ok := want["content-language"]; ok {
|
|
assert.Equal(t, v, get(props.ContentLanguage), "content-language")
|
|
}
|
|
// User metadata (case-insensitive keys from service)
|
|
norm := make(map[string]*string, len(props.Metadata))
|
|
for kk, vv := range props.Metadata {
|
|
norm[strings.ToLower(kk)] = vv
|
|
}
|
|
for k, v := range wantUserMeta {
|
|
pv, ok := norm[strings.ToLower(k)]
|
|
if assert.True(t, ok, fmt.Sprintf("missing user metadata key %q", k)) {
|
|
if pv == nil {
|
|
assert.Equal(t, v, "", k)
|
|
} else {
|
|
assert.Equal(t, v, *pv, k)
|
|
}
|
|
} else {
|
|
// Log available keys for diagnostics
|
|
keys := make([]string, 0, len(props.Metadata))
|
|
for kk := range props.Metadata {
|
|
keys = append(keys, kk)
|
|
}
|
|
t.Logf("available user metadata keys: %v", keys)
|
|
}
|
|
}
|
|
}
|
|
|
|
// helper to read blob tags for an object
|
|
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
|
ao := o.(*Object)
|
|
blb := ao.getBlobSVC()
|
|
resp, err := blb.GetTags(ctx, nil)
|
|
require.NoError(t, err)
|
|
out := make(map[string]string)
|
|
for _, tag := range resp.BlobTagSet {
|
|
if tag.Key != nil {
|
|
k := *tag.Key
|
|
v := ""
|
|
if tag.Value != nil {
|
|
v = *tag.Value
|
|
}
|
|
out[k] = v
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Test metadata across different write paths
|
|
func (f *Fs) testMetadataPaths(t *testing.T) {
|
|
ctx := context.Background()
|
|
if testing.Short() {
|
|
t.Skip("skipping in short mode")
|
|
}
|
|
|
|
// Common expected metadata and headers
|
|
baseMeta := fs.Metadata{
|
|
"cache-control": "no-cache",
|
|
"content-disposition": "inline",
|
|
"content-language": "en-US",
|
|
// Note: Don't set content-encoding here to avoid download decoding differences
|
|
// We will set a custom user metadata key
|
|
"potato": "royal",
|
|
// and modtime
|
|
"mtime": fstest.Time("2009-05-06T04:05:06.499999999Z").Format(time.RFC3339Nano),
|
|
}
|
|
|
|
// Singlepart upload
|
|
t.Run("PutSinglepart", func(t *testing.T) {
|
|
// size less than chunk size
|
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
|
item := fstest.NewItem("meta-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
// override content-type via metadata mapping
|
|
meta := fs.Metadata{}
|
|
meta.Merge(baseMeta)
|
|
meta["content-type"] = "text/plain"
|
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
|
defer func() { _ = obj.Remove(ctx) }()
|
|
|
|
props := getProps(ctx, t, obj)
|
|
assertHeadersAndMetadata(t, props, map[string]string{
|
|
"content-type": "text/plain",
|
|
"cache-control": "no-cache",
|
|
"content-disposition": "inline",
|
|
"content-language": "en-US",
|
|
}, map[string]string{
|
|
"potato": "royal",
|
|
})
|
|
_ = http.StatusOK // keep import for parity but don't inspect RawResponse
|
|
})
|
|
|
|
// Multipart upload
|
|
t.Run("PutMultipart", func(t *testing.T) {
|
|
// size greater than chunk size to force multipart
|
|
contents := random.String(int(f.opt.ChunkSize + 1024))
|
|
item := fstest.NewItem("meta-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
meta := fs.Metadata{}
|
|
meta.Merge(baseMeta)
|
|
meta["content-type"] = "application/json"
|
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
|
defer func() { _ = obj.Remove(ctx) }()
|
|
|
|
props := getProps(ctx, t, obj)
|
|
assertHeadersAndMetadata(t, props, map[string]string{
|
|
"content-type": "application/json",
|
|
"cache-control": "no-cache",
|
|
"content-disposition": "inline",
|
|
"content-language": "en-US",
|
|
}, map[string]string{
|
|
"potato": "royal",
|
|
})
|
|
|
|
// Tags: Singlepart upload
|
|
t.Run("PutSinglepartTags", func(t *testing.T) {
|
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
|
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
meta := fs.Metadata{
|
|
"x-ms-tags": "env=dev,team=sync",
|
|
}
|
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", meta)
|
|
defer func() { _ = obj.Remove(ctx) }()
|
|
|
|
tags := getTagsMap(ctx, t, obj)
|
|
assert.Equal(t, "dev", tags["env"])
|
|
assert.Equal(t, "sync", tags["team"])
|
|
})
|
|
|
|
// Tags: Multipart upload
|
|
t.Run("PutMultipartTags", func(t *testing.T) {
|
|
contents := random.String(int(f.opt.ChunkSize + 2048))
|
|
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
meta := fs.Metadata{
|
|
"x-ms-tags": "project=alpha,release=2025-08",
|
|
}
|
|
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "application/octet-stream", meta)
|
|
defer func() { _ = obj.Remove(ctx) }()
|
|
|
|
tags := getTagsMap(ctx, t, obj)
|
|
assert.Equal(t, "alpha", tags["project"])
|
|
assert.Equal(t, "2025-08", tags["release"])
|
|
})
|
|
})
|
|
|
|
// Singlepart copy with metadata-set mapping; omit content-type to exercise fallback
|
|
t.Run("CopySinglepart", func(t *testing.T) {
|
|
// create small source
|
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
|
srcItem := fstest.NewItem("meta-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
|
defer func() { _ = srcObj.Remove(ctx) }()
|
|
|
|
// set mapping via MetadataSet
|
|
ctx2, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
ci.MetadataSet = fs.Metadata{
|
|
"cache-control": "private, max-age=60",
|
|
"content-disposition": "attachment; filename=foo.txt",
|
|
"content-language": "fr",
|
|
// no content-type: should fallback to source
|
|
"potato": "maris",
|
|
}
|
|
|
|
// do copy
|
|
dstName := "meta-copy-single-dst.txt"
|
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
|
require.NoError(t, err)
|
|
defer func() { _ = dst.Remove(ctx2) }()
|
|
|
|
props := getProps(ctx2, t, dst)
|
|
// content-type should fallback to source (text/plain)
|
|
assertHeadersAndMetadata(t, props, map[string]string{
|
|
"content-type": "text/plain",
|
|
"cache-control": "private, max-age=60",
|
|
"content-disposition": "attachment; filename=foo.txt",
|
|
"content-language": "fr",
|
|
}, map[string]string{
|
|
"potato": "maris",
|
|
})
|
|
// mtime should be populated on copy when --metadata is used
|
|
// and should equal the source ModTime (RFC3339Nano)
|
|
// Read user metadata (case-insensitive)
|
|
m := props.Metadata
|
|
var gotMtime string
|
|
for k, v := range m {
|
|
if strings.EqualFold(k, "mtime") && v != nil {
|
|
gotMtime = *v
|
|
break
|
|
}
|
|
}
|
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
|
// parse and compare times ignoring formatting differences
|
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
|
require.NoError(t, err)
|
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
|
}
|
|
})
|
|
|
|
// CopySinglepart with only --metadata (no MetadataSet) must inject mtime and preserve src content-type
|
|
t.Run("CopySinglepart_MetadataOnly", func(t *testing.T) {
|
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
|
srcItem := fstest.NewItem("meta-copy-single-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
|
defer func() { _ = srcObj.Remove(ctx) }()
|
|
|
|
ctx2, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
|
|
dstName := "meta-copy-single-only-dst.txt"
|
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
|
require.NoError(t, err)
|
|
defer func() { _ = dst.Remove(ctx2) }()
|
|
|
|
props := getProps(ctx2, t, dst)
|
|
assertHeadersAndMetadata(t, props, map[string]string{
|
|
"content-type": "text/plain",
|
|
}, map[string]string{})
|
|
// Assert mtime injected
|
|
m := props.Metadata
|
|
var gotMtime string
|
|
for k, v := range m {
|
|
if strings.EqualFold(k, "mtime") && v != nil {
|
|
gotMtime = *v
|
|
break
|
|
}
|
|
}
|
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
|
require.NoError(t, err)
|
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
|
}
|
|
})
|
|
|
|
// Multipart copy with metadata-set mapping; omit content-type to exercise fallback
|
|
t.Run("CopyMultipart", func(t *testing.T) {
|
|
// create large source to force multipart
|
|
contents := random.String(int(f.opt.CopyCutoff + 1024))
|
|
srcItem := fstest.NewItem("meta-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
|
defer func() { _ = srcObj.Remove(ctx) }()
|
|
|
|
// set mapping via MetadataSet
|
|
ctx2, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
ci.MetadataSet = fs.Metadata{
|
|
"cache-control": "max-age=0, no-cache",
|
|
// omit content-type to trigger fallback
|
|
"content-language": "de",
|
|
"potato": "desiree",
|
|
}
|
|
|
|
dstName := "meta-copy-multi-dst.txt"
|
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
|
require.NoError(t, err)
|
|
defer func() { _ = dst.Remove(ctx2) }()
|
|
|
|
props := getProps(ctx2, t, dst)
|
|
// content-type should fallback to source (application/octet-stream)
|
|
assertHeadersAndMetadata(t, props, map[string]string{
|
|
"content-type": "application/octet-stream",
|
|
"cache-control": "max-age=0, no-cache",
|
|
"content-language": "de",
|
|
}, map[string]string{
|
|
"potato": "desiree",
|
|
})
|
|
// mtime should be populated on copy when --metadata is used
|
|
m := props.Metadata
|
|
var gotMtime string
|
|
for k, v := range m {
|
|
if strings.EqualFold(k, "mtime") && v != nil {
|
|
gotMtime = *v
|
|
break
|
|
}
|
|
}
|
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
|
require.NoError(t, err)
|
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
|
}
|
|
})
|
|
|
|
// CopyMultipart with only --metadata must inject mtime and preserve src content-type
|
|
t.Run("CopyMultipart_MetadataOnly", func(t *testing.T) {
|
|
contents := random.String(int(f.opt.CopyCutoff + 2048))
|
|
srcItem := fstest.NewItem("meta-copy-multi-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
|
defer func() { _ = srcObj.Remove(ctx) }()
|
|
|
|
ctx2, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
|
|
dstName := "meta-copy-multi-only-dst.txt"
|
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
|
require.NoError(t, err)
|
|
defer func() { _ = dst.Remove(ctx2) }()
|
|
|
|
props := getProps(ctx2, t, dst)
|
|
assertHeadersAndMetadata(t, props, map[string]string{
|
|
"content-type": "application/octet-stream",
|
|
}, map[string]string{})
|
|
m := props.Metadata
|
|
var gotMtime string
|
|
for k, v := range m {
|
|
if strings.EqualFold(k, "mtime") && v != nil {
|
|
gotMtime = *v
|
|
break
|
|
}
|
|
}
|
|
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
|
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
|
require.NoError(t, err)
|
|
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
|
}
|
|
})
|
|
|
|
// Tags: Singlepart copy
|
|
t.Run("CopySinglepartTags", func(t *testing.T) {
|
|
// create small source
|
|
contents := random.String(int(f.opt.ChunkSize / 2))
|
|
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
|
defer func() { _ = srcObj.Remove(ctx) }()
|
|
|
|
// set mapping via MetadataSet including tags
|
|
ctx2, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
ci.MetadataSet = fs.Metadata{
|
|
"x-ms-tags": "copy=single,mode=test",
|
|
}
|
|
|
|
dstName := "tags-copy-single-dst.txt"
|
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
|
require.NoError(t, err)
|
|
defer func() { _ = dst.Remove(ctx2) }()
|
|
|
|
tags := getTagsMap(ctx2, t, dst)
|
|
assert.Equal(t, "single", tags["copy"])
|
|
assert.Equal(t, "test", tags["mode"])
|
|
})
|
|
|
|
// Tags: Multipart copy
|
|
t.Run("CopyMultipartTags", func(t *testing.T) {
|
|
// create large source to force multipart
|
|
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
|
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
|
defer func() { _ = srcObj.Remove(ctx) }()
|
|
|
|
ctx2, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
ci.MetadataSet = fs.Metadata{
|
|
"x-ms-tags": "copy=multi,mode=test",
|
|
}
|
|
|
|
dstName := "tags-copy-multi-dst.txt"
|
|
dst, err := f.Copy(ctx2, srcObj, dstName)
|
|
require.NoError(t, err)
|
|
defer func() { _ = dst.Remove(ctx2) }()
|
|
|
|
tags := getTagsMap(ctx2, t, dst)
|
|
assert.Equal(t, "multi", tags["copy"])
|
|
assert.Equal(t, "test", tags["mode"])
|
|
})
|
|
|
|
// Negative: invalid x-ms-tags must error
|
|
t.Run("InvalidXMsTags", func(t *testing.T) {
|
|
contents := random.String(32)
|
|
item := fstest.NewItem("tags-invalid.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
|
// construct ObjectInfo with invalid x-ms-tags
|
|
buf := strings.NewReader(contents)
|
|
// Build obj info with metadata
|
|
meta := fs.Metadata{
|
|
"x-ms-tags": "badpair-without-equals",
|
|
}
|
|
// force metadata on
|
|
ctx2, ci := fs.AddConfig(ctx)
|
|
ci.Metadata = true
|
|
obji := object.NewStaticObjectInfo(item.Path, item.ModTime, int64(len(contents)), true, nil, nil)
|
|
obji = obji.WithMetadata(meta).WithMimeType("text/plain")
|
|
_, err := f.Put(ctx2, buf, obji)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "invalid tag")
|
|
})
|
|
}
|