vendor: switch to using go1.11 modules

This commit is contained in:
Nick Craig-Wood
2018-08-28 16:08:48 +01:00
parent 5c75453aba
commit da1682a30e
6142 changed files with 390 additions and 5155875 deletions
-58
View File
@@ -1,58 +0,0 @@
package s3
import (
"bytes"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/awstesting/unit"
)
func BenchmarkPresign_GetObject(b *testing.B) {
sess := unit.Session
svc := New(sess)
for i := 0; i < b.N; i++ {
req, _ := svc.GetObjectRequest(&GetObjectInput{
Bucket: aws.String("mock-bucket"),
Key: aws.String("mock-key"),
})
u, h, err := req.PresignRequest(15 * time.Minute)
if err != nil {
b.Fatalf("expect no error, got %v", err)
}
if len(u) == 0 {
b.Fatalf("expect url, got none")
}
if len(h) != 0 {
b.Fatalf("no signed headers, got %v", h)
}
}
}
func BenchmarkPresign_PutObject(b *testing.B) {
sess := unit.Session
svc := New(sess)
body := make([]byte, 1024*1024*20)
for i := 0; i < b.N; i++ {
req, _ := svc.PutObjectRequest(&PutObjectInput{
Bucket: aws.String("mock-bucket"),
Key: aws.String("mock-key"),
Body: bytes.NewReader(body),
})
u, h, err := req.PresignRequest(15 * time.Minute)
if err != nil {
b.Fatalf("expect no error, got %v", err)
}
if len(u) == 0 {
b.Fatalf("expect url, got none")
}
if len(h) == 0 {
b.Fatalf("expect signed header, got none")
}
}
}
-523
View File
@@ -1,523 +0,0 @@
package s3
import (
"bytes"
"crypto/md5"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/sdkio"
)
type errorReader struct{}
func (errorReader) Read([]byte) (int, error) {
return 0, fmt.Errorf("errorReader error")
}
func (errorReader) Seek(int64, int) (int64, error) {
return 0, nil
}
func TestComputeBodyHases(t *testing.T) {
bodyContent := []byte("bodyContent goes here")
cases := []struct {
Req *request.Request
ExpectMD5 string
ExpectSHA256 string
Error string
DisableContentMD5 bool
Presigned bool
}{
{
Req: &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
Body: bytes.NewReader(bodyContent),
},
ExpectMD5: "CqD6NNPvoNOBT/5pkjtzOw==",
ExpectSHA256: "3ff09c8b42a58a905e27835919ede45b61722e7cd400f30101bd9ed1a69a1825",
},
{
Req: &request.Request{
HTTPRequest: &http.Request{
Header: func() http.Header {
h := http.Header{}
h.Set(contentMD5Header, "MD5AlreadySet")
return h
}(),
},
Body: bytes.NewReader(bodyContent),
},
ExpectMD5: "MD5AlreadySet",
ExpectSHA256: "3ff09c8b42a58a905e27835919ede45b61722e7cd400f30101bd9ed1a69a1825",
},
{
Req: &request.Request{
HTTPRequest: &http.Request{
Header: func() http.Header {
h := http.Header{}
h.Set(contentSha256Header, "SHA256AlreadySet")
return h
}(),
},
Body: bytes.NewReader(bodyContent),
},
ExpectMD5: "CqD6NNPvoNOBT/5pkjtzOw==",
ExpectSHA256: "SHA256AlreadySet",
},
{
Req: &request.Request{
HTTPRequest: &http.Request{
Header: func() http.Header {
h := http.Header{}
h.Set(contentMD5Header, "MD5AlreadySet")
h.Set(contentSha256Header, "SHA256AlreadySet")
return h
}(),
},
Body: bytes.NewReader(bodyContent),
},
ExpectMD5: "MD5AlreadySet",
ExpectSHA256: "SHA256AlreadySet",
},
{
Req: &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
// Non-seekable reader
Body: aws.ReadSeekCloser(bytes.NewBuffer(bodyContent)),
},
ExpectMD5: "",
ExpectSHA256: "",
},
{
Req: &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
// Empty seekable body
Body: aws.ReadSeekCloser(bytes.NewReader(nil)),
},
ExpectMD5: "1B2M2Y8AsgTpgAmY7PhCfg==",
ExpectSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
},
{
Req: &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
// failure while reading reader
Body: errorReader{},
},
ExpectMD5: "",
ExpectSHA256: "",
Error: "errorReader error",
},
{
// Disabled ContentMD5 validation
Req: &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
Body: bytes.NewReader(bodyContent),
},
ExpectMD5: "",
ExpectSHA256: "",
DisableContentMD5: true,
},
{
// Disabled ContentMD5 validation
Req: &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
Body: bytes.NewReader(bodyContent),
},
ExpectMD5: "",
ExpectSHA256: "",
Presigned: true,
},
}
for i, c := range cases {
c.Req.Config.S3DisableContentMD5Validation = aws.Bool(c.DisableContentMD5)
if c.Presigned {
c.Req.ExpireTime = 10 * time.Minute
}
computeBodyHashes(c.Req)
if e, a := c.ExpectMD5, c.Req.HTTPRequest.Header.Get(contentMD5Header); e != a {
t.Errorf("%d, expect %v md5, got %v", i, e, a)
}
if e, a := c.ExpectSHA256, c.Req.HTTPRequest.Header.Get(contentSha256Header); e != a {
t.Errorf("%d, expect %v sha256, got %v", i, e, a)
}
if len(c.Error) != 0 {
if c.Req.Error == nil {
t.Fatalf("%d, expect error, got none", i)
}
if e, a := c.Error, c.Req.Error.Error(); !strings.Contains(a, e) {
t.Errorf("%d, expect %v error to be in %v", i, e, a)
}
} else if c.Req.Error != nil {
t.Errorf("%d, expect no error, got %v", i, c.Req.Error)
}
}
}
func BenchmarkComputeBodyHashes(b *testing.B) {
body := bytes.NewReader(make([]byte, 2*1024))
req := &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
Body: body,
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
computeBodyHashes(req)
if req.Error != nil {
b.Fatalf("expect no error, got %v", req.Error)
}
req.HTTPRequest.Header = http.Header{}
body.Seek(0, sdkio.SeekStart)
}
}
func TestAskForTxEncodingAppendMD5(t *testing.T) {
cases := []struct {
DisableContentMD5 bool
Presigned bool
}{
{DisableContentMD5: true},
{DisableContentMD5: false},
{Presigned: true},
}
for i, c := range cases {
req := &request.Request{
HTTPRequest: &http.Request{
Header: http.Header{},
},
Config: aws.Config{
S3DisableContentMD5Validation: aws.Bool(c.DisableContentMD5),
},
}
if c.Presigned {
req.ExpireTime = 10 * time.Minute
}
askForTxEncodingAppendMD5(req)
v := req.HTTPRequest.Header.Get(amzTeHeader)
expectHeader := !(c.DisableContentMD5 || c.Presigned)
if e, a := expectHeader, len(v) != 0; e != a {
t.Errorf("%d, expect %t disable content MD5, got %t, %s", i, e, a, v)
}
}
}
func TestUseMD5ValidationReader(t *testing.T) {
body := []byte("create a really cool md5 checksum of me")
bodySum := md5.Sum(body)
bodyWithSum := append(body, bodySum[:]...)
emptyBodySum := md5.Sum([]byte{})
cases := []struct {
Req *request.Request
Error string
Validate func(outupt interface{}) error
}{
{
// Positive: Use Validation reader
Req: &request.Request{
HTTPResponse: &http.Response{
Header: func() http.Header {
h := http.Header{}
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
return h
}(),
},
Data: &GetObjectOutput{
Body: ioutil.NopCloser(bytes.NewReader(bodyWithSum)),
ContentLength: aws.Int64(int64(len(bodyWithSum))),
},
},
Validate: func(output interface{}) error {
getObjOut := output.(*GetObjectOutput)
reader, ok := getObjOut.Body.(*md5ValidationReader)
if !ok {
return fmt.Errorf("expect %T updated body reader, got %T",
(*md5ValidationReader)(nil), getObjOut.Body)
}
if reader.rawReader == nil {
return fmt.Errorf("expect rawReader not to be nil")
}
if reader.payload == nil {
return fmt.Errorf("expect payload not to be nil")
}
if e, a := int64(len(bodyWithSum)-md5.Size), reader.payloadLen; e != a {
return fmt.Errorf("expect %v payload len, got %v", e, a)
}
if reader.hash == nil {
return fmt.Errorf("expect hash not to be nil")
}
return nil
},
},
{
// Positive: Use Validation reader, empty object
Req: &request.Request{
HTTPResponse: &http.Response{
Header: func() http.Header {
h := http.Header{}
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
return h
}(),
},
Data: &GetObjectOutput{
Body: ioutil.NopCloser(bytes.NewReader(emptyBodySum[:])),
ContentLength: aws.Int64(int64(len(emptyBodySum[:]))),
},
},
Validate: func(output interface{}) error {
getObjOut := output.(*GetObjectOutput)
reader, ok := getObjOut.Body.(*md5ValidationReader)
if !ok {
return fmt.Errorf("expect %T updated body reader, got %T",
(*md5ValidationReader)(nil), getObjOut.Body)
}
if reader.rawReader == nil {
return fmt.Errorf("expect rawReader not to be nil")
}
if reader.payload == nil {
return fmt.Errorf("expect payload not to be nil")
}
if e, a := int64(len(emptyBodySum)-md5.Size), reader.payloadLen; e != a {
return fmt.Errorf("expect %v payload len, got %v", e, a)
}
if reader.hash == nil {
return fmt.Errorf("expect hash not to be nil")
}
return nil
},
},
{
// Negative: amzTxEncoding header not set
Req: &request.Request{
HTTPResponse: &http.Response{
Header: http.Header{},
},
Data: &GetObjectOutput{
Body: ioutil.NopCloser(bytes.NewReader(body)),
ContentLength: aws.Int64(int64(len(body))),
},
},
Validate: func(output interface{}) error {
getObjOut := output.(*GetObjectOutput)
reader, ok := getObjOut.Body.(*md5ValidationReader)
if ok {
return fmt.Errorf("expect body reader not to be %T",
reader)
}
return nil
},
},
{
// Negative: Not GetObjectOutput type.
Req: &request.Request{
Operation: &request.Operation{
Name: "PutObject",
},
HTTPResponse: &http.Response{
Header: func() http.Header {
h := http.Header{}
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
return h
}(),
},
Data: &PutObjectOutput{},
},
Error: "header received on unsupported API",
Validate: func(output interface{}) error {
_, ok := output.(*PutObjectOutput)
if !ok {
return fmt.Errorf("expect %T output not to change, got %T",
(*PutObjectOutput)(nil), output)
}
return nil
},
},
{
// Negative: invalid content length.
Req: &request.Request{
HTTPResponse: &http.Response{
Header: func() http.Header {
h := http.Header{}
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
return h
}(),
},
Data: &GetObjectOutput{
Body: ioutil.NopCloser(bytes.NewReader(bodyWithSum)),
ContentLength: aws.Int64(-1),
},
},
Error: "invalid Content-Length -1",
Validate: func(output interface{}) error {
getObjOut := output.(*GetObjectOutput)
reader, ok := getObjOut.Body.(*md5ValidationReader)
if ok {
return fmt.Errorf("expect body reader not to be %T",
reader)
}
return nil
},
},
{
// Negative: invalid content length, < md5.Size.
Req: &request.Request{
HTTPResponse: &http.Response{
Header: func() http.Header {
h := http.Header{}
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
return h
}(),
},
Data: &GetObjectOutput{
Body: ioutil.NopCloser(bytes.NewReader(make([]byte, 5))),
ContentLength: aws.Int64(5),
},
},
Error: "invalid Content-Length 5",
Validate: func(output interface{}) error {
getObjOut := output.(*GetObjectOutput)
reader, ok := getObjOut.Body.(*md5ValidationReader)
if ok {
return fmt.Errorf("expect body reader not to be %T",
reader)
}
return nil
},
},
}
for i, c := range cases {
useMD5ValidationReader(c.Req)
if len(c.Error) != 0 {
if c.Req.Error == nil {
t.Fatalf("%d, expect error, got none", i)
}
if e, a := c.Error, c.Req.Error.Error(); !strings.Contains(a, e) {
t.Errorf("%d, expect %v error to be in %v", i, e, a)
}
} else if c.Req.Error != nil {
t.Errorf("%d, expect no error, got %v", i, c.Req.Error)
}
if c.Validate != nil {
if err := c.Validate(c.Req.Data); err != nil {
t.Errorf("%d, expect Data to validate, got %v", i, err)
}
}
}
}
func TestReaderMD5Validation(t *testing.T) {
body := []byte("create a really cool md5 checksum of me")
bodySum := md5.Sum(body)
bodyWithSum := append(body, bodySum[:]...)
emptyBodySum := md5.Sum([]byte{})
badBodySum := append(body, emptyBodySum[:]...)
cases := []struct {
Content []byte
ContentReader io.ReadCloser
PayloadLen int64
Error string
}{
{
Content: bodyWithSum,
PayloadLen: int64(len(body)),
},
{
Content: emptyBodySum[:],
PayloadLen: 0,
},
{
Content: badBodySum,
PayloadLen: int64(len(body)),
Error: "expected MD5 checksum",
},
{
Content: emptyBodySum[:len(emptyBodySum)-2],
PayloadLen: 0,
Error: "unexpected EOF",
},
{
Content: body,
PayloadLen: int64(len(body) * 2),
Error: "unexpected EOF",
},
{
ContentReader: ioutil.NopCloser(errorReader{}),
PayloadLen: int64(len(body)),
Error: "errorReader error",
},
}
for i, c := range cases {
reader := c.ContentReader
if reader == nil {
reader = ioutil.NopCloser(bytes.NewReader(c.Content))
}
v := newMD5ValidationReader(reader, c.PayloadLen)
var actual bytes.Buffer
n, err := io.Copy(&actual, v)
if len(c.Error) != 0 {
if err == nil {
t.Errorf("%d, expect error, got none", i)
}
if e, a := c.Error, err.Error(); !strings.Contains(a, e) {
t.Errorf("%d, expect %v error to be in %v", i, e, a)
}
continue
} else if err != nil {
t.Errorf("%d, expect no error, got %v", i, err)
continue
}
if e, a := c.PayloadLen, n; e != a {
t.Errorf("%d, expect %v len, got %v", i, e, a)
}
if e, a := c.Content[:c.PayloadLen], actual.Bytes(); !bytes.Equal(e, a) {
t.Errorf("%d, expect:\n%v\nactual:\n%v", i, e, a)
}
}
}
-141
View File
@@ -1,141 +0,0 @@
package s3_test
import (
"bytes"
"io/ioutil"
"net/http"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
var s3LocationTests = []struct {
body string
loc string
}{
{`<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`, ``},
{`<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">EU</LocationConstraint>`, `EU`},
}
func TestGetBucketLocation(t *testing.T) {
for _, test := range s3LocationTests {
s := s3.New(unit.Session)
s.Handlers.Send.Clear()
s.Handlers.Send.PushBack(func(r *request.Request) {
reader := ioutil.NopCloser(bytes.NewReader([]byte(test.body)))
r.HTTPResponse = &http.Response{StatusCode: 200, Body: reader}
})
resp, err := s.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: aws.String("bucket")})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if test.loc == "" {
if v := resp.LocationConstraint; v != nil {
t.Errorf("expect location constraint to be nil, got %s", *v)
}
} else {
if e, a := test.loc, *resp.LocationConstraint; e != a {
t.Errorf("expect %s location constraint, got %v", e, a)
}
}
}
}
func TestNormalizeBucketLocation(t *testing.T) {
cases := []struct {
In, Out string
}{
{"", "us-east-1"},
{"EU", "eu-west-1"},
{"us-east-1", "us-east-1"},
{"something", "something"},
}
for i, c := range cases {
actual := s3.NormalizeBucketLocation(c.In)
if e, a := c.Out, actual; e != a {
t.Errorf("%d, expect %s bucket location, got %s", i, e, a)
}
}
}
func TestWithNormalizeBucketLocation(t *testing.T) {
req := &request.Request{}
req.ApplyOptions(s3.WithNormalizeBucketLocation)
cases := []struct {
In, Out string
}{
{"", "us-east-1"},
{"EU", "eu-west-1"},
{"us-east-1", "us-east-1"},
{"something", "something"},
}
for i, c := range cases {
req.Data = &s3.GetBucketLocationOutput{
LocationConstraint: aws.String(c.In),
}
req.Handlers.Unmarshal.Run(req)
v := req.Data.(*s3.GetBucketLocationOutput).LocationConstraint
if e, a := c.Out, aws.StringValue(v); e != a {
t.Errorf("%d, expect %s bucket location, got %s", i, e, a)
}
}
}
func TestPopulateLocationConstraint(t *testing.T) {
s := s3.New(unit.Session)
in := &s3.CreateBucketInput{
Bucket: aws.String("bucket"),
}
req, _ := s.CreateBucketRequest(in)
if err := req.Build(); err != nil {
t.Fatalf("expect no error, got %v", err)
}
v, _ := awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")
if e, a := "mock-region", *(v[0].(*string)); e != a {
t.Errorf("expect %s location constraint, got %s", e, a)
}
if v := in.CreateBucketConfiguration; v != nil {
// don't modify original params
t.Errorf("expect create bucket Configuration to be nil, got %s", *v)
}
}
func TestNoPopulateLocationConstraintIfProvided(t *testing.T) {
s := s3.New(unit.Session)
req, _ := s.CreateBucketRequest(&s3.CreateBucketInput{
Bucket: aws.String("bucket"),
CreateBucketConfiguration: &s3.CreateBucketConfiguration{},
})
if err := req.Build(); err != nil {
t.Fatalf("expect no error, got %v", err)
}
v, _ := awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")
if l := len(v); l != 0 {
t.Errorf("expect no values, got %d", l)
}
}
func TestNoPopulateLocationConstraintIfClassic(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{Region: aws.String("us-east-1")})
req, _ := s.CreateBucketRequest(&s3.CreateBucketInput{
Bucket: aws.String("bucket"),
})
if err := req.Build(); err != nil {
t.Fatalf("expect no error, got %v", err)
}
v, _ := awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")
if l := len(v); l != 0 {
t.Errorf("expect no values, got %d", l)
}
}
-171
View File
@@ -1,171 +0,0 @@
package s3_test
import (
"crypto/md5"
"encoding/base64"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
func assertMD5(t *testing.T, req *request.Request) {
err := req.Build()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
b, _ := ioutil.ReadAll(req.HTTPRequest.Body)
out := md5.Sum(b)
if len(b) == 0 {
t.Error("expected non-empty value")
}
if e, a := base64.StdEncoding.EncodeToString(out[:]), req.HTTPRequest.Header.Get("Content-MD5"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
func TestMD5InPutBucketCors(t *testing.T) {
svc := s3.New(unit.Session)
req, _ := svc.PutBucketCorsRequest(&s3.PutBucketCorsInput{
Bucket: aws.String("bucketname"),
CORSConfiguration: &s3.CORSConfiguration{
CORSRules: []*s3.CORSRule{
{
AllowedMethods: []*string{aws.String("GET")},
AllowedOrigins: []*string{aws.String("*")},
},
},
},
})
assertMD5(t, req)
}
func TestMD5InPutBucketLifecycle(t *testing.T) {
svc := s3.New(unit.Session)
req, _ := svc.PutBucketLifecycleRequest(&s3.PutBucketLifecycleInput{
Bucket: aws.String("bucketname"),
LifecycleConfiguration: &s3.LifecycleConfiguration{
Rules: []*s3.Rule{
{
ID: aws.String("ID"),
Prefix: aws.String("Prefix"),
Status: aws.String("Enabled"),
},
},
},
})
assertMD5(t, req)
}
func TestMD5InPutBucketPolicy(t *testing.T) {
svc := s3.New(unit.Session)
req, _ := svc.PutBucketPolicyRequest(&s3.PutBucketPolicyInput{
Bucket: aws.String("bucketname"),
Policy: aws.String("{}"),
})
assertMD5(t, req)
}
func TestMD5InPutBucketTagging(t *testing.T) {
svc := s3.New(unit.Session)
req, _ := svc.PutBucketTaggingRequest(&s3.PutBucketTaggingInput{
Bucket: aws.String("bucketname"),
Tagging: &s3.Tagging{
TagSet: []*s3.Tag{
{Key: aws.String("KEY"), Value: aws.String("VALUE")},
},
},
})
assertMD5(t, req)
}
func TestMD5InDeleteObjects(t *testing.T) {
svc := s3.New(unit.Session)
req, _ := svc.DeleteObjectsRequest(&s3.DeleteObjectsInput{
Bucket: aws.String("bucketname"),
Delete: &s3.Delete{
Objects: []*s3.ObjectIdentifier{
{Key: aws.String("key")},
},
},
})
assertMD5(t, req)
}
func TestMD5InPutBucketLifecycleConfiguration(t *testing.T) {
svc := s3.New(unit.Session)
req, _ := svc.PutBucketLifecycleConfigurationRequest(&s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String("bucketname"),
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: []*s3.LifecycleRule{
{Prefix: aws.String("prefix"), Status: aws.String(s3.ExpirationStatusEnabled)},
},
},
})
assertMD5(t, req)
}
const (
metaKeyPrefix = `X-Amz-Meta-`
utf8KeySuffix = `My-Info`
utf8Value = "hello-世界\u0444"
)
func TestPutObjectMetadataWithUnicode(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if e, a := utf8Value, r.Header.Get(metaKeyPrefix+utf8KeySuffix); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}))
svc := s3.New(unit.Session, &aws.Config{
Endpoint: aws.String(server.URL),
DisableSSL: aws.Bool(true),
})
_, err := svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String("my_bucket"),
Key: aws.String("my_key"),
Body: strings.NewReader(""),
Metadata: func() map[string]*string {
v := map[string]*string{}
v[utf8KeySuffix] = aws.String(utf8Value)
return v
}(),
})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
}
func TestGetObjectMetadataWithUnicode(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set(metaKeyPrefix+utf8KeySuffix, utf8Value)
}))
svc := s3.New(unit.Session, &aws.Config{
Endpoint: aws.String(server.URL),
DisableSSL: aws.Bool(true),
})
resp, err := svc.GetObject(&s3.GetObjectInput{
Bucket: aws.String("my_bucket"),
Key: aws.String("my_key"),
})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
resp.Body.Close()
if e, a := utf8Value, *resp.Metadata[utf8KeySuffix]; e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
-297
View File
@@ -1,297 +0,0 @@
package s3
import (
"bytes"
"encoding/csv"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"reflect"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/corehandlers"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi"
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamtest"
"github.com/aws/aws-sdk-go/private/protocol/restxml"
)
func mockSelectObjectContentEventStreamsOutputEvents() ([]SelectObjectContentEventStreamEvent, []eventstream.Message) {
expectEvents := []SelectObjectContentEventStreamEvent{
&ContinuationEvent{},
&EndEvent{},
&ProgressEvent{
Details: &Progress{
BytesProcessed: aws.Int64(123),
BytesReturned: aws.Int64(456),
BytesScanned: aws.Int64(789),
},
},
&RecordsEvent{
Payload: []byte("abc123"),
},
&StatsEvent{
Details: &Stats{
BytesProcessed: aws.Int64(123),
BytesReturned: aws.Int64(456),
BytesScanned: aws.Int64(789),
},
},
}
var marshalers request.HandlerList
marshalers.PushBackNamed(restxml.BuildHandler)
payloadMarshaler := protocol.HandlerPayloadMarshal{
Marshalers: marshalers,
}
eventMsgs := []eventstream.Message{
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Cont"),
},
},
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("End"),
},
},
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Progress"),
},
},
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[2]),
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Records"),
},
},
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[3]),
},
{
Headers: eventstream.Headers{
eventstreamtest.EventMessageTypeHeader,
{
Name: eventstreamapi.EventTypeHeader,
Value: eventstream.StringValue("Stats"),
},
},
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[4]),
},
}
return expectEvents, eventMsgs
}
func TestSelectObjectContentEventStream(t *testing.T) {
expectEvents, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
sess, cleanupFn, err := eventstreamtest.SetupEventStreamSession(t,
eventstreamtest.ServeEventStream{
T: t,
Events: eventMsgs,
},
false,
)
if err != nil {
t.Fatalf("expect no error, %v", err)
}
defer cleanupFn()
svc := New(sess)
params := &SelectObjectContentInput{}
resp, err := svc.SelectObjectContent(params)
if err != nil {
t.Fatalf("expect no error got, %v", err)
}
defer resp.EventStream.Close()
var i int
for event := range resp.EventStream.Events() {
if event == nil {
t.Errorf("%d, expect event, got nil", i)
}
if e, a := expectEvents[i], event; !reflect.DeepEqual(e, a) {
t.Errorf("%d, expect %T %v, got %T %v", i, e, e, a, a)
}
i++
}
if err := resp.EventStream.Err(); err != nil {
t.Errorf("expect no error, %v", err)
}
}
func TestSelectObjectContentEventStream_Close(t *testing.T) {
_, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
sess, cleanupFn, err := eventstreamtest.SetupEventStreamSession(t,
eventstreamtest.ServeEventStream{
T: t,
Events: eventMsgs,
},
false,
)
if err != nil {
t.Fatalf("expect no error, %v", err)
}
defer cleanupFn()
svc := New(sess)
params := &SelectObjectContentInput{}
resp, err := svc.SelectObjectContent(params)
if err != nil {
t.Fatalf("expect no error got, %v", err)
}
resp.EventStream.Close()
if err := resp.EventStream.Err(); err != nil {
t.Errorf("expect no error, %v", err)
}
}
func ExampleSelectObjectContentEventStream() {
sess := session.Must(session.NewSession())
svc := New(sess)
/*
Example myObjectKey CSV content:
name,number
gopher,0
ᵷodɥǝɹ,1
*/
// Make the Select Object Content API request using the object uploaded.
resp, err := svc.SelectObjectContent(&SelectObjectContentInput{
Bucket: aws.String("myBucket"),
Key: aws.String("myObjectKey"),
Expression: aws.String("SELECT name FROM S3Object WHERE cast(number as int) < 1"),
ExpressionType: aws.String(ExpressionTypeSql),
InputSerialization: &InputSerialization{
CSV: &CSVInput{
FileHeaderInfo: aws.String(FileHeaderInfoUse),
},
},
OutputSerialization: &OutputSerialization{
CSV: &CSVOutput{},
},
})
if err != nil {
fmt.Fprintf(os.Stderr, "failed making API request, %v\n", err)
return
}
defer resp.EventStream.Close()
results, resultWriter := io.Pipe()
go func() {
defer resultWriter.Close()
for event := range resp.EventStream.Events() {
switch e := event.(type) {
case *RecordsEvent:
resultWriter.Write(e.Payload)
case *StatsEvent:
fmt.Printf("Processed %d bytes\n", e.Details.BytesProcessed)
}
}
}()
// Printout the results
resReader := csv.NewReader(results)
for {
record, err := resReader.Read()
if err == io.EOF {
break
}
fmt.Println(record)
}
if err := resp.EventStream.Err(); err != nil {
fmt.Fprintf(os.Stderr, "reading from event stream failed, %v\n", err)
}
}
func BenchmarkSelectObjectContentEventStream(b *testing.B) {
_, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
var buf bytes.Buffer
encoder := eventstream.NewEncoder(&buf)
for _, msg := range eventMsgs {
if err := encoder.Encode(msg); err != nil {
b.Fatalf("failed to encode message, %v", err)
}
}
stream := &loopReader{source: bytes.NewReader(buf.Bytes())}
sess := unit.Session
svc := New(sess, &aws.Config{
Endpoint: aws.String("https://example.com"),
DisableParamValidation: aws.Bool(true),
})
svc.Handlers.Send.Swap(corehandlers.SendHandler.Name,
request.NamedHandler{Name: "mockSend",
Fn: func(r *request.Request) {
r.HTTPResponse = &http.Response{
Status: "200 OK",
StatusCode: 200,
Header: http.Header{},
Body: ioutil.NopCloser(stream),
}
},
},
)
params := &SelectObjectContentInput{}
resp, err := svc.SelectObjectContent(params)
if err != nil {
b.Fatalf("failed to create request, %v", err)
}
defer resp.EventStream.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err = resp.EventStream.Err(); err != nil {
b.Fatalf("expect no error, got %v", err)
}
event := <-resp.EventStream.Events()
if event == nil {
b.Fatalf("expect event, got nil, %v, %d", resp.EventStream.Err(), i)
}
}
}
type loopReader struct {
source *bytes.Reader
}
func (c *loopReader) Read(p []byte) (int, error) {
if c.source.Len() == 0 {
c.source.Seek(0, 0)
}
return c.source.Read(p)
}
File diff suppressed because it is too large Load Diff
-179
View File
@@ -1,179 +0,0 @@
package s3_test
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
type s3BucketTest struct {
bucket string
url string
errCode string
}
var (
sslTests = []s3BucketTest{
{"abc", "https://abc.s3.mock-region.amazonaws.com/", ""},
{"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""},
{"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""},
{"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""},
}
nosslTests = []s3BucketTest{
{"a.b.c", "http://a.b.c.s3.mock-region.amazonaws.com/", ""},
{"a..bc", "http://s3.mock-region.amazonaws.com/a..bc", ""},
}
forcepathTests = []s3BucketTest{
{"abc", "https://s3.mock-region.amazonaws.com/abc", ""},
{"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""},
{"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""},
{"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""},
}
accelerateTests = []s3BucketTest{
{"abc", "https://abc.s3-accelerate.amazonaws.com/", ""},
{"a.b.c", "https://s3.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
{"a$b$c", "https://s3.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
}
accelerateNoSSLTests = []s3BucketTest{
{"abc", "http://abc.s3-accelerate.amazonaws.com/", ""},
{"a.b.c", "http://a.b.c.s3-accelerate.amazonaws.com/", ""},
{"a$b$c", "http://s3.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
}
accelerateDualstack = []s3BucketTest{
{"abc", "https://abc.s3-accelerate.dualstack.amazonaws.com/", ""},
{"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
{"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
}
)
func runTests(t *testing.T, svc *s3.S3, tests []s3BucketTest) {
for i, test := range tests {
req, _ := svc.ListObjectsRequest(&s3.ListObjectsInput{Bucket: &test.bucket})
req.Build()
if e, a := test.url, req.HTTPRequest.URL.String(); e != a {
t.Errorf("%d, expect url %s, got %s", i, e, a)
}
if test.errCode != "" {
if err := req.Error; err == nil {
t.Fatalf("%d, expect no error", i)
}
if a, e := req.Error.(awserr.Error).Code(), test.errCode; !strings.Contains(a, e) {
t.Errorf("%d, expect error code to contain %q, got %q", i, e, a)
}
}
}
}
func TestAccelerateBucketBuild(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{S3UseAccelerate: aws.Bool(true)})
runTests(t, s, accelerateTests)
}
func TestAccelerateNoSSLBucketBuild(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{S3UseAccelerate: aws.Bool(true), DisableSSL: aws.Bool(true)})
runTests(t, s, accelerateNoSSLTests)
}
func TestAccelerateDualstackBucketBuild(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{
S3UseAccelerate: aws.Bool(true),
UseDualStack: aws.Bool(true),
})
runTests(t, s, accelerateDualstack)
}
func TestHostStyleBucketBuild(t *testing.T) {
s := s3.New(unit.Session)
runTests(t, s, sslTests)
}
func TestHostStyleBucketBuildNoSSL(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{DisableSSL: aws.Bool(true)})
runTests(t, s, nosslTests)
}
func TestPathStyleBucketBuild(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{S3ForcePathStyle: aws.Bool(true)})
runTests(t, s, forcepathTests)
}
func TestHostStyleBucketGetBucketLocation(t *testing.T) {
s := s3.New(unit.Session)
req, _ := s.GetBucketLocationRequest(&s3.GetBucketLocationInput{
Bucket: aws.String("bucket"),
})
req.Build()
if req.Error != nil {
t.Fatalf("expect no error, got %v", req.Error)
}
u, _ := url.Parse(req.HTTPRequest.URL.String())
if e, a := "bucket", u.Host; strings.Contains(a, e) {
t.Errorf("expect %s to not be in %s", e, a)
}
if e, a := "bucket", u.Path; !strings.Contains(a, e) {
t.Errorf("expect %s to be in %s", e, a)
}
}
func TestVirtualHostStyleSuite(t *testing.T) {
f, err := os.Open(filepath.Join("testdata", "virtual_host.json"))
if err != nil {
t.Fatalf("expect no error, %v", err)
}
cases := []struct {
Bucket string
Region string
UseDualStack bool
UseS3Accelerate bool
ConfiguredAddressingStyle string
ExpectedURI string
}{}
decoder := json.NewDecoder(f)
if err := decoder.Decode(&cases); err != nil {
t.Fatalf("expect no error, %v", err)
}
const testPathStyle = "path"
for i, c := range cases {
svc := s3.New(unit.Session, &aws.Config{
Region: &c.Region,
UseDualStack: &c.UseDualStack,
S3UseAccelerate: &c.UseS3Accelerate,
S3ForcePathStyle: aws.Bool(c.ConfiguredAddressingStyle == testPathStyle),
})
req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{
Bucket: &c.Bucket,
})
req.Build()
if req.Error != nil {
t.Fatalf("expect no error, got %v", req.Error)
}
// Trim trailing '/' that are added by the SDK but not in the tests.
actualURI := strings.TrimRightFunc(
req.HTTPRequest.URL.String(),
func(r rune) bool { return r == '/' },
)
if e, a := c.ExpectedURI, actualURI; e != a {
t.Errorf("%d, expect\n%s\nurl to be\n%s", i, e, a)
}
}
}
@@ -1,83 +0,0 @@
// +build go1.6
package s3_test
import (
"bytes"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
func TestAdd100Continue_Added(t *testing.T) {
svc := s3.New(unit.Session)
r, _ := svc.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("dest"),
Body: bytes.NewReader(make([]byte, 1024*1024*5)),
})
err := r.Sign()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := "100-Continue", r.HTTPRequest.Header.Get("Expect"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
func TestAdd100Continue_SkipDisabled(t *testing.T) {
svc := s3.New(unit.Session, aws.NewConfig().WithS3Disable100Continue(true))
r, _ := svc.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("dest"),
Body: bytes.NewReader(make([]byte, 1024*1024*5)),
})
err := r.Sign()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if r.HTTPRequest.Header.Get("Expect") != "" {
t.Errorf("expected empty value, but received %s", r.HTTPRequest.Header.Get("Expect"))
}
}
func TestAdd100Continue_SkipNonPUT(t *testing.T) {
svc := s3.New(unit.Session)
r, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("dest"),
})
err := r.Sign()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if r.HTTPRequest.Header.Get("Expect") != "" {
t.Errorf("expected empty value, but received %s", r.HTTPRequest.Header.Get("Expect"))
}
}
func TestAdd100Continue_SkipTooSmall(t *testing.T) {
svc := s3.New(unit.Session)
r, _ := svc.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("dest"),
Body: bytes.NewReader(make([]byte, 1024*1024*1)),
})
err := r.Sign()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if r.HTTPRequest.Header.Get("Expect") != "" {
t.Errorf("expected empty value, but received %s", r.HTTPRequest.Header.Get("Expect"))
}
}
-190
View File
@@ -1,190 +0,0 @@
package s3crypto
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"io"
)
// AESCBC is a symmetric crypto algorithm. This algorithm
// requires a padder due to CBC needing to be of the same block
// size. AES CBC is vulnerable to Padding Oracle attacks and
// so should be avoided when possible.
type aesCBC struct {
encrypter cipher.BlockMode
decrypter cipher.BlockMode
padder Padder
}
// newAESCBC creates a new AES CBC cipher. Expects keys to be of
// the correct size.
func newAESCBC(cd CipherData, padder Padder) (Cipher, error) {
block, err := aes.NewCipher(cd.Key)
if err != nil {
return nil, err
}
encrypter := cipher.NewCBCEncrypter(block, cd.IV)
decrypter := cipher.NewCBCDecrypter(block, cd.IV)
return &aesCBC{encrypter, decrypter, padder}, nil
}
// Encrypt will encrypt the data using AES CBC by returning
// an io.Reader. The io.Reader will encrypt the data as Read
// is called.
func (c *aesCBC) Encrypt(src io.Reader) io.Reader {
reader := &cbcEncryptReader{
encrypter: c.encrypter,
src: src,
padder: c.padder,
}
return reader
}
type cbcEncryptReader struct {
encrypter cipher.BlockMode
src io.Reader
padder Padder
size int
buf bytes.Buffer
}
// Read will read from our io.Reader and encrypt the data as necessary.
// Due to padding, we have to do some logic that when we encounter an
// end of file to pad properly.
func (reader *cbcEncryptReader) Read(data []byte) (int, error) {
n, err := reader.src.Read(data)
reader.size += n
blockSize := reader.encrypter.BlockSize()
reader.buf.Write(data[:n])
if err == io.EOF {
b := make([]byte, getSliceSize(blockSize, reader.buf.Len(), len(data)))
n, err = reader.buf.Read(b)
if err != nil && err != io.EOF {
return n, err
}
// The buffer is now empty, we can now pad the data
if reader.buf.Len() == 0 {
b, err = reader.padder.Pad(b[:n], reader.size)
if err != nil {
return n, err
}
n = len(b)
err = io.EOF
}
// We only want to encrypt if we have read anything
if n > 0 {
reader.encrypter.CryptBlocks(data, b)
}
return n, err
}
if err != nil {
return n, err
}
if size := reader.buf.Len(); size >= blockSize {
nBlocks := size / blockSize
if size > len(data) {
nBlocks = len(data) / blockSize
}
if nBlocks > 0 {
b := make([]byte, nBlocks*blockSize)
n, _ = reader.buf.Read(b)
reader.encrypter.CryptBlocks(data, b[:n])
}
} else {
n = 0
}
return n, nil
}
// Decrypt will decrypt the data using AES CBC
func (c *aesCBC) Decrypt(src io.Reader) io.Reader {
return &cbcDecryptReader{
decrypter: c.decrypter,
src: src,
padder: c.padder,
}
}
type cbcDecryptReader struct {
decrypter cipher.BlockMode
src io.Reader
padder Padder
buf bytes.Buffer
}
// Read will read from our io.Reader and decrypt the data as necessary.
// Due to padding, we have to do some logic that when we encounter an
// end of file to pad properly.
func (reader *cbcDecryptReader) Read(data []byte) (int, error) {
n, err := reader.src.Read(data)
blockSize := reader.decrypter.BlockSize()
reader.buf.Write(data[:n])
if err == io.EOF {
b := make([]byte, getSliceSize(blockSize, reader.buf.Len(), len(data)))
n, err = reader.buf.Read(b)
if err != nil && err != io.EOF {
return n, err
}
// We only want to decrypt if we have read anything
if n > 0 {
reader.decrypter.CryptBlocks(data, b)
}
if reader.buf.Len() == 0 {
b, err = reader.padder.Unpad(data[:n])
n = len(b)
if err != nil {
return n, err
}
err = io.EOF
}
return n, err
}
if err != nil {
return n, err
}
if size := reader.buf.Len(); size >= blockSize {
nBlocks := size / blockSize
if size > len(data) {
nBlocks = len(data) / blockSize
}
// The last block is always padded. This will allow us to unpad
// when we receive an io.EOF error
nBlocks -= blockSize
if nBlocks > 0 {
b := make([]byte, nBlocks*blockSize)
n, _ = reader.buf.Read(b)
reader.decrypter.CryptBlocks(data, b[:n])
} else {
n = 0
}
}
return n, nil
}
// getSliceSize will return the correct amount of bytes we need to
// read with regards to padding.
func getSliceSize(blockSize, bufSize, dataSize int) int {
size := bufSize
if bufSize > dataSize {
size = dataSize
}
size = size - (size % blockSize) - blockSize
if size <= 0 {
size = blockSize
}
return size
}
@@ -1,73 +0,0 @@
package s3crypto
import (
"io"
"strings"
)
const (
cbcKeySize = 32
cbcNonceSize = 16
)
type cbcContentCipherBuilder struct {
generator CipherDataGenerator
padder Padder
}
// AESCBCContentCipherBuilder returns a new encryption only mode structure with a specific cipher
// for the master key
func AESCBCContentCipherBuilder(generator CipherDataGenerator, padder Padder) ContentCipherBuilder {
return cbcContentCipherBuilder{generator: generator, padder: padder}
}
func (builder cbcContentCipherBuilder) ContentCipher() (ContentCipher, error) {
cd, err := builder.generator.GenerateCipherData(cbcKeySize, cbcNonceSize)
if err != nil {
return nil, err
}
cd.Padder = builder.padder
return newAESCBCContentCipher(cd)
}
// newAESCBCContentCipher will create a new aes cbc content cipher. If the cipher data's
// will set the CEK algorithm if it hasn't been set.
func newAESCBCContentCipher(cd CipherData) (ContentCipher, error) {
if len(cd.CEKAlgorithm) == 0 {
cd.CEKAlgorithm = strings.Join([]string{AESCBC, cd.Padder.Name()}, "/")
}
cipher, err := newAESCBC(cd, cd.Padder)
if err != nil {
return nil, err
}
return &aesCBCContentCipher{
CipherData: cd,
Cipher: cipher,
}, nil
}
// aesCBCContentCipher will use AES CBC for the main cipher.
type aesCBCContentCipher struct {
CipherData CipherData
Cipher Cipher
}
// EncryptContents will generate a random key and iv and encrypt the data using cbc
func (cc *aesCBCContentCipher) EncryptContents(src io.Reader) (io.Reader, error) {
return cc.Cipher.Encrypt(src), nil
}
// DecryptContents will use the symmetric key provider to instantiate a new CBC cipher.
// We grab a decrypt reader from CBC and wrap it in a CryptoReadCloser. The only error
// expected here is when the key or iv is of invalid length.
func (cc *aesCBCContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser, error) {
reader := cc.Cipher.Decrypt(src)
return &CryptoReadCloser{Body: src, Decrypter: reader}, nil
}
// GetCipherData returns cipher data
func (cc aesCBCContentCipher) GetCipherData() CipherData {
return cc.CipherData
}
@@ -1,20 +0,0 @@
package s3crypto_test
import (
"testing"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
func TestAESCBCBuilder(t *testing.T) {
generator := mockGenerator{}
builder := s3crypto.AESCBCContentCipherBuilder(generator, s3crypto.NoPadder)
if builder == nil {
t.Fatal(builder)
}
_, err := builder.ContentCipher()
if err != nil {
t.Fatal(err)
}
}
-29
View File
@@ -1,29 +0,0 @@
package s3crypto
const (
pkcs5BlockSize = 16
)
var aescbcPadding = aescbcPadder{pkcs7Padder{16}}
// AESCBCPadder is used to pad AES encrypted and decrypted data.
// Altough it uses the pkcs5Padder, it isn't following the RFC
// for PKCS5. The only reason why it is called pkcs5Padder is
// due to the Name returning PKCS5Padding.
var AESCBCPadder = Padder(aescbcPadding)
type aescbcPadder struct {
padder pkcs7Padder
}
func (padder aescbcPadder) Pad(b []byte, n int) ([]byte, error) {
return padder.padder.Pad(b, n)
}
func (padder aescbcPadder) Unpad(b []byte) ([]byte, error) {
return padder.padder.Unpad(b)
}
func (padder aescbcPadder) Name() string {
return "PKCS5Padding"
}
@@ -1,41 +0,0 @@
package s3crypto
import (
"bytes"
"fmt"
"testing"
)
func TestAESCBCPadding(t *testing.T) {
for i := 0; i < 16; i++ {
input := make([]byte, i)
expected := append(input, bytes.Repeat([]byte{byte(16 - i)}, 16-i)...)
b, err := AESCBCPadder.Pad(input, len(input))
if err != nil {
t.Fatal("Expected error to be nil but received " + err.Error())
}
if len(b) != len(expected) {
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
}
if bytes.Compare(b, expected) != 0 {
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
}
}
}
func TestAESCBCUnpadding(t *testing.T) {
for i := 0; i < 16; i++ {
expected := make([]byte, i)
input := append(expected, bytes.Repeat([]byte{byte(16 - i)}, 16-i)...)
b, err := AESCBCPadder.Unpad(input)
if err != nil {
t.Fatal("Error received, was expecting nil: " + err.Error())
}
if len(b) != len(expected) {
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
}
if bytes.Compare(b, expected) != 0 {
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
}
}
}
-501
View File
@@ -1,501 +0,0 @@
package s3crypto
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"testing"
)
func TestAESCBCEncryptDecrypt(t *testing.T) {
var testCases = []struct {
key string
iv string
plaintext string
ciphertext string
decodeHex bool
padder Padder
}{
// Test vectors from RFC 3602: https://tools.ietf.org/html/rfc3602
{
"06a9214036b8a15b512e03d534120006",
"3dafba429d9eb430b422da802c9fac41",
"Single block msg",
"e353779c1079aeb82708942dbe77181a",
false,
NoPadder,
},
{
"c286696d887c9aa0611bbb3e2025a45a",
"562e17996d093d28ddb3ba695a2e6f58",
"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f",
"d296cd94c2cccf8a3a863028b5e1dc0a7586602d253cfff91b8266bea6d61ab1",
true,
NoPadder,
},
{
"6c3ea0477630ce21a2ce334aa746c2cd",
"c782dc4c098c66cbd9cd27d825682c81",
"This is a 48-byte message (exactly 3 AES blocks)",
"d0a02b3836451753d493665d33f0e8862dea54cdb293abc7506939276772f8d5021c19216bad525c8579695d83ba2684",
false,
NoPadder,
},
{
"56e47a38c5598974bc46903dba290349",
"8ce82eefbea0da3c44699ed7db51b7d9",
"a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedf",
"c30e32ffedc0774e6aff6af0869f71aa0f3af07a9a31a9c684db207eb0ef8e4e35907aa632c3ffdf868bb7b29d3d46ad83ce9f9a102ee99d49a53e87f4c3da55",
true,
NoPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"",
"B012949BA07D1A6DCE9DEE67274D41AB",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41",
"8A11ABA68A566132FFE04DB336621D41",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141",
"97D0896E41DFDB5CEA4A9EB70A938CFD",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141",
"8464EAD45FA2D8790E8741E32C28083F",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141",
"1E656D6E2745BA9F154FAF136B2BC73D",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141",
"0B6031C4B230DAC6BD6D3F195645B287",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141",
"5D09FEB6462BB489489A7E18FD341D9D",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141",
"85745E398F2FD1050C2CE8F8614DA369",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141",
"7BE52933970BA7B0FC6FB3FC37648205",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141",
"ED3A1E134EF36CCFE60C8123B4272F89",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141",
"C3B7C9E177E1052FC736F65FC1E74209",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141",
"C3A8B53F7F57F0B9D346FA99810A3C28",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141",
"D16B1ECE5BF00AF919E139E99775FF06",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141",
"B258F4DF57FFCA1EFCF8D76140F05139",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141",
"3CD2282DE24A2CF9E23326CC3DC9077A",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141",
"3010232E7C752A3B4C9EE428B4C4FE88",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A22BC4E6D03BFD2418DD412D1ED1B31AF",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A5427BBD4A4D50776989441370E3B5B16",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A7FF985F55567D1B25EA40E23BB4CB1FE",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A0835E548C7370D8F8D9925C0E6B54727",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ADC0CF1436399E67BC1122B31CB596649",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A3D096F0DEAFF91938B82E5D404B0B065",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217AD56ABA897A355CF307CCB74226243192",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A151284F950B1B1DBCAD6D9E7900DF4E6",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217AEF85A612514121C299A1D87116C4A182",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A67F157569BFB4013EA3AD16DB8C69AD6",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217AF8520D191F6ACBD88B2140588B91C697",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ADD8BBAA71745669B96F2683E2F5AEC35",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217AFB2D4282817D7EC6B33EFAD7AA14A3C5",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A459B89E7E0DAF3DA654576B60B2DA7CE",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A65759F23F9789D05B23D5DBAA9E32036",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217A03C78FBD5E2CB08B3B6D181E23FBDE79",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA013D941FBBDE56C106C482CD022F290F",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA0645D313AC3C29B79DB1AA2E00A5B393",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA2ED0FD8048053BF22EBE501D82C4B3F1",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAC57D706C7866A01D6E913F98AE57EE54",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAB7FC1241FAFDFE45C4FF982D5DC1DAEF",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA7063EA296922DE8BDFD3B29D786C5F91",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA3A4603475F4AFDBFADC6E7FA908188B1",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA3365C63C2AF2A6C8FB4D0E9ED3C6FDA3",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA78BCC1874C0B7EB52645FC8F03B9C9CF",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA9B7A31397718EECB89B9E9CCCD729326",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAB15EA8A67E9E9FADB4249710277F3D4F",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA94641D6A076193C660632CEA3F9CB02C",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAB2170A08417BE77F0EAA9110F4790E12",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA4E30F1CD7B2256ABD57DC3DAB05376C9",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA9909B7B93D01BDAAC22D15AF34DF1EEF",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"4141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAD97F5D1206F00E5C7225CAD81CCD4027",
true,
AESCBCPadder,
},
{
"11111111111111111111111111111111",
"22222222222222222222222222222222",
"414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA570CBB001A0C87558906B60C884AB5F41DA97CEF2A9401BC6DD0D22A54DBAD6D",
true,
AESCBCPadder,
},
}
for i, testCase := range testCases {
key, _ := hex.DecodeString(testCase.key)
iv, _ := hex.DecodeString(testCase.iv)
cd := CipherData{
Key: key,
IV: iv,
}
cbc, err := newAESCBC(cd, testCase.padder)
if err != nil {
t.Fatal(fmt.Sprintf("Case %d: Expected no error for cipher creation, but received: %v", i, err.Error()))
}
plaintext := []byte(testCase.plaintext)
if testCase.decodeHex {
plaintext, _ = hex.DecodeString(testCase.plaintext)
}
cipherdata := cbc.Encrypt(bytes.NewReader(plaintext))
ciphertext := []byte{}
b := make([]byte, 19)
err = nil
n := 0
for err != io.EOF {
n, err = cipherdata.Read(b)
ciphertext = append(ciphertext, b[:n]...)
}
if err != io.EOF {
t.Fatal(fmt.Sprintf("Case %d: Expected no error during io reading, but received: %v", i, err.Error()))
}
expectedData, _ := hex.DecodeString(testCase.ciphertext)
if bytes.Compare(expectedData, ciphertext) != 0 {
t.Log("\n", ciphertext, "\n", expectedData)
t.Fatal(fmt.Sprintf("Case %d: AES CBC encryption fails. Data is not the same", i))
}
plaindata := cbc.Decrypt(bytes.NewReader(ciphertext))
plaintextDecrypted := []byte{}
err = nil
for err != io.EOF {
n, err = plaindata.Read(b)
plaintextDecrypted = append(plaintextDecrypted, b[:n]...)
}
if err != io.EOF {
t.Fatal(fmt.Sprintf("Case %d: Expected no error during io reading, but received: %v", i, err.Error()))
}
if bytes.Compare(plaintext, plaintextDecrypted) != 0 {
t.Log("\n", plaintext, "\n", plaintextDecrypted)
t.Fatal(fmt.Sprintf("Case %d: AES CBC decryption fails. Data is not the same", i))
}
}
}
-105
View File
@@ -1,105 +0,0 @@
package s3crypto
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"io"
"io/ioutil"
)
// AESGCM Symmetric encryption algorithm. Since Golang designed this
// with only TLS in mind. We have to load it all into memory meaning
// this isn't streamed.
type aesGCM struct {
aead cipher.AEAD
nonce []byte
}
// newAESGCM creates a new AES GCM cipher. Expects keys to be of
// the correct size.
//
// Example:
//
// cd := &s3crypto.CipherData{
// Key: key,
// "IV": iv,
// }
// cipher, err := s3crypto.newAESGCM(cd)
func newAESGCM(cd CipherData) (Cipher, error) {
block, err := aes.NewCipher(cd.Key)
if err != nil {
return nil, err
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
return &aesGCM{aesgcm, cd.IV}, nil
}
// Encrypt will encrypt the data using AES GCM
// Tag will be included as the last 16 bytes of the slice
func (c *aesGCM) Encrypt(src io.Reader) io.Reader {
reader := &gcmEncryptReader{
encrypter: c.aead,
nonce: c.nonce,
src: src,
}
return reader
}
type gcmEncryptReader struct {
encrypter cipher.AEAD
nonce []byte
src io.Reader
buf *bytes.Buffer
}
func (reader *gcmEncryptReader) Read(data []byte) (int, error) {
if reader.buf == nil {
b, err := ioutil.ReadAll(reader.src)
if err != nil {
return len(b), err
}
b = reader.encrypter.Seal(b[:0], reader.nonce, b, nil)
reader.buf = bytes.NewBuffer(b)
}
return reader.buf.Read(data)
}
// Decrypt will decrypt the data using AES GCM
func (c *aesGCM) Decrypt(src io.Reader) io.Reader {
return &gcmDecryptReader{
decrypter: c.aead,
nonce: c.nonce,
src: src,
}
}
type gcmDecryptReader struct {
decrypter cipher.AEAD
nonce []byte
src io.Reader
buf *bytes.Buffer
}
func (reader *gcmDecryptReader) Read(data []byte) (int, error) {
if reader.buf == nil {
b, err := ioutil.ReadAll(reader.src)
if err != nil {
return len(b), err
}
b, err = reader.decrypter.Open(b[:0], reader.nonce, b, nil)
if err != nil {
return len(b), err
}
reader.buf = bytes.NewBuffer(b)
}
return reader.buf.Read(data)
}
@@ -1,68 +0,0 @@
package s3crypto
import (
"io"
)
const (
gcmKeySize = 32
gcmNonceSize = 12
)
type gcmContentCipherBuilder struct {
generator CipherDataGenerator
}
// AESGCMContentCipherBuilder returns a new encryption only mode structure with a specific cipher
// for the master key
func AESGCMContentCipherBuilder(generator CipherDataGenerator) ContentCipherBuilder {
return gcmContentCipherBuilder{generator}
}
func (builder gcmContentCipherBuilder) ContentCipher() (ContentCipher, error) {
cd, err := builder.generator.GenerateCipherData(gcmKeySize, gcmNonceSize)
if err != nil {
return nil, err
}
return newAESGCMContentCipher(cd)
}
func newAESGCMContentCipher(cd CipherData) (ContentCipher, error) {
cd.CEKAlgorithm = AESGCMNoPadding
cd.TagLength = "128"
cipher, err := newAESGCM(cd)
if err != nil {
return nil, err
}
return &aesGCMContentCipher{
CipherData: cd,
Cipher: cipher,
}, nil
}
// AESGCMContentCipher will use AES GCM for the main cipher.
type aesGCMContentCipher struct {
CipherData CipherData
Cipher Cipher
}
// EncryptContents will generate a random key and iv and encrypt the data using cbc
func (cc *aesGCMContentCipher) EncryptContents(src io.Reader) (io.Reader, error) {
return cc.Cipher.Encrypt(src), nil
}
// DecryptContents will use the symmetric key provider to instantiate a new GCM cipher.
// We grab a decrypt reader from gcm and wrap it in a CryptoReadCloser. The only error
// expected here is when the key or iv is of invalid length.
func (cc *aesGCMContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser, error) {
reader := cc.Cipher.Decrypt(src)
return &CryptoReadCloser{Body: src, Decrypter: reader}, nil
}
// GetCipherData returns cipher data
func (cc aesGCMContentCipher) GetCipherData() CipherData {
return cc.CipherData
}
@@ -1,28 +0,0 @@
package s3crypto_test
import (
"testing"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
func TestAESGCMContentCipherBuilder(t *testing.T) {
generator := mockGenerator{}
if builder := s3crypto.AESGCMContentCipherBuilder(generator); builder == nil {
t.Error("expected non-nil value")
}
}
func TestAESGCMContentCipherNewEncryptor(t *testing.T) {
generator := mockGenerator{}
builder := s3crypto.AESGCMContentCipherBuilder(generator)
cipher, err := builder.ContentCipher()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if cipher == nil {
t.Errorf("expected non-nil vaue")
}
}
-81
View File
@@ -1,81 +0,0 @@
package s3crypto
import (
"bytes"
"encoding/hex"
"io/ioutil"
"testing"
)
// AES GCM
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_128_Test_0(t *testing.T) {
iv, _ := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
plaintext, _ := hex.DecodeString("2db5168e932556f8089a0622981d017d")
expected, _ := hex.DecodeString("fa4362189661d163fcd6a56d8bf0405ad636ac1bbedd5cc3ee727dc2ab4a9489")
tag, _ := hex.DecodeString("d636ac1bbedd5cc3ee727dc2ab4a9489")
aesgcmTest(t, iv, key, plaintext, expected, tag)
}
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_104_Test_3(t *testing.T) {
iv, _ := hex.DecodeString("4742357c335913153ff0eb0f")
key, _ := hex.DecodeString("e5a0eb92cc2b064e1bc80891faf1fab5e9a17a9c3a984e25416720e30e6c2b21")
plaintext, _ := hex.DecodeString("8499893e16b0ba8b007d54665a")
expected, _ := hex.DecodeString("eb8e6175f1fe38eb1acf95fd5188a8b74bb74fda553e91020a23deed45")
tag, _ := hex.DecodeString("88a8b74bb74fda553e91020a23deed45")
aesgcmTest(t, iv, key, plaintext, expected, tag)
}
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_256_Test_6(t *testing.T) {
iv, _ := hex.DecodeString("a291484c3de8bec6b47f525f")
key, _ := hex.DecodeString("37f39137416bafde6f75022a7a527cc593b6000a83ff51ec04871a0ff5360e4e")
plaintext, _ := hex.DecodeString("fafd94cede8b5a0730394bec68a8e77dba288d6ccaa8e1563a81d6e7ccc7fc97")
expected, _ := hex.DecodeString("44dc868006b21d49284016565ffb3979cc4271d967628bf7cdaf86db888e92e501a2b578aa2f41ec6379a44a31cc019c")
tag, _ := hex.DecodeString("01a2b578aa2f41ec6379a44a31cc019c")
aesgcmTest(t, iv, key, plaintext, expected, tag)
}
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_408_Test_8(t *testing.T) {
iv, _ := hex.DecodeString("92f258071d79af3e63672285")
key, _ := hex.DecodeString("595f259c55abe00ae07535ca5d9b09d6efb9f7e9abb64605c337acbd6b14fc7e")
plaintext, _ := hex.DecodeString("a6fee33eb110a2d769bbc52b0f36969c287874f665681477a25fc4c48015c541fbe2394133ba490a34ee2dd67b898177849a91")
expected, _ := hex.DecodeString("bbca4a9e09ae9690c0f6f8d405e53dccd666aa9c5fa13c8758bc30abe1ddd1bcce0d36a1eaaaaffef20cd3c5970b9673f8a65c26ccecb9976fd6ac9c2c0f372c52c821")
tag, _ := hex.DecodeString("26ccecb9976fd6ac9c2c0f372c52c821")
aesgcmTest(t, iv, key, plaintext, expected, tag)
}
func aesgcmTest(t *testing.T, iv, key, plaintext, expected, tag []byte) {
cd := CipherData{
Key: key,
IV: iv,
}
gcm, err := newAESGCM(cd)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
cipherdata := gcm.Encrypt(bytes.NewReader(plaintext))
ciphertext, err := ioutil.ReadAll(cipherdata)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
// splitting tag and ciphertext
etag := ciphertext[len(ciphertext)-16:]
if !bytes.Equal(etag, tag) {
t.Errorf("expected tags to be equivalent")
}
if !bytes.Equal(ciphertext, expected) {
t.Errorf("expected ciphertext to be equivalent")
}
data := gcm.Decrypt(bytes.NewReader(ciphertext))
text, err := ioutil.ReadAll(data)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !bytes.Equal(plaintext, text) {
t.Errorf("expected ciphertext to be equivalent")
}
}
-43
View File
@@ -1,43 +0,0 @@
package s3crypto
import (
"io"
)
// Cipher interface allows for either encryption and decryption of an object
type Cipher interface {
Encrypter
Decrypter
}
// Encrypter interface with only the encrypt method
type Encrypter interface {
Encrypt(io.Reader) io.Reader
}
// Decrypter interface with only the decrypt method
type Decrypter interface {
Decrypt(io.Reader) io.Reader
}
// CryptoReadCloser handles closing of the body and allowing reads from the decrypted
// content.
type CryptoReadCloser struct {
Body io.ReadCloser
Decrypter io.Reader
isClosed bool
}
// Close lets the CryptoReadCloser satisfy io.ReadCloser interface
func (rc *CryptoReadCloser) Close() error {
rc.isClosed = true
return rc.Body.Close()
}
// Read lets the CryptoReadCloser satisfy io.ReadCloser interface
func (rc *CryptoReadCloser) Read(b []byte) (int, error) {
if rc.isClosed {
return 0, io.EOF
}
return rc.Decrypter.Read(b)
}
-31
View File
@@ -1,31 +0,0 @@
package s3crypto
import "io"
// ContentCipherBuilder is a builder interface that builds
// ciphers for each request.
type ContentCipherBuilder interface {
ContentCipher() (ContentCipher, error)
}
// ContentCipher deals with encrypting and decrypting content
type ContentCipher interface {
EncryptContents(io.Reader) (io.Reader, error)
DecryptContents(io.ReadCloser) (io.ReadCloser, error)
GetCipherData() CipherData
}
// CipherData is used for content encryption. It is used for storing the
// metadata of the encrypted content.
type CipherData struct {
Key []byte
IV []byte
WrapAlgorithm string
CEKAlgorithm string
TagLength string
MaterialDescription MaterialDescription
// EncryptedKey should be populated when calling GenerateCipherData
EncryptedKey []byte
Padder Padder
}
-40
View File
@@ -1,40 +0,0 @@
package s3crypto_test
import (
"io/ioutil"
"strings"
"testing"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
func TestCryptoReadCloserRead(t *testing.T) {
expectedStr := "HELLO WORLD "
str := strings.NewReader(expectedStr)
rc := &s3crypto.CryptoReadCloser{Body: ioutil.NopCloser(str), Decrypter: str}
b, err := ioutil.ReadAll(rc)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if expectedStr != string(b) {
t.Errorf("expected %s, but received %s", expectedStr, string(b))
}
}
func TestCryptoReadCloserClose(t *testing.T) {
data := "HELLO WORLD "
expectedStr := ""
str := strings.NewReader(data)
rc := &s3crypto.CryptoReadCloser{Body: ioutil.NopCloser(str), Decrypter: str}
rc.Close()
b, err := ioutil.ReadAll(rc)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if expectedStr != string(b) {
t.Errorf("expected %s, but received %s", expectedStr, string(b))
}
}
-111
View File
@@ -1,111 +0,0 @@
package s3crypto
import (
"encoding/base64"
"strconv"
"strings"
"github.com/aws/aws-sdk-go/aws/awserr"
)
func (client *DecryptionClient) contentCipherFromEnvelope(env Envelope) (ContentCipher, error) {
wrap, err := client.wrapFromEnvelope(env)
if err != nil {
return nil, err
}
return client.cekFromEnvelope(env, wrap)
}
func (client *DecryptionClient) wrapFromEnvelope(env Envelope) (CipherDataDecrypter, error) {
f, ok := client.WrapRegistry[env.WrapAlg]
if !ok || f == nil {
return nil, awserr.New(
"InvalidWrapAlgorithmError",
"wrap algorithm isn't supported, "+env.WrapAlg,
nil,
)
}
return f(env)
}
// AESGCMNoPadding is the constant value that is used to specify
// the CEK algorithm consiting of AES GCM with no padding.
const AESGCMNoPadding = "AES/GCM/NoPadding"
// AESCBC is the string constant that signifies the AES CBC algorithm cipher.
const AESCBC = "AES/CBC"
func (client *DecryptionClient) cekFromEnvelope(env Envelope, decrypter CipherDataDecrypter) (ContentCipher, error) {
f, ok := client.CEKRegistry[env.CEKAlg]
if !ok || f == nil {
return nil, awserr.New(
"InvalidCEKAlgorithmError",
"cek algorithm isn't supported, "+env.CEKAlg,
nil,
)
}
key, err := base64.StdEncoding.DecodeString(env.CipherKey)
if err != nil {
return nil, err
}
iv, err := base64.StdEncoding.DecodeString(env.IV)
if err != nil {
return nil, err
}
key, err = decrypter.DecryptKey(key)
if err != nil {
return nil, err
}
cd := CipherData{
Key: key,
IV: iv,
CEKAlgorithm: env.CEKAlg,
Padder: client.getPadder(env.CEKAlg),
}
return f(cd)
}
// getPadder will return an unpadder with checking the cek algorithm specific padder.
// If there wasn't a cek algorithm specific padder, we check the padder itself.
// We return a no unpadder, if no unpadder was found. This means any customization
// either contained padding within the cipher implementation, and to maintain
// backwards compatility we will simply not unpad anything.
func (client *DecryptionClient) getPadder(cekAlg string) Padder {
padder, ok := client.PadderRegistry[cekAlg]
if !ok {
padder, ok = client.PadderRegistry[cekAlg[strings.LastIndex(cekAlg, "/")+1:]]
if !ok {
return NoPadder
}
}
return padder
}
func encodeMeta(reader hashReader, cd CipherData) (Envelope, error) {
iv := base64.StdEncoding.EncodeToString(cd.IV)
key := base64.StdEncoding.EncodeToString(cd.EncryptedKey)
md5 := reader.GetValue()
contentLength := reader.GetContentLength()
md5Str := base64.StdEncoding.EncodeToString(md5)
matdesc, err := cd.MaterialDescription.encodeDescription()
if err != nil {
return Envelope{}, err
}
return Envelope{
CipherKey: key,
IV: iv,
MatDesc: string(matdesc),
WrapAlg: cd.WrapAlgorithm,
CEKAlg: cd.CEKAlgorithm,
TagLen: cd.TagLength,
UnencryptedMD5: md5Str,
UnencryptedContentLen: strconv.FormatInt(contentLength, 10),
}, nil
}
@@ -1,267 +0,0 @@
package s3crypto
import (
"encoding/base64"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/kms"
)
func TestWrapFactory(t *testing.T) {
c := DecryptionClient{
WrapRegistry: map[string]WrapEntry{
KMSWrap: (kmsKeyHandler{
kms: kms.New(unit.Session),
}).decryptHandler,
},
CEKRegistry: map[string]CEKEntry{
AESGCMNoPadding: newAESGCMContentCipher,
},
}
env := Envelope{
WrapAlg: KMSWrap,
MatDesc: `{"kms_cmk_id":""}`,
}
wrap, err := c.wrapFromEnvelope(env)
w, ok := wrap.(*kmsKeyHandler)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if wrap == nil {
t.Error("expected non-nil value")
}
if !ok {
t.Errorf("expected kmsKeyHandler, but received %v", *w)
}
}
func TestWrapFactoryErrorNoWrap(t *testing.T) {
c := DecryptionClient{
WrapRegistry: map[string]WrapEntry{
KMSWrap: (kmsKeyHandler{
kms: kms.New(unit.Session),
}).decryptHandler,
},
CEKRegistry: map[string]CEKEntry{
AESGCMNoPadding: newAESGCMContentCipher,
},
}
env := Envelope{
WrapAlg: "none",
MatDesc: `{"kms_cmk_id":""}`,
}
wrap, err := c.wrapFromEnvelope(env)
if err == nil {
t.Error("expected error, but received none")
}
if wrap != nil {
t.Errorf("expected nil wrap value, received %v", wrap)
}
}
func TestWrapFactoryCustomEntry(t *testing.T) {
c := DecryptionClient{
WrapRegistry: map[string]WrapEntry{
"custom": (kmsKeyHandler{
kms: kms.New(unit.Session),
}).decryptHandler,
},
CEKRegistry: map[string]CEKEntry{
AESGCMNoPadding: newAESGCMContentCipher,
},
}
env := Envelope{
WrapAlg: "custom",
MatDesc: `{"kms_cmk_id":""}`,
}
wrap, err := c.wrapFromEnvelope(env)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if wrap == nil {
t.Errorf("expected nil wrap value, received %v", wrap)
}
}
func TestCEKFactory(t *testing.T) {
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
keyB64 := base64.URLEncoding.EncodeToString(key)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
defer ts.Close()
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
c := DecryptionClient{
WrapRegistry: map[string]WrapEntry{
KMSWrap: (kmsKeyHandler{
kms: kms.New(sess),
}).decryptHandler,
},
CEKRegistry: map[string]CEKEntry{
AESGCMNoPadding: newAESGCMContentCipher,
},
PadderRegistry: map[string]Padder{
NoPadder.Name(): NoPadder,
},
}
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
ivB64 := base64.URLEncoding.EncodeToString(iv)
cipherKey, err := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
cipherKeyB64 := base64.URLEncoding.EncodeToString(cipherKey)
env := Envelope{
WrapAlg: KMSWrap,
CEKAlg: AESGCMNoPadding,
CipherKey: cipherKeyB64,
IV: ivB64,
MatDesc: `{"kms_cmk_id":""}`,
}
wrap, err := c.wrapFromEnvelope(env)
cek, err := c.cekFromEnvelope(env, wrap)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if cek == nil {
t.Errorf("expected non-nil cek")
}
}
func TestCEKFactoryNoCEK(t *testing.T) {
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
keyB64 := base64.URLEncoding.EncodeToString(key)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
defer ts.Close()
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
c := DecryptionClient{
WrapRegistry: map[string]WrapEntry{
KMSWrap: (kmsKeyHandler{
kms: kms.New(sess),
}).decryptHandler,
},
CEKRegistry: map[string]CEKEntry{
AESGCMNoPadding: newAESGCMContentCipher,
},
PadderRegistry: map[string]Padder{
NoPadder.Name(): NoPadder,
},
}
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
ivB64 := base64.URLEncoding.EncodeToString(iv)
cipherKey, err := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
cipherKeyB64 := base64.URLEncoding.EncodeToString(cipherKey)
env := Envelope{
WrapAlg: KMSWrap,
CEKAlg: "none",
CipherKey: cipherKeyB64,
IV: ivB64,
MatDesc: `{"kms_cmk_id":""}`,
}
wrap, err := c.wrapFromEnvelope(env)
cek, err := c.cekFromEnvelope(env, wrap)
if err == nil {
t.Error("expected error, but received none")
}
if cek != nil {
t.Errorf("expected nil cek value, received %v", wrap)
}
}
func TestCEKFactoryCustomEntry(t *testing.T) {
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
keyB64 := base64.URLEncoding.EncodeToString(key)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
defer ts.Close()
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
c := DecryptionClient{
WrapRegistry: map[string]WrapEntry{
KMSWrap: (kmsKeyHandler{
kms: kms.New(sess),
}).decryptHandler,
},
CEKRegistry: map[string]CEKEntry{
"custom": newAESGCMContentCipher,
},
PadderRegistry: map[string]Padder{},
}
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
ivB64 := base64.URLEncoding.EncodeToString(iv)
cipherKey, err := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
cipherKeyB64 := base64.URLEncoding.EncodeToString(cipherKey)
env := Envelope{
WrapAlg: KMSWrap,
CEKAlg: "custom",
CipherKey: cipherKeyB64,
IV: ivB64,
MatDesc: `{"kms_cmk_id":""}`,
}
wrap, err := c.wrapFromEnvelope(env)
cek, err := c.cekFromEnvelope(env, wrap)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if cek == nil {
t.Errorf("expected non-nil cek")
}
}
@@ -1,135 +0,0 @@
package s3crypto
import (
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/kms"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
)
// WrapEntry is builder that return a proper key decrypter and error
type WrapEntry func(Envelope) (CipherDataDecrypter, error)
// CEKEntry is a builder thatn returns a proper content decrypter and error
type CEKEntry func(CipherData) (ContentCipher, error)
// DecryptionClient is an S3 crypto client. The decryption client
// will handle all get object requests from Amazon S3.
// Supported key wrapping algorithms:
// *AWS KMS
//
// Supported content ciphers:
// * AES/GCM
// * AES/CBC
type DecryptionClient struct {
S3Client s3iface.S3API
// LoadStrategy is used to load the metadata either from the metadata of the object
// or from a separate file in s3.
//
// Defaults to our default load strategy.
LoadStrategy LoadStrategy
WrapRegistry map[string]WrapEntry
CEKRegistry map[string]CEKEntry
PadderRegistry map[string]Padder
}
// NewDecryptionClient instantiates a new S3 crypto client
//
// Example:
// sess := session.New()
// svc := s3crypto.NewDecryptionClient(sess, func(svc *s3crypto.DecryptionClient{
// // Custom client options here
// }))
func NewDecryptionClient(prov client.ConfigProvider, options ...func(*DecryptionClient)) *DecryptionClient {
s3client := s3.New(prov)
client := &DecryptionClient{
S3Client: s3client,
LoadStrategy: defaultV2LoadStrategy{
client: s3client,
},
WrapRegistry: map[string]WrapEntry{
KMSWrap: (kmsKeyHandler{
kms: kms.New(prov),
}).decryptHandler,
},
CEKRegistry: map[string]CEKEntry{
AESGCMNoPadding: newAESGCMContentCipher,
strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): newAESCBCContentCipher,
},
PadderRegistry: map[string]Padder{
strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): AESCBCPadder,
"NoPadding": NoPadder,
},
}
for _, option := range options {
option(client)
}
return client
}
// GetObjectRequest will make a request to s3 and retrieve the object. In this process
// decryption will be done. The SDK only supports V2 reads of KMS and GCM.
//
// Example:
// sess := session.New()
// svc := s3crypto.NewDecryptionClient(sess)
// req, out := svc.GetObjectRequest(&s3.GetObjectInput {
// Key: aws.String("testKey"),
// Bucket: aws.String("testBucket"),
// })
// err := req.Send()
func (c *DecryptionClient) GetObjectRequest(input *s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput) {
req, out := c.S3Client.GetObjectRequest(input)
req.Handlers.Unmarshal.PushBack(func(r *request.Request) {
env, err := c.LoadStrategy.Load(r)
if err != nil {
r.Error = err
out.Body.Close()
return
}
// If KMS should return the correct CEK algorithm with the proper
// KMS key provider
cipher, err := c.contentCipherFromEnvelope(env)
if err != nil {
r.Error = err
out.Body.Close()
return
}
reader, err := cipher.DecryptContents(out.Body)
if err != nil {
r.Error = err
out.Body.Close()
return
}
out.Body = reader
})
return req, out
}
// GetObject is a wrapper for GetObjectRequest
func (c *DecryptionClient) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
req, out := c.GetObjectRequest(input)
return out, req.Send()
}
// GetObjectWithContext is a wrapper for GetObjectRequest with the additional
// context, and request options support.
//
// GetObjectWithContext is the same as GetObject with the additional support for
// Context input parameters. The Context must not be nil. A nil Context will
// cause a panic. Use the Context to add deadlining, timeouts, ect. In the future
// this may create sub-contexts for individual underlying requests.
func (c *DecryptionClient) GetObjectWithContext(ctx aws.Context, input *s3.GetObjectInput, opts ...request.Option) (*s3.GetObjectOutput, error) {
req, out := c.GetObjectRequest(input)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return out, req.Send()
}
@@ -1,251 +0,0 @@
package s3crypto_test
import (
"bytes"
"encoding/base64"
"encoding/hex"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
func TestGetObjectGCM(t *testing.T) {
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
keyB64 := base64.StdEncoding.EncodeToString(key)
// This is our KMS response
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
defer ts.Close()
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
c := s3crypto.NewDecryptionClient(sess)
if c == nil {
t.Error("expected non-nil value")
}
input := &s3.GetObjectInput{
Key: aws.String("test"),
Bucket: aws.String("test"),
}
req, out := c.GetObjectRequest(input)
req.Handlers.Send.Clear()
req.Handlers.Send.PushBack(func(r *request.Request) {
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
b, err := hex.DecodeString("fa4362189661d163fcd6a56d8bf0405ad636ac1bbedd5cc3ee727dc2ab4a9489")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
r.HTTPResponse = &http.Response{
StatusCode: 200,
Header: http.Header{
http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"SpFRES0JyU8BLZSKo51SrwILK4lhtZsWiMNjgO4WmoK+joMwZPG7Hw=="},
http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{base64.URLEncoding.EncodeToString(iv)},
http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"arn:aws:kms:us-east-1:172259396726:key/a22a4b30-79f4-4b3d-bab4-a26d327a231b"}`},
http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap},
http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{s3crypto.AESGCMNoPadding},
http.CanonicalHeaderKey("x-amz-meta-x-amz-tag-len"): []string{"128"},
},
Body: ioutil.NopCloser(bytes.NewBuffer(b)),
}
out.Metadata = make(map[string]*string)
out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap)
})
err := req.Send()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
b, err := ioutil.ReadAll(out.Body)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
expected, err := hex.DecodeString("2db5168e932556f8089a0622981d017d")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !bytes.Equal(expected, b) {
t.Error("expected bytes to be equivalent")
}
}
func TestGetObjectCBC(t *testing.T) {
key, _ := hex.DecodeString("898be9cc5004ed0fa6e117c9a3099d31")
keyB64 := base64.StdEncoding.EncodeToString(key)
// This is our KMS response
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
defer ts.Close()
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
c := s3crypto.NewDecryptionClient(sess)
if c == nil {
t.Error("expected non-nil value")
}
input := &s3.GetObjectInput{
Key: aws.String("test"),
Bucket: aws.String("test"),
}
req, out := c.GetObjectRequest(input)
req.Handlers.Send.Clear()
req.Handlers.Send.PushBack(func(r *request.Request) {
iv, err := hex.DecodeString("9dea7621945988f96491083849b068df")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
b, err := hex.DecodeString("e232cd6ef50047801ee681ec30f61d53cfd6b0bca02fd03c1b234baa10ea82ac9dab8b960926433a19ce6dea08677e34")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
r.HTTPResponse = &http.Response{
StatusCode: 200,
Header: http.Header{
http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"SpFRES0JyU8BLZSKo51SrwILK4lhtZsWiMNjgO4WmoK+joMwZPG7Hw=="},
http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{base64.URLEncoding.EncodeToString(iv)},
http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"arn:aws:kms:us-east-1:172259396726:key/a22a4b30-79f4-4b3d-bab4-a26d327a231b"}`},
http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap},
http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{strings.Join([]string{s3crypto.AESCBC, s3crypto.AESCBCPadder.Name()}, "/")},
},
Body: ioutil.NopCloser(bytes.NewBuffer(b)),
}
out.Metadata = make(map[string]*string)
out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap)
})
err := req.Send()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
b, err := ioutil.ReadAll(out.Body)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
expected, err := hex.DecodeString("0397f4f6820b1f9386f14403be5ac16e50213bd473b4874b9bcbf5f318ee686b1d")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !bytes.Equal(expected, b) {
t.Error("expected bytes to be equivalent")
}
}
func TestGetObjectCBC2(t *testing.T) {
key, _ := hex.DecodeString("8d70e92489c4e6cfb12261b4d17f4b85826da687fc8742fcf9f87fadb5b4cb89")
keyB64 := base64.StdEncoding.EncodeToString(key)
// This is our KMS response
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
defer ts.Close()
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
c := s3crypto.NewDecryptionClient(sess)
if c == nil {
t.Error("expected non-nil value")
}
input := &s3.GetObjectInput{
Key: aws.String("test"),
Bucket: aws.String("test"),
}
req, out := c.GetObjectRequest(input)
req.Handlers.Send.Clear()
req.Handlers.Send.PushBack(func(r *request.Request) {
b, err := hex.DecodeString("fd0c71ecb7ed16a9bf42ea5f75501d416df608f190890c3b4d8897f24744cd7f9ea4a0b212e60634302450e1c5378f047ff753ccefe365d411c36339bf22e301fae4c3a6226719a4b93dc74c1af79d0296659b5d56c0892315f2c7cc30190220db1eaafae3920d6d9c65d0aa366499afc17af493454e141c6e0fbdeb6a990cb4")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
r.HTTPResponse = &http.Response{
StatusCode: 200,
Header: http.Header{
http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"AQEDAHikdGvcj7Gil5VqAR/JWvvPp3ue26+t2vhWy4lL2hg4mAAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDCcy43wCR0bSsnzTrAIBEIA7WdD2jxC3tCrK6TOdiEfbIN64m+UN7Velz4y0LRra5jn2U1CDClacwIpiBYuDp5ymPKO+ZqUGE0WEf20="},
http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{"EMMWJY8ZLcK/9FOj3iCpng=="},
http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"arn:aws:kms:us-east-1:172259396726:key/a22a4b30-79f4-4b3d-bab4-a26d327a231b"}`},
http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap},
http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{strings.Join([]string{s3crypto.AESCBC, s3crypto.AESCBCPadder.Name()}, "/")},
},
Body: ioutil.NopCloser(bytes.NewBuffer(b)),
}
fmt.Println("HEADER", r.HTTPResponse.Header)
out.Metadata = make(map[string]*string)
out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap)
})
err := req.Send()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
b, err := ioutil.ReadAll(out.Body)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
expected, err := hex.DecodeString("a6ccd3482f5ce25c9ddeb69437cd0acbc0bdda2ef8696d90781de2b35704543529871b2032e68ef1c5baed1769aba8d420d1aca181341b49b8b3587a6580cdf1d809c68f06735f7735c16691f4b70c967d68fc08195b81ad71bcc4df452fd0a5799c1e1234f92f1cd929fc072167ccf9f2ac85b93170932b32")
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !bytes.Equal(expected, b) {
t.Error("expected bytes to be equivalent")
}
}
func TestGetObjectWithContext(t *testing.T) {
c := s3crypto.NewDecryptionClient(unit.Session)
ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
ctx.Error = fmt.Errorf("context canceled")
close(ctx.DoneCh)
input := s3.GetObjectInput{
Key: aws.String("test"),
Bucket: aws.String("test"),
}
_, err := c.GetObjectWithContext(ctx, &input)
if err == nil {
t.Fatalf("expected error, did not get one")
}
aerr := err.(awserr.Error)
if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
t.Errorf("expected error code %q, got %q", e, a)
}
if e, a := "canceled", aerr.Message(); !strings.Contains(a, e) {
t.Errorf("expected error message to contain %q, but did not %q", e, a)
}
}
-66
View File
@@ -1,66 +0,0 @@
/*
Package s3crypto provides encryption to S3 using KMS and AES GCM.
Keyproviders are interfaces that handle masterkeys. Masterkeys are used to encrypt and decrypt the randomly
generated cipher keys. The SDK currently uses KMS to do this. A user does not need to provide a master key
since all that information is hidden in KMS.
Modes are interfaces that handle content encryption and decryption. It is an abstraction layer that instantiates
the ciphers. If content is being encrypted we generate the key and iv of the cipher. For decryption, we use the
metadata stored either on the object or an instruction file object to decrypt the contents.
Ciphers are interfaces that handle encryption and decryption of data. This may be key wrap ciphers or content
ciphers.
Creating an S3 cryptography client
cmkID := "<some key ID>"
sess := session.New()
// Create the KeyProvider
handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID)
// Create an encryption and decryption client
// We need to pass the session here so S3 can use it. In addition, any decryption that
// occurs will use the KMS client.
svc := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler))
svc := s3crypto.NewDecryptionClient(sess)
Configuration of the S3 cryptography client
cfg := s3crypto.EncryptionConfig{
// Save instruction files to separate objects
SaveStrategy: NewS3SaveStrategy(session.New(), ""),
// Change instruction file suffix to .example
InstructionFileSuffix: ".example",
// Set temp folder path
TempFolderPath: "/path/to/tmp/folder/",
// Any content less than the minimum file size will use memory
// instead of writing the contents to a temp file.
MinFileSize: int64(1024 * 1024 * 1024),
}
The default SaveStrategy is to the object's header.
The InstructionFileSuffix defaults to .instruction. Careful here though, if you do this, be sure you know
what that suffix is in grabbing data. All requests will look for fooKey.example instead of fooKey.instruction.
This suffix only affects gets and not puts. Put uses the keyprovider's suffix.
Registration of new wrap or cek algorithms are also supported by the SDK. Let's say we want to support `AES Wrap`
and `AES CTR`. Let's assume we have already defined the functionality.
svc := s3crypto.NewDecryptionClient(sess)
svc.WrapRegistry["AESWrap"] = NewAESWrap
svc.CEKRegistry["AES/CTR/NoPadding"] = NewAESCTR
We have now registered these new algorithms to the decryption client. When the client calls `GetObject` and sees
the wrap as `AESWrap` then it'll use that wrap algorithm. This is also true for `AES/CTR/NoPadding`.
For encryption adding a custom content cipher builder and key handler will allow for encryption of custom
defined ciphers.
// Our wrap algorithm, AESWrap
handler := NewAESWrap(key, iv)
// Our content cipher builder, AESCTRContentCipherBuilder
svc := s3crypto.NewEncryptionClient(sess, NewAESCTRContentCipherBuilder(handler))
*/
package s3crypto
@@ -1,146 +0,0 @@
package s3crypto
import (
"encoding/hex"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/sdkio"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
)
// DefaultMinFileSize is used to check whether we want to write to a temp file
// or store the data in memory.
const DefaultMinFileSize = 1024 * 512 * 5
// EncryptionClient is an S3 crypto client. By default the SDK will use Authentication mode which
// will use KMS for key wrapping and AES GCM for content encryption.
// AES GCM will load all data into memory. However, the rest of the content algorithms
// do not load the entire contents into memory.
type EncryptionClient struct {
S3Client s3iface.S3API
ContentCipherBuilder ContentCipherBuilder
// SaveStrategy will dictate where the envelope is saved.
//
// Defaults to the object's metadata
SaveStrategy SaveStrategy
// TempFolderPath is used to store temp files when calling PutObject.
// Temporary files are needed to compute the X-Amz-Content-Sha256 header.
TempFolderPath string
// MinFileSize is the minimum size for the content to write to a
// temporary file instead of using memory.
MinFileSize int64
}
// NewEncryptionClient instantiates a new S3 crypto client
//
// Example:
// cmkID := "arn:aws:kms:region:000000000000:key/00000000-0000-0000-0000-000000000000"
// sess := session.New()
// handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID)
// svc := s3crypto.New(sess, s3crypto.AESGCMContentCipherBuilder(handler))
func NewEncryptionClient(prov client.ConfigProvider, builder ContentCipherBuilder, options ...func(*EncryptionClient)) *EncryptionClient {
client := &EncryptionClient{
S3Client: s3.New(prov),
ContentCipherBuilder: builder,
SaveStrategy: HeaderV2SaveStrategy{},
MinFileSize: DefaultMinFileSize,
}
for _, option := range options {
option(client)
}
return client
}
// PutObjectRequest creates a temp file to encrypt the contents into. It then streams
// that data to S3.
//
// Example:
// svc := s3crypto.New(session.New(), s3crypto.AESGCMContentCipherBuilder(handler))
// req, out := svc.PutObjectRequest(&s3.PutObjectInput {
// Key: aws.String("testKey"),
// Bucket: aws.String("testBucket"),
// Body: strings.NewReader("test data"),
// })
// err := req.Send()
func (c *EncryptionClient) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) {
req, out := c.S3Client.PutObjectRequest(input)
// Get Size of file
n, err := aws.SeekerLen(input.Body)
if err != nil {
req.Error = err
return req, out
}
dst, err := getWriterStore(req, c.TempFolderPath, n >= c.MinFileSize)
if err != nil {
req.Error = err
return req, out
}
encryptor, err := c.ContentCipherBuilder.ContentCipher()
req.Handlers.Build.PushFront(func(r *request.Request) {
if err != nil {
r.Error = err
return
}
md5 := newMD5Reader(input.Body)
sha := newSHA256Writer(dst)
reader, err := encryptor.EncryptContents(md5)
if err != nil {
r.Error = err
return
}
_, err = io.Copy(sha, reader)
if err != nil {
r.Error = err
return
}
data := encryptor.GetCipherData()
env, err := encodeMeta(md5, data)
if err != nil {
r.Error = err
return
}
shaHex := hex.EncodeToString(sha.GetValue())
req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", shaHex)
dst.Seek(0, sdkio.SeekStart)
input.Body = dst
err = c.SaveStrategy.Save(env, r)
r.Error = err
})
return req, out
}
// PutObject is a wrapper for PutObjectRequest
func (c *EncryptionClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
req, out := c.PutObjectRequest(input)
return out, req.Send()
}
// PutObjectWithContext is a wrapper for PutObjectRequest with the additional
// context, and request options support.
//
// PutObjectWithContext is the same as PutObject with the additional support for
// Context input parameters. The Context must not be nil. A nil Context will
// cause a panic. Use the Context to add deadlining, timeouts, ect. In the future
// this may create sub-contexts for individual underlying requests.
func (c *EncryptionClient) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInput, opts ...request.Option) (*s3.PutObjectOutput, error) {
req, out := c.PutObjectRequest(input)
req.SetContext(ctx)
req.ApplyOptions(opts...)
return out, req.Send()
}
@@ -1,111 +0,0 @@
package s3crypto_test
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/kms"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
func TestDefaultConfigValues(t *testing.T) {
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
svc := kms.New(sess)
handler := s3crypto.NewKMSKeyGenerator(svc, "testid")
c := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler))
if c == nil {
t.Error("expected non-vil client value")
}
if c.ContentCipherBuilder == nil {
t.Error("expected non-vil content cipher builder value")
}
if c.SaveStrategy == nil {
t.Error("expected non-vil save strategy value")
}
}
func TestPutObject(t *testing.T) {
size := 1024 * 1024
data := make([]byte, size)
expected := bytes.Repeat([]byte{1}, size)
generator := mockGenerator{}
cb := mockCipherBuilder{generator}
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
c := s3crypto.NewEncryptionClient(sess, cb)
if c == nil {
t.Error("expected non-vil client value")
}
input := &s3.PutObjectInput{
Key: aws.String("test"),
Bucket: aws.String("test"),
Body: bytes.NewReader(data),
}
req, _ := c.PutObjectRequest(input)
req.Handlers.Send.Clear()
req.Handlers.Send.PushBack(func(r *request.Request) {
r.Error = errors.New("stop")
r.HTTPResponse = &http.Response{
StatusCode: 200,
}
})
err := req.Send()
if e, a := "stop", err.Error(); e != a {
t.Errorf("expected %s error, but received %s", e, a)
}
b, err := ioutil.ReadAll(req.HTTPRequest.Body)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !bytes.Equal(expected, b) {
t.Error("expected bytes to be equivalent, but received otherwise")
}
}
func TestPutObjectWithContext(t *testing.T) {
generator := mockGenerator{}
cb := mockCipherBuilder{generator}
c := s3crypto.NewEncryptionClient(unit.Session, cb)
ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
ctx.Error = fmt.Errorf("context canceled")
close(ctx.DoneCh)
input := s3.PutObjectInput{
Bucket: aws.String("test"),
Key: aws.String("test"),
Body: bytes.NewReader([]byte{}),
}
_, err := c.PutObjectWithContext(ctx, &input)
if err == nil {
t.Fatalf("expected error, did not get one")
}
aerr := err.(awserr.Error)
if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
t.Errorf("expected error code %q, got %q", e, a)
}
if e, a := "canceled", aerr.Message(); !strings.Contains(a, e) {
t.Errorf("expected error message to contain %q, but did not %q", e, a)
}
}
-37
View File
@@ -1,37 +0,0 @@
package s3crypto
// DefaultInstructionKeySuffix is appended to the end of the instruction file key when
// grabbing or saving to S3
const DefaultInstructionKeySuffix = ".instruction"
const (
metaHeader = "x-amz-meta"
keyV1Header = "x-amz-key"
keyV2Header = keyV1Header + "-v2"
ivHeader = "x-amz-iv"
matDescHeader = "x-amz-matdesc"
cekAlgorithmHeader = "x-amz-cek-alg"
wrapAlgorithmHeader = "x-amz-wrap-alg"
tagLengthHeader = "x-amz-tag-len"
unencryptedMD5Header = "x-amz-unencrypted-content-md5"
unencryptedContentLengthHeader = "x-amz-unencrypted-content-length"
)
// Envelope encryption starts off by generating a random symmetric key using
// AES GCM. The SDK generates a random IV based off the encryption cipher
// chosen. The master key that was provided, whether by the user or KMS, will be used
// to encrypt the randomly generated symmetric key and base64 encode the iv. This will
// allow for decryption of that same data later.
type Envelope struct {
// IV is the randomly generated IV base64 encoded.
IV string `json:"x-amz-iv"`
// CipherKey is the randomly generated cipher key.
CipherKey string `json:"x-amz-key-v2"`
// MaterialDesc is a description to distinguish from other envelopes.
MatDesc string `json:"x-amz-matdesc"`
WrapAlg string `json:"x-amz-wrap-alg"`
CEKAlg string `json:"x-amz-cek-alg"`
TagLen string `json:"x-amz-tag-len"`
UnencryptedMD5 string `json:"x-amz-unencrypted-content-md5"`
UnencryptedContentLen string `json:"x-amz-unencrypted-content-length"`
}
-61
View File
@@ -1,61 +0,0 @@
package s3crypto
import (
"crypto/md5"
"crypto/sha256"
"hash"
"io"
)
// hashReader is used for calculating SHA256 when following the sigv4 specification.
// Additionally this used for calculating the unencrypted MD5.
type hashReader interface {
GetValue() []byte
GetContentLength() int64
}
type sha256Writer struct {
sha256 []byte
hash hash.Hash
out io.Writer
}
func newSHA256Writer(f io.Writer) *sha256Writer {
return &sha256Writer{hash: sha256.New(), out: f}
}
func (r *sha256Writer) Write(b []byte) (int, error) {
r.hash.Write(b)
return r.out.Write(b)
}
func (r *sha256Writer) GetValue() []byte {
return r.hash.Sum(nil)
}
type md5Reader struct {
contentLength int64
hash hash.Hash
body io.Reader
}
func newMD5Reader(body io.Reader) *md5Reader {
return &md5Reader{hash: md5.New(), body: body}
}
func (w *md5Reader) Read(b []byte) (int, error) {
n, err := w.body.Read(b)
if err != nil && err != io.EOF {
return n, err
}
w.contentLength += int64(n)
w.hash.Write(b[:n])
return n, err
}
func (w *md5Reader) GetValue() []byte {
return w.hash.Sum(nil)
}
func (w *md5Reader) GetContentLength() int64 {
return w.contentLength
}
-29
View File
@@ -1,29 +0,0 @@
package s3crypto
import (
"bytes"
"encoding/hex"
"testing"
)
// From Go stdlib encoding/sha256 test cases
func TestSHA256(t *testing.T) {
sha := newSHA256Writer(nil)
expected, _ := hex.DecodeString("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
b := sha.GetValue()
if !bytes.Equal(expected, b) {
t.Errorf("expected equivalent sha values, but received otherwise")
}
}
func TestSHA256_Case2(t *testing.T) {
sha := newSHA256Writer(bytes.NewBuffer([]byte{}))
sha.Write([]byte("hello"))
expected, _ := hex.DecodeString("2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")
b := sha.GetValue()
if !bytes.Equal(expected, b) {
t.Errorf("expected equivalent sha values, but received otherwise")
}
}
-70
View File
@@ -1,70 +0,0 @@
package s3crypto
import (
"errors"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/aws/aws-sdk-go/aws/request"
)
func getWriterStore(req *request.Request, path string, useTempFile bool) (io.ReadWriteSeeker, error) {
if !useTempFile {
return &bytesReadWriteSeeker{}, nil
}
// Create temp file to be used later for calculating the SHA256 header
f, err := ioutil.TempFile(path, "")
if err != nil {
return nil, err
}
req.Handlers.Send.PushBack(func(r *request.Request) {
// Close the temp file and cleanup
f.Close()
fpath := filepath.Join(path, f.Name())
os.Remove(fpath)
})
return f, nil
}
type bytesReadWriteSeeker struct {
buf []byte
i int64
}
// Copied from Go stdlib bytes.Reader
func (ws *bytesReadWriteSeeker) Read(b []byte) (int, error) {
if ws.i >= int64(len(ws.buf)) {
return 0, io.EOF
}
n := copy(b, ws.buf[ws.i:])
ws.i += int64(n)
return n, nil
}
func (ws *bytesReadWriteSeeker) Write(b []byte) (int, error) {
ws.buf = append(ws.buf, b...)
return len(b), nil
}
// Copied from Go stdlib bytes.Reader
func (ws *bytesReadWriteSeeker) Seek(offset int64, whence int) (int64, error) {
var abs int64
switch whence {
case 0:
abs = offset
case 1:
abs = int64(ws.i) + offset
case 2:
abs = int64(len(ws.buf)) + offset
default:
return 0, errors.New("bytes.Reader.Seek: invalid whence")
}
if abs < 0 {
return 0, errors.New("bytes.Reader.Seek: negative position")
}
ws.i = abs
return abs, nil
}
-84
View File
@@ -1,84 +0,0 @@
package s3crypto
import (
"bytes"
"testing"
"github.com/aws/aws-sdk-go/internal/sdkio"
)
func TestBytesReadWriteSeeker_Read(t *testing.T) {
b := &bytesReadWriteSeeker{[]byte{1, 2, 3}, 0}
expected := []byte{1, 2, 3}
buf := make([]byte, 3)
n, err := b.Read(buf)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := 3, n; e != a {
t.Errorf("expected %d, but received %d", e, a)
}
if !bytes.Equal(expected, buf) {
t.Error("expected equivalent byte slices, but received otherwise")
}
}
func TestBytesReadWriteSeeker_Write(t *testing.T) {
b := &bytesReadWriteSeeker{}
expected := []byte{1, 2, 3}
buf := make([]byte, 3)
n, err := b.Write([]byte{1, 2, 3})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := 3, n; e != a {
t.Errorf("expected %d, but received %d", e, a)
}
n, err = b.Read(buf)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := 3, n; e != a {
t.Errorf("expected %d, but received %d", e, a)
}
if !bytes.Equal(expected, buf) {
t.Error("expected equivalent byte slices, but received otherwise")
}
}
func TestBytesReadWriteSeeker_Seek(t *testing.T) {
b := &bytesReadWriteSeeker{[]byte{1, 2, 3}, 0}
expected := []byte{2, 3}
m, err := b.Seek(1, sdkio.SeekStart)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := 1, int(m); e != a {
t.Errorf("expected %d, but received %d", e, a)
}
buf := make([]byte, 3)
n, err := b.Read(buf)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := 2, n; e != a {
t.Errorf("expected %d, but received %d", e, a)
}
if !bytes.Equal(expected, buf[:n]) {
t.Error("expected equivalent byte slices, but received otherwise")
}
}
-21
View File
@@ -1,21 +0,0 @@
package s3crypto
import "crypto/rand"
// CipherDataGenerator handles generating proper key and IVs of proper size for the
// content cipher. CipherDataGenerator will also encrypt the key and store it in
// the CipherData.
type CipherDataGenerator interface {
GenerateCipherData(int, int) (CipherData, error)
}
// CipherDataDecrypter is a handler to decrypt keys from the envelope.
type CipherDataDecrypter interface {
DecryptKey([]byte) ([]byte, error)
}
func generateBytes(n int) []byte {
b := make([]byte, n)
rand.Read(b)
return b
}
@@ -1,20 +0,0 @@
package s3crypto
import (
"testing"
)
func TestGenerateBytes(t *testing.T) {
b := generateBytes(5)
if e, a := 5, len(b); e != a {
t.Errorf("expected %d, but received %d", e, a)
}
b = generateBytes(0)
if e, a := 0, len(b); e != a {
t.Errorf("expected %d, but received %d", e, a)
}
b = generateBytes(1024)
if e, a := 1024, len(b); e != a {
t.Errorf("expected %d, but received %d", e, a)
}
}
-114
View File
@@ -1,114 +0,0 @@
package s3crypto
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/kms"
"github.com/aws/aws-sdk-go/service/kms/kmsiface"
)
const (
// KMSWrap is a constant used during decryption to build a KMS key handler.
KMSWrap = "kms"
)
// kmsKeyHandler will make calls to KMS to get the masterkey
type kmsKeyHandler struct {
kms kmsiface.KMSAPI
cmkID *string
CipherData
}
// NewKMSKeyGenerator builds a new KMS key provider using the customer key ID and material
// description.
//
// Example:
// sess := session.New(&aws.Config{})
// cmkID := "arn to key"
// matdesc := s3crypto.MaterialDescription{}
// handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID)
func NewKMSKeyGenerator(kmsClient kmsiface.KMSAPI, cmkID string) CipherDataGenerator {
return NewKMSKeyGeneratorWithMatDesc(kmsClient, cmkID, MaterialDescription{})
}
// NewKMSKeyGeneratorWithMatDesc builds a new KMS key provider using the customer key ID and material
// description.
//
// Example:
// sess := session.New(&aws.Config{})
// cmkID := "arn to key"
// matdesc := s3crypto.MaterialDescription{}
// handler, err := s3crypto.NewKMSKeyGeneratorWithMatDesc(kms.New(sess), cmkID, matdesc)
func NewKMSKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGenerator {
if matdesc == nil {
matdesc = MaterialDescription{}
}
matdesc["kms_cmk_id"] = &cmkID
// These values are read only making them thread safe
kp := &kmsKeyHandler{
kms: kmsClient,
cmkID: &cmkID,
}
// These values are read only making them thread safe
kp.CipherData.WrapAlgorithm = KMSWrap
kp.CipherData.MaterialDescription = matdesc
return kp
}
// decryptHandler initializes a KMS keyprovider with a material description. This
// is used with Decrypting kms content, due to the cmkID being in the material description.
func (kp kmsKeyHandler) decryptHandler(env Envelope) (CipherDataDecrypter, error) {
m := MaterialDescription{}
err := m.decodeDescription([]byte(env.MatDesc))
if err != nil {
return nil, err
}
cmkID, ok := m["kms_cmk_id"]
if !ok {
return nil, awserr.New("MissingCMKIDError", "Material description is missing CMK ID", nil)
}
kp.CipherData.MaterialDescription = m
kp.cmkID = cmkID
kp.WrapAlgorithm = KMSWrap
return &kp, nil
}
// DecryptKey makes a call to KMS to decrypt the key.
func (kp *kmsKeyHandler) DecryptKey(key []byte) ([]byte, error) {
out, err := kp.kms.Decrypt(&kms.DecryptInput{
EncryptionContext: map[string]*string(kp.CipherData.MaterialDescription),
CiphertextBlob: key,
GrantTokens: []*string{},
})
if err != nil {
return nil, err
}
return out.Plaintext, nil
}
// GenerateCipherData makes a call to KMS to generate a data key, Upon making
// the call, it also sets the encrypted key.
func (kp *kmsKeyHandler) GenerateCipherData(keySize, ivSize int) (CipherData, error) {
out, err := kp.kms.GenerateDataKey(&kms.GenerateDataKeyInput{
EncryptionContext: kp.CipherData.MaterialDescription,
KeyId: kp.cmkID,
KeySpec: aws.String("AES_256"),
})
if err != nil {
return CipherData{}, err
}
iv := generateBytes(ivSize)
cd := CipherData{
Key: out.Plaintext,
IV: iv,
WrapAlgorithm: KMSWrap,
MaterialDescription: kp.CipherData.MaterialDescription,
EncryptedKey: out.CiphertextBlob,
}
return cd, nil
}
@@ -1,125 +0,0 @@
package s3crypto
import (
"bytes"
"encoding/base64"
"encoding/hex"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/kms"
)
func TestBuildKMSEncryptHandler(t *testing.T) {
svc := kms.New(unit.Session)
handler := NewKMSKeyGenerator(svc, "testid")
if handler == nil {
t.Error("expected non-nil handler")
}
}
func TestBuildKMSEncryptHandlerWithMatDesc(t *testing.T) {
svc := kms.New(unit.Session)
handler := NewKMSKeyGeneratorWithMatDesc(svc, "testid", MaterialDescription{
"Testing": aws.String("123"),
})
if handler == nil {
t.Error("expected non-nil handler")
}
kmsHandler := handler.(*kmsKeyHandler)
expected := MaterialDescription{
"kms_cmk_id": aws.String("testid"),
"Testing": aws.String("123"),
}
if !reflect.DeepEqual(expected, kmsHandler.CipherData.MaterialDescription) {
t.Errorf("expected %v, but received %v", expected, kmsHandler.CipherData.MaterialDescription)
}
}
func TestKMSGenerateCipherData(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, `{"CiphertextBlob":"AQEDAHhqBCCY1MSimw8gOGcUma79cn4ANvTtQyv9iuBdbcEF1QAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ6IcN5E4wVbk38MNAIBEIA7oF1E3lS7FY9DkoxPc/UmJsEwHzL82zMqoLwXIvi8LQHr8If4Lv6zKqY8u0+JRgSVoqCvZDx3p8Cn6nM=","KeyId":"arn:aws:kms:us-west-2:042062605278:key/c80a5cdb-8d09-4f9f-89ee-df01b2e3870a","Plaintext":"6tmyz9JLBE2yIuU7iXpArqpDVle172WSmxjcO6GNT7E="}`)
}))
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
svc := kms.New(sess)
handler := NewKMSKeyGenerator(svc, "testid")
keySize := 32
ivSize := 16
cd, err := handler.GenerateCipherData(keySize, ivSize)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if keySize != len(cd.Key) {
t.Errorf("expected %d, but received %d", keySize, len(cd.Key))
}
if ivSize != len(cd.IV) {
t.Errorf("expected %d, but received %d", ivSize, len(cd.IV))
}
}
func TestKMSDecrypt(t *testing.T) {
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
keyB64 := base64.URLEncoding.EncodeToString(key)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
handler, err := (kmsKeyHandler{kms: kms.New(sess)}).decryptHandler(Envelope{MatDesc: `{"kms_cmk_id":"test"}`})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
plaintextKey, err := handler.DecryptKey([]byte{1, 2, 3, 4})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !bytes.Equal(key, plaintextKey) {
t.Errorf("expected %v, but received %v", key, plaintextKey)
}
}
func TestKMSDecryptBadJSON(t *testing.T) {
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
keyB64 := base64.URLEncoding.EncodeToString(key)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
}))
sess := unit.Session.Copy(&aws.Config{
MaxRetries: aws.Int(0),
Endpoint: aws.String(ts.URL),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Region: aws.String("us-west-2"),
})
_, err := (kmsKeyHandler{kms: kms.New(sess)}).decryptHandler(Envelope{MatDesc: `{"kms_cmk_id":"test"`})
if err == nil {
t.Errorf("expected error, but received none")
}
}
-18
View File
@@ -1,18 +0,0 @@
package s3crypto
import (
"encoding/json"
)
// MaterialDescription is used to identify how and what master
// key has been used.
type MaterialDescription map[string]*string
func (md *MaterialDescription) encodeDescription() ([]byte, error) {
v, err := json.Marshal(&md)
return v, err
}
func (md *MaterialDescription) decodeDescription(b []byte) error {
return json.Unmarshal(b, &md)
}
-35
View File
@@ -1,35 +0,0 @@
package s3crypto
import (
"reflect"
"testing"
"github.com/aws/aws-sdk-go/aws"
)
func TestEncodeMaterialDescription(t *testing.T) {
md := MaterialDescription{}
md["foo"] = aws.String("bar")
b, err := md.encodeDescription()
expected := `{"foo":"bar"}`
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if expected != string(b) {
t.Errorf("expected %s, but received %s", expected, string(b))
}
}
func TestDecodeMaterialDescription(t *testing.T) {
md := MaterialDescription{}
json := `{"foo":"bar"}`
err := md.decodeDescription([]byte(json))
expected := MaterialDescription{
"foo": aws.String("bar"),
}
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !reflect.DeepEqual(expected, md) {
t.Error("expected material description to be equivalent, but received otherwise")
}
}
-70
View File
@@ -1,70 +0,0 @@
package s3crypto_test
import (
"bytes"
"io"
"io/ioutil"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
type mockGenerator struct {
}
func (m mockGenerator) GenerateCipherData(keySize, ivSize int) (s3crypto.CipherData, error) {
cd := s3crypto.CipherData{
Key: make([]byte, keySize),
IV: make([]byte, ivSize),
}
return cd, nil
}
func (m mockGenerator) EncryptKey(key []byte) ([]byte, error) {
size := len(key)
b := bytes.Repeat([]byte{1}, size)
return b, nil
}
func (m mockGenerator) DecryptKey(key []byte) ([]byte, error) {
return make([]byte, 16), nil
}
type mockCipherBuilder struct {
generator s3crypto.CipherDataGenerator
}
func (builder mockCipherBuilder) ContentCipher() (s3crypto.ContentCipher, error) {
cd, err := builder.generator.GenerateCipherData(32, 16)
if err != nil {
return nil, err
}
return &mockContentCipher{cd}, nil
}
type mockContentCipher struct {
cd s3crypto.CipherData
}
func (cipher *mockContentCipher) GetCipherData() s3crypto.CipherData {
return cipher.cd
}
func (cipher *mockContentCipher) EncryptContents(src io.Reader) (io.Reader, error) {
b, err := ioutil.ReadAll(src)
if err != nil {
return nil, err
}
size := len(b)
b = bytes.Repeat([]byte{1}, size)
return bytes.NewReader(b), nil
}
func (cipher *mockContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser, error) {
b, err := ioutil.ReadAll(src)
if err != nil {
return nil, err
}
size := len(b)
return ioutil.NopCloser(bytes.NewReader(make([]byte, size))), nil
}
-35
View File
@@ -1,35 +0,0 @@
package s3crypto
// Padder handles padding of crypto data
type Padder interface {
// Pad will pad the byte array.
// The second parameter is NOT how many
// bytes to pad by, but how many bytes
// have been read prior to the padding.
// This allows for streamable padding.
Pad([]byte, int) ([]byte, error)
// Unpad will unpad the byte bytes. Unpad
// methods must be constant time.
Unpad([]byte) ([]byte, error)
// Name returns the name of the padder.
// This is used when decrypting on
// instantiating new padders.
Name() string
}
// NoPadder does not pad anything
var NoPadder = Padder(noPadder{})
type noPadder struct{}
func (padder noPadder) Pad(b []byte, n int) ([]byte, error) {
return b, nil
}
func (padder noPadder) Unpad(b []byte) ([]byte, error) {
return b, nil
}
func (padder noPadder) Name() string {
return "NoPadding"
}
-80
View File
@@ -1,80 +0,0 @@
package s3crypto
// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Portions Licensed under the MIT License. Copyright (c) 2016 Carl Jackson
import (
"bytes"
"crypto/subtle"
"github.com/aws/aws-sdk-go/aws/awserr"
)
const (
pkcs7MaxPaddingSize = 255
)
type pkcs7Padder struct {
blockSize int
}
// NewPKCS7Padder follows the RFC 2315: https://www.ietf.org/rfc/rfc2315.txt
// PKCS7 padding is subject to side-channel attacks and timing attacks. For
// the most secure data, use an authenticated crypto algorithm.
func NewPKCS7Padder(blockSize int) Padder {
return pkcs7Padder{blockSize}
}
var errPKCS7Padding = awserr.New("InvalidPadding", "invalid padding", nil)
// Pad will pad the data relative to how many bytes have been read.
// Pad follows the PKCS7 standard.
func (padder pkcs7Padder) Pad(buf []byte, n int) ([]byte, error) {
if padder.blockSize < 1 || padder.blockSize > pkcs7MaxPaddingSize {
return nil, awserr.New("InvalidBlockSize", "block size must be between 1 and 255", nil)
}
size := padder.blockSize - (n % padder.blockSize)
pad := bytes.Repeat([]byte{byte(size)}, size)
buf = append(buf, pad...)
return buf, nil
}
// Unpad will unpad the correct amount of bytes based off
// of the PKCS7 standard
func (padder pkcs7Padder) Unpad(buf []byte) ([]byte, error) {
if len(buf) == 0 {
return nil, errPKCS7Padding
}
// Here be dragons. We're attempting to check the padding in constant
// time. The only piece of information here which is public is len(buf).
// This code is modeled loosely after tls1_cbc_remove_padding from
// OpenSSL.
padLen := buf[len(buf)-1]
toCheck := pkcs7MaxPaddingSize
good := 1
if toCheck > len(buf) {
toCheck = len(buf)
}
for i := 0; i < toCheck; i++ {
b := buf[len(buf)-1-i]
outOfRange := subtle.ConstantTimeLessOrEq(int(padLen), i)
equal := subtle.ConstantTimeByteEq(padLen, b)
good &= subtle.ConstantTimeSelect(outOfRange, 1, equal)
}
good &= subtle.ConstantTimeLessOrEq(1, int(padLen))
good &= subtle.ConstantTimeLessOrEq(int(padLen), len(buf))
if good != 1 {
return nil, errPKCS7Padding
}
return buf[:len(buf)-int(padLen)], nil
}
func (padder pkcs7Padder) Name() string {
return "PKCS7Padding"
}
@@ -1,57 +0,0 @@
package s3crypto_test
import (
"bytes"
"fmt"
"testing"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
func padTest(size int, t *testing.T) {
padder := s3crypto.NewPKCS7Padder(size)
for i := 0; i < size; i++ {
input := make([]byte, i)
expected := append(input, bytes.Repeat([]byte{byte(size - i)}, size-i)...)
b, err := padder.Pad(input, len(input))
if err != nil {
t.Fatal("Expected error to be nil but received " + err.Error())
}
if len(b) != len(expected) {
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
}
if bytes.Compare(b, expected) != 0 {
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
}
}
}
func unpadTest(size int, t *testing.T) {
padder := s3crypto.NewPKCS7Padder(size)
for i := 0; i < size; i++ {
expected := make([]byte, i)
input := append(expected, bytes.Repeat([]byte{byte(size - i)}, size-i)...)
b, err := padder.Unpad(input)
if err != nil {
t.Fatal("Error received, was expecting nil: " + err.Error())
}
if len(b) != len(expected) {
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
}
if bytes.Compare(b, expected) != 0 {
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
}
}
}
func TestPKCS7Padding(t *testing.T) {
padTest(10, t)
padTest(16, t)
padTest(255, t)
}
func TestPKCS7Unpadding(t *testing.T) {
unpadTest(10, t)
unpadTest(16, t)
unpadTest(255, t)
}
-145
View File
@@ -1,145 +0,0 @@
package s3crypto
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
)
// SaveStrategy is how the data's metadata wants to be saved
type SaveStrategy interface {
Save(Envelope, *request.Request) error
}
// S3SaveStrategy will save the metadata to a separate instruction file in S3
type S3SaveStrategy struct {
Client *s3.S3
InstructionFileSuffix string
}
// Save will save the envelope contents to s3.
func (strat S3SaveStrategy) Save(env Envelope, req *request.Request) error {
input := req.Params.(*s3.PutObjectInput)
b, err := json.Marshal(env)
if err != nil {
return err
}
instInput := s3.PutObjectInput{
Bucket: input.Bucket,
Body: bytes.NewReader(b),
}
if strat.InstructionFileSuffix == "" {
instInput.Key = aws.String(*input.Key + DefaultInstructionKeySuffix)
} else {
instInput.Key = aws.String(*input.Key + strat.InstructionFileSuffix)
}
_, err = strat.Client.PutObject(&instInput)
return err
}
// HeaderV2SaveStrategy will save the metadata of the crypto contents to the header of
// the object.
type HeaderV2SaveStrategy struct{}
// Save will save the envelope to the request's header.
func (strat HeaderV2SaveStrategy) Save(env Envelope, req *request.Request) error {
input := req.Params.(*s3.PutObjectInput)
if input.Metadata == nil {
input.Metadata = map[string]*string{}
}
input.Metadata[http.CanonicalHeaderKey(keyV2Header)] = &env.CipherKey
input.Metadata[http.CanonicalHeaderKey(ivHeader)] = &env.IV
input.Metadata[http.CanonicalHeaderKey(matDescHeader)] = &env.MatDesc
input.Metadata[http.CanonicalHeaderKey(wrapAlgorithmHeader)] = &env.WrapAlg
input.Metadata[http.CanonicalHeaderKey(cekAlgorithmHeader)] = &env.CEKAlg
input.Metadata[http.CanonicalHeaderKey(unencryptedMD5Header)] = &env.UnencryptedMD5
input.Metadata[http.CanonicalHeaderKey(unencryptedContentLengthHeader)] = &env.UnencryptedContentLen
if len(env.TagLen) > 0 {
input.Metadata[http.CanonicalHeaderKey(tagLengthHeader)] = &env.TagLen
}
return nil
}
// LoadStrategy ...
type LoadStrategy interface {
Load(*request.Request) (Envelope, error)
}
// S3LoadStrategy will load the instruction file from s3
type S3LoadStrategy struct {
Client *s3.S3
InstructionFileSuffix string
}
// Load from a given instruction file suffix
func (load S3LoadStrategy) Load(req *request.Request) (Envelope, error) {
env := Envelope{}
if load.InstructionFileSuffix == "" {
load.InstructionFileSuffix = DefaultInstructionKeySuffix
}
input := req.Params.(*s3.GetObjectInput)
out, err := load.Client.GetObject(&s3.GetObjectInput{
Key: aws.String(strings.Join([]string{*input.Key, load.InstructionFileSuffix}, "")),
Bucket: input.Bucket,
})
if err != nil {
return env, err
}
b, err := ioutil.ReadAll(out.Body)
if err != nil {
return env, err
}
err = json.Unmarshal(b, &env)
return env, err
}
// HeaderV2LoadStrategy will load the envelope from the metadata
type HeaderV2LoadStrategy struct{}
// Load from a given object's header
func (load HeaderV2LoadStrategy) Load(req *request.Request) (Envelope, error) {
env := Envelope{}
env.CipherKey = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, keyV2Header}, "-"))
env.IV = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, ivHeader}, "-"))
env.MatDesc = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, matDescHeader}, "-"))
env.WrapAlg = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, wrapAlgorithmHeader}, "-"))
env.CEKAlg = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, cekAlgorithmHeader}, "-"))
env.TagLen = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, tagLengthHeader}, "-"))
env.UnencryptedMD5 = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, unencryptedMD5Header}, "-"))
env.UnencryptedContentLen = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, unencryptedContentLengthHeader}, "-"))
return env, nil
}
type defaultV2LoadStrategy struct {
client *s3.S3
suffix string
}
func (load defaultV2LoadStrategy) Load(req *request.Request) (Envelope, error) {
if value := req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, keyV2Header}, "-")); value != "" {
strat := HeaderV2LoadStrategy{}
return strat.Load(req)
} else if value = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, keyV1Header}, "-")); value != "" {
return Envelope{}, awserr.New("V1NotSupportedError", "The AWS SDK for Go does not support version 1", nil)
}
strat := S3LoadStrategy{
Client: load.client,
InstructionFileSuffix: load.suffix,
}
return strat.Load(req)
}
-77
View File
@@ -1,77 +0,0 @@
package s3crypto_test
import (
"reflect"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
)
func TestHeaderV2SaveStrategy(t *testing.T) {
cases := []struct {
env s3crypto.Envelope
expected map[string]*string
}{
{
s3crypto.Envelope{
CipherKey: "Foo",
IV: "Bar",
MatDesc: "{}",
WrapAlg: s3crypto.KMSWrap,
CEKAlg: s3crypto.AESGCMNoPadding,
TagLen: "128",
UnencryptedMD5: "hello",
UnencryptedContentLen: "0",
},
map[string]*string{
"X-Amz-Key-V2": aws.String("Foo"),
"X-Amz-Iv": aws.String("Bar"),
"X-Amz-Matdesc": aws.String("{}"),
"X-Amz-Wrap-Alg": aws.String(s3crypto.KMSWrap),
"X-Amz-Cek-Alg": aws.String(s3crypto.AESGCMNoPadding),
"X-Amz-Tag-Len": aws.String("128"),
"X-Amz-Unencrypted-Content-Md5": aws.String("hello"),
"X-Amz-Unencrypted-Content-Length": aws.String("0"),
},
},
{
s3crypto.Envelope{
CipherKey: "Foo",
IV: "Bar",
MatDesc: "{}",
WrapAlg: s3crypto.KMSWrap,
CEKAlg: s3crypto.AESGCMNoPadding,
UnencryptedMD5: "hello",
UnencryptedContentLen: "0",
},
map[string]*string{
"X-Amz-Key-V2": aws.String("Foo"),
"X-Amz-Iv": aws.String("Bar"),
"X-Amz-Matdesc": aws.String("{}"),
"X-Amz-Wrap-Alg": aws.String(s3crypto.KMSWrap),
"X-Amz-Cek-Alg": aws.String(s3crypto.AESGCMNoPadding),
"X-Amz-Unencrypted-Content-Md5": aws.String("hello"),
"X-Amz-Unencrypted-Content-Length": aws.String("0"),
},
},
}
for _, c := range cases {
params := &s3.PutObjectInput{}
req := &request.Request{
Params: params,
}
strat := s3crypto.HeaderV2SaveStrategy{}
err := strat.Save(c.env, req)
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if !reflect.DeepEqual(c.expected, params.Metadata) {
t.Errorf("expected %v, but received %v", c.expected, params.Metadata)
}
}
}
-116
View File
@@ -1,116 +0,0 @@
// +build go1.7
package s3manager
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
)
// #1790 bug
func TestBatchDeleteContext(t *testing.T) {
cases := []struct {
objects []BatchDeleteObject
size int
expected int
ctx aws.Context
closeAt int
errCheck func(error) (string, bool)
}{
{
[]BatchDeleteObject{
{
Object: &s3.DeleteObjectInput{
Key: aws.String("1"),
Bucket: aws.String("bucket1"),
},
},
{
Object: &s3.DeleteObjectInput{
Key: aws.String("2"),
Bucket: aws.String("bucket2"),
},
},
{
Object: &s3.DeleteObjectInput{
Key: aws.String("3"),
Bucket: aws.String("bucket3"),
},
},
{
Object: &s3.DeleteObjectInput{
Key: aws.String("4"),
Bucket: aws.String("bucket4"),
},
},
},
1,
0,
aws.BackgroundContext(),
0,
func(err error) (string, bool) {
batchErr, ok := err.(*BatchError)
if !ok {
return "not BatchError type", false
}
errs := batchErr.Errors
if len(errs) != 4 {
return fmt.Sprintf("expected 1, but received %d", len(errs)), false
}
for _, tempErr := range errs {
aerr, ok := tempErr.OrigErr.(awserr.Error)
if !ok {
return "not awserr.Error type", false
}
if code := aerr.Code(); code != request.CanceledErrorCode {
return fmt.Sprintf("expected %q, but received %q", request.CanceledErrorCode, code), false
}
}
return "", true
},
},
}
count := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
count++
}))
svc := &mockS3Client{S3: buildS3SvcClient(server.URL)}
for i, c := range cases {
ctx, done := context.WithCancel(c.ctx)
defer done()
if i == c.closeAt {
done()
}
batcher := BatchDelete{
Client: svc,
BatchSize: c.size,
}
err := batcher.Delete(ctx, &DeleteObjectsIterator{Objects: c.objects})
if msg, ok := c.errCheck(err); !ok {
t.Error(msg)
}
if count != c.expected {
t.Errorf("Case %d: expected %d, but received %d", i, c.expected, count)
}
count = 0
}
}
File diff suppressed because it is too large Load Diff
@@ -1,96 +0,0 @@
package s3manager
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
func testSetupGetBucketRegionServer(region string, statusCode int, incHeader bool) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if incHeader {
w.Header().Set(bucketRegionHeader, region)
}
w.WriteHeader(statusCode)
}))
}
var testGetBucketRegionCases = []struct {
RespRegion string
StatusCode int
HintRegion string
ExpectReqRegion string
}{
{"bucket-region", 301, "hint-region", ""},
{"bucket-region", 403, "hint-region", ""},
{"bucket-region", 200, "hint-region", ""},
{"bucket-region", 200, "", "default-region"},
}
func TestGetBucketRegion_Exists(t *testing.T) {
for i, c := range testGetBucketRegionCases {
server := testSetupGetBucketRegionServer(c.RespRegion, c.StatusCode, true)
sess := unit.Session.Copy()
sess.Config.Region = aws.String("default-region")
sess.Config.Endpoint = aws.String(server.URL)
sess.Config.DisableSSL = aws.Bool(true)
ctx := aws.BackgroundContext()
region, err := GetBucketRegion(ctx, sess, "bucket", c.HintRegion)
if err != nil {
t.Fatalf("%d, expect no error, got %v", i, err)
}
if e, a := c.RespRegion, region; e != a {
t.Errorf("%d, expect %q region, got %q", i, e, a)
}
}
}
func TestGetBucketRegion_NotExists(t *testing.T) {
server := testSetupGetBucketRegionServer("ignore-region", 404, false)
sess := unit.Session.Copy()
sess.Config.Endpoint = aws.String(server.URL)
sess.Config.DisableSSL = aws.Bool(true)
ctx := aws.BackgroundContext()
region, err := GetBucketRegion(ctx, sess, "bucket", "hint-region")
if err == nil {
t.Fatalf("expect error, but did not get one")
}
aerr := err.(awserr.Error)
if e, a := "NotFound", aerr.Code(); e != a {
t.Errorf("expect %s error code, got %s", e, a)
}
if len(region) != 0 {
t.Errorf("expect region not to be set, got %q", region)
}
}
func TestGetBucketRegionWithClient(t *testing.T) {
for i, c := range testGetBucketRegionCases {
server := testSetupGetBucketRegionServer(c.RespRegion, c.StatusCode, true)
svc := s3.New(unit.Session, &aws.Config{
Region: aws.String("hint-region"),
Endpoint: aws.String(server.URL),
DisableSSL: aws.Bool(true),
})
ctx := aws.BackgroundContext()
region, err := GetBucketRegionWithClient(ctx, svc, "bucket")
if err != nil {
t.Fatalf("%d, expect no error, got %v", i, err)
}
if e, a := c.RespRegion, region; e != a {
t.Errorf("%d, expect %q region, got %q", i, e, a)
}
}
}
-658
View File
@@ -1,658 +0,0 @@
package s3manager_test
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"reflect"
"regexp"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
func dlLoggingSvc(data []byte) (*s3.S3, *[]string, *[]string) {
var m sync.Mutex
names := []string{}
ranges := []string{}
svc := s3.New(unit.Session)
svc.Handlers.Send.Clear()
svc.Handlers.Send.PushBack(func(r *request.Request) {
m.Lock()
defer m.Unlock()
names = append(names, r.Operation.Name)
ranges = append(ranges, *r.Params.(*s3.GetObjectInput).Range)
rerng := regexp.MustCompile(`bytes=(\d+)-(\d+)`)
rng := rerng.FindStringSubmatch(r.HTTPRequest.Header.Get("Range"))
start, _ := strconv.ParseInt(rng[1], 10, 64)
fin, _ := strconv.ParseInt(rng[2], 10, 64)
fin++
if fin > int64(len(data)) {
fin = int64(len(data))
}
bodyBytes := data[start:fin]
r.HTTPResponse = &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)),
Header: http.Header{},
}
r.HTTPResponse.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d",
start, fin-1, len(data)))
r.HTTPResponse.Header.Set("Content-Length", fmt.Sprintf("%d", len(bodyBytes)))
})
return svc, &names, &ranges
}
func dlLoggingSvcNoChunk(data []byte) (*s3.S3, *[]string) {
var m sync.Mutex
names := []string{}
svc := s3.New(unit.Session)
svc.Handlers.Send.Clear()
svc.Handlers.Send.PushBack(func(r *request.Request) {
m.Lock()
defer m.Unlock()
names = append(names, r.Operation.Name)
r.HTTPResponse = &http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewReader(data[:])),
Header: http.Header{},
}
r.HTTPResponse.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
})
return svc, &names
}
func dlLoggingSvcNoContentRangeLength(data []byte, states []int) (*s3.S3, *[]string) {
var m sync.Mutex
names := []string{}
var index int = 0
svc := s3.New(unit.Session)
svc.Handlers.Send.Clear()
svc.Handlers.Send.PushBack(func(r *request.Request) {
m.Lock()
defer m.Unlock()
names = append(names, r.Operation.Name)
r.HTTPResponse = &http.Response{
StatusCode: states[index],
Body: ioutil.NopCloser(bytes.NewReader(data[:])),
Header: http.Header{},
}
index++
})
return svc, &names
}
func dlLoggingSvcContentRangeTotalAny(data []byte, states []int) (*s3.S3, *[]string) {
var m sync.Mutex
names := []string{}
ranges := []string{}
var index int = 0
svc := s3.New(unit.Session)
svc.Handlers.Send.Clear()
svc.Handlers.Send.PushBack(func(r *request.Request) {
m.Lock()
defer m.Unlock()
names = append(names, r.Operation.Name)
ranges = append(ranges, *r.Params.(*s3.GetObjectInput).Range)
rerng := regexp.MustCompile(`bytes=(\d+)-(\d+)`)
rng := rerng.FindStringSubmatch(r.HTTPRequest.Header.Get("Range"))
start, _ := strconv.ParseInt(rng[1], 10, 64)
fin, _ := strconv.ParseInt(rng[2], 10, 64)
fin++
if fin >= int64(len(data)) {
fin = int64(len(data))
}
// Setting start and finish to 0 because this state of 1 is suppose to
// be an error state of 416
if index == len(states)-1 {
start = 0
fin = 0
}
bodyBytes := data[start:fin]
r.HTTPResponse = &http.Response{
StatusCode: states[index],
Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)),
Header: http.Header{},
}
r.HTTPResponse.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/*",
start, fin-1))
index++
})
return svc, &names
}
func dlLoggingSvcWithErrReader(cases []testErrReader) (*s3.S3, *[]string) {
var m sync.Mutex
names := []string{}
var index int = 0
svc := s3.New(unit.Session, &aws.Config{
MaxRetries: aws.Int(len(cases) - 1),
})
svc.Handlers.Send.Clear()
svc.Handlers.Send.PushBack(func(r *request.Request) {
m.Lock()
defer m.Unlock()
names = append(names, r.Operation.Name)
c := cases[index]
r.HTTPResponse = &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(&c),
Header: http.Header{},
}
r.HTTPResponse.Header.Set("Content-Range",
fmt.Sprintf("bytes %d-%d/%d", 0, c.Len-1, c.Len))
r.HTTPResponse.Header.Set("Content-Length", fmt.Sprintf("%d", c.Len))
index++
})
return svc, &names
}
func TestDownloadOrder(t *testing.T) {
s, names, ranges := dlLoggingSvc(buf12MB)
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(len(buf12MB)), n; e != a {
t.Errorf("expect %d buffer length, got %d", e, a)
}
expectCalls := []string{"GetObject", "GetObject", "GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
expectRngs := []string{"bytes=0-5242879", "bytes=5242880-10485759", "bytes=10485760-15728639"}
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v ranges, got %v", e, a)
}
count := 0
for _, b := range w.Bytes() {
count += int(b)
}
if count != 0 {
t.Errorf("expect 0 count, got %d", count)
}
}
func TestDownloadZero(t *testing.T) {
s, names, ranges := dlLoggingSvc([]byte{})
d := s3manager.NewDownloaderWithClient(s)
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if n != 0 {
t.Errorf("expect 0 bytes read, got %d", n)
}
expectCalls := []string{"GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
expectRngs := []string{"bytes=0-5242879"}
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v ranges, got %v", e, a)
}
}
func TestDownloadSetPartSize(t *testing.T) {
s, names, ranges := dlLoggingSvc([]byte{1, 2, 3})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
d.PartSize = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(3), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject", "GetObject", "GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
expectRngs := []string{"bytes=0-0", "bytes=1-1", "bytes=2-2"}
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v ranges, got %v", e, a)
}
expectBytes := []byte{1, 2, 3}
if e, a := expectBytes, w.Bytes(); !reflect.DeepEqual(e, a) {
t.Errorf("expect %v bytes, got %v", e, a)
}
}
func TestDownloadError(t *testing.T) {
s, names, _ := dlLoggingSvc([]byte{1, 2, 3})
num := 0
s.Handlers.Send.PushBack(func(r *request.Request) {
num++
if num > 1 {
r.HTTPResponse.StatusCode = 400
r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
}
})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
d.PartSize = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err == nil {
t.Fatalf("expect error, got none")
}
aerr := err.(awserr.Error)
if e, a := "BadRequest", aerr.Code(); e != a {
t.Errorf("expect %s error code, got %s", e, a)
}
if e, a := int64(1), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject", "GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
expectBytes := []byte{1}
if e, a := expectBytes, w.Bytes(); !reflect.DeepEqual(e, a) {
t.Errorf("expect %v bytes, got %v", e, a)
}
}
func TestDownloadNonChunk(t *testing.T) {
s, names := dlLoggingSvcNoChunk(buf2MB)
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(len(buf2MB)), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
count := 0
for _, b := range w.Bytes() {
count += int(b)
}
if count != 0 {
t.Errorf("expect 0 count, got %d", count)
}
}
func TestDownloadNoContentRangeLength(t *testing.T) {
s, names := dlLoggingSvcNoContentRangeLength(buf2MB, []int{200, 416})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(len(buf2MB)), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject", "GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
count := 0
for _, b := range w.Bytes() {
count += int(b)
}
if count != 0 {
t.Errorf("expect 0 count, got %d", count)
}
}
func TestDownloadContentRangeTotalAny(t *testing.T) {
s, names := dlLoggingSvcContentRangeTotalAny(buf2MB, []int{200, 416})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(len(buf2MB)), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject", "GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
count := 0
for _, b := range w.Bytes() {
count += int(b)
}
if count != 0 {
t.Errorf("expect 0 count, got %d", count)
}
}
func TestDownloadPartBodyRetry_SuccessRetry(t *testing.T) {
s, names := dlLoggingSvcWithErrReader([]testErrReader{
{Buf: []byte("ab"), Len: 3, Err: io.ErrUnexpectedEOF},
{Buf: []byte("123"), Len: 3, Err: io.EOF},
})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(3), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject", "GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
if e, a := "123", string(w.Bytes()); e != a {
t.Errorf("expect %q response, got %q", e, a)
}
}
func TestDownloadPartBodyRetry_SuccessNoRetry(t *testing.T) {
s, names := dlLoggingSvcWithErrReader([]testErrReader{
{Buf: []byte("abc"), Len: 3, Err: io.EOF},
})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(3), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
if e, a := "abc", string(w.Bytes()); e != a {
t.Errorf("expect %q response, got %q", e, a)
}
}
func TestDownloadPartBodyRetry_FailRetry(t *testing.T) {
s, names := dlLoggingSvcWithErrReader([]testErrReader{
{Buf: []byte("ab"), Len: 3, Err: io.ErrUnexpectedEOF},
})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 1
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
})
if err == nil {
t.Fatalf("expect error, got none")
}
if e, a := "unexpected EOF", err.Error(); !strings.Contains(a, e) {
t.Errorf("expect %q error message to be in %q", e, a)
}
if e, a := int64(2), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
if e, a := "ab", string(w.Bytes()); e != a {
t.Errorf("expect %q response, got %q", e, a)
}
}
func TestDownloadWithContextCanceled(t *testing.T) {
d := s3manager.NewDownloader(unit.Session)
params := s3.GetObjectInput{
Bucket: aws.String("Bucket"),
Key: aws.String("Key"),
}
ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
ctx.Error = fmt.Errorf("context canceled")
close(ctx.DoneCh)
w := &aws.WriteAtBuffer{}
_, err := d.DownloadWithContext(ctx, w, &params)
if err == nil {
t.Fatalf("expected error, did not get one")
}
aerr := err.(awserr.Error)
if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
t.Errorf("expected error code %q, got %q", e, a)
}
if e, a := "canceled", aerr.Message(); !strings.Contains(a, e) {
t.Errorf("expected error message to contain %q, but did not %q", e, a)
}
}
func TestDownload_WithRange(t *testing.T) {
s, names, ranges := dlLoggingSvc([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
d.Concurrency = 10 // should be ignored
d.PartSize = 1 // should be ignored
})
w := &aws.WriteAtBuffer{}
n, err := d.Download(w, &s3.GetObjectInput{
Bucket: aws.String("bucket"),
Key: aws.String("key"),
Range: aws.String("bytes=2-6"),
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
if e, a := int64(5), n; e != a {
t.Errorf("expect %d bytes read, got %d", e, a)
}
expectCalls := []string{"GetObject"}
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v API calls, got %v", e, a)
}
expectRngs := []string{"bytes=2-6"}
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
t.Errorf("expect %v ranges, got %v", e, a)
}
expectBytes := []byte{2, 3, 4, 5, 6}
if e, a := expectBytes, w.Bytes(); !reflect.DeepEqual(e, a) {
t.Errorf("expect %v bytes, got %v", e, a)
}
}
func TestDownload_WithFailure(t *testing.T) {
svc := s3.New(unit.Session)
svc.Handlers.Send.Clear()
first := true
svc.Handlers.Send.PushBack(func(r *request.Request) {
if first {
first = false
body := bytes.NewReader(make([]byte, s3manager.DefaultDownloadPartSize))
r.HTTPResponse = &http.Response{
StatusCode: http.StatusOK,
Status: http.StatusText(http.StatusOK),
ContentLength: int64(body.Len()),
Body: ioutil.NopCloser(body),
Header: http.Header{},
}
r.HTTPResponse.Header.Set("Content-Length", strconv.Itoa(body.Len()))
r.HTTPResponse.Header.Set("Content-Range",
fmt.Sprintf("bytes 0-%d/%d", body.Len()-1, body.Len()*10))
return
}
// Give a chance for the multipart chunks to be queued up
time.Sleep(1 * time.Second)
r.HTTPResponse = &http.Response{
Header: http.Header{},
Body: ioutil.NopCloser(&bytes.Buffer{}),
}
r.Error = awserr.New("ConnectionError", "some connection error", nil)
r.Retryable = aws.Bool(false)
})
start := time.Now()
d := s3manager.NewDownloaderWithClient(svc, func(d *s3manager.Downloader) {
d.Concurrency = 2
})
w := &aws.WriteAtBuffer{}
params := s3.GetObjectInput{
Bucket: aws.String("Bucket"),
Key: aws.String("Key"),
}
// Expect this request to exit quickly after failure
_, err := d.Download(w, &params)
if err == nil {
t.Fatalf("expect error, got none")
}
limit := start.Add(5 * time.Second)
dur := time.Now().Sub(start)
if time.Now().After(limit) {
t.Errorf("expect time to be less than %v, took %v", limit, dur)
}
}
type testErrReader struct {
Buf []byte
Err error
Len int64
off int
}
func (r *testErrReader) Read(p []byte) (int, error) {
to := len(r.Buf) - r.off
n := copy(p, r.Buf[r.off:to])
r.off += n
if n < len(p) {
return n, r.Err
}
return n, nil
}
@@ -1,26 +0,0 @@
// Package s3manageriface provides an interface for the s3manager package
package s3manageriface
import (
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
// DownloaderAPI is the interface type for s3manager.Downloader.
type DownloaderAPI interface {
Download(io.WriterAt, *s3.GetObjectInput, ...func(*s3manager.Downloader)) (int64, error)
DownloadWithContext(aws.Context, io.WriterAt, *s3.GetObjectInput, ...func(*s3manager.Downloader)) (int64, error)
}
var _ DownloaderAPI = (*s3manager.Downloader)(nil)
// UploaderAPI is the interface type for s3manager.Uploader.
type UploaderAPI interface {
Upload(*s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
UploadWithContext(aws.Context, *s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
}
var _ UploaderAPI = (*s3manager.Uploader)(nil)
-4
View File
@@ -1,4 +0,0 @@
package s3manager_test
var buf12MB = make([]byte, 1024*1024*12)
var buf2MB = make([]byte, 1024*1024*2)
File diff suppressed because it is too large Load Diff
-111
View File
@@ -1,111 +0,0 @@
package s3_test
import (
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
func TestSSECustomerKeyOverHTTPError(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{DisableSSL: aws.Bool(true)})
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
Bucket: aws.String("bucket"),
CopySource: aws.String("bucket/source"),
Key: aws.String("dest"),
SSECustomerKey: aws.String("key"),
})
err := req.Build()
if err == nil {
t.Error("expected an error")
}
if e, a := "ConfigError", err.(awserr.Error).Code(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if !strings.Contains(err.(awserr.Error).Message(), "cannot send SSE keys over HTTP") {
t.Errorf("expected error to contain 'cannot send SSE keys over HTTP', but received %s", err.(awserr.Error).Message())
}
}
func TestCopySourceSSECustomerKeyOverHTTPError(t *testing.T) {
s := s3.New(unit.Session, &aws.Config{DisableSSL: aws.Bool(true)})
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
Bucket: aws.String("bucket"),
CopySource: aws.String("bucket/source"),
Key: aws.String("dest"),
CopySourceSSECustomerKey: aws.String("key"),
})
err := req.Build()
if err == nil {
t.Error("expected an error")
}
if e, a := "ConfigError", err.(awserr.Error).Code(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if !strings.Contains(err.(awserr.Error).Message(), "cannot send SSE keys over HTTP") {
t.Errorf("expected error to contain 'cannot send SSE keys over HTTP', but received %s", err.(awserr.Error).Message())
}
}
func TestComputeSSEKeys(t *testing.T) {
s := s3.New(unit.Session)
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
Bucket: aws.String("bucket"),
CopySource: aws.String("bucket/source"),
Key: aws.String("dest"),
SSECustomerKey: aws.String("key"),
CopySourceSSECustomerKey: aws.String("key"),
})
err := req.Build()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "PG4LipwVIkqCKLmpjKFTHQ==", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key-md5"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "PG4LipwVIkqCKLmpjKFTHQ==", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key-md5"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
func TestComputeSSEKeysShortcircuit(t *testing.T) {
s := s3.New(unit.Session)
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
Bucket: aws.String("bucket"),
CopySource: aws.String("bucket/source"),
Key: aws.String("dest"),
SSECustomerKey: aws.String("key"),
CopySourceSSECustomerKey: aws.String("key"),
SSECustomerKeyMD5: aws.String("MD5"),
CopySourceSSECustomerKeyMD5: aws.String("MD5"),
})
err := req.Build()
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "MD5", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key-md5"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "MD5", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key-md5"); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
-166
View File
@@ -1,166 +0,0 @@
package s3_test
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
const errMsg = `<Error><Code>ErrorCode</Code><Message>message body</Message><RequestId>requestID</RequestId><HostId>hostID=</HostId></Error>`
var lastModifiedTime = time.Date(2009, 11, 23, 0, 0, 0, 0, time.UTC)
func TestCopyObjectNoError(t *testing.T) {
const successMsg = `
<?xml version="1.0" encoding="UTF-8"?>
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2009-11-23T0:00:00Z</LastModified><ETag>&quot;1da64c7f13d1e8dbeaea40b905fd586c&quot;</ETag></CopyObjectResult>`
res, err := newCopyTestSvc(successMsg).CopyObject(&s3.CopyObjectInput{
Bucket: aws.String("bucketname"),
CopySource: aws.String("bucketname/exists.txt"),
Key: aws.String("destination.txt"),
})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := fmt.Sprintf(`%q`, "1da64c7f13d1e8dbeaea40b905fd586c"), *res.CopyObjectResult.ETag; e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := lastModifiedTime, *res.CopyObjectResult.LastModified; !e.Equal(a) {
t.Errorf("expected %v, but received %v", e, a)
}
}
func TestCopyObjectError(t *testing.T) {
_, err := newCopyTestSvc(errMsg).CopyObject(&s3.CopyObjectInput{
Bucket: aws.String("bucketname"),
CopySource: aws.String("bucketname/doesnotexist.txt"),
Key: aws.String("destination.txt"),
})
if err == nil {
t.Error("expected error, but received none")
}
e := err.(awserr.Error)
if e, a := "ErrorCode", e.Code(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "message body", e.Message(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
func TestUploadPartCopySuccess(t *testing.T) {
const successMsg = `
<?xml version="1.0" encoding="UTF-8"?>
<UploadPartCopyResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2009-11-23T0:00:00Z</LastModified><ETag>&quot;1da64c7f13d1e8dbeaea40b905fd586c&quot;</ETag></UploadPartCopyResult>`
res, err := newCopyTestSvc(successMsg).UploadPartCopy(&s3.UploadPartCopyInput{
Bucket: aws.String("bucketname"),
CopySource: aws.String("bucketname/doesnotexist.txt"),
Key: aws.String("destination.txt"),
PartNumber: aws.Int64(0),
UploadId: aws.String("uploadID"),
})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := fmt.Sprintf(`%q`, "1da64c7f13d1e8dbeaea40b905fd586c"), *res.CopyPartResult.ETag; e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := lastModifiedTime, *res.CopyPartResult.LastModified; !e.Equal(a) {
t.Errorf("expected %v, but received %v", e, a)
}
}
func TestUploadPartCopyError(t *testing.T) {
_, err := newCopyTestSvc(errMsg).UploadPartCopy(&s3.UploadPartCopyInput{
Bucket: aws.String("bucketname"),
CopySource: aws.String("bucketname/doesnotexist.txt"),
Key: aws.String("destination.txt"),
PartNumber: aws.Int64(0),
UploadId: aws.String("uploadID"),
})
if err == nil {
t.Error("expected an error")
}
e := err.(awserr.Error)
if e, a := "ErrorCode", e.Code(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "message body", e.Message(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
func TestCompleteMultipartUploadSuccess(t *testing.T) {
const successMsg = `
<?xml version="1.0" encoding="UTF-8"?>
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Location>locationName</Location><Bucket>bucketName</Bucket><Key>keyName</Key><ETag>"etagVal"</ETag></CompleteMultipartUploadResult>`
res, err := newCopyTestSvc(successMsg).CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String("bucketname"),
Key: aws.String("key"),
UploadId: aws.String("uploadID"),
})
if err != nil {
t.Errorf("expected no error, but received %v", err)
}
if e, a := `"etagVal"`, *res.ETag; e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "bucketName", *res.Bucket; e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "keyName", *res.Key; e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "locationName", *res.Location; e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
func TestCompleteMultipartUploadError(t *testing.T) {
_, err := newCopyTestSvc(errMsg).CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String("bucketname"),
Key: aws.String("key"),
UploadId: aws.String("uploadID"),
})
if err == nil {
t.Error("expected an error")
}
e := err.(awserr.Error)
if e, a := "ErrorCode", e.Code(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
if e, a := "message body", e.Message(); e != a {
t.Errorf("expected %s, but received %s", e, a)
}
}
func newCopyTestSvc(errMsg string) *s3.S3 {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, errMsg, http.StatusOK)
}))
return s3.New(unit.Session, aws.NewConfig().
WithEndpoint(server.URL).
WithDisableSSL(true).
WithMaxRetries(0).
WithS3ForcePathStyle(true))
}
-178
View File
@@ -1,178 +0,0 @@
[
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.us-west-1.amazonaws.com",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-with-number-1",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-with-number-1.s3.us-west-1.amazonaws.com",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.cn-north-1.amazonaws.com.cn",
"Region": "cn-north-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "BucketName",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.amazonaws.com/BucketName",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "BucketName",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.amazonaws.com/BucketName",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket_name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket_name",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket.name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket.name",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "-bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/-bucket-name",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name-",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket-name-",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "aa",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/aa",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
"Region": "us-west-1",
"UseDualstack": false,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.dualstack.us-east-1.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3.dualstack.us-west-2.amazonaws.com",
"Region": "us-west-2",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket.name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://s3.dualstack.us-west-2.amazonaws.com/bucket.name",
"Region": "us-west-2",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "default",
"ExpectedUri": "https://bucket-name.s3-accelerate.dualstack.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://s3.amazonaws.com/bucket-name",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": false,
"UseS3Accelerate": true
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://s3.dualstack.us-east-1.amazonaws.com/bucket-name",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": false
},
{
"Bucket": "bucket-name",
"ConfiguredAddressingStyle": "path",
"ExpectedUri": "https://bucket-name.s3-accelerate.dualstack.amazonaws.com",
"Region": "us-east-1",
"UseDualstack": true,
"UseS3Accelerate": true
}
]
@@ -1,40 +0,0 @@
package s3
import (
"net/http"
"testing"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting"
)
func TestUnmarhsalErrorLeak(t *testing.T) {
req := &request.Request{
HTTPRequest: &http.Request{
Header: make(http.Header),
Body: &awstesting.ReadCloser{Size: 2048},
},
}
req.HTTPResponse = &http.Response{
Body: &awstesting.ReadCloser{Size: 2048},
Header: http.Header{
"X-Amzn-Requestid": []string{"1"},
},
StatusCode: http.StatusOK,
}
reader := req.HTTPResponse.Body.(*awstesting.ReadCloser)
unmarshalError(req)
if req.Error == nil {
t.Error("expected an error, but received none")
}
if !reader.Closed {
t.Error("expected reader to be closed")
}
if e, a := 0, reader.Size; e != a {
t.Errorf("expected %d, but received %d", e, a)
}
}
-253
View File
@@ -1,253 +0,0 @@
package s3_test
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/awstesting/unit"
"github.com/aws/aws-sdk-go/service/s3"
)
type testErrorCase struct {
RespFn func() *http.Response
ReqID, HostID string
Code, Msg string
WithoutStatusMsg bool
}
var testUnmarshalCases = []testErrorCase{
{
RespFn: func() *http.Response {
return &http.Response{
StatusCode: 301,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(bytes.NewReader(nil)),
ContentLength: -1,
}
},
ReqID: "abc123",
HostID: "321cba",
Code: "BucketRegionError", Msg: "incorrect region, the bucket is not in 'mock-region' region",
},
{
RespFn: func() *http.Response {
return &http.Response{
StatusCode: 403,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(bytes.NewReader(nil)),
ContentLength: 0,
}
},
ReqID: "abc123",
HostID: "321cba",
Code: "Forbidden", Msg: "Forbidden",
},
{
RespFn: func() *http.Response {
return &http.Response{
StatusCode: 400,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(bytes.NewReader(nil)),
ContentLength: 0,
}
},
ReqID: "abc123",
HostID: "321cba",
Code: "BadRequest", Msg: "Bad Request",
},
{
RespFn: func() *http.Response {
return &http.Response{
StatusCode: 404,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(bytes.NewReader(nil)),
ContentLength: 0,
}
},
ReqID: "abc123",
HostID: "321cba",
Code: "NotFound", Msg: "Not Found",
},
{
// SDK only reads request ID and host ID from the header. The values
// in message body are ignored.
RespFn: func() *http.Response {
body := `<Error><Code>SomeException</Code><Message>Exception message</Message><RequestId>ignored-request-id</RequestId><HostId>ignored-host-id</HostId></Error>`
return &http.Response{
StatusCode: 500,
Header: http.Header{
"X-Amz-Request-Id": []string{"taken-request-id"},
"X-Amz-Id-2": []string{"taken-host-id"},
},
Body: ioutil.NopCloser(strings.NewReader(body)),
ContentLength: int64(len(body)),
}
},
ReqID: "taken-request-id",
HostID: "taken-host-id",
Code: "SomeException", Msg: "Exception message",
},
{
RespFn: func() *http.Response {
return &http.Response{
StatusCode: 404,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(bytes.NewReader(nil)),
ContentLength: -1,
}
},
ReqID: "abc123",
HostID: "321cba",
Code: "NotFound", Msg: "Not Found", WithoutStatusMsg: true,
},
{
RespFn: func() *http.Response {
return &http.Response{
StatusCode: 404,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(bytes.NewReader(nil)),
ContentLength: -1,
}
},
ReqID: "abc123",
HostID: "321cba",
Code: "NotFound", Msg: "Not Found",
},
}
func TestUnmarshalError(t *testing.T) {
for i, c := range testUnmarshalCases {
s := s3.New(unit.Session)
s.Handlers.Send.Clear()
s.Handlers.Send.PushBack(func(r *request.Request) {
r.HTTPResponse = c.RespFn()
if !c.WithoutStatusMsg {
r.HTTPResponse.Status = fmt.Sprintf("%d%s",
r.HTTPResponse.StatusCode,
http.StatusText(r.HTTPResponse.StatusCode))
}
})
_, err := s.PutBucketAcl(&s3.PutBucketAclInput{
Bucket: aws.String("bucket"), ACL: aws.String("public-read"),
})
if err == nil {
t.Fatalf("%d, expected error, got nil", i)
}
if e, a := c.Code, err.(awserr.Error).Code(); e != a {
t.Errorf("%d, Code: expect %s, got %s", i, e, a)
}
if e, a := c.Msg, err.(awserr.Error).Message(); e != a {
t.Errorf("%d, Message: expect %s, got %s", i, e, a)
}
if e, a := c.ReqID, err.(awserr.RequestFailure).RequestID(); e != a {
t.Errorf("%d, RequestID: expect %s, got %s", i, e, a)
}
if e, a := c.HostID, err.(s3.RequestFailure).HostID(); e != a {
t.Errorf("%d, HostID: expect %s, got %s", i, e, a)
}
}
}
const completeMultiResp = `
163
<?xml version="1.0" encoding="UTF-8"?>
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Location>https://bucket.s3-us-west-2.amazonaws.com/key</Location><Bucket>bucket</Bucket><Key>key</Key><ETag>&quot;a7d414b9133d6483d9a1c4e04e856e3b-2&quot;</ETag></CompleteMultipartUploadResult>
0
`
func Test200NoErrorUnmarshalError(t *testing.T) {
s := s3.New(unit.Session)
s.Handlers.Send.Clear()
s.Handlers.Send.PushBack(func(r *request.Request) {
r.HTTPResponse = &http.Response{
StatusCode: 200,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(strings.NewReader(completeMultiResp)),
ContentLength: -1,
}
r.HTTPResponse.Status = http.StatusText(r.HTTPResponse.StatusCode)
})
_, err := s.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String("bucket"), Key: aws.String("key"),
UploadId: aws.String("id"),
MultipartUpload: &s3.CompletedMultipartUpload{Parts: []*s3.CompletedPart{
{ETag: aws.String("etag"), PartNumber: aws.Int64(1)},
}},
})
if err != nil {
t.Fatalf("expect no error, got %v", err)
}
}
const completeMultiErrResp = `<Error><Code>SomeException</Code><Message>Exception message</Message></Error>`
func Test200WithErrorUnmarshalError(t *testing.T) {
s := s3.New(unit.Session)
s.Handlers.Send.Clear()
s.Handlers.Send.PushBack(func(r *request.Request) {
r.HTTPResponse = &http.Response{
StatusCode: 200,
Header: http.Header{
"X-Amz-Request-Id": []string{"abc123"},
"X-Amz-Id-2": []string{"321cba"},
},
Body: ioutil.NopCloser(strings.NewReader(completeMultiErrResp)),
ContentLength: -1,
}
r.HTTPResponse.Status = http.StatusText(r.HTTPResponse.StatusCode)
})
_, err := s.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String("bucket"), Key: aws.String("key"),
UploadId: aws.String("id"),
MultipartUpload: &s3.CompletedMultipartUpload{Parts: []*s3.CompletedPart{
{ETag: aws.String("etag"), PartNumber: aws.Int64(1)},
}},
})
if err == nil {
t.Fatalf("expected error, got nil")
}
if e, a := "SomeException", err.(awserr.Error).Code(); e != a {
t.Errorf("Code: expect %s, got %s", e, a)
}
if e, a := "Exception message", err.(awserr.Error).Message(); e != a {
t.Errorf("Message: expect %s, got %s", e, a)
}
if e, a := "abc123", err.(s3.RequestFailure).RequestID(); e != a {
t.Errorf("RequestID: expect %s, got %s", e, a)
}
if e, a := "321cba", err.(s3.RequestFailure).HostID(); e != a {
t.Errorf("HostID: expect %s, got %s", e, a)
}
}