vendor: switch to using go1.11 modules
This commit is contained in:
-58
@@ -1,58 +0,0 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
)
|
||||
|
||||
func BenchmarkPresign_GetObject(b *testing.B) {
|
||||
sess := unit.Session
|
||||
svc := New(sess)
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
req, _ := svc.GetObjectRequest(&GetObjectInput{
|
||||
Bucket: aws.String("mock-bucket"),
|
||||
Key: aws.String("mock-key"),
|
||||
})
|
||||
|
||||
u, h, err := req.PresignRequest(15 * time.Minute)
|
||||
if err != nil {
|
||||
b.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if len(u) == 0 {
|
||||
b.Fatalf("expect url, got none")
|
||||
}
|
||||
if len(h) != 0 {
|
||||
b.Fatalf("no signed headers, got %v", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPresign_PutObject(b *testing.B) {
|
||||
sess := unit.Session
|
||||
svc := New(sess)
|
||||
|
||||
body := make([]byte, 1024*1024*20)
|
||||
for i := 0; i < b.N; i++ {
|
||||
req, _ := svc.PutObjectRequest(&PutObjectInput{
|
||||
Bucket: aws.String("mock-bucket"),
|
||||
Key: aws.String("mock-key"),
|
||||
Body: bytes.NewReader(body),
|
||||
})
|
||||
|
||||
u, h, err := req.PresignRequest(15 * time.Minute)
|
||||
if err != nil {
|
||||
b.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if len(u) == 0 {
|
||||
b.Fatalf("expect url, got none")
|
||||
}
|
||||
if len(h) == 0 {
|
||||
b.Fatalf("expect signed header, got none")
|
||||
}
|
||||
}
|
||||
}
|
||||
-523
@@ -1,523 +0,0 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/internal/sdkio"
|
||||
)
|
||||
|
||||
type errorReader struct{}
|
||||
|
||||
func (errorReader) Read([]byte) (int, error) {
|
||||
return 0, fmt.Errorf("errorReader error")
|
||||
}
|
||||
func (errorReader) Seek(int64, int) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func TestComputeBodyHases(t *testing.T) {
|
||||
bodyContent := []byte("bodyContent goes here")
|
||||
|
||||
cases := []struct {
|
||||
Req *request.Request
|
||||
ExpectMD5 string
|
||||
ExpectSHA256 string
|
||||
Error string
|
||||
DisableContentMD5 bool
|
||||
Presigned bool
|
||||
}{
|
||||
{
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
Body: bytes.NewReader(bodyContent),
|
||||
},
|
||||
ExpectMD5: "CqD6NNPvoNOBT/5pkjtzOw==",
|
||||
ExpectSHA256: "3ff09c8b42a58a905e27835919ede45b61722e7cd400f30101bd9ed1a69a1825",
|
||||
},
|
||||
{
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(contentMD5Header, "MD5AlreadySet")
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Body: bytes.NewReader(bodyContent),
|
||||
},
|
||||
ExpectMD5: "MD5AlreadySet",
|
||||
ExpectSHA256: "3ff09c8b42a58a905e27835919ede45b61722e7cd400f30101bd9ed1a69a1825",
|
||||
},
|
||||
{
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(contentSha256Header, "SHA256AlreadySet")
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Body: bytes.NewReader(bodyContent),
|
||||
},
|
||||
ExpectMD5: "CqD6NNPvoNOBT/5pkjtzOw==",
|
||||
ExpectSHA256: "SHA256AlreadySet",
|
||||
},
|
||||
{
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(contentMD5Header, "MD5AlreadySet")
|
||||
h.Set(contentSha256Header, "SHA256AlreadySet")
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Body: bytes.NewReader(bodyContent),
|
||||
},
|
||||
ExpectMD5: "MD5AlreadySet",
|
||||
ExpectSHA256: "SHA256AlreadySet",
|
||||
},
|
||||
{
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
// Non-seekable reader
|
||||
Body: aws.ReadSeekCloser(bytes.NewBuffer(bodyContent)),
|
||||
},
|
||||
ExpectMD5: "",
|
||||
ExpectSHA256: "",
|
||||
},
|
||||
{
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
// Empty seekable body
|
||||
Body: aws.ReadSeekCloser(bytes.NewReader(nil)),
|
||||
},
|
||||
ExpectMD5: "1B2M2Y8AsgTpgAmY7PhCfg==",
|
||||
ExpectSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
{
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
// failure while reading reader
|
||||
Body: errorReader{},
|
||||
},
|
||||
ExpectMD5: "",
|
||||
ExpectSHA256: "",
|
||||
Error: "errorReader error",
|
||||
},
|
||||
{
|
||||
// Disabled ContentMD5 validation
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
Body: bytes.NewReader(bodyContent),
|
||||
},
|
||||
ExpectMD5: "",
|
||||
ExpectSHA256: "",
|
||||
DisableContentMD5: true,
|
||||
},
|
||||
{
|
||||
// Disabled ContentMD5 validation
|
||||
Req: &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
Body: bytes.NewReader(bodyContent),
|
||||
},
|
||||
ExpectMD5: "",
|
||||
ExpectSHA256: "",
|
||||
Presigned: true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
c.Req.Config.S3DisableContentMD5Validation = aws.Bool(c.DisableContentMD5)
|
||||
|
||||
if c.Presigned {
|
||||
c.Req.ExpireTime = 10 * time.Minute
|
||||
}
|
||||
computeBodyHashes(c.Req)
|
||||
|
||||
if e, a := c.ExpectMD5, c.Req.HTTPRequest.Header.Get(contentMD5Header); e != a {
|
||||
t.Errorf("%d, expect %v md5, got %v", i, e, a)
|
||||
}
|
||||
|
||||
if e, a := c.ExpectSHA256, c.Req.HTTPRequest.Header.Get(contentSha256Header); e != a {
|
||||
t.Errorf("%d, expect %v sha256, got %v", i, e, a)
|
||||
}
|
||||
|
||||
if len(c.Error) != 0 {
|
||||
if c.Req.Error == nil {
|
||||
t.Fatalf("%d, expect error, got none", i)
|
||||
}
|
||||
if e, a := c.Error, c.Req.Error.Error(); !strings.Contains(a, e) {
|
||||
t.Errorf("%d, expect %v error to be in %v", i, e, a)
|
||||
}
|
||||
|
||||
} else if c.Req.Error != nil {
|
||||
t.Errorf("%d, expect no error, got %v", i, c.Req.Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkComputeBodyHashes(b *testing.B) {
|
||||
body := bytes.NewReader(make([]byte, 2*1024))
|
||||
req := &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
Body: body,
|
||||
}
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
computeBodyHashes(req)
|
||||
if req.Error != nil {
|
||||
b.Fatalf("expect no error, got %v", req.Error)
|
||||
}
|
||||
|
||||
req.HTTPRequest.Header = http.Header{}
|
||||
body.Seek(0, sdkio.SeekStart)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAskForTxEncodingAppendMD5(t *testing.T) {
|
||||
cases := []struct {
|
||||
DisableContentMD5 bool
|
||||
Presigned bool
|
||||
}{
|
||||
{DisableContentMD5: true},
|
||||
{DisableContentMD5: false},
|
||||
{Presigned: true},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
req := &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: http.Header{},
|
||||
},
|
||||
Config: aws.Config{
|
||||
S3DisableContentMD5Validation: aws.Bool(c.DisableContentMD5),
|
||||
},
|
||||
}
|
||||
if c.Presigned {
|
||||
req.ExpireTime = 10 * time.Minute
|
||||
}
|
||||
|
||||
askForTxEncodingAppendMD5(req)
|
||||
|
||||
v := req.HTTPRequest.Header.Get(amzTeHeader)
|
||||
|
||||
expectHeader := !(c.DisableContentMD5 || c.Presigned)
|
||||
|
||||
if e, a := expectHeader, len(v) != 0; e != a {
|
||||
t.Errorf("%d, expect %t disable content MD5, got %t, %s", i, e, a, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUseMD5ValidationReader(t *testing.T) {
|
||||
body := []byte("create a really cool md5 checksum of me")
|
||||
bodySum := md5.Sum(body)
|
||||
bodyWithSum := append(body, bodySum[:]...)
|
||||
|
||||
emptyBodySum := md5.Sum([]byte{})
|
||||
|
||||
cases := []struct {
|
||||
Req *request.Request
|
||||
Error string
|
||||
Validate func(outupt interface{}) error
|
||||
}{
|
||||
{
|
||||
// Positive: Use Validation reader
|
||||
Req: &request.Request{
|
||||
HTTPResponse: &http.Response{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Data: &GetObjectOutput{
|
||||
Body: ioutil.NopCloser(bytes.NewReader(bodyWithSum)),
|
||||
ContentLength: aws.Int64(int64(len(bodyWithSum))),
|
||||
},
|
||||
},
|
||||
Validate: func(output interface{}) error {
|
||||
getObjOut := output.(*GetObjectOutput)
|
||||
reader, ok := getObjOut.Body.(*md5ValidationReader)
|
||||
if !ok {
|
||||
return fmt.Errorf("expect %T updated body reader, got %T",
|
||||
(*md5ValidationReader)(nil), getObjOut.Body)
|
||||
}
|
||||
|
||||
if reader.rawReader == nil {
|
||||
return fmt.Errorf("expect rawReader not to be nil")
|
||||
}
|
||||
if reader.payload == nil {
|
||||
return fmt.Errorf("expect payload not to be nil")
|
||||
}
|
||||
if e, a := int64(len(bodyWithSum)-md5.Size), reader.payloadLen; e != a {
|
||||
return fmt.Errorf("expect %v payload len, got %v", e, a)
|
||||
}
|
||||
if reader.hash == nil {
|
||||
return fmt.Errorf("expect hash not to be nil")
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// Positive: Use Validation reader, empty object
|
||||
Req: &request.Request{
|
||||
HTTPResponse: &http.Response{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Data: &GetObjectOutput{
|
||||
Body: ioutil.NopCloser(bytes.NewReader(emptyBodySum[:])),
|
||||
ContentLength: aws.Int64(int64(len(emptyBodySum[:]))),
|
||||
},
|
||||
},
|
||||
Validate: func(output interface{}) error {
|
||||
getObjOut := output.(*GetObjectOutput)
|
||||
reader, ok := getObjOut.Body.(*md5ValidationReader)
|
||||
if !ok {
|
||||
return fmt.Errorf("expect %T updated body reader, got %T",
|
||||
(*md5ValidationReader)(nil), getObjOut.Body)
|
||||
}
|
||||
|
||||
if reader.rawReader == nil {
|
||||
return fmt.Errorf("expect rawReader not to be nil")
|
||||
}
|
||||
if reader.payload == nil {
|
||||
return fmt.Errorf("expect payload not to be nil")
|
||||
}
|
||||
if e, a := int64(len(emptyBodySum)-md5.Size), reader.payloadLen; e != a {
|
||||
return fmt.Errorf("expect %v payload len, got %v", e, a)
|
||||
}
|
||||
if reader.hash == nil {
|
||||
return fmt.Errorf("expect hash not to be nil")
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// Negative: amzTxEncoding header not set
|
||||
Req: &request.Request{
|
||||
HTTPResponse: &http.Response{
|
||||
Header: http.Header{},
|
||||
},
|
||||
Data: &GetObjectOutput{
|
||||
Body: ioutil.NopCloser(bytes.NewReader(body)),
|
||||
ContentLength: aws.Int64(int64(len(body))),
|
||||
},
|
||||
},
|
||||
Validate: func(output interface{}) error {
|
||||
getObjOut := output.(*GetObjectOutput)
|
||||
reader, ok := getObjOut.Body.(*md5ValidationReader)
|
||||
if ok {
|
||||
return fmt.Errorf("expect body reader not to be %T",
|
||||
reader)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// Negative: Not GetObjectOutput type.
|
||||
Req: &request.Request{
|
||||
Operation: &request.Operation{
|
||||
Name: "PutObject",
|
||||
},
|
||||
HTTPResponse: &http.Response{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Data: &PutObjectOutput{},
|
||||
},
|
||||
Error: "header received on unsupported API",
|
||||
Validate: func(output interface{}) error {
|
||||
_, ok := output.(*PutObjectOutput)
|
||||
if !ok {
|
||||
return fmt.Errorf("expect %T output not to change, got %T",
|
||||
(*PutObjectOutput)(nil), output)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// Negative: invalid content length.
|
||||
Req: &request.Request{
|
||||
HTTPResponse: &http.Response{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Data: &GetObjectOutput{
|
||||
Body: ioutil.NopCloser(bytes.NewReader(bodyWithSum)),
|
||||
ContentLength: aws.Int64(-1),
|
||||
},
|
||||
},
|
||||
Error: "invalid Content-Length -1",
|
||||
Validate: func(output interface{}) error {
|
||||
getObjOut := output.(*GetObjectOutput)
|
||||
reader, ok := getObjOut.Body.(*md5ValidationReader)
|
||||
if ok {
|
||||
return fmt.Errorf("expect body reader not to be %T",
|
||||
reader)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// Negative: invalid content length, < md5.Size.
|
||||
Req: &request.Request{
|
||||
HTTPResponse: &http.Response{
|
||||
Header: func() http.Header {
|
||||
h := http.Header{}
|
||||
h.Set(amzTxEncodingHeader, appendMD5TxEncoding)
|
||||
return h
|
||||
}(),
|
||||
},
|
||||
Data: &GetObjectOutput{
|
||||
Body: ioutil.NopCloser(bytes.NewReader(make([]byte, 5))),
|
||||
ContentLength: aws.Int64(5),
|
||||
},
|
||||
},
|
||||
Error: "invalid Content-Length 5",
|
||||
Validate: func(output interface{}) error {
|
||||
getObjOut := output.(*GetObjectOutput)
|
||||
reader, ok := getObjOut.Body.(*md5ValidationReader)
|
||||
if ok {
|
||||
return fmt.Errorf("expect body reader not to be %T",
|
||||
reader)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
useMD5ValidationReader(c.Req)
|
||||
if len(c.Error) != 0 {
|
||||
if c.Req.Error == nil {
|
||||
t.Fatalf("%d, expect error, got none", i)
|
||||
}
|
||||
if e, a := c.Error, c.Req.Error.Error(); !strings.Contains(a, e) {
|
||||
t.Errorf("%d, expect %v error to be in %v", i, e, a)
|
||||
}
|
||||
} else if c.Req.Error != nil {
|
||||
t.Errorf("%d, expect no error, got %v", i, c.Req.Error)
|
||||
}
|
||||
|
||||
if c.Validate != nil {
|
||||
if err := c.Validate(c.Req.Data); err != nil {
|
||||
t.Errorf("%d, expect Data to validate, got %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderMD5Validation(t *testing.T) {
|
||||
body := []byte("create a really cool md5 checksum of me")
|
||||
bodySum := md5.Sum(body)
|
||||
bodyWithSum := append(body, bodySum[:]...)
|
||||
emptyBodySum := md5.Sum([]byte{})
|
||||
badBodySum := append(body, emptyBodySum[:]...)
|
||||
|
||||
cases := []struct {
|
||||
Content []byte
|
||||
ContentReader io.ReadCloser
|
||||
PayloadLen int64
|
||||
Error string
|
||||
}{
|
||||
{
|
||||
Content: bodyWithSum,
|
||||
PayloadLen: int64(len(body)),
|
||||
},
|
||||
{
|
||||
Content: emptyBodySum[:],
|
||||
PayloadLen: 0,
|
||||
},
|
||||
{
|
||||
Content: badBodySum,
|
||||
PayloadLen: int64(len(body)),
|
||||
Error: "expected MD5 checksum",
|
||||
},
|
||||
{
|
||||
Content: emptyBodySum[:len(emptyBodySum)-2],
|
||||
PayloadLen: 0,
|
||||
Error: "unexpected EOF",
|
||||
},
|
||||
{
|
||||
Content: body,
|
||||
PayloadLen: int64(len(body) * 2),
|
||||
Error: "unexpected EOF",
|
||||
},
|
||||
{
|
||||
ContentReader: ioutil.NopCloser(errorReader{}),
|
||||
PayloadLen: int64(len(body)),
|
||||
Error: "errorReader error",
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
reader := c.ContentReader
|
||||
if reader == nil {
|
||||
reader = ioutil.NopCloser(bytes.NewReader(c.Content))
|
||||
}
|
||||
v := newMD5ValidationReader(reader, c.PayloadLen)
|
||||
|
||||
var actual bytes.Buffer
|
||||
n, err := io.Copy(&actual, v)
|
||||
if len(c.Error) != 0 {
|
||||
if err == nil {
|
||||
t.Errorf("%d, expect error, got none", i)
|
||||
}
|
||||
if e, a := c.Error, err.Error(); !strings.Contains(a, e) {
|
||||
t.Errorf("%d, expect %v error to be in %v", i, e, a)
|
||||
}
|
||||
continue
|
||||
} else if err != nil {
|
||||
t.Errorf("%d, expect no error, got %v", i, err)
|
||||
continue
|
||||
}
|
||||
if e, a := c.PayloadLen, n; e != a {
|
||||
t.Errorf("%d, expect %v len, got %v", i, e, a)
|
||||
}
|
||||
|
||||
if e, a := c.Content[:c.PayloadLen], actual.Bytes(); !bytes.Equal(e, a) {
|
||||
t.Errorf("%d, expect:\n%v\nactual:\n%v", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
-141
@@ -1,141 +0,0 @@
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awsutil"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
var s3LocationTests = []struct {
|
||||
body string
|
||||
loc string
|
||||
}{
|
||||
{`<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`, ``},
|
||||
{`<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">EU</LocationConstraint>`, `EU`},
|
||||
}
|
||||
|
||||
func TestGetBucketLocation(t *testing.T) {
|
||||
for _, test := range s3LocationTests {
|
||||
s := s3.New(unit.Session)
|
||||
s.Handlers.Send.Clear()
|
||||
s.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
reader := ioutil.NopCloser(bytes.NewReader([]byte(test.body)))
|
||||
r.HTTPResponse = &http.Response{StatusCode: 200, Body: reader}
|
||||
})
|
||||
|
||||
resp, err := s.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: aws.String("bucket")})
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if test.loc == "" {
|
||||
if v := resp.LocationConstraint; v != nil {
|
||||
t.Errorf("expect location constraint to be nil, got %s", *v)
|
||||
}
|
||||
} else {
|
||||
if e, a := test.loc, *resp.LocationConstraint; e != a {
|
||||
t.Errorf("expect %s location constraint, got %v", e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeBucketLocation(t *testing.T) {
|
||||
cases := []struct {
|
||||
In, Out string
|
||||
}{
|
||||
{"", "us-east-1"},
|
||||
{"EU", "eu-west-1"},
|
||||
{"us-east-1", "us-east-1"},
|
||||
{"something", "something"},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
actual := s3.NormalizeBucketLocation(c.In)
|
||||
if e, a := c.Out, actual; e != a {
|
||||
t.Errorf("%d, expect %s bucket location, got %s", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNormalizeBucketLocation(t *testing.T) {
|
||||
req := &request.Request{}
|
||||
req.ApplyOptions(s3.WithNormalizeBucketLocation)
|
||||
|
||||
cases := []struct {
|
||||
In, Out string
|
||||
}{
|
||||
{"", "us-east-1"},
|
||||
{"EU", "eu-west-1"},
|
||||
{"us-east-1", "us-east-1"},
|
||||
{"something", "something"},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
req.Data = &s3.GetBucketLocationOutput{
|
||||
LocationConstraint: aws.String(c.In),
|
||||
}
|
||||
req.Handlers.Unmarshal.Run(req)
|
||||
|
||||
v := req.Data.(*s3.GetBucketLocationOutput).LocationConstraint
|
||||
if e, a := c.Out, aws.StringValue(v); e != a {
|
||||
t.Errorf("%d, expect %s bucket location, got %s", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPopulateLocationConstraint(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
in := &s3.CreateBucketInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
}
|
||||
req, _ := s.CreateBucketRequest(in)
|
||||
if err := req.Build(); err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
|
||||
v, _ := awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")
|
||||
if e, a := "mock-region", *(v[0].(*string)); e != a {
|
||||
t.Errorf("expect %s location constraint, got %s", e, a)
|
||||
}
|
||||
if v := in.CreateBucketConfiguration; v != nil {
|
||||
// don't modify original params
|
||||
t.Errorf("expect create bucket Configuration to be nil, got %s", *v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoPopulateLocationConstraintIfProvided(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
req, _ := s.CreateBucketRequest(&s3.CreateBucketInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
CreateBucketConfiguration: &s3.CreateBucketConfiguration{},
|
||||
})
|
||||
if err := req.Build(); err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
v, _ := awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")
|
||||
if l := len(v); l != 0 {
|
||||
t.Errorf("expect no values, got %d", l)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoPopulateLocationConstraintIfClassic(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{Region: aws.String("us-east-1")})
|
||||
req, _ := s.CreateBucketRequest(&s3.CreateBucketInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
})
|
||||
if err := req.Build(); err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
v, _ := awsutil.ValuesAtPath(req.Params, "CreateBucketConfiguration.LocationConstraint")
|
||||
if l := len(v); l != 0 {
|
||||
t.Errorf("expect no values, got %d", l)
|
||||
}
|
||||
}
|
||||
-171
@@ -1,171 +0,0 @@
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func assertMD5(t *testing.T, req *request.Request) {
|
||||
err := req.Build()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
b, _ := ioutil.ReadAll(req.HTTPRequest.Body)
|
||||
out := md5.Sum(b)
|
||||
if len(b) == 0 {
|
||||
t.Error("expected non-empty value")
|
||||
}
|
||||
if e, a := base64.StdEncoding.EncodeToString(out[:]), req.HTTPRequest.Header.Get("Content-MD5"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMD5InPutBucketCors(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
req, _ := svc.PutBucketCorsRequest(&s3.PutBucketCorsInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
CORSConfiguration: &s3.CORSConfiguration{
|
||||
CORSRules: []*s3.CORSRule{
|
||||
{
|
||||
AllowedMethods: []*string{aws.String("GET")},
|
||||
AllowedOrigins: []*string{aws.String("*")},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
assertMD5(t, req)
|
||||
}
|
||||
|
||||
func TestMD5InPutBucketLifecycle(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
req, _ := svc.PutBucketLifecycleRequest(&s3.PutBucketLifecycleInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
LifecycleConfiguration: &s3.LifecycleConfiguration{
|
||||
Rules: []*s3.Rule{
|
||||
{
|
||||
ID: aws.String("ID"),
|
||||
Prefix: aws.String("Prefix"),
|
||||
Status: aws.String("Enabled"),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
assertMD5(t, req)
|
||||
}
|
||||
|
||||
func TestMD5InPutBucketPolicy(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
req, _ := svc.PutBucketPolicyRequest(&s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
Policy: aws.String("{}"),
|
||||
})
|
||||
assertMD5(t, req)
|
||||
}
|
||||
|
||||
func TestMD5InPutBucketTagging(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
req, _ := svc.PutBucketTaggingRequest(&s3.PutBucketTaggingInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
Tagging: &s3.Tagging{
|
||||
TagSet: []*s3.Tag{
|
||||
{Key: aws.String("KEY"), Value: aws.String("VALUE")},
|
||||
},
|
||||
},
|
||||
})
|
||||
assertMD5(t, req)
|
||||
}
|
||||
|
||||
func TestMD5InDeleteObjects(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
req, _ := svc.DeleteObjectsRequest(&s3.DeleteObjectsInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
Delete: &s3.Delete{
|
||||
Objects: []*s3.ObjectIdentifier{
|
||||
{Key: aws.String("key")},
|
||||
},
|
||||
},
|
||||
})
|
||||
assertMD5(t, req)
|
||||
}
|
||||
|
||||
func TestMD5InPutBucketLifecycleConfiguration(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
req, _ := svc.PutBucketLifecycleConfigurationRequest(&s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
|
||||
Rules: []*s3.LifecycleRule{
|
||||
{Prefix: aws.String("prefix"), Status: aws.String(s3.ExpirationStatusEnabled)},
|
||||
},
|
||||
},
|
||||
})
|
||||
assertMD5(t, req)
|
||||
}
|
||||
|
||||
const (
|
||||
metaKeyPrefix = `X-Amz-Meta-`
|
||||
utf8KeySuffix = `My-Info`
|
||||
utf8Value = "hello-世界\u0444"
|
||||
)
|
||||
|
||||
func TestPutObjectMetadataWithUnicode(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if e, a := utf8Value, r.Header.Get(metaKeyPrefix+utf8KeySuffix); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}))
|
||||
svc := s3.New(unit.Session, &aws.Config{
|
||||
Endpoint: aws.String(server.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
})
|
||||
|
||||
_, err := svc.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String("my_bucket"),
|
||||
Key: aws.String("my_key"),
|
||||
Body: strings.NewReader(""),
|
||||
Metadata: func() map[string]*string {
|
||||
v := map[string]*string{}
|
||||
v[utf8KeySuffix] = aws.String(utf8Value)
|
||||
return v
|
||||
}(),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectMetadataWithUnicode(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set(metaKeyPrefix+utf8KeySuffix, utf8Value)
|
||||
}))
|
||||
svc := s3.New(unit.Session, &aws.Config{
|
||||
Endpoint: aws.String(server.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
})
|
||||
|
||||
resp, err := svc.GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String("my_bucket"),
|
||||
Key: aws.String("my_key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if e, a := utf8Value, *resp.Metadata[utf8KeySuffix]; e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
|
||||
}
|
||||
-297
@@ -1,297 +0,0 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/corehandlers"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/private/protocol"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/eventstream"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamtest"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/restxml"
|
||||
)
|
||||
|
||||
func mockSelectObjectContentEventStreamsOutputEvents() ([]SelectObjectContentEventStreamEvent, []eventstream.Message) {
|
||||
|
||||
expectEvents := []SelectObjectContentEventStreamEvent{
|
||||
&ContinuationEvent{},
|
||||
&EndEvent{},
|
||||
&ProgressEvent{
|
||||
Details: &Progress{
|
||||
BytesProcessed: aws.Int64(123),
|
||||
BytesReturned: aws.Int64(456),
|
||||
BytesScanned: aws.Int64(789),
|
||||
},
|
||||
},
|
||||
&RecordsEvent{
|
||||
Payload: []byte("abc123"),
|
||||
},
|
||||
&StatsEvent{
|
||||
Details: &Stats{
|
||||
BytesProcessed: aws.Int64(123),
|
||||
BytesReturned: aws.Int64(456),
|
||||
BytesScanned: aws.Int64(789),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var marshalers request.HandlerList
|
||||
marshalers.PushBackNamed(restxml.BuildHandler)
|
||||
payloadMarshaler := protocol.HandlerPayloadMarshal{
|
||||
Marshalers: marshalers,
|
||||
}
|
||||
|
||||
eventMsgs := []eventstream.Message{
|
||||
{
|
||||
Headers: eventstream.Headers{
|
||||
eventstreamtest.EventMessageTypeHeader,
|
||||
{
|
||||
Name: eventstreamapi.EventTypeHeader,
|
||||
Value: eventstream.StringValue("Cont"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Headers: eventstream.Headers{
|
||||
eventstreamtest.EventMessageTypeHeader,
|
||||
{
|
||||
Name: eventstreamapi.EventTypeHeader,
|
||||
Value: eventstream.StringValue("End"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Headers: eventstream.Headers{
|
||||
eventstreamtest.EventMessageTypeHeader,
|
||||
{
|
||||
Name: eventstreamapi.EventTypeHeader,
|
||||
Value: eventstream.StringValue("Progress"),
|
||||
},
|
||||
},
|
||||
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[2]),
|
||||
},
|
||||
{
|
||||
Headers: eventstream.Headers{
|
||||
eventstreamtest.EventMessageTypeHeader,
|
||||
{
|
||||
Name: eventstreamapi.EventTypeHeader,
|
||||
Value: eventstream.StringValue("Records"),
|
||||
},
|
||||
},
|
||||
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[3]),
|
||||
},
|
||||
{
|
||||
Headers: eventstream.Headers{
|
||||
eventstreamtest.EventMessageTypeHeader,
|
||||
{
|
||||
Name: eventstreamapi.EventTypeHeader,
|
||||
Value: eventstream.StringValue("Stats"),
|
||||
},
|
||||
},
|
||||
Payload: eventstreamtest.MarshalEventPayload(payloadMarshaler, expectEvents[4]),
|
||||
},
|
||||
}
|
||||
|
||||
return expectEvents, eventMsgs
|
||||
}
|
||||
|
||||
func TestSelectObjectContentEventStream(t *testing.T) {
|
||||
expectEvents, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
|
||||
sess, cleanupFn, err := eventstreamtest.SetupEventStreamSession(t,
|
||||
eventstreamtest.ServeEventStream{
|
||||
T: t,
|
||||
Events: eventMsgs,
|
||||
},
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, %v", err)
|
||||
}
|
||||
defer cleanupFn()
|
||||
|
||||
svc := New(sess)
|
||||
|
||||
params := &SelectObjectContentInput{}
|
||||
resp, err := svc.SelectObjectContent(params)
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error got, %v", err)
|
||||
}
|
||||
defer resp.EventStream.Close()
|
||||
|
||||
var i int
|
||||
for event := range resp.EventStream.Events() {
|
||||
if event == nil {
|
||||
t.Errorf("%d, expect event, got nil", i)
|
||||
}
|
||||
if e, a := expectEvents[i], event; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("%d, expect %T %v, got %T %v", i, e, e, a, a)
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
if err := resp.EventStream.Err(); err != nil {
|
||||
t.Errorf("expect no error, %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectObjectContentEventStream_Close(t *testing.T) {
|
||||
_, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
|
||||
sess, cleanupFn, err := eventstreamtest.SetupEventStreamSession(t,
|
||||
eventstreamtest.ServeEventStream{
|
||||
T: t,
|
||||
Events: eventMsgs,
|
||||
},
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, %v", err)
|
||||
}
|
||||
defer cleanupFn()
|
||||
|
||||
svc := New(sess)
|
||||
|
||||
params := &SelectObjectContentInput{}
|
||||
resp, err := svc.SelectObjectContent(params)
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error got, %v", err)
|
||||
}
|
||||
|
||||
resp.EventStream.Close()
|
||||
|
||||
if err := resp.EventStream.Err(); err != nil {
|
||||
t.Errorf("expect no error, %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func ExampleSelectObjectContentEventStream() {
|
||||
sess := session.Must(session.NewSession())
|
||||
svc := New(sess)
|
||||
|
||||
/*
|
||||
Example myObjectKey CSV content:
|
||||
|
||||
name,number
|
||||
gopher,0
|
||||
ᵷodɥǝɹ,1
|
||||
*/
|
||||
|
||||
// Make the Select Object Content API request using the object uploaded.
|
||||
resp, err := svc.SelectObjectContent(&SelectObjectContentInput{
|
||||
Bucket: aws.String("myBucket"),
|
||||
Key: aws.String("myObjectKey"),
|
||||
Expression: aws.String("SELECT name FROM S3Object WHERE cast(number as int) < 1"),
|
||||
ExpressionType: aws.String(ExpressionTypeSql),
|
||||
InputSerialization: &InputSerialization{
|
||||
CSV: &CSVInput{
|
||||
FileHeaderInfo: aws.String(FileHeaderInfoUse),
|
||||
},
|
||||
},
|
||||
OutputSerialization: &OutputSerialization{
|
||||
CSV: &CSVOutput{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed making API request, %v\n", err)
|
||||
return
|
||||
}
|
||||
defer resp.EventStream.Close()
|
||||
|
||||
results, resultWriter := io.Pipe()
|
||||
go func() {
|
||||
defer resultWriter.Close()
|
||||
for event := range resp.EventStream.Events() {
|
||||
switch e := event.(type) {
|
||||
case *RecordsEvent:
|
||||
resultWriter.Write(e.Payload)
|
||||
case *StatsEvent:
|
||||
fmt.Printf("Processed %d bytes\n", e.Details.BytesProcessed)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Printout the results
|
||||
resReader := csv.NewReader(results)
|
||||
for {
|
||||
record, err := resReader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
fmt.Println(record)
|
||||
}
|
||||
|
||||
if err := resp.EventStream.Err(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "reading from event stream failed, %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSelectObjectContentEventStream(b *testing.B) {
|
||||
_, eventMsgs := mockSelectObjectContentEventStreamsOutputEvents()
|
||||
var buf bytes.Buffer
|
||||
encoder := eventstream.NewEncoder(&buf)
|
||||
for _, msg := range eventMsgs {
|
||||
if err := encoder.Encode(msg); err != nil {
|
||||
b.Fatalf("failed to encode message, %v", err)
|
||||
}
|
||||
}
|
||||
stream := &loopReader{source: bytes.NewReader(buf.Bytes())}
|
||||
|
||||
sess := unit.Session
|
||||
svc := New(sess, &aws.Config{
|
||||
Endpoint: aws.String("https://example.com"),
|
||||
DisableParamValidation: aws.Bool(true),
|
||||
})
|
||||
svc.Handlers.Send.Swap(corehandlers.SendHandler.Name,
|
||||
request.NamedHandler{Name: "mockSend",
|
||||
Fn: func(r *request.Request) {
|
||||
r.HTTPResponse = &http.Response{
|
||||
Status: "200 OK",
|
||||
StatusCode: 200,
|
||||
Header: http.Header{},
|
||||
Body: ioutil.NopCloser(stream),
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
params := &SelectObjectContentInput{}
|
||||
resp, err := svc.SelectObjectContent(params)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create request, %v", err)
|
||||
}
|
||||
defer resp.EventStream.Close()
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
if err = resp.EventStream.Err(); err != nil {
|
||||
b.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
event := <-resp.EventStream.Events()
|
||||
if event == nil {
|
||||
b.Fatalf("expect event, got nil, %v, %d", resp.EventStream.Err(), i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type loopReader struct {
|
||||
source *bytes.Reader
|
||||
}
|
||||
|
||||
func (c *loopReader) Read(p []byte) (int, error) {
|
||||
if c.source.Len() == 0 {
|
||||
c.source.Seek(0, 0)
|
||||
}
|
||||
|
||||
return c.source.Read(p)
|
||||
}
|
||||
-2237
File diff suppressed because it is too large
Load Diff
-179
@@ -1,179 +0,0 @@
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
type s3BucketTest struct {
|
||||
bucket string
|
||||
url string
|
||||
errCode string
|
||||
}
|
||||
|
||||
var (
|
||||
sslTests = []s3BucketTest{
|
||||
{"abc", "https://abc.s3.mock-region.amazonaws.com/", ""},
|
||||
{"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""},
|
||||
{"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""},
|
||||
{"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""},
|
||||
}
|
||||
|
||||
nosslTests = []s3BucketTest{
|
||||
{"a.b.c", "http://a.b.c.s3.mock-region.amazonaws.com/", ""},
|
||||
{"a..bc", "http://s3.mock-region.amazonaws.com/a..bc", ""},
|
||||
}
|
||||
|
||||
forcepathTests = []s3BucketTest{
|
||||
{"abc", "https://s3.mock-region.amazonaws.com/abc", ""},
|
||||
{"a$b$c", "https://s3.mock-region.amazonaws.com/a%24b%24c", ""},
|
||||
{"a.b.c", "https://s3.mock-region.amazonaws.com/a.b.c", ""},
|
||||
{"a..bc", "https://s3.mock-region.amazonaws.com/a..bc", ""},
|
||||
}
|
||||
|
||||
accelerateTests = []s3BucketTest{
|
||||
{"abc", "https://abc.s3-accelerate.amazonaws.com/", ""},
|
||||
{"a.b.c", "https://s3.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
|
||||
{"a$b$c", "https://s3.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
|
||||
}
|
||||
|
||||
accelerateNoSSLTests = []s3BucketTest{
|
||||
{"abc", "http://abc.s3-accelerate.amazonaws.com/", ""},
|
||||
{"a.b.c", "http://a.b.c.s3-accelerate.amazonaws.com/", ""},
|
||||
{"a$b$c", "http://s3.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
|
||||
}
|
||||
|
||||
accelerateDualstack = []s3BucketTest{
|
||||
{"abc", "https://abc.s3-accelerate.dualstack.amazonaws.com/", ""},
|
||||
{"a.b.c", "https://s3.dualstack.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
|
||||
{"a$b$c", "https://s3.dualstack.mock-region.amazonaws.com/%7BBucket%7D", "InvalidParameterException"},
|
||||
}
|
||||
)
|
||||
|
||||
func runTests(t *testing.T, svc *s3.S3, tests []s3BucketTest) {
|
||||
for i, test := range tests {
|
||||
req, _ := svc.ListObjectsRequest(&s3.ListObjectsInput{Bucket: &test.bucket})
|
||||
req.Build()
|
||||
if e, a := test.url, req.HTTPRequest.URL.String(); e != a {
|
||||
t.Errorf("%d, expect url %s, got %s", i, e, a)
|
||||
}
|
||||
if test.errCode != "" {
|
||||
if err := req.Error; err == nil {
|
||||
t.Fatalf("%d, expect no error", i)
|
||||
}
|
||||
if a, e := req.Error.(awserr.Error).Code(), test.errCode; !strings.Contains(a, e) {
|
||||
t.Errorf("%d, expect error code to contain %q, got %q", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccelerateBucketBuild(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{S3UseAccelerate: aws.Bool(true)})
|
||||
runTests(t, s, accelerateTests)
|
||||
}
|
||||
|
||||
func TestAccelerateNoSSLBucketBuild(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{S3UseAccelerate: aws.Bool(true), DisableSSL: aws.Bool(true)})
|
||||
runTests(t, s, accelerateNoSSLTests)
|
||||
}
|
||||
|
||||
func TestAccelerateDualstackBucketBuild(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{
|
||||
S3UseAccelerate: aws.Bool(true),
|
||||
UseDualStack: aws.Bool(true),
|
||||
})
|
||||
runTests(t, s, accelerateDualstack)
|
||||
}
|
||||
|
||||
func TestHostStyleBucketBuild(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
runTests(t, s, sslTests)
|
||||
}
|
||||
|
||||
func TestHostStyleBucketBuildNoSSL(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{DisableSSL: aws.Bool(true)})
|
||||
runTests(t, s, nosslTests)
|
||||
}
|
||||
|
||||
func TestPathStyleBucketBuild(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{S3ForcePathStyle: aws.Bool(true)})
|
||||
runTests(t, s, forcepathTests)
|
||||
}
|
||||
|
||||
func TestHostStyleBucketGetBucketLocation(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
req, _ := s.GetBucketLocationRequest(&s3.GetBucketLocationInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
})
|
||||
|
||||
req.Build()
|
||||
if req.Error != nil {
|
||||
t.Fatalf("expect no error, got %v", req.Error)
|
||||
}
|
||||
u, _ := url.Parse(req.HTTPRequest.URL.String())
|
||||
if e, a := "bucket", u.Host; strings.Contains(a, e) {
|
||||
t.Errorf("expect %s to not be in %s", e, a)
|
||||
}
|
||||
if e, a := "bucket", u.Path; !strings.Contains(a, e) {
|
||||
t.Errorf("expect %s to be in %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualHostStyleSuite(t *testing.T) {
|
||||
f, err := os.Open(filepath.Join("testdata", "virtual_host.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, %v", err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
Bucket string
|
||||
Region string
|
||||
UseDualStack bool
|
||||
UseS3Accelerate bool
|
||||
ConfiguredAddressingStyle string
|
||||
|
||||
ExpectedURI string
|
||||
}{}
|
||||
|
||||
decoder := json.NewDecoder(f)
|
||||
if err := decoder.Decode(&cases); err != nil {
|
||||
t.Fatalf("expect no error, %v", err)
|
||||
}
|
||||
|
||||
const testPathStyle = "path"
|
||||
for i, c := range cases {
|
||||
svc := s3.New(unit.Session, &aws.Config{
|
||||
Region: &c.Region,
|
||||
UseDualStack: &c.UseDualStack,
|
||||
S3UseAccelerate: &c.UseS3Accelerate,
|
||||
S3ForcePathStyle: aws.Bool(c.ConfiguredAddressingStyle == testPathStyle),
|
||||
})
|
||||
|
||||
req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{
|
||||
Bucket: &c.Bucket,
|
||||
})
|
||||
req.Build()
|
||||
if req.Error != nil {
|
||||
t.Fatalf("expect no error, got %v", req.Error)
|
||||
}
|
||||
|
||||
// Trim trailing '/' that are added by the SDK but not in the tests.
|
||||
actualURI := strings.TrimRightFunc(
|
||||
req.HTTPRequest.URL.String(),
|
||||
func(r rune) bool { return r == '/' },
|
||||
)
|
||||
if e, a := c.ExpectedURI, actualURI; e != a {
|
||||
t.Errorf("%d, expect\n%s\nurl to be\n%s", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
// +build go1.6
|
||||
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func TestAdd100Continue_Added(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
r, _ := svc.PutObjectRequest(&s3.PutObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("dest"),
|
||||
Body: bytes.NewReader(make([]byte, 1024*1024*5)),
|
||||
})
|
||||
|
||||
err := r.Sign()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if e, a := "100-Continue", r.HTTPRequest.Header.Get("Expect"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdd100Continue_SkipDisabled(t *testing.T) {
|
||||
svc := s3.New(unit.Session, aws.NewConfig().WithS3Disable100Continue(true))
|
||||
r, _ := svc.PutObjectRequest(&s3.PutObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("dest"),
|
||||
Body: bytes.NewReader(make([]byte, 1024*1024*5)),
|
||||
})
|
||||
|
||||
err := r.Sign()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if r.HTTPRequest.Header.Get("Expect") != "" {
|
||||
t.Errorf("expected empty value, but received %s", r.HTTPRequest.Header.Get("Expect"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdd100Continue_SkipNonPUT(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
r, _ := svc.GetObjectRequest(&s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("dest"),
|
||||
})
|
||||
|
||||
err := r.Sign()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if r.HTTPRequest.Header.Get("Expect") != "" {
|
||||
t.Errorf("expected empty value, but received %s", r.HTTPRequest.Header.Get("Expect"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdd100Continue_SkipTooSmall(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
r, _ := svc.PutObjectRequest(&s3.PutObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("dest"),
|
||||
Body: bytes.NewReader(make([]byte, 1024*1024*1)),
|
||||
})
|
||||
|
||||
err := r.Sign()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if r.HTTPRequest.Header.Get("Expect") != "" {
|
||||
t.Errorf("expected empty value, but received %s", r.HTTPRequest.Header.Get("Expect"))
|
||||
}
|
||||
}
|
||||
-190
@@ -1,190 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"io"
|
||||
)
|
||||
|
||||
// AESCBC is a symmetric crypto algorithm. This algorithm
|
||||
// requires a padder due to CBC needing to be of the same block
|
||||
// size. AES CBC is vulnerable to Padding Oracle attacks and
|
||||
// so should be avoided when possible.
|
||||
type aesCBC struct {
|
||||
encrypter cipher.BlockMode
|
||||
decrypter cipher.BlockMode
|
||||
padder Padder
|
||||
}
|
||||
|
||||
// newAESCBC creates a new AES CBC cipher. Expects keys to be of
|
||||
// the correct size.
|
||||
func newAESCBC(cd CipherData, padder Padder) (Cipher, error) {
|
||||
block, err := aes.NewCipher(cd.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
encrypter := cipher.NewCBCEncrypter(block, cd.IV)
|
||||
decrypter := cipher.NewCBCDecrypter(block, cd.IV)
|
||||
|
||||
return &aesCBC{encrypter, decrypter, padder}, nil
|
||||
}
|
||||
|
||||
// Encrypt will encrypt the data using AES CBC by returning
|
||||
// an io.Reader. The io.Reader will encrypt the data as Read
|
||||
// is called.
|
||||
func (c *aesCBC) Encrypt(src io.Reader) io.Reader {
|
||||
reader := &cbcEncryptReader{
|
||||
encrypter: c.encrypter,
|
||||
src: src,
|
||||
padder: c.padder,
|
||||
}
|
||||
return reader
|
||||
}
|
||||
|
||||
type cbcEncryptReader struct {
|
||||
encrypter cipher.BlockMode
|
||||
src io.Reader
|
||||
padder Padder
|
||||
size int
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
// Read will read from our io.Reader and encrypt the data as necessary.
|
||||
// Due to padding, we have to do some logic that when we encounter an
|
||||
// end of file to pad properly.
|
||||
func (reader *cbcEncryptReader) Read(data []byte) (int, error) {
|
||||
n, err := reader.src.Read(data)
|
||||
reader.size += n
|
||||
blockSize := reader.encrypter.BlockSize()
|
||||
reader.buf.Write(data[:n])
|
||||
|
||||
if err == io.EOF {
|
||||
b := make([]byte, getSliceSize(blockSize, reader.buf.Len(), len(data)))
|
||||
n, err = reader.buf.Read(b)
|
||||
if err != nil && err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
// The buffer is now empty, we can now pad the data
|
||||
if reader.buf.Len() == 0 {
|
||||
b, err = reader.padder.Pad(b[:n], reader.size)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
n = len(b)
|
||||
err = io.EOF
|
||||
}
|
||||
// We only want to encrypt if we have read anything
|
||||
if n > 0 {
|
||||
reader.encrypter.CryptBlocks(data, b)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if size := reader.buf.Len(); size >= blockSize {
|
||||
nBlocks := size / blockSize
|
||||
if size > len(data) {
|
||||
nBlocks = len(data) / blockSize
|
||||
}
|
||||
|
||||
if nBlocks > 0 {
|
||||
b := make([]byte, nBlocks*blockSize)
|
||||
n, _ = reader.buf.Read(b)
|
||||
reader.encrypter.CryptBlocks(data, b[:n])
|
||||
}
|
||||
} else {
|
||||
n = 0
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Decrypt will decrypt the data using AES CBC
|
||||
func (c *aesCBC) Decrypt(src io.Reader) io.Reader {
|
||||
return &cbcDecryptReader{
|
||||
decrypter: c.decrypter,
|
||||
src: src,
|
||||
padder: c.padder,
|
||||
}
|
||||
}
|
||||
|
||||
type cbcDecryptReader struct {
|
||||
decrypter cipher.BlockMode
|
||||
src io.Reader
|
||||
padder Padder
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
// Read will read from our io.Reader and decrypt the data as necessary.
|
||||
// Due to padding, we have to do some logic that when we encounter an
|
||||
// end of file to pad properly.
|
||||
func (reader *cbcDecryptReader) Read(data []byte) (int, error) {
|
||||
n, err := reader.src.Read(data)
|
||||
blockSize := reader.decrypter.BlockSize()
|
||||
reader.buf.Write(data[:n])
|
||||
|
||||
if err == io.EOF {
|
||||
b := make([]byte, getSliceSize(blockSize, reader.buf.Len(), len(data)))
|
||||
n, err = reader.buf.Read(b)
|
||||
if err != nil && err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
// We only want to decrypt if we have read anything
|
||||
if n > 0 {
|
||||
reader.decrypter.CryptBlocks(data, b)
|
||||
}
|
||||
|
||||
if reader.buf.Len() == 0 {
|
||||
b, err = reader.padder.Unpad(data[:n])
|
||||
n = len(b)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
err = io.EOF
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
|
||||
if size := reader.buf.Len(); size >= blockSize {
|
||||
nBlocks := size / blockSize
|
||||
if size > len(data) {
|
||||
nBlocks = len(data) / blockSize
|
||||
}
|
||||
// The last block is always padded. This will allow us to unpad
|
||||
// when we receive an io.EOF error
|
||||
nBlocks -= blockSize
|
||||
|
||||
if nBlocks > 0 {
|
||||
b := make([]byte, nBlocks*blockSize)
|
||||
n, _ = reader.buf.Read(b)
|
||||
reader.decrypter.CryptBlocks(data, b[:n])
|
||||
} else {
|
||||
n = 0
|
||||
}
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// getSliceSize will return the correct amount of bytes we need to
|
||||
// read with regards to padding.
|
||||
func getSliceSize(blockSize, bufSize, dataSize int) int {
|
||||
size := bufSize
|
||||
if bufSize > dataSize {
|
||||
size = dataSize
|
||||
}
|
||||
size = size - (size % blockSize) - blockSize
|
||||
if size <= 0 {
|
||||
size = blockSize
|
||||
}
|
||||
|
||||
return size
|
||||
}
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
cbcKeySize = 32
|
||||
cbcNonceSize = 16
|
||||
)
|
||||
|
||||
type cbcContentCipherBuilder struct {
|
||||
generator CipherDataGenerator
|
||||
padder Padder
|
||||
}
|
||||
|
||||
// AESCBCContentCipherBuilder returns a new encryption only mode structure with a specific cipher
|
||||
// for the master key
|
||||
func AESCBCContentCipherBuilder(generator CipherDataGenerator, padder Padder) ContentCipherBuilder {
|
||||
return cbcContentCipherBuilder{generator: generator, padder: padder}
|
||||
}
|
||||
|
||||
func (builder cbcContentCipherBuilder) ContentCipher() (ContentCipher, error) {
|
||||
cd, err := builder.generator.GenerateCipherData(cbcKeySize, cbcNonceSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cd.Padder = builder.padder
|
||||
return newAESCBCContentCipher(cd)
|
||||
}
|
||||
|
||||
// newAESCBCContentCipher will create a new aes cbc content cipher. If the cipher data's
|
||||
// will set the CEK algorithm if it hasn't been set.
|
||||
func newAESCBCContentCipher(cd CipherData) (ContentCipher, error) {
|
||||
if len(cd.CEKAlgorithm) == 0 {
|
||||
cd.CEKAlgorithm = strings.Join([]string{AESCBC, cd.Padder.Name()}, "/")
|
||||
}
|
||||
cipher, err := newAESCBC(cd, cd.Padder)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &aesCBCContentCipher{
|
||||
CipherData: cd,
|
||||
Cipher: cipher,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// aesCBCContentCipher will use AES CBC for the main cipher.
|
||||
type aesCBCContentCipher struct {
|
||||
CipherData CipherData
|
||||
Cipher Cipher
|
||||
}
|
||||
|
||||
// EncryptContents will generate a random key and iv and encrypt the data using cbc
|
||||
func (cc *aesCBCContentCipher) EncryptContents(src io.Reader) (io.Reader, error) {
|
||||
return cc.Cipher.Encrypt(src), nil
|
||||
}
|
||||
|
||||
// DecryptContents will use the symmetric key provider to instantiate a new CBC cipher.
|
||||
// We grab a decrypt reader from CBC and wrap it in a CryptoReadCloser. The only error
|
||||
// expected here is when the key or iv is of invalid length.
|
||||
func (cc *aesCBCContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser, error) {
|
||||
reader := cc.Cipher.Decrypt(src)
|
||||
return &CryptoReadCloser{Body: src, Decrypter: reader}, nil
|
||||
}
|
||||
|
||||
// GetCipherData returns cipher data
|
||||
func (cc aesCBCContentCipher) GetCipherData() CipherData {
|
||||
return cc.CipherData
|
||||
}
|
||||
Generated
Vendored
-20
@@ -1,20 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
func TestAESCBCBuilder(t *testing.T) {
|
||||
generator := mockGenerator{}
|
||||
builder := s3crypto.AESCBCContentCipherBuilder(generator, s3crypto.NoPadder)
|
||||
if builder == nil {
|
||||
t.Fatal(builder)
|
||||
}
|
||||
|
||||
_, err := builder.ContentCipher()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
const (
|
||||
pkcs5BlockSize = 16
|
||||
)
|
||||
|
||||
var aescbcPadding = aescbcPadder{pkcs7Padder{16}}
|
||||
|
||||
// AESCBCPadder is used to pad AES encrypted and decrypted data.
|
||||
// Altough it uses the pkcs5Padder, it isn't following the RFC
|
||||
// for PKCS5. The only reason why it is called pkcs5Padder is
|
||||
// due to the Name returning PKCS5Padding.
|
||||
var AESCBCPadder = Padder(aescbcPadding)
|
||||
|
||||
type aescbcPadder struct {
|
||||
padder pkcs7Padder
|
||||
}
|
||||
|
||||
func (padder aescbcPadder) Pad(b []byte, n int) ([]byte, error) {
|
||||
return padder.padder.Pad(b, n)
|
||||
}
|
||||
|
||||
func (padder aescbcPadder) Unpad(b []byte) ([]byte, error) {
|
||||
return padder.padder.Unpad(b)
|
||||
}
|
||||
|
||||
func (padder aescbcPadder) Name() string {
|
||||
return "PKCS5Padding"
|
||||
}
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAESCBCPadding(t *testing.T) {
|
||||
for i := 0; i < 16; i++ {
|
||||
input := make([]byte, i)
|
||||
expected := append(input, bytes.Repeat([]byte{byte(16 - i)}, 16-i)...)
|
||||
b, err := AESCBCPadder.Pad(input, len(input))
|
||||
if err != nil {
|
||||
t.Fatal("Expected error to be nil but received " + err.Error())
|
||||
}
|
||||
if len(b) != len(expected) {
|
||||
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
|
||||
}
|
||||
if bytes.Compare(b, expected) != 0 {
|
||||
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAESCBCUnpadding(t *testing.T) {
|
||||
for i := 0; i < 16; i++ {
|
||||
expected := make([]byte, i)
|
||||
input := append(expected, bytes.Repeat([]byte{byte(16 - i)}, 16-i)...)
|
||||
b, err := AESCBCPadder.Unpad(input)
|
||||
if err != nil {
|
||||
t.Fatal("Error received, was expecting nil: " + err.Error())
|
||||
}
|
||||
if len(b) != len(expected) {
|
||||
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
|
||||
}
|
||||
if bytes.Compare(b, expected) != 0 {
|
||||
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
|
||||
}
|
||||
}
|
||||
}
|
||||
-501
@@ -1,501 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAESCBCEncryptDecrypt(t *testing.T) {
|
||||
var testCases = []struct {
|
||||
key string
|
||||
iv string
|
||||
plaintext string
|
||||
ciphertext string
|
||||
decodeHex bool
|
||||
padder Padder
|
||||
}{
|
||||
// Test vectors from RFC 3602: https://tools.ietf.org/html/rfc3602
|
||||
{
|
||||
"06a9214036b8a15b512e03d534120006",
|
||||
"3dafba429d9eb430b422da802c9fac41",
|
||||
"Single block msg",
|
||||
"e353779c1079aeb82708942dbe77181a",
|
||||
false,
|
||||
NoPadder,
|
||||
},
|
||||
{
|
||||
"c286696d887c9aa0611bbb3e2025a45a",
|
||||
"562e17996d093d28ddb3ba695a2e6f58",
|
||||
"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f",
|
||||
"d296cd94c2cccf8a3a863028b5e1dc0a7586602d253cfff91b8266bea6d61ab1",
|
||||
true,
|
||||
NoPadder,
|
||||
},
|
||||
{
|
||||
"6c3ea0477630ce21a2ce334aa746c2cd",
|
||||
"c782dc4c098c66cbd9cd27d825682c81",
|
||||
"This is a 48-byte message (exactly 3 AES blocks)",
|
||||
"d0a02b3836451753d493665d33f0e8862dea54cdb293abc7506939276772f8d5021c19216bad525c8579695d83ba2684",
|
||||
false,
|
||||
NoPadder,
|
||||
},
|
||||
{
|
||||
"56e47a38c5598974bc46903dba290349",
|
||||
"8ce82eefbea0da3c44699ed7db51b7d9",
|
||||
"a0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedf",
|
||||
"c30e32ffedc0774e6aff6af0869f71aa0f3af07a9a31a9c684db207eb0ef8e4e35907aa632c3ffdf868bb7b29d3d46ad83ce9f9a102ee99d49a53e87f4c3da55",
|
||||
true,
|
||||
NoPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"",
|
||||
"B012949BA07D1A6DCE9DEE67274D41AB",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41",
|
||||
"8A11ABA68A566132FFE04DB336621D41",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141",
|
||||
"97D0896E41DFDB5CEA4A9EB70A938CFD",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141",
|
||||
"8464EAD45FA2D8790E8741E32C28083F",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141",
|
||||
"1E656D6E2745BA9F154FAF136B2BC73D",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141",
|
||||
"0B6031C4B230DAC6BD6D3F195645B287",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141",
|
||||
"5D09FEB6462BB489489A7E18FD341D9D",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141",
|
||||
"85745E398F2FD1050C2CE8F8614DA369",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141",
|
||||
"7BE52933970BA7B0FC6FB3FC37648205",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141",
|
||||
"ED3A1E134EF36CCFE60C8123B4272F89",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141",
|
||||
"C3B7C9E177E1052FC736F65FC1E74209",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141",
|
||||
"C3A8B53F7F57F0B9D346FA99810A3C28",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141",
|
||||
"D16B1ECE5BF00AF919E139E99775FF06",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141",
|
||||
"B258F4DF57FFCA1EFCF8D76140F05139",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141",
|
||||
"3CD2282DE24A2CF9E23326CC3DC9077A",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141",
|
||||
"3010232E7C752A3B4C9EE428B4C4FE88",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A22BC4E6D03BFD2418DD412D1ED1B31AF",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A5427BBD4A4D50776989441370E3B5B16",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A7FF985F55567D1B25EA40E23BB4CB1FE",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A0835E548C7370D8F8D9925C0E6B54727",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ADC0CF1436399E67BC1122B31CB596649",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A3D096F0DEAFF91938B82E5D404B0B065",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217AD56ABA897A355CF307CCB74226243192",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A151284F950B1B1DBCAD6D9E7900DF4E6",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217AEF85A612514121C299A1D87116C4A182",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A67F157569BFB4013EA3AD16DB8C69AD6",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217AF8520D191F6ACBD88B2140588B91C697",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ADD8BBAA71745669B96F2683E2F5AEC35",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217AFB2D4282817D7EC6B33EFAD7AA14A3C5",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A459B89E7E0DAF3DA654576B60B2DA7CE",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A65759F23F9789D05B23D5DBAA9E32036",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217A03C78FBD5E2CB08B3B6D181E23FBDE79",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA013D941FBBDE56C106C482CD022F290F",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA0645D313AC3C29B79DB1AA2E00A5B393",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA2ED0FD8048053BF22EBE501D82C4B3F1",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAC57D706C7866A01D6E913F98AE57EE54",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAB7FC1241FAFDFE45C4FF982D5DC1DAEF",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA7063EA296922DE8BDFD3B29D786C5F91",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA3A4603475F4AFDBFADC6E7FA908188B1",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA3365C63C2AF2A6C8FB4D0E9ED3C6FDA3",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA78BCC1874C0B7EB52645FC8F03B9C9CF",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA9B7A31397718EECB89B9E9CCCD729326",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAB15EA8A67E9E9FADB4249710277F3D4F",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA94641D6A076193C660632CEA3F9CB02C",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAB2170A08417BE77F0EAA9110F4790E12",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA4E30F1CD7B2256ABD57DC3DAB05376C9",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"41414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA9909B7B93D01BDAAC22D15AF34DF1EEF",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"4141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CAD97F5D1206F00E5C7225CAD81CCD4027",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
{
|
||||
"11111111111111111111111111111111",
|
||||
"22222222222222222222222222222222",
|
||||
"414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141",
|
||||
"C3304FA46097CBBA59085416764A217ACEF79EE1163A2F52888F87A3979EB3CA570CBB001A0C87558906B60C884AB5F41DA97CEF2A9401BC6DD0D22A54DBAD6D",
|
||||
true,
|
||||
AESCBCPadder,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
key, _ := hex.DecodeString(testCase.key)
|
||||
iv, _ := hex.DecodeString(testCase.iv)
|
||||
cd := CipherData{
|
||||
Key: key,
|
||||
IV: iv,
|
||||
}
|
||||
|
||||
cbc, err := newAESCBC(cd, testCase.padder)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Sprintf("Case %d: Expected no error for cipher creation, but received: %v", i, err.Error()))
|
||||
}
|
||||
|
||||
plaintext := []byte(testCase.plaintext)
|
||||
if testCase.decodeHex {
|
||||
plaintext, _ = hex.DecodeString(testCase.plaintext)
|
||||
}
|
||||
|
||||
cipherdata := cbc.Encrypt(bytes.NewReader(plaintext))
|
||||
ciphertext := []byte{}
|
||||
b := make([]byte, 19)
|
||||
err = nil
|
||||
n := 0
|
||||
for err != io.EOF {
|
||||
n, err = cipherdata.Read(b)
|
||||
ciphertext = append(ciphertext, b[:n]...)
|
||||
}
|
||||
|
||||
if err != io.EOF {
|
||||
t.Fatal(fmt.Sprintf("Case %d: Expected no error during io reading, but received: %v", i, err.Error()))
|
||||
}
|
||||
|
||||
expectedData, _ := hex.DecodeString(testCase.ciphertext)
|
||||
if bytes.Compare(expectedData, ciphertext) != 0 {
|
||||
t.Log("\n", ciphertext, "\n", expectedData)
|
||||
t.Fatal(fmt.Sprintf("Case %d: AES CBC encryption fails. Data is not the same", i))
|
||||
}
|
||||
|
||||
plaindata := cbc.Decrypt(bytes.NewReader(ciphertext))
|
||||
plaintextDecrypted := []byte{}
|
||||
err = nil
|
||||
for err != io.EOF {
|
||||
n, err = plaindata.Read(b)
|
||||
plaintextDecrypted = append(plaintextDecrypted, b[:n]...)
|
||||
}
|
||||
if err != io.EOF {
|
||||
t.Fatal(fmt.Sprintf("Case %d: Expected no error during io reading, but received: %v", i, err.Error()))
|
||||
}
|
||||
|
||||
if bytes.Compare(plaintext, plaintextDecrypted) != 0 {
|
||||
t.Log("\n", plaintext, "\n", plaintextDecrypted)
|
||||
t.Fatal(fmt.Sprintf("Case %d: AES CBC decryption fails. Data is not the same", i))
|
||||
}
|
||||
}
|
||||
}
|
||||
-105
@@ -1,105 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
// AESGCM Symmetric encryption algorithm. Since Golang designed this
|
||||
// with only TLS in mind. We have to load it all into memory meaning
|
||||
// this isn't streamed.
|
||||
type aesGCM struct {
|
||||
aead cipher.AEAD
|
||||
nonce []byte
|
||||
}
|
||||
|
||||
// newAESGCM creates a new AES GCM cipher. Expects keys to be of
|
||||
// the correct size.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// cd := &s3crypto.CipherData{
|
||||
// Key: key,
|
||||
// "IV": iv,
|
||||
// }
|
||||
// cipher, err := s3crypto.newAESGCM(cd)
|
||||
func newAESGCM(cd CipherData) (Cipher, error) {
|
||||
block, err := aes.NewCipher(cd.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
aesgcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &aesGCM{aesgcm, cd.IV}, nil
|
||||
}
|
||||
|
||||
// Encrypt will encrypt the data using AES GCM
|
||||
// Tag will be included as the last 16 bytes of the slice
|
||||
func (c *aesGCM) Encrypt(src io.Reader) io.Reader {
|
||||
reader := &gcmEncryptReader{
|
||||
encrypter: c.aead,
|
||||
nonce: c.nonce,
|
||||
src: src,
|
||||
}
|
||||
return reader
|
||||
}
|
||||
|
||||
type gcmEncryptReader struct {
|
||||
encrypter cipher.AEAD
|
||||
nonce []byte
|
||||
src io.Reader
|
||||
buf *bytes.Buffer
|
||||
}
|
||||
|
||||
func (reader *gcmEncryptReader) Read(data []byte) (int, error) {
|
||||
if reader.buf == nil {
|
||||
b, err := ioutil.ReadAll(reader.src)
|
||||
if err != nil {
|
||||
return len(b), err
|
||||
}
|
||||
b = reader.encrypter.Seal(b[:0], reader.nonce, b, nil)
|
||||
reader.buf = bytes.NewBuffer(b)
|
||||
}
|
||||
|
||||
return reader.buf.Read(data)
|
||||
}
|
||||
|
||||
// Decrypt will decrypt the data using AES GCM
|
||||
func (c *aesGCM) Decrypt(src io.Reader) io.Reader {
|
||||
return &gcmDecryptReader{
|
||||
decrypter: c.aead,
|
||||
nonce: c.nonce,
|
||||
src: src,
|
||||
}
|
||||
}
|
||||
|
||||
type gcmDecryptReader struct {
|
||||
decrypter cipher.AEAD
|
||||
nonce []byte
|
||||
src io.Reader
|
||||
buf *bytes.Buffer
|
||||
}
|
||||
|
||||
func (reader *gcmDecryptReader) Read(data []byte) (int, error) {
|
||||
if reader.buf == nil {
|
||||
b, err := ioutil.ReadAll(reader.src)
|
||||
if err != nil {
|
||||
return len(b), err
|
||||
}
|
||||
b, err = reader.decrypter.Open(b[:0], reader.nonce, b, nil)
|
||||
if err != nil {
|
||||
return len(b), err
|
||||
}
|
||||
|
||||
reader.buf = bytes.NewBuffer(b)
|
||||
}
|
||||
|
||||
return reader.buf.Read(data)
|
||||
}
|
||||
-68
@@ -1,68 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
gcmKeySize = 32
|
||||
gcmNonceSize = 12
|
||||
)
|
||||
|
||||
type gcmContentCipherBuilder struct {
|
||||
generator CipherDataGenerator
|
||||
}
|
||||
|
||||
// AESGCMContentCipherBuilder returns a new encryption only mode structure with a specific cipher
|
||||
// for the master key
|
||||
func AESGCMContentCipherBuilder(generator CipherDataGenerator) ContentCipherBuilder {
|
||||
return gcmContentCipherBuilder{generator}
|
||||
}
|
||||
|
||||
func (builder gcmContentCipherBuilder) ContentCipher() (ContentCipher, error) {
|
||||
cd, err := builder.generator.GenerateCipherData(gcmKeySize, gcmNonceSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return newAESGCMContentCipher(cd)
|
||||
}
|
||||
|
||||
func newAESGCMContentCipher(cd CipherData) (ContentCipher, error) {
|
||||
cd.CEKAlgorithm = AESGCMNoPadding
|
||||
cd.TagLength = "128"
|
||||
|
||||
cipher, err := newAESGCM(cd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &aesGCMContentCipher{
|
||||
CipherData: cd,
|
||||
Cipher: cipher,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AESGCMContentCipher will use AES GCM for the main cipher.
|
||||
type aesGCMContentCipher struct {
|
||||
CipherData CipherData
|
||||
Cipher Cipher
|
||||
}
|
||||
|
||||
// EncryptContents will generate a random key and iv and encrypt the data using cbc
|
||||
func (cc *aesGCMContentCipher) EncryptContents(src io.Reader) (io.Reader, error) {
|
||||
return cc.Cipher.Encrypt(src), nil
|
||||
}
|
||||
|
||||
// DecryptContents will use the symmetric key provider to instantiate a new GCM cipher.
|
||||
// We grab a decrypt reader from gcm and wrap it in a CryptoReadCloser. The only error
|
||||
// expected here is when the key or iv is of invalid length.
|
||||
func (cc *aesGCMContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser, error) {
|
||||
reader := cc.Cipher.Decrypt(src)
|
||||
return &CryptoReadCloser{Body: src, Decrypter: reader}, nil
|
||||
}
|
||||
|
||||
// GetCipherData returns cipher data
|
||||
func (cc aesGCMContentCipher) GetCipherData() CipherData {
|
||||
return cc.CipherData
|
||||
}
|
||||
Generated
Vendored
-28
@@ -1,28 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
func TestAESGCMContentCipherBuilder(t *testing.T) {
|
||||
generator := mockGenerator{}
|
||||
if builder := s3crypto.AESGCMContentCipherBuilder(generator); builder == nil {
|
||||
t.Error("expected non-nil value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAESGCMContentCipherNewEncryptor(t *testing.T) {
|
||||
generator := mockGenerator{}
|
||||
builder := s3crypto.AESGCMContentCipherBuilder(generator)
|
||||
cipher, err := builder.ContentCipher()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if cipher == nil {
|
||||
t.Errorf("expected non-nil vaue")
|
||||
}
|
||||
}
|
||||
-81
@@ -1,81 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// AES GCM
|
||||
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_128_Test_0(t *testing.T) {
|
||||
iv, _ := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
|
||||
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
plaintext, _ := hex.DecodeString("2db5168e932556f8089a0622981d017d")
|
||||
expected, _ := hex.DecodeString("fa4362189661d163fcd6a56d8bf0405ad636ac1bbedd5cc3ee727dc2ab4a9489")
|
||||
tag, _ := hex.DecodeString("d636ac1bbedd5cc3ee727dc2ab4a9489")
|
||||
aesgcmTest(t, iv, key, plaintext, expected, tag)
|
||||
}
|
||||
|
||||
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_104_Test_3(t *testing.T) {
|
||||
iv, _ := hex.DecodeString("4742357c335913153ff0eb0f")
|
||||
key, _ := hex.DecodeString("e5a0eb92cc2b064e1bc80891faf1fab5e9a17a9c3a984e25416720e30e6c2b21")
|
||||
plaintext, _ := hex.DecodeString("8499893e16b0ba8b007d54665a")
|
||||
expected, _ := hex.DecodeString("eb8e6175f1fe38eb1acf95fd5188a8b74bb74fda553e91020a23deed45")
|
||||
tag, _ := hex.DecodeString("88a8b74bb74fda553e91020a23deed45")
|
||||
aesgcmTest(t, iv, key, plaintext, expected, tag)
|
||||
}
|
||||
|
||||
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_256_Test_6(t *testing.T) {
|
||||
iv, _ := hex.DecodeString("a291484c3de8bec6b47f525f")
|
||||
key, _ := hex.DecodeString("37f39137416bafde6f75022a7a527cc593b6000a83ff51ec04871a0ff5360e4e")
|
||||
plaintext, _ := hex.DecodeString("fafd94cede8b5a0730394bec68a8e77dba288d6ccaa8e1563a81d6e7ccc7fc97")
|
||||
expected, _ := hex.DecodeString("44dc868006b21d49284016565ffb3979cc4271d967628bf7cdaf86db888e92e501a2b578aa2f41ec6379a44a31cc019c")
|
||||
tag, _ := hex.DecodeString("01a2b578aa2f41ec6379a44a31cc019c")
|
||||
aesgcmTest(t, iv, key, plaintext, expected, tag)
|
||||
}
|
||||
|
||||
func TestAES_GCM_NIST_gcmEncryptExtIV256_PTLen_408_Test_8(t *testing.T) {
|
||||
iv, _ := hex.DecodeString("92f258071d79af3e63672285")
|
||||
key, _ := hex.DecodeString("595f259c55abe00ae07535ca5d9b09d6efb9f7e9abb64605c337acbd6b14fc7e")
|
||||
plaintext, _ := hex.DecodeString("a6fee33eb110a2d769bbc52b0f36969c287874f665681477a25fc4c48015c541fbe2394133ba490a34ee2dd67b898177849a91")
|
||||
expected, _ := hex.DecodeString("bbca4a9e09ae9690c0f6f8d405e53dccd666aa9c5fa13c8758bc30abe1ddd1bcce0d36a1eaaaaffef20cd3c5970b9673f8a65c26ccecb9976fd6ac9c2c0f372c52c821")
|
||||
tag, _ := hex.DecodeString("26ccecb9976fd6ac9c2c0f372c52c821")
|
||||
aesgcmTest(t, iv, key, plaintext, expected, tag)
|
||||
}
|
||||
|
||||
func aesgcmTest(t *testing.T, iv, key, plaintext, expected, tag []byte) {
|
||||
cd := CipherData{
|
||||
Key: key,
|
||||
IV: iv,
|
||||
}
|
||||
gcm, err := newAESGCM(cd)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
cipherdata := gcm.Encrypt(bytes.NewReader(plaintext))
|
||||
|
||||
ciphertext, err := ioutil.ReadAll(cipherdata)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
// splitting tag and ciphertext
|
||||
etag := ciphertext[len(ciphertext)-16:]
|
||||
if !bytes.Equal(etag, tag) {
|
||||
t.Errorf("expected tags to be equivalent")
|
||||
}
|
||||
if !bytes.Equal(ciphertext, expected) {
|
||||
t.Errorf("expected ciphertext to be equivalent")
|
||||
}
|
||||
|
||||
data := gcm.Decrypt(bytes.NewReader(ciphertext))
|
||||
text, err := ioutil.ReadAll(data)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if !bytes.Equal(plaintext, text) {
|
||||
t.Errorf("expected ciphertext to be equivalent")
|
||||
}
|
||||
}
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// Cipher interface allows for either encryption and decryption of an object
|
||||
type Cipher interface {
|
||||
Encrypter
|
||||
Decrypter
|
||||
}
|
||||
|
||||
// Encrypter interface with only the encrypt method
|
||||
type Encrypter interface {
|
||||
Encrypt(io.Reader) io.Reader
|
||||
}
|
||||
|
||||
// Decrypter interface with only the decrypt method
|
||||
type Decrypter interface {
|
||||
Decrypt(io.Reader) io.Reader
|
||||
}
|
||||
|
||||
// CryptoReadCloser handles closing of the body and allowing reads from the decrypted
|
||||
// content.
|
||||
type CryptoReadCloser struct {
|
||||
Body io.ReadCloser
|
||||
Decrypter io.Reader
|
||||
isClosed bool
|
||||
}
|
||||
|
||||
// Close lets the CryptoReadCloser satisfy io.ReadCloser interface
|
||||
func (rc *CryptoReadCloser) Close() error {
|
||||
rc.isClosed = true
|
||||
return rc.Body.Close()
|
||||
}
|
||||
|
||||
// Read lets the CryptoReadCloser satisfy io.ReadCloser interface
|
||||
func (rc *CryptoReadCloser) Read(b []byte) (int, error) {
|
||||
if rc.isClosed {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return rc.Decrypter.Read(b)
|
||||
}
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import "io"
|
||||
|
||||
// ContentCipherBuilder is a builder interface that builds
|
||||
// ciphers for each request.
|
||||
type ContentCipherBuilder interface {
|
||||
ContentCipher() (ContentCipher, error)
|
||||
}
|
||||
|
||||
// ContentCipher deals with encrypting and decrypting content
|
||||
type ContentCipher interface {
|
||||
EncryptContents(io.Reader) (io.Reader, error)
|
||||
DecryptContents(io.ReadCloser) (io.ReadCloser, error)
|
||||
GetCipherData() CipherData
|
||||
}
|
||||
|
||||
// CipherData is used for content encryption. It is used for storing the
|
||||
// metadata of the encrypted content.
|
||||
type CipherData struct {
|
||||
Key []byte
|
||||
IV []byte
|
||||
WrapAlgorithm string
|
||||
CEKAlgorithm string
|
||||
TagLength string
|
||||
MaterialDescription MaterialDescription
|
||||
// EncryptedKey should be populated when calling GenerateCipherData
|
||||
EncryptedKey []byte
|
||||
|
||||
Padder Padder
|
||||
}
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
func TestCryptoReadCloserRead(t *testing.T) {
|
||||
expectedStr := "HELLO WORLD "
|
||||
str := strings.NewReader(expectedStr)
|
||||
rc := &s3crypto.CryptoReadCloser{Body: ioutil.NopCloser(str), Decrypter: str}
|
||||
|
||||
b, err := ioutil.ReadAll(rc)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if expectedStr != string(b) {
|
||||
t.Errorf("expected %s, but received %s", expectedStr, string(b))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCryptoReadCloserClose(t *testing.T) {
|
||||
data := "HELLO WORLD "
|
||||
expectedStr := ""
|
||||
|
||||
str := strings.NewReader(data)
|
||||
rc := &s3crypto.CryptoReadCloser{Body: ioutil.NopCloser(str), Decrypter: str}
|
||||
rc.Close()
|
||||
|
||||
b, err := ioutil.ReadAll(rc)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if expectedStr != string(b) {
|
||||
t.Errorf("expected %s, but received %s", expectedStr, string(b))
|
||||
}
|
||||
}
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
)
|
||||
|
||||
func (client *DecryptionClient) contentCipherFromEnvelope(env Envelope) (ContentCipher, error) {
|
||||
wrap, err := client.wrapFromEnvelope(env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client.cekFromEnvelope(env, wrap)
|
||||
}
|
||||
|
||||
func (client *DecryptionClient) wrapFromEnvelope(env Envelope) (CipherDataDecrypter, error) {
|
||||
f, ok := client.WrapRegistry[env.WrapAlg]
|
||||
if !ok || f == nil {
|
||||
return nil, awserr.New(
|
||||
"InvalidWrapAlgorithmError",
|
||||
"wrap algorithm isn't supported, "+env.WrapAlg,
|
||||
nil,
|
||||
)
|
||||
}
|
||||
return f(env)
|
||||
}
|
||||
|
||||
// AESGCMNoPadding is the constant value that is used to specify
|
||||
// the CEK algorithm consiting of AES GCM with no padding.
|
||||
const AESGCMNoPadding = "AES/GCM/NoPadding"
|
||||
|
||||
// AESCBC is the string constant that signifies the AES CBC algorithm cipher.
|
||||
const AESCBC = "AES/CBC"
|
||||
|
||||
func (client *DecryptionClient) cekFromEnvelope(env Envelope, decrypter CipherDataDecrypter) (ContentCipher, error) {
|
||||
f, ok := client.CEKRegistry[env.CEKAlg]
|
||||
if !ok || f == nil {
|
||||
return nil, awserr.New(
|
||||
"InvalidCEKAlgorithmError",
|
||||
"cek algorithm isn't supported, "+env.CEKAlg,
|
||||
nil,
|
||||
)
|
||||
}
|
||||
|
||||
key, err := base64.StdEncoding.DecodeString(env.CipherKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
iv, err := base64.StdEncoding.DecodeString(env.IV)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err = decrypter.DecryptKey(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cd := CipherData{
|
||||
Key: key,
|
||||
IV: iv,
|
||||
CEKAlgorithm: env.CEKAlg,
|
||||
Padder: client.getPadder(env.CEKAlg),
|
||||
}
|
||||
return f(cd)
|
||||
}
|
||||
|
||||
// getPadder will return an unpadder with checking the cek algorithm specific padder.
|
||||
// If there wasn't a cek algorithm specific padder, we check the padder itself.
|
||||
// We return a no unpadder, if no unpadder was found. This means any customization
|
||||
// either contained padding within the cipher implementation, and to maintain
|
||||
// backwards compatility we will simply not unpad anything.
|
||||
func (client *DecryptionClient) getPadder(cekAlg string) Padder {
|
||||
padder, ok := client.PadderRegistry[cekAlg]
|
||||
if !ok {
|
||||
padder, ok = client.PadderRegistry[cekAlg[strings.LastIndex(cekAlg, "/")+1:]]
|
||||
if !ok {
|
||||
return NoPadder
|
||||
}
|
||||
}
|
||||
return padder
|
||||
}
|
||||
|
||||
func encodeMeta(reader hashReader, cd CipherData) (Envelope, error) {
|
||||
iv := base64.StdEncoding.EncodeToString(cd.IV)
|
||||
key := base64.StdEncoding.EncodeToString(cd.EncryptedKey)
|
||||
|
||||
md5 := reader.GetValue()
|
||||
contentLength := reader.GetContentLength()
|
||||
|
||||
md5Str := base64.StdEncoding.EncodeToString(md5)
|
||||
matdesc, err := cd.MaterialDescription.encodeDescription()
|
||||
if err != nil {
|
||||
return Envelope{}, err
|
||||
}
|
||||
|
||||
return Envelope{
|
||||
CipherKey: key,
|
||||
IV: iv,
|
||||
MatDesc: string(matdesc),
|
||||
WrapAlg: cd.WrapAlgorithm,
|
||||
CEKAlg: cd.CEKAlgorithm,
|
||||
TagLen: cd.TagLength,
|
||||
UnencryptedMD5: md5Str,
|
||||
UnencryptedContentLen: strconv.FormatInt(contentLength, 10),
|
||||
}, nil
|
||||
}
|
||||
-267
@@ -1,267 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/kms"
|
||||
)
|
||||
|
||||
func TestWrapFactory(t *testing.T) {
|
||||
c := DecryptionClient{
|
||||
WrapRegistry: map[string]WrapEntry{
|
||||
KMSWrap: (kmsKeyHandler{
|
||||
kms: kms.New(unit.Session),
|
||||
}).decryptHandler,
|
||||
},
|
||||
CEKRegistry: map[string]CEKEntry{
|
||||
AESGCMNoPadding: newAESGCMContentCipher,
|
||||
},
|
||||
}
|
||||
env := Envelope{
|
||||
WrapAlg: KMSWrap,
|
||||
MatDesc: `{"kms_cmk_id":""}`,
|
||||
}
|
||||
wrap, err := c.wrapFromEnvelope(env)
|
||||
w, ok := wrap.(*kmsKeyHandler)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if wrap == nil {
|
||||
t.Error("expected non-nil value")
|
||||
}
|
||||
if !ok {
|
||||
t.Errorf("expected kmsKeyHandler, but received %v", *w)
|
||||
}
|
||||
}
|
||||
func TestWrapFactoryErrorNoWrap(t *testing.T) {
|
||||
c := DecryptionClient{
|
||||
WrapRegistry: map[string]WrapEntry{
|
||||
KMSWrap: (kmsKeyHandler{
|
||||
kms: kms.New(unit.Session),
|
||||
}).decryptHandler,
|
||||
},
|
||||
CEKRegistry: map[string]CEKEntry{
|
||||
AESGCMNoPadding: newAESGCMContentCipher,
|
||||
},
|
||||
}
|
||||
env := Envelope{
|
||||
WrapAlg: "none",
|
||||
MatDesc: `{"kms_cmk_id":""}`,
|
||||
}
|
||||
wrap, err := c.wrapFromEnvelope(env)
|
||||
|
||||
if err == nil {
|
||||
t.Error("expected error, but received none")
|
||||
}
|
||||
if wrap != nil {
|
||||
t.Errorf("expected nil wrap value, received %v", wrap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapFactoryCustomEntry(t *testing.T) {
|
||||
c := DecryptionClient{
|
||||
WrapRegistry: map[string]WrapEntry{
|
||||
"custom": (kmsKeyHandler{
|
||||
kms: kms.New(unit.Session),
|
||||
}).decryptHandler,
|
||||
},
|
||||
CEKRegistry: map[string]CEKEntry{
|
||||
AESGCMNoPadding: newAESGCMContentCipher,
|
||||
},
|
||||
}
|
||||
env := Envelope{
|
||||
WrapAlg: "custom",
|
||||
MatDesc: `{"kms_cmk_id":""}`,
|
||||
}
|
||||
wrap, err := c.wrapFromEnvelope(env)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if wrap == nil {
|
||||
t.Errorf("expected nil wrap value, received %v", wrap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCEKFactory(t *testing.T) {
|
||||
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
keyB64 := base64.URLEncoding.EncodeToString(key)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
c := DecryptionClient{
|
||||
WrapRegistry: map[string]WrapEntry{
|
||||
KMSWrap: (kmsKeyHandler{
|
||||
kms: kms.New(sess),
|
||||
}).decryptHandler,
|
||||
},
|
||||
CEKRegistry: map[string]CEKEntry{
|
||||
AESGCMNoPadding: newAESGCMContentCipher,
|
||||
},
|
||||
PadderRegistry: map[string]Padder{
|
||||
NoPadder.Name(): NoPadder,
|
||||
},
|
||||
}
|
||||
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
ivB64 := base64.URLEncoding.EncodeToString(iv)
|
||||
|
||||
cipherKey, err := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
cipherKeyB64 := base64.URLEncoding.EncodeToString(cipherKey)
|
||||
|
||||
env := Envelope{
|
||||
WrapAlg: KMSWrap,
|
||||
CEKAlg: AESGCMNoPadding,
|
||||
CipherKey: cipherKeyB64,
|
||||
IV: ivB64,
|
||||
MatDesc: `{"kms_cmk_id":""}`,
|
||||
}
|
||||
wrap, err := c.wrapFromEnvelope(env)
|
||||
cek, err := c.cekFromEnvelope(env, wrap)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if cek == nil {
|
||||
t.Errorf("expected non-nil cek")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCEKFactoryNoCEK(t *testing.T) {
|
||||
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
keyB64 := base64.URLEncoding.EncodeToString(key)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
c := DecryptionClient{
|
||||
WrapRegistry: map[string]WrapEntry{
|
||||
KMSWrap: (kmsKeyHandler{
|
||||
kms: kms.New(sess),
|
||||
}).decryptHandler,
|
||||
},
|
||||
CEKRegistry: map[string]CEKEntry{
|
||||
AESGCMNoPadding: newAESGCMContentCipher,
|
||||
},
|
||||
PadderRegistry: map[string]Padder{
|
||||
NoPadder.Name(): NoPadder,
|
||||
},
|
||||
}
|
||||
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
ivB64 := base64.URLEncoding.EncodeToString(iv)
|
||||
|
||||
cipherKey, err := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
cipherKeyB64 := base64.URLEncoding.EncodeToString(cipherKey)
|
||||
|
||||
env := Envelope{
|
||||
WrapAlg: KMSWrap,
|
||||
CEKAlg: "none",
|
||||
CipherKey: cipherKeyB64,
|
||||
IV: ivB64,
|
||||
MatDesc: `{"kms_cmk_id":""}`,
|
||||
}
|
||||
wrap, err := c.wrapFromEnvelope(env)
|
||||
cek, err := c.cekFromEnvelope(env, wrap)
|
||||
|
||||
if err == nil {
|
||||
t.Error("expected error, but received none")
|
||||
}
|
||||
if cek != nil {
|
||||
t.Errorf("expected nil cek value, received %v", wrap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCEKFactoryCustomEntry(t *testing.T) {
|
||||
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
keyB64 := base64.URLEncoding.EncodeToString(key)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
c := DecryptionClient{
|
||||
WrapRegistry: map[string]WrapEntry{
|
||||
KMSWrap: (kmsKeyHandler{
|
||||
kms: kms.New(sess),
|
||||
}).decryptHandler,
|
||||
},
|
||||
CEKRegistry: map[string]CEKEntry{
|
||||
"custom": newAESGCMContentCipher,
|
||||
},
|
||||
PadderRegistry: map[string]Padder{},
|
||||
}
|
||||
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
ivB64 := base64.URLEncoding.EncodeToString(iv)
|
||||
|
||||
cipherKey, err := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
cipherKeyB64 := base64.URLEncoding.EncodeToString(cipherKey)
|
||||
|
||||
env := Envelope{
|
||||
WrapAlg: KMSWrap,
|
||||
CEKAlg: "custom",
|
||||
CipherKey: cipherKeyB64,
|
||||
IV: ivB64,
|
||||
MatDesc: `{"kms_cmk_id":""}`,
|
||||
}
|
||||
wrap, err := c.wrapFromEnvelope(env)
|
||||
cek, err := c.cekFromEnvelope(env, wrap)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if cek == nil {
|
||||
t.Errorf("expected non-nil cek")
|
||||
}
|
||||
}
|
||||
-135
@@ -1,135 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/client"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/service/kms"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3iface"
|
||||
)
|
||||
|
||||
// WrapEntry is builder that return a proper key decrypter and error
|
||||
type WrapEntry func(Envelope) (CipherDataDecrypter, error)
|
||||
|
||||
// CEKEntry is a builder thatn returns a proper content decrypter and error
|
||||
type CEKEntry func(CipherData) (ContentCipher, error)
|
||||
|
||||
// DecryptionClient is an S3 crypto client. The decryption client
|
||||
// will handle all get object requests from Amazon S3.
|
||||
// Supported key wrapping algorithms:
|
||||
// *AWS KMS
|
||||
//
|
||||
// Supported content ciphers:
|
||||
// * AES/GCM
|
||||
// * AES/CBC
|
||||
type DecryptionClient struct {
|
||||
S3Client s3iface.S3API
|
||||
// LoadStrategy is used to load the metadata either from the metadata of the object
|
||||
// or from a separate file in s3.
|
||||
//
|
||||
// Defaults to our default load strategy.
|
||||
LoadStrategy LoadStrategy
|
||||
|
||||
WrapRegistry map[string]WrapEntry
|
||||
CEKRegistry map[string]CEKEntry
|
||||
PadderRegistry map[string]Padder
|
||||
}
|
||||
|
||||
// NewDecryptionClient instantiates a new S3 crypto client
|
||||
//
|
||||
// Example:
|
||||
// sess := session.New()
|
||||
// svc := s3crypto.NewDecryptionClient(sess, func(svc *s3crypto.DecryptionClient{
|
||||
// // Custom client options here
|
||||
// }))
|
||||
func NewDecryptionClient(prov client.ConfigProvider, options ...func(*DecryptionClient)) *DecryptionClient {
|
||||
s3client := s3.New(prov)
|
||||
client := &DecryptionClient{
|
||||
S3Client: s3client,
|
||||
LoadStrategy: defaultV2LoadStrategy{
|
||||
client: s3client,
|
||||
},
|
||||
WrapRegistry: map[string]WrapEntry{
|
||||
KMSWrap: (kmsKeyHandler{
|
||||
kms: kms.New(prov),
|
||||
}).decryptHandler,
|
||||
},
|
||||
CEKRegistry: map[string]CEKEntry{
|
||||
AESGCMNoPadding: newAESGCMContentCipher,
|
||||
strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): newAESCBCContentCipher,
|
||||
},
|
||||
PadderRegistry: map[string]Padder{
|
||||
strings.Join([]string{AESCBC, AESCBCPadder.Name()}, "/"): AESCBCPadder,
|
||||
"NoPadding": NoPadder,
|
||||
},
|
||||
}
|
||||
for _, option := range options {
|
||||
option(client)
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
// GetObjectRequest will make a request to s3 and retrieve the object. In this process
|
||||
// decryption will be done. The SDK only supports V2 reads of KMS and GCM.
|
||||
//
|
||||
// Example:
|
||||
// sess := session.New()
|
||||
// svc := s3crypto.NewDecryptionClient(sess)
|
||||
// req, out := svc.GetObjectRequest(&s3.GetObjectInput {
|
||||
// Key: aws.String("testKey"),
|
||||
// Bucket: aws.String("testBucket"),
|
||||
// })
|
||||
// err := req.Send()
|
||||
func (c *DecryptionClient) GetObjectRequest(input *s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput) {
|
||||
req, out := c.S3Client.GetObjectRequest(input)
|
||||
req.Handlers.Unmarshal.PushBack(func(r *request.Request) {
|
||||
env, err := c.LoadStrategy.Load(r)
|
||||
if err != nil {
|
||||
r.Error = err
|
||||
out.Body.Close()
|
||||
return
|
||||
}
|
||||
|
||||
// If KMS should return the correct CEK algorithm with the proper
|
||||
// KMS key provider
|
||||
cipher, err := c.contentCipherFromEnvelope(env)
|
||||
if err != nil {
|
||||
r.Error = err
|
||||
out.Body.Close()
|
||||
return
|
||||
}
|
||||
|
||||
reader, err := cipher.DecryptContents(out.Body)
|
||||
if err != nil {
|
||||
r.Error = err
|
||||
out.Body.Close()
|
||||
return
|
||||
}
|
||||
out.Body = reader
|
||||
})
|
||||
return req, out
|
||||
}
|
||||
|
||||
// GetObject is a wrapper for GetObjectRequest
|
||||
func (c *DecryptionClient) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
|
||||
req, out := c.GetObjectRequest(input)
|
||||
return out, req.Send()
|
||||
}
|
||||
|
||||
// GetObjectWithContext is a wrapper for GetObjectRequest with the additional
|
||||
// context, and request options support.
|
||||
//
|
||||
// GetObjectWithContext is the same as GetObject with the additional support for
|
||||
// Context input parameters. The Context must not be nil. A nil Context will
|
||||
// cause a panic. Use the Context to add deadlining, timeouts, ect. In the future
|
||||
// this may create sub-contexts for individual underlying requests.
|
||||
func (c *DecryptionClient) GetObjectWithContext(ctx aws.Context, input *s3.GetObjectInput, opts ...request.Option) (*s3.GetObjectOutput, error) {
|
||||
req, out := c.GetObjectRequest(input)
|
||||
req.SetContext(ctx)
|
||||
req.ApplyOptions(opts...)
|
||||
return out, req.Send()
|
||||
}
|
||||
-251
@@ -1,251 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/awstesting"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
func TestGetObjectGCM(t *testing.T) {
|
||||
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
keyB64 := base64.StdEncoding.EncodeToString(key)
|
||||
// This is our KMS response
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
c := s3crypto.NewDecryptionClient(sess)
|
||||
if c == nil {
|
||||
t.Error("expected non-nil value")
|
||||
}
|
||||
input := &s3.GetObjectInput{
|
||||
Key: aws.String("test"),
|
||||
Bucket: aws.String("test"),
|
||||
}
|
||||
req, out := c.GetObjectRequest(input)
|
||||
req.Handlers.Send.Clear()
|
||||
req.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
iv, err := hex.DecodeString("0d18e06c7c725ac9e362e1ce")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
b, err := hex.DecodeString("fa4362189661d163fcd6a56d8bf0405ad636ac1bbedd5cc3ee727dc2ab4a9489")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
Header: http.Header{
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"SpFRES0JyU8BLZSKo51SrwILK4lhtZsWiMNjgO4WmoK+joMwZPG7Hw=="},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{base64.URLEncoding.EncodeToString(iv)},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"arn:aws:kms:us-east-1:172259396726:key/a22a4b30-79f4-4b3d-bab4-a26d327a231b"}`},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{s3crypto.AESGCMNoPadding},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-tag-len"): []string{"128"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewBuffer(b)),
|
||||
}
|
||||
out.Metadata = make(map[string]*string)
|
||||
out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap)
|
||||
})
|
||||
err := req.Send()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
b, err := ioutil.ReadAll(out.Body)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
expected, err := hex.DecodeString("2db5168e932556f8089a0622981d017d")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(expected, b) {
|
||||
t.Error("expected bytes to be equivalent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectCBC(t *testing.T) {
|
||||
key, _ := hex.DecodeString("898be9cc5004ed0fa6e117c9a3099d31")
|
||||
keyB64 := base64.StdEncoding.EncodeToString(key)
|
||||
// This is our KMS response
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
c := s3crypto.NewDecryptionClient(sess)
|
||||
if c == nil {
|
||||
t.Error("expected non-nil value")
|
||||
}
|
||||
input := &s3.GetObjectInput{
|
||||
Key: aws.String("test"),
|
||||
Bucket: aws.String("test"),
|
||||
}
|
||||
req, out := c.GetObjectRequest(input)
|
||||
req.Handlers.Send.Clear()
|
||||
req.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
iv, err := hex.DecodeString("9dea7621945988f96491083849b068df")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
b, err := hex.DecodeString("e232cd6ef50047801ee681ec30f61d53cfd6b0bca02fd03c1b234baa10ea82ac9dab8b960926433a19ce6dea08677e34")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
Header: http.Header{
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"SpFRES0JyU8BLZSKo51SrwILK4lhtZsWiMNjgO4WmoK+joMwZPG7Hw=="},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{base64.URLEncoding.EncodeToString(iv)},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"arn:aws:kms:us-east-1:172259396726:key/a22a4b30-79f4-4b3d-bab4-a26d327a231b"}`},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{strings.Join([]string{s3crypto.AESCBC, s3crypto.AESCBCPadder.Name()}, "/")},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewBuffer(b)),
|
||||
}
|
||||
out.Metadata = make(map[string]*string)
|
||||
out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap)
|
||||
})
|
||||
err := req.Send()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
b, err := ioutil.ReadAll(out.Body)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
expected, err := hex.DecodeString("0397f4f6820b1f9386f14403be5ac16e50213bd473b4874b9bcbf5f318ee686b1d")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(expected, b) {
|
||||
t.Error("expected bytes to be equivalent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectCBC2(t *testing.T) {
|
||||
key, _ := hex.DecodeString("8d70e92489c4e6cfb12261b4d17f4b85826da687fc8742fcf9f87fadb5b4cb89")
|
||||
keyB64 := base64.StdEncoding.EncodeToString(key)
|
||||
// This is our KMS response
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
c := s3crypto.NewDecryptionClient(sess)
|
||||
if c == nil {
|
||||
t.Error("expected non-nil value")
|
||||
}
|
||||
input := &s3.GetObjectInput{
|
||||
Key: aws.String("test"),
|
||||
Bucket: aws.String("test"),
|
||||
}
|
||||
req, out := c.GetObjectRequest(input)
|
||||
req.Handlers.Send.Clear()
|
||||
req.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
b, err := hex.DecodeString("fd0c71ecb7ed16a9bf42ea5f75501d416df608f190890c3b4d8897f24744cd7f9ea4a0b212e60634302450e1c5378f047ff753ccefe365d411c36339bf22e301fae4c3a6226719a4b93dc74c1af79d0296659b5d56c0892315f2c7cc30190220db1eaafae3920d6d9c65d0aa366499afc17af493454e141c6e0fbdeb6a990cb4")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
Header: http.Header{
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-key-v2"): []string{"AQEDAHikdGvcj7Gil5VqAR/JWvvPp3ue26+t2vhWy4lL2hg4mAAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDCcy43wCR0bSsnzTrAIBEIA7WdD2jxC3tCrK6TOdiEfbIN64m+UN7Velz4y0LRra5jn2U1CDClacwIpiBYuDp5ymPKO+ZqUGE0WEf20="},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-iv"): []string{"EMMWJY8ZLcK/9FOj3iCpng=="},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-matdesc"): []string{`{"kms_cmk_id":"arn:aws:kms:us-east-1:172259396726:key/a22a4b30-79f4-4b3d-bab4-a26d327a231b"}`},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-wrap-alg"): []string{s3crypto.KMSWrap},
|
||||
http.CanonicalHeaderKey("x-amz-meta-x-amz-cek-alg"): []string{strings.Join([]string{s3crypto.AESCBC, s3crypto.AESCBCPadder.Name()}, "/")},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewBuffer(b)),
|
||||
}
|
||||
fmt.Println("HEADER", r.HTTPResponse.Header)
|
||||
out.Metadata = make(map[string]*string)
|
||||
out.Metadata["x-amz-wrap-alg"] = aws.String(s3crypto.KMSWrap)
|
||||
})
|
||||
err := req.Send()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
b, err := ioutil.ReadAll(out.Body)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
expected, err := hex.DecodeString("a6ccd3482f5ce25c9ddeb69437cd0acbc0bdda2ef8696d90781de2b35704543529871b2032e68ef1c5baed1769aba8d420d1aca181341b49b8b3587a6580cdf1d809c68f06735f7735c16691f4b70c967d68fc08195b81ad71bcc4df452fd0a5799c1e1234f92f1cd929fc072167ccf9f2ac85b93170932b32")
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(expected, b) {
|
||||
t.Error("expected bytes to be equivalent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectWithContext(t *testing.T) {
|
||||
c := s3crypto.NewDecryptionClient(unit.Session)
|
||||
|
||||
ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
|
||||
ctx.Error = fmt.Errorf("context canceled")
|
||||
close(ctx.DoneCh)
|
||||
|
||||
input := s3.GetObjectInput{
|
||||
Key: aws.String("test"),
|
||||
Bucket: aws.String("test"),
|
||||
}
|
||||
_, err := c.GetObjectWithContext(ctx, &input)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, did not get one")
|
||||
}
|
||||
aerr := err.(awserr.Error)
|
||||
if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
|
||||
t.Errorf("expected error code %q, got %q", e, a)
|
||||
}
|
||||
if e, a := "canceled", aerr.Message(); !strings.Contains(a, e) {
|
||||
t.Errorf("expected error message to contain %q, but did not %q", e, a)
|
||||
}
|
||||
}
|
||||
-66
@@ -1,66 +0,0 @@
|
||||
/*
|
||||
Package s3crypto provides encryption to S3 using KMS and AES GCM.
|
||||
|
||||
Keyproviders are interfaces that handle masterkeys. Masterkeys are used to encrypt and decrypt the randomly
|
||||
generated cipher keys. The SDK currently uses KMS to do this. A user does not need to provide a master key
|
||||
since all that information is hidden in KMS.
|
||||
|
||||
Modes are interfaces that handle content encryption and decryption. It is an abstraction layer that instantiates
|
||||
the ciphers. If content is being encrypted we generate the key and iv of the cipher. For decryption, we use the
|
||||
metadata stored either on the object or an instruction file object to decrypt the contents.
|
||||
|
||||
Ciphers are interfaces that handle encryption and decryption of data. This may be key wrap ciphers or content
|
||||
ciphers.
|
||||
|
||||
Creating an S3 cryptography client
|
||||
|
||||
cmkID := "<some key ID>"
|
||||
sess := session.New()
|
||||
// Create the KeyProvider
|
||||
handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID)
|
||||
|
||||
// Create an encryption and decryption client
|
||||
// We need to pass the session here so S3 can use it. In addition, any decryption that
|
||||
// occurs will use the KMS client.
|
||||
svc := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler))
|
||||
svc := s3crypto.NewDecryptionClient(sess)
|
||||
|
||||
Configuration of the S3 cryptography client
|
||||
|
||||
cfg := s3crypto.EncryptionConfig{
|
||||
// Save instruction files to separate objects
|
||||
SaveStrategy: NewS3SaveStrategy(session.New(), ""),
|
||||
// Change instruction file suffix to .example
|
||||
InstructionFileSuffix: ".example",
|
||||
// Set temp folder path
|
||||
TempFolderPath: "/path/to/tmp/folder/",
|
||||
// Any content less than the minimum file size will use memory
|
||||
// instead of writing the contents to a temp file.
|
||||
MinFileSize: int64(1024 * 1024 * 1024),
|
||||
}
|
||||
|
||||
The default SaveStrategy is to the object's header.
|
||||
|
||||
The InstructionFileSuffix defaults to .instruction. Careful here though, if you do this, be sure you know
|
||||
what that suffix is in grabbing data. All requests will look for fooKey.example instead of fooKey.instruction.
|
||||
This suffix only affects gets and not puts. Put uses the keyprovider's suffix.
|
||||
|
||||
Registration of new wrap or cek algorithms are also supported by the SDK. Let's say we want to support `AES Wrap`
|
||||
and `AES CTR`. Let's assume we have already defined the functionality.
|
||||
|
||||
svc := s3crypto.NewDecryptionClient(sess)
|
||||
svc.WrapRegistry["AESWrap"] = NewAESWrap
|
||||
svc.CEKRegistry["AES/CTR/NoPadding"] = NewAESCTR
|
||||
|
||||
We have now registered these new algorithms to the decryption client. When the client calls `GetObject` and sees
|
||||
the wrap as `AESWrap` then it'll use that wrap algorithm. This is also true for `AES/CTR/NoPadding`.
|
||||
|
||||
For encryption adding a custom content cipher builder and key handler will allow for encryption of custom
|
||||
defined ciphers.
|
||||
|
||||
// Our wrap algorithm, AESWrap
|
||||
handler := NewAESWrap(key, iv)
|
||||
// Our content cipher builder, AESCTRContentCipherBuilder
|
||||
svc := s3crypto.NewEncryptionClient(sess, NewAESCTRContentCipherBuilder(handler))
|
||||
*/
|
||||
package s3crypto
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/client"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/internal/sdkio"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3iface"
|
||||
)
|
||||
|
||||
// DefaultMinFileSize is used to check whether we want to write to a temp file
|
||||
// or store the data in memory.
|
||||
const DefaultMinFileSize = 1024 * 512 * 5
|
||||
|
||||
// EncryptionClient is an S3 crypto client. By default the SDK will use Authentication mode which
|
||||
// will use KMS for key wrapping and AES GCM for content encryption.
|
||||
// AES GCM will load all data into memory. However, the rest of the content algorithms
|
||||
// do not load the entire contents into memory.
|
||||
type EncryptionClient struct {
|
||||
S3Client s3iface.S3API
|
||||
ContentCipherBuilder ContentCipherBuilder
|
||||
// SaveStrategy will dictate where the envelope is saved.
|
||||
//
|
||||
// Defaults to the object's metadata
|
||||
SaveStrategy SaveStrategy
|
||||
// TempFolderPath is used to store temp files when calling PutObject.
|
||||
// Temporary files are needed to compute the X-Amz-Content-Sha256 header.
|
||||
TempFolderPath string
|
||||
// MinFileSize is the minimum size for the content to write to a
|
||||
// temporary file instead of using memory.
|
||||
MinFileSize int64
|
||||
}
|
||||
|
||||
// NewEncryptionClient instantiates a new S3 crypto client
|
||||
//
|
||||
// Example:
|
||||
// cmkID := "arn:aws:kms:region:000000000000:key/00000000-0000-0000-0000-000000000000"
|
||||
// sess := session.New()
|
||||
// handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID)
|
||||
// svc := s3crypto.New(sess, s3crypto.AESGCMContentCipherBuilder(handler))
|
||||
func NewEncryptionClient(prov client.ConfigProvider, builder ContentCipherBuilder, options ...func(*EncryptionClient)) *EncryptionClient {
|
||||
client := &EncryptionClient{
|
||||
S3Client: s3.New(prov),
|
||||
ContentCipherBuilder: builder,
|
||||
SaveStrategy: HeaderV2SaveStrategy{},
|
||||
MinFileSize: DefaultMinFileSize,
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
option(client)
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
// PutObjectRequest creates a temp file to encrypt the contents into. It then streams
|
||||
// that data to S3.
|
||||
//
|
||||
// Example:
|
||||
// svc := s3crypto.New(session.New(), s3crypto.AESGCMContentCipherBuilder(handler))
|
||||
// req, out := svc.PutObjectRequest(&s3.PutObjectInput {
|
||||
// Key: aws.String("testKey"),
|
||||
// Bucket: aws.String("testBucket"),
|
||||
// Body: strings.NewReader("test data"),
|
||||
// })
|
||||
// err := req.Send()
|
||||
func (c *EncryptionClient) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) {
|
||||
req, out := c.S3Client.PutObjectRequest(input)
|
||||
|
||||
// Get Size of file
|
||||
n, err := aws.SeekerLen(input.Body)
|
||||
if err != nil {
|
||||
req.Error = err
|
||||
return req, out
|
||||
}
|
||||
|
||||
dst, err := getWriterStore(req, c.TempFolderPath, n >= c.MinFileSize)
|
||||
if err != nil {
|
||||
req.Error = err
|
||||
return req, out
|
||||
}
|
||||
|
||||
encryptor, err := c.ContentCipherBuilder.ContentCipher()
|
||||
req.Handlers.Build.PushFront(func(r *request.Request) {
|
||||
if err != nil {
|
||||
r.Error = err
|
||||
return
|
||||
}
|
||||
|
||||
md5 := newMD5Reader(input.Body)
|
||||
sha := newSHA256Writer(dst)
|
||||
reader, err := encryptor.EncryptContents(md5)
|
||||
if err != nil {
|
||||
r.Error = err
|
||||
return
|
||||
}
|
||||
|
||||
_, err = io.Copy(sha, reader)
|
||||
if err != nil {
|
||||
r.Error = err
|
||||
return
|
||||
}
|
||||
|
||||
data := encryptor.GetCipherData()
|
||||
env, err := encodeMeta(md5, data)
|
||||
if err != nil {
|
||||
r.Error = err
|
||||
return
|
||||
}
|
||||
|
||||
shaHex := hex.EncodeToString(sha.GetValue())
|
||||
req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", shaHex)
|
||||
|
||||
dst.Seek(0, sdkio.SeekStart)
|
||||
input.Body = dst
|
||||
|
||||
err = c.SaveStrategy.Save(env, r)
|
||||
r.Error = err
|
||||
})
|
||||
|
||||
return req, out
|
||||
}
|
||||
|
||||
// PutObject is a wrapper for PutObjectRequest
|
||||
func (c *EncryptionClient) PutObject(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
|
||||
req, out := c.PutObjectRequest(input)
|
||||
return out, req.Send()
|
||||
}
|
||||
|
||||
// PutObjectWithContext is a wrapper for PutObjectRequest with the additional
|
||||
// context, and request options support.
|
||||
//
|
||||
// PutObjectWithContext is the same as PutObject with the additional support for
|
||||
// Context input parameters. The Context must not be nil. A nil Context will
|
||||
// cause a panic. Use the Context to add deadlining, timeouts, ect. In the future
|
||||
// this may create sub-contexts for individual underlying requests.
|
||||
func (c *EncryptionClient) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInput, opts ...request.Option) (*s3.PutObjectOutput, error) {
|
||||
req, out := c.PutObjectRequest(input)
|
||||
req.SetContext(ctx)
|
||||
req.ApplyOptions(opts...)
|
||||
return out, req.Send()
|
||||
}
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/awstesting"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/kms"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
func TestDefaultConfigValues(t *testing.T) {
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
svc := kms.New(sess)
|
||||
handler := s3crypto.NewKMSKeyGenerator(svc, "testid")
|
||||
|
||||
c := s3crypto.NewEncryptionClient(sess, s3crypto.AESGCMContentCipherBuilder(handler))
|
||||
|
||||
if c == nil {
|
||||
t.Error("expected non-vil client value")
|
||||
}
|
||||
if c.ContentCipherBuilder == nil {
|
||||
t.Error("expected non-vil content cipher builder value")
|
||||
}
|
||||
if c.SaveStrategy == nil {
|
||||
t.Error("expected non-vil save strategy value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObject(t *testing.T) {
|
||||
size := 1024 * 1024
|
||||
data := make([]byte, size)
|
||||
expected := bytes.Repeat([]byte{1}, size)
|
||||
generator := mockGenerator{}
|
||||
cb := mockCipherBuilder{generator}
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
c := s3crypto.NewEncryptionClient(sess, cb)
|
||||
if c == nil {
|
||||
t.Error("expected non-vil client value")
|
||||
}
|
||||
input := &s3.PutObjectInput{
|
||||
Key: aws.String("test"),
|
||||
Bucket: aws.String("test"),
|
||||
Body: bytes.NewReader(data),
|
||||
}
|
||||
req, _ := c.PutObjectRequest(input)
|
||||
req.Handlers.Send.Clear()
|
||||
req.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
r.Error = errors.New("stop")
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
}
|
||||
})
|
||||
err := req.Send()
|
||||
if e, a := "stop", err.Error(); e != a {
|
||||
t.Errorf("expected %s error, but received %s", e, a)
|
||||
}
|
||||
b, err := ioutil.ReadAll(req.HTTPRequest.Body)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if !bytes.Equal(expected, b) {
|
||||
t.Error("expected bytes to be equivalent, but received otherwise")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectWithContext(t *testing.T) {
|
||||
generator := mockGenerator{}
|
||||
cb := mockCipherBuilder{generator}
|
||||
|
||||
c := s3crypto.NewEncryptionClient(unit.Session, cb)
|
||||
|
||||
ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
|
||||
ctx.Error = fmt.Errorf("context canceled")
|
||||
close(ctx.DoneCh)
|
||||
|
||||
input := s3.PutObjectInput{
|
||||
Bucket: aws.String("test"),
|
||||
Key: aws.String("test"),
|
||||
Body: bytes.NewReader([]byte{}),
|
||||
}
|
||||
_, err := c.PutObjectWithContext(ctx, &input)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, did not get one")
|
||||
}
|
||||
aerr := err.(awserr.Error)
|
||||
if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
|
||||
t.Errorf("expected error code %q, got %q", e, a)
|
||||
}
|
||||
if e, a := "canceled", aerr.Message(); !strings.Contains(a, e) {
|
||||
t.Errorf("expected error message to contain %q, but did not %q", e, a)
|
||||
}
|
||||
}
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
// DefaultInstructionKeySuffix is appended to the end of the instruction file key when
|
||||
// grabbing or saving to S3
|
||||
const DefaultInstructionKeySuffix = ".instruction"
|
||||
|
||||
const (
|
||||
metaHeader = "x-amz-meta"
|
||||
keyV1Header = "x-amz-key"
|
||||
keyV2Header = keyV1Header + "-v2"
|
||||
ivHeader = "x-amz-iv"
|
||||
matDescHeader = "x-amz-matdesc"
|
||||
cekAlgorithmHeader = "x-amz-cek-alg"
|
||||
wrapAlgorithmHeader = "x-amz-wrap-alg"
|
||||
tagLengthHeader = "x-amz-tag-len"
|
||||
unencryptedMD5Header = "x-amz-unencrypted-content-md5"
|
||||
unencryptedContentLengthHeader = "x-amz-unencrypted-content-length"
|
||||
)
|
||||
|
||||
// Envelope encryption starts off by generating a random symmetric key using
|
||||
// AES GCM. The SDK generates a random IV based off the encryption cipher
|
||||
// chosen. The master key that was provided, whether by the user or KMS, will be used
|
||||
// to encrypt the randomly generated symmetric key and base64 encode the iv. This will
|
||||
// allow for decryption of that same data later.
|
||||
type Envelope struct {
|
||||
// IV is the randomly generated IV base64 encoded.
|
||||
IV string `json:"x-amz-iv"`
|
||||
// CipherKey is the randomly generated cipher key.
|
||||
CipherKey string `json:"x-amz-key-v2"`
|
||||
// MaterialDesc is a description to distinguish from other envelopes.
|
||||
MatDesc string `json:"x-amz-matdesc"`
|
||||
WrapAlg string `json:"x-amz-wrap-alg"`
|
||||
CEKAlg string `json:"x-amz-cek-alg"`
|
||||
TagLen string `json:"x-amz-tag-len"`
|
||||
UnencryptedMD5 string `json:"x-amz-unencrypted-content-md5"`
|
||||
UnencryptedContentLen string `json:"x-amz-unencrypted-content-length"`
|
||||
}
|
||||
-61
@@ -1,61 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
"io"
|
||||
)
|
||||
|
||||
// hashReader is used for calculating SHA256 when following the sigv4 specification.
|
||||
// Additionally this used for calculating the unencrypted MD5.
|
||||
type hashReader interface {
|
||||
GetValue() []byte
|
||||
GetContentLength() int64
|
||||
}
|
||||
|
||||
type sha256Writer struct {
|
||||
sha256 []byte
|
||||
hash hash.Hash
|
||||
out io.Writer
|
||||
}
|
||||
|
||||
func newSHA256Writer(f io.Writer) *sha256Writer {
|
||||
return &sha256Writer{hash: sha256.New(), out: f}
|
||||
}
|
||||
func (r *sha256Writer) Write(b []byte) (int, error) {
|
||||
r.hash.Write(b)
|
||||
return r.out.Write(b)
|
||||
}
|
||||
|
||||
func (r *sha256Writer) GetValue() []byte {
|
||||
return r.hash.Sum(nil)
|
||||
}
|
||||
|
||||
type md5Reader struct {
|
||||
contentLength int64
|
||||
hash hash.Hash
|
||||
body io.Reader
|
||||
}
|
||||
|
||||
func newMD5Reader(body io.Reader) *md5Reader {
|
||||
return &md5Reader{hash: md5.New(), body: body}
|
||||
}
|
||||
|
||||
func (w *md5Reader) Read(b []byte) (int, error) {
|
||||
n, err := w.body.Read(b)
|
||||
if err != nil && err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
w.contentLength += int64(n)
|
||||
w.hash.Write(b[:n])
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *md5Reader) GetValue() []byte {
|
||||
return w.hash.Sum(nil)
|
||||
}
|
||||
|
||||
func (w *md5Reader) GetContentLength() int64 {
|
||||
return w.contentLength
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// From Go stdlib encoding/sha256 test cases
|
||||
func TestSHA256(t *testing.T) {
|
||||
sha := newSHA256Writer(nil)
|
||||
expected, _ := hex.DecodeString("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||
b := sha.GetValue()
|
||||
|
||||
if !bytes.Equal(expected, b) {
|
||||
t.Errorf("expected equivalent sha values, but received otherwise")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSHA256_Case2(t *testing.T) {
|
||||
sha := newSHA256Writer(bytes.NewBuffer([]byte{}))
|
||||
sha.Write([]byte("hello"))
|
||||
expected, _ := hex.DecodeString("2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")
|
||||
b := sha.GetValue()
|
||||
|
||||
if !bytes.Equal(expected, b) {
|
||||
t.Errorf("expected equivalent sha values, but received otherwise")
|
||||
}
|
||||
}
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
)
|
||||
|
||||
func getWriterStore(req *request.Request, path string, useTempFile bool) (io.ReadWriteSeeker, error) {
|
||||
if !useTempFile {
|
||||
return &bytesReadWriteSeeker{}, nil
|
||||
}
|
||||
// Create temp file to be used later for calculating the SHA256 header
|
||||
f, err := ioutil.TempFile(path, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
// Close the temp file and cleanup
|
||||
f.Close()
|
||||
fpath := filepath.Join(path, f.Name())
|
||||
os.Remove(fpath)
|
||||
})
|
||||
return f, nil
|
||||
}
|
||||
|
||||
type bytesReadWriteSeeker struct {
|
||||
buf []byte
|
||||
i int64
|
||||
}
|
||||
|
||||
// Copied from Go stdlib bytes.Reader
|
||||
func (ws *bytesReadWriteSeeker) Read(b []byte) (int, error) {
|
||||
if ws.i >= int64(len(ws.buf)) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(b, ws.buf[ws.i:])
|
||||
ws.i += int64(n)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (ws *bytesReadWriteSeeker) Write(b []byte) (int, error) {
|
||||
ws.buf = append(ws.buf, b...)
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
// Copied from Go stdlib bytes.Reader
|
||||
func (ws *bytesReadWriteSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
var abs int64
|
||||
switch whence {
|
||||
case 0:
|
||||
abs = offset
|
||||
case 1:
|
||||
abs = int64(ws.i) + offset
|
||||
case 2:
|
||||
abs = int64(len(ws.buf)) + offset
|
||||
default:
|
||||
return 0, errors.New("bytes.Reader.Seek: invalid whence")
|
||||
}
|
||||
if abs < 0 {
|
||||
return 0, errors.New("bytes.Reader.Seek: negative position")
|
||||
}
|
||||
ws.i = abs
|
||||
return abs, nil
|
||||
}
|
||||
-84
@@ -1,84 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/internal/sdkio"
|
||||
)
|
||||
|
||||
func TestBytesReadWriteSeeker_Read(t *testing.T) {
|
||||
b := &bytesReadWriteSeeker{[]byte{1, 2, 3}, 0}
|
||||
expected := []byte{1, 2, 3}
|
||||
buf := make([]byte, 3)
|
||||
n, err := b.Read(buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if e, a := 3, n; e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
|
||||
if !bytes.Equal(expected, buf) {
|
||||
t.Error("expected equivalent byte slices, but received otherwise")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesReadWriteSeeker_Write(t *testing.T) {
|
||||
b := &bytesReadWriteSeeker{}
|
||||
expected := []byte{1, 2, 3}
|
||||
buf := make([]byte, 3)
|
||||
n, err := b.Write([]byte{1, 2, 3})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if e, a := 3, n; e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
|
||||
n, err = b.Read(buf)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if e, a := 3, n; e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
|
||||
if !bytes.Equal(expected, buf) {
|
||||
t.Error("expected equivalent byte slices, but received otherwise")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesReadWriteSeeker_Seek(t *testing.T) {
|
||||
b := &bytesReadWriteSeeker{[]byte{1, 2, 3}, 0}
|
||||
expected := []byte{2, 3}
|
||||
m, err := b.Seek(1, sdkio.SeekStart)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if e, a := 1, int(m); e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
|
||||
buf := make([]byte, 3)
|
||||
n, err := b.Read(buf)
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if e, a := 2, n; e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
|
||||
if !bytes.Equal(expected, buf[:n]) {
|
||||
t.Error("expected equivalent byte slices, but received otherwise")
|
||||
}
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import "crypto/rand"
|
||||
|
||||
// CipherDataGenerator handles generating proper key and IVs of proper size for the
|
||||
// content cipher. CipherDataGenerator will also encrypt the key and store it in
|
||||
// the CipherData.
|
||||
type CipherDataGenerator interface {
|
||||
GenerateCipherData(int, int) (CipherData, error)
|
||||
}
|
||||
|
||||
// CipherDataDecrypter is a handler to decrypt keys from the envelope.
|
||||
type CipherDataDecrypter interface {
|
||||
DecryptKey([]byte) ([]byte, error)
|
||||
}
|
||||
|
||||
func generateBytes(n int) []byte {
|
||||
b := make([]byte, n)
|
||||
rand.Read(b)
|
||||
return b
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateBytes(t *testing.T) {
|
||||
b := generateBytes(5)
|
||||
if e, a := 5, len(b); e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
b = generateBytes(0)
|
||||
if e, a := 0, len(b); e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
b = generateBytes(1024)
|
||||
if e, a := 1024, len(b); e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
}
|
||||
-114
@@ -1,114 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/service/kms"
|
||||
"github.com/aws/aws-sdk-go/service/kms/kmsiface"
|
||||
)
|
||||
|
||||
const (
|
||||
// KMSWrap is a constant used during decryption to build a KMS key handler.
|
||||
KMSWrap = "kms"
|
||||
)
|
||||
|
||||
// kmsKeyHandler will make calls to KMS to get the masterkey
|
||||
type kmsKeyHandler struct {
|
||||
kms kmsiface.KMSAPI
|
||||
cmkID *string
|
||||
|
||||
CipherData
|
||||
}
|
||||
|
||||
// NewKMSKeyGenerator builds a new KMS key provider using the customer key ID and material
|
||||
// description.
|
||||
//
|
||||
// Example:
|
||||
// sess := session.New(&aws.Config{})
|
||||
// cmkID := "arn to key"
|
||||
// matdesc := s3crypto.MaterialDescription{}
|
||||
// handler := s3crypto.NewKMSKeyGenerator(kms.New(sess), cmkID)
|
||||
func NewKMSKeyGenerator(kmsClient kmsiface.KMSAPI, cmkID string) CipherDataGenerator {
|
||||
return NewKMSKeyGeneratorWithMatDesc(kmsClient, cmkID, MaterialDescription{})
|
||||
}
|
||||
|
||||
// NewKMSKeyGeneratorWithMatDesc builds a new KMS key provider using the customer key ID and material
|
||||
// description.
|
||||
//
|
||||
// Example:
|
||||
// sess := session.New(&aws.Config{})
|
||||
// cmkID := "arn to key"
|
||||
// matdesc := s3crypto.MaterialDescription{}
|
||||
// handler, err := s3crypto.NewKMSKeyGeneratorWithMatDesc(kms.New(sess), cmkID, matdesc)
|
||||
func NewKMSKeyGeneratorWithMatDesc(kmsClient kmsiface.KMSAPI, cmkID string, matdesc MaterialDescription) CipherDataGenerator {
|
||||
if matdesc == nil {
|
||||
matdesc = MaterialDescription{}
|
||||
}
|
||||
matdesc["kms_cmk_id"] = &cmkID
|
||||
|
||||
// These values are read only making them thread safe
|
||||
kp := &kmsKeyHandler{
|
||||
kms: kmsClient,
|
||||
cmkID: &cmkID,
|
||||
}
|
||||
// These values are read only making them thread safe
|
||||
kp.CipherData.WrapAlgorithm = KMSWrap
|
||||
kp.CipherData.MaterialDescription = matdesc
|
||||
return kp
|
||||
}
|
||||
|
||||
// decryptHandler initializes a KMS keyprovider with a material description. This
|
||||
// is used with Decrypting kms content, due to the cmkID being in the material description.
|
||||
func (kp kmsKeyHandler) decryptHandler(env Envelope) (CipherDataDecrypter, error) {
|
||||
m := MaterialDescription{}
|
||||
err := m.decodeDescription([]byte(env.MatDesc))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cmkID, ok := m["kms_cmk_id"]
|
||||
if !ok {
|
||||
return nil, awserr.New("MissingCMKIDError", "Material description is missing CMK ID", nil)
|
||||
}
|
||||
|
||||
kp.CipherData.MaterialDescription = m
|
||||
kp.cmkID = cmkID
|
||||
kp.WrapAlgorithm = KMSWrap
|
||||
return &kp, nil
|
||||
}
|
||||
|
||||
// DecryptKey makes a call to KMS to decrypt the key.
|
||||
func (kp *kmsKeyHandler) DecryptKey(key []byte) ([]byte, error) {
|
||||
out, err := kp.kms.Decrypt(&kms.DecryptInput{
|
||||
EncryptionContext: map[string]*string(kp.CipherData.MaterialDescription),
|
||||
CiphertextBlob: key,
|
||||
GrantTokens: []*string{},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Plaintext, nil
|
||||
}
|
||||
|
||||
// GenerateCipherData makes a call to KMS to generate a data key, Upon making
|
||||
// the call, it also sets the encrypted key.
|
||||
func (kp *kmsKeyHandler) GenerateCipherData(keySize, ivSize int) (CipherData, error) {
|
||||
out, err := kp.kms.GenerateDataKey(&kms.GenerateDataKeyInput{
|
||||
EncryptionContext: kp.CipherData.MaterialDescription,
|
||||
KeyId: kp.cmkID,
|
||||
KeySpec: aws.String("AES_256"),
|
||||
})
|
||||
if err != nil {
|
||||
return CipherData{}, err
|
||||
}
|
||||
|
||||
iv := generateBytes(ivSize)
|
||||
cd := CipherData{
|
||||
Key: out.Plaintext,
|
||||
IV: iv,
|
||||
WrapAlgorithm: KMSWrap,
|
||||
MaterialDescription: kp.CipherData.MaterialDescription,
|
||||
EncryptedKey: out.CiphertextBlob,
|
||||
}
|
||||
return cd, nil
|
||||
}
|
||||
-125
@@ -1,125 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/kms"
|
||||
)
|
||||
|
||||
func TestBuildKMSEncryptHandler(t *testing.T) {
|
||||
svc := kms.New(unit.Session)
|
||||
handler := NewKMSKeyGenerator(svc, "testid")
|
||||
if handler == nil {
|
||||
t.Error("expected non-nil handler")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildKMSEncryptHandlerWithMatDesc(t *testing.T) {
|
||||
svc := kms.New(unit.Session)
|
||||
handler := NewKMSKeyGeneratorWithMatDesc(svc, "testid", MaterialDescription{
|
||||
"Testing": aws.String("123"),
|
||||
})
|
||||
if handler == nil {
|
||||
t.Error("expected non-nil handler")
|
||||
}
|
||||
|
||||
kmsHandler := handler.(*kmsKeyHandler)
|
||||
expected := MaterialDescription{
|
||||
"kms_cmk_id": aws.String("testid"),
|
||||
"Testing": aws.String("123"),
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(expected, kmsHandler.CipherData.MaterialDescription) {
|
||||
t.Errorf("expected %v, but received %v", expected, kmsHandler.CipherData.MaterialDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKMSGenerateCipherData(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, `{"CiphertextBlob":"AQEDAHhqBCCY1MSimw8gOGcUma79cn4ANvTtQyv9iuBdbcEF1QAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDJ6IcN5E4wVbk38MNAIBEIA7oF1E3lS7FY9DkoxPc/UmJsEwHzL82zMqoLwXIvi8LQHr8If4Lv6zKqY8u0+JRgSVoqCvZDx3p8Cn6nM=","KeyId":"arn:aws:kms:us-west-2:042062605278:key/c80a5cdb-8d09-4f9f-89ee-df01b2e3870a","Plaintext":"6tmyz9JLBE2yIuU7iXpArqpDVle172WSmxjcO6GNT7E="}`)
|
||||
}))
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
svc := kms.New(sess)
|
||||
handler := NewKMSKeyGenerator(svc, "testid")
|
||||
|
||||
keySize := 32
|
||||
ivSize := 16
|
||||
|
||||
cd, err := handler.GenerateCipherData(keySize, ivSize)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if keySize != len(cd.Key) {
|
||||
t.Errorf("expected %d, but received %d", keySize, len(cd.Key))
|
||||
}
|
||||
if ivSize != len(cd.IV) {
|
||||
t.Errorf("expected %d, but received %d", ivSize, len(cd.IV))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKMSDecrypt(t *testing.T) {
|
||||
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
keyB64 := base64.URLEncoding.EncodeToString(key)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
handler, err := (kmsKeyHandler{kms: kms.New(sess)}).decryptHandler(Envelope{MatDesc: `{"kms_cmk_id":"test"}`})
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
plaintextKey, err := handler.DecryptKey([]byte{1, 2, 3, 4})
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(key, plaintextKey) {
|
||||
t.Errorf("expected %v, but received %v", key, plaintextKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKMSDecryptBadJSON(t *testing.T) {
|
||||
key, _ := hex.DecodeString("31bdadd96698c204aa9ce1448ea94ae1fb4a9a0b3c9d773b51bb1822666b8f22")
|
||||
keyB64 := base64.URLEncoding.EncodeToString(key)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintln(w, fmt.Sprintf("%s%s%s", `{"KeyId":"test-key-id","Plaintext":"`, keyB64, `"}`))
|
||||
}))
|
||||
|
||||
sess := unit.Session.Copy(&aws.Config{
|
||||
MaxRetries: aws.Int(0),
|
||||
Endpoint: aws.String(ts.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Region: aws.String("us-west-2"),
|
||||
})
|
||||
|
||||
_, err := (kmsKeyHandler{kms: kms.New(sess)}).decryptHandler(Envelope{MatDesc: `{"kms_cmk_id":"test"`})
|
||||
if err == nil {
|
||||
t.Errorf("expected error, but received none")
|
||||
}
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// MaterialDescription is used to identify how and what master
|
||||
// key has been used.
|
||||
type MaterialDescription map[string]*string
|
||||
|
||||
func (md *MaterialDescription) encodeDescription() ([]byte, error) {
|
||||
v, err := json.Marshal(&md)
|
||||
return v, err
|
||||
}
|
||||
|
||||
func (md *MaterialDescription) decodeDescription(b []byte) error {
|
||||
return json.Unmarshal(b, &md)
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
)
|
||||
|
||||
func TestEncodeMaterialDescription(t *testing.T) {
|
||||
md := MaterialDescription{}
|
||||
md["foo"] = aws.String("bar")
|
||||
b, err := md.encodeDescription()
|
||||
expected := `{"foo":"bar"}`
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if expected != string(b) {
|
||||
t.Errorf("expected %s, but received %s", expected, string(b))
|
||||
}
|
||||
}
|
||||
func TestDecodeMaterialDescription(t *testing.T) {
|
||||
md := MaterialDescription{}
|
||||
json := `{"foo":"bar"}`
|
||||
err := md.decodeDescription([]byte(json))
|
||||
expected := MaterialDescription{
|
||||
"foo": aws.String("bar"),
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(expected, md) {
|
||||
t.Error("expected material description to be equivalent, but received otherwise")
|
||||
}
|
||||
}
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
type mockGenerator struct {
|
||||
}
|
||||
|
||||
func (m mockGenerator) GenerateCipherData(keySize, ivSize int) (s3crypto.CipherData, error) {
|
||||
cd := s3crypto.CipherData{
|
||||
Key: make([]byte, keySize),
|
||||
IV: make([]byte, ivSize),
|
||||
}
|
||||
return cd, nil
|
||||
}
|
||||
|
||||
func (m mockGenerator) EncryptKey(key []byte) ([]byte, error) {
|
||||
size := len(key)
|
||||
b := bytes.Repeat([]byte{1}, size)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (m mockGenerator) DecryptKey(key []byte) ([]byte, error) {
|
||||
return make([]byte, 16), nil
|
||||
|
||||
}
|
||||
|
||||
type mockCipherBuilder struct {
|
||||
generator s3crypto.CipherDataGenerator
|
||||
}
|
||||
|
||||
func (builder mockCipherBuilder) ContentCipher() (s3crypto.ContentCipher, error) {
|
||||
cd, err := builder.generator.GenerateCipherData(32, 16)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &mockContentCipher{cd}, nil
|
||||
}
|
||||
|
||||
type mockContentCipher struct {
|
||||
cd s3crypto.CipherData
|
||||
}
|
||||
|
||||
func (cipher *mockContentCipher) GetCipherData() s3crypto.CipherData {
|
||||
return cipher.cd
|
||||
}
|
||||
|
||||
func (cipher *mockContentCipher) EncryptContents(src io.Reader) (io.Reader, error) {
|
||||
b, err := ioutil.ReadAll(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := len(b)
|
||||
b = bytes.Repeat([]byte{1}, size)
|
||||
return bytes.NewReader(b), nil
|
||||
}
|
||||
|
||||
func (cipher *mockContentCipher) DecryptContents(src io.ReadCloser) (io.ReadCloser, error) {
|
||||
b, err := ioutil.ReadAll(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := len(b)
|
||||
return ioutil.NopCloser(bytes.NewReader(make([]byte, size))), nil
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
// Padder handles padding of crypto data
|
||||
type Padder interface {
|
||||
// Pad will pad the byte array.
|
||||
// The second parameter is NOT how many
|
||||
// bytes to pad by, but how many bytes
|
||||
// have been read prior to the padding.
|
||||
// This allows for streamable padding.
|
||||
Pad([]byte, int) ([]byte, error)
|
||||
// Unpad will unpad the byte bytes. Unpad
|
||||
// methods must be constant time.
|
||||
Unpad([]byte) ([]byte, error)
|
||||
// Name returns the name of the padder.
|
||||
// This is used when decrypting on
|
||||
// instantiating new padders.
|
||||
Name() string
|
||||
}
|
||||
|
||||
// NoPadder does not pad anything
|
||||
var NoPadder = Padder(noPadder{})
|
||||
|
||||
type noPadder struct{}
|
||||
|
||||
func (padder noPadder) Pad(b []byte, n int) ([]byte, error) {
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (padder noPadder) Unpad(b []byte) ([]byte, error) {
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (padder noPadder) Name() string {
|
||||
return "NoPadding"
|
||||
}
|
||||
-80
@@ -1,80 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
//
|
||||
// Portions Licensed under the MIT License. Copyright (c) 2016 Carl Jackson
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/subtle"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
)
|
||||
|
||||
const (
|
||||
pkcs7MaxPaddingSize = 255
|
||||
)
|
||||
|
||||
type pkcs7Padder struct {
|
||||
blockSize int
|
||||
}
|
||||
|
||||
// NewPKCS7Padder follows the RFC 2315: https://www.ietf.org/rfc/rfc2315.txt
|
||||
// PKCS7 padding is subject to side-channel attacks and timing attacks. For
|
||||
// the most secure data, use an authenticated crypto algorithm.
|
||||
func NewPKCS7Padder(blockSize int) Padder {
|
||||
return pkcs7Padder{blockSize}
|
||||
}
|
||||
|
||||
var errPKCS7Padding = awserr.New("InvalidPadding", "invalid padding", nil)
|
||||
|
||||
// Pad will pad the data relative to how many bytes have been read.
|
||||
// Pad follows the PKCS7 standard.
|
||||
func (padder pkcs7Padder) Pad(buf []byte, n int) ([]byte, error) {
|
||||
if padder.blockSize < 1 || padder.blockSize > pkcs7MaxPaddingSize {
|
||||
return nil, awserr.New("InvalidBlockSize", "block size must be between 1 and 255", nil)
|
||||
}
|
||||
size := padder.blockSize - (n % padder.blockSize)
|
||||
pad := bytes.Repeat([]byte{byte(size)}, size)
|
||||
buf = append(buf, pad...)
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// Unpad will unpad the correct amount of bytes based off
|
||||
// of the PKCS7 standard
|
||||
func (padder pkcs7Padder) Unpad(buf []byte) ([]byte, error) {
|
||||
if len(buf) == 0 {
|
||||
return nil, errPKCS7Padding
|
||||
}
|
||||
|
||||
// Here be dragons. We're attempting to check the padding in constant
|
||||
// time. The only piece of information here which is public is len(buf).
|
||||
// This code is modeled loosely after tls1_cbc_remove_padding from
|
||||
// OpenSSL.
|
||||
padLen := buf[len(buf)-1]
|
||||
toCheck := pkcs7MaxPaddingSize
|
||||
good := 1
|
||||
if toCheck > len(buf) {
|
||||
toCheck = len(buf)
|
||||
}
|
||||
for i := 0; i < toCheck; i++ {
|
||||
b := buf[len(buf)-1-i]
|
||||
|
||||
outOfRange := subtle.ConstantTimeLessOrEq(int(padLen), i)
|
||||
equal := subtle.ConstantTimeByteEq(padLen, b)
|
||||
good &= subtle.ConstantTimeSelect(outOfRange, 1, equal)
|
||||
}
|
||||
|
||||
good &= subtle.ConstantTimeLessOrEq(1, int(padLen))
|
||||
good &= subtle.ConstantTimeLessOrEq(int(padLen), len(buf))
|
||||
|
||||
if good != 1 {
|
||||
return nil, errPKCS7Padding
|
||||
}
|
||||
|
||||
return buf[:len(buf)-int(padLen)], nil
|
||||
}
|
||||
|
||||
func (padder pkcs7Padder) Name() string {
|
||||
return "PKCS7Padding"
|
||||
}
|
||||
-57
@@ -1,57 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
func padTest(size int, t *testing.T) {
|
||||
padder := s3crypto.NewPKCS7Padder(size)
|
||||
for i := 0; i < size; i++ {
|
||||
input := make([]byte, i)
|
||||
expected := append(input, bytes.Repeat([]byte{byte(size - i)}, size-i)...)
|
||||
b, err := padder.Pad(input, len(input))
|
||||
if err != nil {
|
||||
t.Fatal("Expected error to be nil but received " + err.Error())
|
||||
}
|
||||
if len(b) != len(expected) {
|
||||
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
|
||||
}
|
||||
if bytes.Compare(b, expected) != 0 {
|
||||
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func unpadTest(size int, t *testing.T) {
|
||||
padder := s3crypto.NewPKCS7Padder(size)
|
||||
for i := 0; i < size; i++ {
|
||||
expected := make([]byte, i)
|
||||
input := append(expected, bytes.Repeat([]byte{byte(size - i)}, size-i)...)
|
||||
b, err := padder.Unpad(input)
|
||||
if err != nil {
|
||||
t.Fatal("Error received, was expecting nil: " + err.Error())
|
||||
}
|
||||
if len(b) != len(expected) {
|
||||
t.Fatal(fmt.Sprintf("Case %d: data is not of the same length", i))
|
||||
}
|
||||
if bytes.Compare(b, expected) != 0 {
|
||||
t.Fatal(fmt.Sprintf("Expected %v but got %v", expected, b))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPKCS7Padding(t *testing.T) {
|
||||
padTest(10, t)
|
||||
padTest(16, t)
|
||||
padTest(255, t)
|
||||
}
|
||||
|
||||
func TestPKCS7Unpadding(t *testing.T) {
|
||||
unpadTest(10, t)
|
||||
unpadTest(16, t)
|
||||
unpadTest(255, t)
|
||||
}
|
||||
-145
@@ -1,145 +0,0 @@
|
||||
package s3crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
// SaveStrategy is how the data's metadata wants to be saved
|
||||
type SaveStrategy interface {
|
||||
Save(Envelope, *request.Request) error
|
||||
}
|
||||
|
||||
// S3SaveStrategy will save the metadata to a separate instruction file in S3
|
||||
type S3SaveStrategy struct {
|
||||
Client *s3.S3
|
||||
InstructionFileSuffix string
|
||||
}
|
||||
|
||||
// Save will save the envelope contents to s3.
|
||||
func (strat S3SaveStrategy) Save(env Envelope, req *request.Request) error {
|
||||
input := req.Params.(*s3.PutObjectInput)
|
||||
b, err := json.Marshal(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
instInput := s3.PutObjectInput{
|
||||
Bucket: input.Bucket,
|
||||
Body: bytes.NewReader(b),
|
||||
}
|
||||
|
||||
if strat.InstructionFileSuffix == "" {
|
||||
instInput.Key = aws.String(*input.Key + DefaultInstructionKeySuffix)
|
||||
} else {
|
||||
instInput.Key = aws.String(*input.Key + strat.InstructionFileSuffix)
|
||||
}
|
||||
|
||||
_, err = strat.Client.PutObject(&instInput)
|
||||
return err
|
||||
}
|
||||
|
||||
// HeaderV2SaveStrategy will save the metadata of the crypto contents to the header of
|
||||
// the object.
|
||||
type HeaderV2SaveStrategy struct{}
|
||||
|
||||
// Save will save the envelope to the request's header.
|
||||
func (strat HeaderV2SaveStrategy) Save(env Envelope, req *request.Request) error {
|
||||
input := req.Params.(*s3.PutObjectInput)
|
||||
if input.Metadata == nil {
|
||||
input.Metadata = map[string]*string{}
|
||||
}
|
||||
|
||||
input.Metadata[http.CanonicalHeaderKey(keyV2Header)] = &env.CipherKey
|
||||
input.Metadata[http.CanonicalHeaderKey(ivHeader)] = &env.IV
|
||||
input.Metadata[http.CanonicalHeaderKey(matDescHeader)] = &env.MatDesc
|
||||
input.Metadata[http.CanonicalHeaderKey(wrapAlgorithmHeader)] = &env.WrapAlg
|
||||
input.Metadata[http.CanonicalHeaderKey(cekAlgorithmHeader)] = &env.CEKAlg
|
||||
input.Metadata[http.CanonicalHeaderKey(unencryptedMD5Header)] = &env.UnencryptedMD5
|
||||
input.Metadata[http.CanonicalHeaderKey(unencryptedContentLengthHeader)] = &env.UnencryptedContentLen
|
||||
|
||||
if len(env.TagLen) > 0 {
|
||||
input.Metadata[http.CanonicalHeaderKey(tagLengthHeader)] = &env.TagLen
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadStrategy ...
|
||||
type LoadStrategy interface {
|
||||
Load(*request.Request) (Envelope, error)
|
||||
}
|
||||
|
||||
// S3LoadStrategy will load the instruction file from s3
|
||||
type S3LoadStrategy struct {
|
||||
Client *s3.S3
|
||||
InstructionFileSuffix string
|
||||
}
|
||||
|
||||
// Load from a given instruction file suffix
|
||||
func (load S3LoadStrategy) Load(req *request.Request) (Envelope, error) {
|
||||
env := Envelope{}
|
||||
if load.InstructionFileSuffix == "" {
|
||||
load.InstructionFileSuffix = DefaultInstructionKeySuffix
|
||||
}
|
||||
|
||||
input := req.Params.(*s3.GetObjectInput)
|
||||
out, err := load.Client.GetObject(&s3.GetObjectInput{
|
||||
Key: aws.String(strings.Join([]string{*input.Key, load.InstructionFileSuffix}, "")),
|
||||
Bucket: input.Bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return env, err
|
||||
}
|
||||
|
||||
b, err := ioutil.ReadAll(out.Body)
|
||||
if err != nil {
|
||||
return env, err
|
||||
}
|
||||
err = json.Unmarshal(b, &env)
|
||||
return env, err
|
||||
}
|
||||
|
||||
// HeaderV2LoadStrategy will load the envelope from the metadata
|
||||
type HeaderV2LoadStrategy struct{}
|
||||
|
||||
// Load from a given object's header
|
||||
func (load HeaderV2LoadStrategy) Load(req *request.Request) (Envelope, error) {
|
||||
env := Envelope{}
|
||||
env.CipherKey = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, keyV2Header}, "-"))
|
||||
env.IV = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, ivHeader}, "-"))
|
||||
env.MatDesc = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, matDescHeader}, "-"))
|
||||
env.WrapAlg = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, wrapAlgorithmHeader}, "-"))
|
||||
env.CEKAlg = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, cekAlgorithmHeader}, "-"))
|
||||
env.TagLen = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, tagLengthHeader}, "-"))
|
||||
env.UnencryptedMD5 = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, unencryptedMD5Header}, "-"))
|
||||
env.UnencryptedContentLen = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, unencryptedContentLengthHeader}, "-"))
|
||||
return env, nil
|
||||
}
|
||||
|
||||
type defaultV2LoadStrategy struct {
|
||||
client *s3.S3
|
||||
suffix string
|
||||
}
|
||||
|
||||
func (load defaultV2LoadStrategy) Load(req *request.Request) (Envelope, error) {
|
||||
if value := req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, keyV2Header}, "-")); value != "" {
|
||||
strat := HeaderV2LoadStrategy{}
|
||||
return strat.Load(req)
|
||||
} else if value = req.HTTPResponse.Header.Get(strings.Join([]string{metaHeader, keyV1Header}, "-")); value != "" {
|
||||
return Envelope{}, awserr.New("V1NotSupportedError", "The AWS SDK for Go does not support version 1", nil)
|
||||
}
|
||||
|
||||
strat := S3LoadStrategy{
|
||||
Client: load.client,
|
||||
InstructionFileSuffix: load.suffix,
|
||||
}
|
||||
return strat.Load(req)
|
||||
}
|
||||
-77
@@ -1,77 +0,0 @@
|
||||
package s3crypto_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3crypto"
|
||||
)
|
||||
|
||||
func TestHeaderV2SaveStrategy(t *testing.T) {
|
||||
cases := []struct {
|
||||
env s3crypto.Envelope
|
||||
expected map[string]*string
|
||||
}{
|
||||
{
|
||||
s3crypto.Envelope{
|
||||
CipherKey: "Foo",
|
||||
IV: "Bar",
|
||||
MatDesc: "{}",
|
||||
WrapAlg: s3crypto.KMSWrap,
|
||||
CEKAlg: s3crypto.AESGCMNoPadding,
|
||||
TagLen: "128",
|
||||
UnencryptedMD5: "hello",
|
||||
UnencryptedContentLen: "0",
|
||||
},
|
||||
map[string]*string{
|
||||
"X-Amz-Key-V2": aws.String("Foo"),
|
||||
"X-Amz-Iv": aws.String("Bar"),
|
||||
"X-Amz-Matdesc": aws.String("{}"),
|
||||
"X-Amz-Wrap-Alg": aws.String(s3crypto.KMSWrap),
|
||||
"X-Amz-Cek-Alg": aws.String(s3crypto.AESGCMNoPadding),
|
||||
"X-Amz-Tag-Len": aws.String("128"),
|
||||
"X-Amz-Unencrypted-Content-Md5": aws.String("hello"),
|
||||
"X-Amz-Unencrypted-Content-Length": aws.String("0"),
|
||||
},
|
||||
},
|
||||
{
|
||||
s3crypto.Envelope{
|
||||
CipherKey: "Foo",
|
||||
IV: "Bar",
|
||||
MatDesc: "{}",
|
||||
WrapAlg: s3crypto.KMSWrap,
|
||||
CEKAlg: s3crypto.AESGCMNoPadding,
|
||||
UnencryptedMD5: "hello",
|
||||
UnencryptedContentLen: "0",
|
||||
},
|
||||
map[string]*string{
|
||||
"X-Amz-Key-V2": aws.String("Foo"),
|
||||
"X-Amz-Iv": aws.String("Bar"),
|
||||
"X-Amz-Matdesc": aws.String("{}"),
|
||||
"X-Amz-Wrap-Alg": aws.String(s3crypto.KMSWrap),
|
||||
"X-Amz-Cek-Alg": aws.String(s3crypto.AESGCMNoPadding),
|
||||
"X-Amz-Unencrypted-Content-Md5": aws.String("hello"),
|
||||
"X-Amz-Unencrypted-Content-Length": aws.String("0"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
params := &s3.PutObjectInput{}
|
||||
req := &request.Request{
|
||||
Params: params,
|
||||
}
|
||||
strat := s3crypto.HeaderV2SaveStrategy{}
|
||||
err := strat.Save(c.env, req)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(c.expected, params.Metadata) {
|
||||
t.Errorf("expected %v, but received %v", c.expected, params.Metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
// +build go1.7
|
||||
|
||||
package s3manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
// #1790 bug
|
||||
func TestBatchDeleteContext(t *testing.T) {
|
||||
cases := []struct {
|
||||
objects []BatchDeleteObject
|
||||
size int
|
||||
expected int
|
||||
ctx aws.Context
|
||||
closeAt int
|
||||
errCheck func(error) (string, bool)
|
||||
}{
|
||||
{
|
||||
[]BatchDeleteObject{
|
||||
{
|
||||
Object: &s3.DeleteObjectInput{
|
||||
Key: aws.String("1"),
|
||||
Bucket: aws.String("bucket1"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: &s3.DeleteObjectInput{
|
||||
Key: aws.String("2"),
|
||||
Bucket: aws.String("bucket2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: &s3.DeleteObjectInput{
|
||||
Key: aws.String("3"),
|
||||
Bucket: aws.String("bucket3"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Object: &s3.DeleteObjectInput{
|
||||
Key: aws.String("4"),
|
||||
Bucket: aws.String("bucket4"),
|
||||
},
|
||||
},
|
||||
},
|
||||
1,
|
||||
0,
|
||||
aws.BackgroundContext(),
|
||||
0,
|
||||
func(err error) (string, bool) {
|
||||
batchErr, ok := err.(*BatchError)
|
||||
if !ok {
|
||||
return "not BatchError type", false
|
||||
}
|
||||
|
||||
errs := batchErr.Errors
|
||||
if len(errs) != 4 {
|
||||
return fmt.Sprintf("expected 1, but received %d", len(errs)), false
|
||||
}
|
||||
|
||||
for _, tempErr := range errs {
|
||||
aerr, ok := tempErr.OrigErr.(awserr.Error)
|
||||
if !ok {
|
||||
return "not awserr.Error type", false
|
||||
}
|
||||
|
||||
if code := aerr.Code(); code != request.CanceledErrorCode {
|
||||
return fmt.Sprintf("expected %q, but received %q", request.CanceledErrorCode, code), false
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
count := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
count++
|
||||
}))
|
||||
|
||||
svc := &mockS3Client{S3: buildS3SvcClient(server.URL)}
|
||||
for i, c := range cases {
|
||||
ctx, done := context.WithCancel(c.ctx)
|
||||
defer done()
|
||||
if i == c.closeAt {
|
||||
done()
|
||||
}
|
||||
|
||||
batcher := BatchDelete{
|
||||
Client: svc,
|
||||
BatchSize: c.size,
|
||||
}
|
||||
|
||||
err := batcher.Delete(ctx, &DeleteObjectsIterator{Objects: c.objects})
|
||||
|
||||
if msg, ok := c.errCheck(err); !ok {
|
||||
t.Error(msg)
|
||||
}
|
||||
|
||||
if count != c.expected {
|
||||
t.Errorf("Case %d: expected %d, but received %d", i, c.expected, count)
|
||||
}
|
||||
|
||||
count = 0
|
||||
}
|
||||
}
|
||||
-1107
File diff suppressed because it is too large
Load Diff
-96
@@ -1,96 +0,0 @@
|
||||
package s3manager
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func testSetupGetBucketRegionServer(region string, statusCode int, incHeader bool) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if incHeader {
|
||||
w.Header().Set(bucketRegionHeader, region)
|
||||
}
|
||||
w.WriteHeader(statusCode)
|
||||
}))
|
||||
}
|
||||
|
||||
var testGetBucketRegionCases = []struct {
|
||||
RespRegion string
|
||||
StatusCode int
|
||||
HintRegion string
|
||||
ExpectReqRegion string
|
||||
}{
|
||||
{"bucket-region", 301, "hint-region", ""},
|
||||
{"bucket-region", 403, "hint-region", ""},
|
||||
{"bucket-region", 200, "hint-region", ""},
|
||||
{"bucket-region", 200, "", "default-region"},
|
||||
}
|
||||
|
||||
func TestGetBucketRegion_Exists(t *testing.T) {
|
||||
for i, c := range testGetBucketRegionCases {
|
||||
server := testSetupGetBucketRegionServer(c.RespRegion, c.StatusCode, true)
|
||||
|
||||
sess := unit.Session.Copy()
|
||||
sess.Config.Region = aws.String("default-region")
|
||||
sess.Config.Endpoint = aws.String(server.URL)
|
||||
sess.Config.DisableSSL = aws.Bool(true)
|
||||
|
||||
ctx := aws.BackgroundContext()
|
||||
region, err := GetBucketRegion(ctx, sess, "bucket", c.HintRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("%d, expect no error, got %v", i, err)
|
||||
}
|
||||
if e, a := c.RespRegion, region; e != a {
|
||||
t.Errorf("%d, expect %q region, got %q", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBucketRegion_NotExists(t *testing.T) {
|
||||
server := testSetupGetBucketRegionServer("ignore-region", 404, false)
|
||||
|
||||
sess := unit.Session.Copy()
|
||||
sess.Config.Endpoint = aws.String(server.URL)
|
||||
sess.Config.DisableSSL = aws.Bool(true)
|
||||
|
||||
ctx := aws.BackgroundContext()
|
||||
region, err := GetBucketRegion(ctx, sess, "bucket", "hint-region")
|
||||
if err == nil {
|
||||
t.Fatalf("expect error, but did not get one")
|
||||
}
|
||||
aerr := err.(awserr.Error)
|
||||
if e, a := "NotFound", aerr.Code(); e != a {
|
||||
t.Errorf("expect %s error code, got %s", e, a)
|
||||
}
|
||||
if len(region) != 0 {
|
||||
t.Errorf("expect region not to be set, got %q", region)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBucketRegionWithClient(t *testing.T) {
|
||||
for i, c := range testGetBucketRegionCases {
|
||||
server := testSetupGetBucketRegionServer(c.RespRegion, c.StatusCode, true)
|
||||
|
||||
svc := s3.New(unit.Session, &aws.Config{
|
||||
Region: aws.String("hint-region"),
|
||||
Endpoint: aws.String(server.URL),
|
||||
DisableSSL: aws.Bool(true),
|
||||
})
|
||||
|
||||
ctx := aws.BackgroundContext()
|
||||
|
||||
region, err := GetBucketRegionWithClient(ctx, svc, "bucket")
|
||||
if err != nil {
|
||||
t.Fatalf("%d, expect no error, got %v", i, err)
|
||||
}
|
||||
if e, a := c.RespRegion, region; e != a {
|
||||
t.Errorf("%d, expect %q region, got %q", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
-658
@@ -1,658 +0,0 @@
|
||||
package s3manager_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/awstesting"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
func dlLoggingSvc(data []byte) (*s3.S3, *[]string, *[]string) {
|
||||
var m sync.Mutex
|
||||
names := []string{}
|
||||
ranges := []string{}
|
||||
|
||||
svc := s3.New(unit.Session)
|
||||
svc.Handlers.Send.Clear()
|
||||
svc.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
names = append(names, r.Operation.Name)
|
||||
ranges = append(ranges, *r.Params.(*s3.GetObjectInput).Range)
|
||||
|
||||
rerng := regexp.MustCompile(`bytes=(\d+)-(\d+)`)
|
||||
rng := rerng.FindStringSubmatch(r.HTTPRequest.Header.Get("Range"))
|
||||
start, _ := strconv.ParseInt(rng[1], 10, 64)
|
||||
fin, _ := strconv.ParseInt(rng[2], 10, 64)
|
||||
fin++
|
||||
|
||||
if fin > int64(len(data)) {
|
||||
fin = int64(len(data))
|
||||
}
|
||||
|
||||
bodyBytes := data[start:fin]
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)),
|
||||
Header: http.Header{},
|
||||
}
|
||||
r.HTTPResponse.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d",
|
||||
start, fin-1, len(data)))
|
||||
r.HTTPResponse.Header.Set("Content-Length", fmt.Sprintf("%d", len(bodyBytes)))
|
||||
})
|
||||
|
||||
return svc, &names, &ranges
|
||||
}
|
||||
|
||||
func dlLoggingSvcNoChunk(data []byte) (*s3.S3, *[]string) {
|
||||
var m sync.Mutex
|
||||
names := []string{}
|
||||
|
||||
svc := s3.New(unit.Session)
|
||||
svc.Handlers.Send.Clear()
|
||||
svc.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
names = append(names, r.Operation.Name)
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: ioutil.NopCloser(bytes.NewReader(data[:])),
|
||||
Header: http.Header{},
|
||||
}
|
||||
r.HTTPResponse.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
|
||||
})
|
||||
|
||||
return svc, &names
|
||||
}
|
||||
|
||||
func dlLoggingSvcNoContentRangeLength(data []byte, states []int) (*s3.S3, *[]string) {
|
||||
var m sync.Mutex
|
||||
names := []string{}
|
||||
var index int = 0
|
||||
|
||||
svc := s3.New(unit.Session)
|
||||
svc.Handlers.Send.Clear()
|
||||
svc.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
names = append(names, r.Operation.Name)
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: states[index],
|
||||
Body: ioutil.NopCloser(bytes.NewReader(data[:])),
|
||||
Header: http.Header{},
|
||||
}
|
||||
index++
|
||||
})
|
||||
|
||||
return svc, &names
|
||||
}
|
||||
|
||||
func dlLoggingSvcContentRangeTotalAny(data []byte, states []int) (*s3.S3, *[]string) {
|
||||
var m sync.Mutex
|
||||
names := []string{}
|
||||
ranges := []string{}
|
||||
var index int = 0
|
||||
|
||||
svc := s3.New(unit.Session)
|
||||
svc.Handlers.Send.Clear()
|
||||
svc.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
names = append(names, r.Operation.Name)
|
||||
ranges = append(ranges, *r.Params.(*s3.GetObjectInput).Range)
|
||||
|
||||
rerng := regexp.MustCompile(`bytes=(\d+)-(\d+)`)
|
||||
rng := rerng.FindStringSubmatch(r.HTTPRequest.Header.Get("Range"))
|
||||
start, _ := strconv.ParseInt(rng[1], 10, 64)
|
||||
fin, _ := strconv.ParseInt(rng[2], 10, 64)
|
||||
fin++
|
||||
|
||||
if fin >= int64(len(data)) {
|
||||
fin = int64(len(data))
|
||||
}
|
||||
|
||||
// Setting start and finish to 0 because this state of 1 is suppose to
|
||||
// be an error state of 416
|
||||
if index == len(states)-1 {
|
||||
start = 0
|
||||
fin = 0
|
||||
}
|
||||
|
||||
bodyBytes := data[start:fin]
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: states[index],
|
||||
Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)),
|
||||
Header: http.Header{},
|
||||
}
|
||||
r.HTTPResponse.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/*",
|
||||
start, fin-1))
|
||||
index++
|
||||
})
|
||||
|
||||
return svc, &names
|
||||
}
|
||||
|
||||
func dlLoggingSvcWithErrReader(cases []testErrReader) (*s3.S3, *[]string) {
|
||||
var m sync.Mutex
|
||||
names := []string{}
|
||||
var index int = 0
|
||||
|
||||
svc := s3.New(unit.Session, &aws.Config{
|
||||
MaxRetries: aws.Int(len(cases) - 1),
|
||||
})
|
||||
svc.Handlers.Send.Clear()
|
||||
svc.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
names = append(names, r.Operation.Name)
|
||||
|
||||
c := cases[index]
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: ioutil.NopCloser(&c),
|
||||
Header: http.Header{},
|
||||
}
|
||||
r.HTTPResponse.Header.Set("Content-Range",
|
||||
fmt.Sprintf("bytes %d-%d/%d", 0, c.Len-1, c.Len))
|
||||
r.HTTPResponse.Header.Set("Content-Length", fmt.Sprintf("%d", c.Len))
|
||||
index++
|
||||
})
|
||||
|
||||
return svc, &names
|
||||
}
|
||||
|
||||
func TestDownloadOrder(t *testing.T) {
|
||||
s, names, ranges := dlLoggingSvc(buf12MB)
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
})
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(len(buf12MB)), n; e != a {
|
||||
t.Errorf("expect %d buffer length, got %d", e, a)
|
||||
}
|
||||
|
||||
expectCalls := []string{"GetObject", "GetObject", "GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
|
||||
expectRngs := []string{"bytes=0-5242879", "bytes=5242880-10485759", "bytes=10485760-15728639"}
|
||||
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v ranges, got %v", e, a)
|
||||
}
|
||||
|
||||
count := 0
|
||||
for _, b := range w.Bytes() {
|
||||
count += int(b)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("expect 0 count, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadZero(t *testing.T) {
|
||||
s, names, ranges := dlLoggingSvc([]byte{})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s)
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Errorf("expect 0 bytes read, got %d", n)
|
||||
}
|
||||
expectCalls := []string{"GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
|
||||
expectRngs := []string{"bytes=0-5242879"}
|
||||
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v ranges, got %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadSetPartSize(t *testing.T) {
|
||||
s, names, ranges := dlLoggingSvc([]byte{1, 2, 3})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
d.PartSize = 1
|
||||
})
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(3), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject", "GetObject", "GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
expectRngs := []string{"bytes=0-0", "bytes=1-1", "bytes=2-2"}
|
||||
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v ranges, got %v", e, a)
|
||||
}
|
||||
expectBytes := []byte{1, 2, 3}
|
||||
if e, a := expectBytes, w.Bytes(); !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v bytes, got %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadError(t *testing.T) {
|
||||
s, names, _ := dlLoggingSvc([]byte{1, 2, 3})
|
||||
|
||||
num := 0
|
||||
s.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
num++
|
||||
if num > 1 {
|
||||
r.HTTPResponse.StatusCode = 400
|
||||
r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
|
||||
}
|
||||
})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
d.PartSize = 1
|
||||
})
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatalf("expect error, got none")
|
||||
}
|
||||
aerr := err.(awserr.Error)
|
||||
if e, a := "BadRequest", aerr.Code(); e != a {
|
||||
t.Errorf("expect %s error code, got %s", e, a)
|
||||
}
|
||||
if e, a := int64(1), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject", "GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
expectBytes := []byte{1}
|
||||
if e, a := expectBytes, w.Bytes(); !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v bytes, got %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadNonChunk(t *testing.T) {
|
||||
s, names := dlLoggingSvcNoChunk(buf2MB)
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
})
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(len(buf2MB)), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
|
||||
count := 0
|
||||
for _, b := range w.Bytes() {
|
||||
count += int(b)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("expect 0 count, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadNoContentRangeLength(t *testing.T) {
|
||||
s, names := dlLoggingSvcNoContentRangeLength(buf2MB, []int{200, 416})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
})
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(len(buf2MB)), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject", "GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
|
||||
count := 0
|
||||
for _, b := range w.Bytes() {
|
||||
count += int(b)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("expect 0 count, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadContentRangeTotalAny(t *testing.T) {
|
||||
s, names := dlLoggingSvcContentRangeTotalAny(buf2MB, []int{200, 416})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
})
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(len(buf2MB)), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject", "GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
|
||||
count := 0
|
||||
for _, b := range w.Bytes() {
|
||||
count += int(b)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("expect 0 count, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPartBodyRetry_SuccessRetry(t *testing.T) {
|
||||
s, names := dlLoggingSvcWithErrReader([]testErrReader{
|
||||
{Buf: []byte("ab"), Len: 3, Err: io.ErrUnexpectedEOF},
|
||||
{Buf: []byte("123"), Len: 3, Err: io.EOF},
|
||||
})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
})
|
||||
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(3), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject", "GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
if e, a := "123", string(w.Bytes()); e != a {
|
||||
t.Errorf("expect %q response, got %q", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPartBodyRetry_SuccessNoRetry(t *testing.T) {
|
||||
s, names := dlLoggingSvcWithErrReader([]testErrReader{
|
||||
{Buf: []byte("abc"), Len: 3, Err: io.EOF},
|
||||
})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
})
|
||||
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(3), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
if e, a := "abc", string(w.Bytes()); e != a {
|
||||
t.Errorf("expect %q response, got %q", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadPartBodyRetry_FailRetry(t *testing.T) {
|
||||
s, names := dlLoggingSvcWithErrReader([]testErrReader{
|
||||
{Buf: []byte("ab"), Len: 3, Err: io.ErrUnexpectedEOF},
|
||||
})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 1
|
||||
})
|
||||
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatalf("expect error, got none")
|
||||
}
|
||||
if e, a := "unexpected EOF", err.Error(); !strings.Contains(a, e) {
|
||||
t.Errorf("expect %q error message to be in %q", e, a)
|
||||
}
|
||||
if e, a := int64(2), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
if e, a := "ab", string(w.Bytes()); e != a {
|
||||
t.Errorf("expect %q response, got %q", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadWithContextCanceled(t *testing.T) {
|
||||
d := s3manager.NewDownloader(unit.Session)
|
||||
|
||||
params := s3.GetObjectInput{
|
||||
Bucket: aws.String("Bucket"),
|
||||
Key: aws.String("Key"),
|
||||
}
|
||||
|
||||
ctx := &awstesting.FakeContext{DoneCh: make(chan struct{})}
|
||||
ctx.Error = fmt.Errorf("context canceled")
|
||||
close(ctx.DoneCh)
|
||||
|
||||
w := &aws.WriteAtBuffer{}
|
||||
|
||||
_, err := d.DownloadWithContext(ctx, w, ¶ms)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, did not get one")
|
||||
}
|
||||
aerr := err.(awserr.Error)
|
||||
if e, a := request.CanceledErrorCode, aerr.Code(); e != a {
|
||||
t.Errorf("expected error code %q, got %q", e, a)
|
||||
}
|
||||
if e, a := "canceled", aerr.Message(); !strings.Contains(a, e) {
|
||||
t.Errorf("expected error message to contain %q, but did not %q", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownload_WithRange(t *testing.T) {
|
||||
s, names, ranges := dlLoggingSvc([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
|
||||
|
||||
d := s3manager.NewDownloaderWithClient(s, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 10 // should be ignored
|
||||
d.PartSize = 1 // should be ignored
|
||||
})
|
||||
|
||||
w := &aws.WriteAtBuffer{}
|
||||
n, err := d.Download(w, &s3.GetObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
Key: aws.String("key"),
|
||||
Range: aws.String("bytes=2-6"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
if e, a := int64(5), n; e != a {
|
||||
t.Errorf("expect %d bytes read, got %d", e, a)
|
||||
}
|
||||
expectCalls := []string{"GetObject"}
|
||||
if e, a := expectCalls, *names; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v API calls, got %v", e, a)
|
||||
}
|
||||
expectRngs := []string{"bytes=2-6"}
|
||||
if e, a := expectRngs, *ranges; !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v ranges, got %v", e, a)
|
||||
}
|
||||
expectBytes := []byte{2, 3, 4, 5, 6}
|
||||
if e, a := expectBytes, w.Bytes(); !reflect.DeepEqual(e, a) {
|
||||
t.Errorf("expect %v bytes, got %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownload_WithFailure(t *testing.T) {
|
||||
svc := s3.New(unit.Session)
|
||||
svc.Handlers.Send.Clear()
|
||||
|
||||
first := true
|
||||
svc.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
if first {
|
||||
first = false
|
||||
body := bytes.NewReader(make([]byte, s3manager.DefaultDownloadPartSize))
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Status: http.StatusText(http.StatusOK),
|
||||
ContentLength: int64(body.Len()),
|
||||
Body: ioutil.NopCloser(body),
|
||||
Header: http.Header{},
|
||||
}
|
||||
r.HTTPResponse.Header.Set("Content-Length", strconv.Itoa(body.Len()))
|
||||
r.HTTPResponse.Header.Set("Content-Range",
|
||||
fmt.Sprintf("bytes 0-%d/%d", body.Len()-1, body.Len()*10))
|
||||
return
|
||||
}
|
||||
|
||||
// Give a chance for the multipart chunks to be queued up
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
r.HTTPResponse = &http.Response{
|
||||
Header: http.Header{},
|
||||
Body: ioutil.NopCloser(&bytes.Buffer{}),
|
||||
}
|
||||
r.Error = awserr.New("ConnectionError", "some connection error", nil)
|
||||
r.Retryable = aws.Bool(false)
|
||||
})
|
||||
|
||||
start := time.Now()
|
||||
d := s3manager.NewDownloaderWithClient(svc, func(d *s3manager.Downloader) {
|
||||
d.Concurrency = 2
|
||||
})
|
||||
|
||||
w := &aws.WriteAtBuffer{}
|
||||
params := s3.GetObjectInput{
|
||||
Bucket: aws.String("Bucket"),
|
||||
Key: aws.String("Key"),
|
||||
}
|
||||
|
||||
// Expect this request to exit quickly after failure
|
||||
_, err := d.Download(w, ¶ms)
|
||||
if err == nil {
|
||||
t.Fatalf("expect error, got none")
|
||||
}
|
||||
|
||||
limit := start.Add(5 * time.Second)
|
||||
dur := time.Now().Sub(start)
|
||||
if time.Now().After(limit) {
|
||||
t.Errorf("expect time to be less than %v, took %v", limit, dur)
|
||||
}
|
||||
}
|
||||
|
||||
type testErrReader struct {
|
||||
Buf []byte
|
||||
Err error
|
||||
Len int64
|
||||
|
||||
off int
|
||||
}
|
||||
|
||||
func (r *testErrReader) Read(p []byte) (int, error) {
|
||||
to := len(r.Buf) - r.off
|
||||
|
||||
n := copy(p, r.Buf[r.off:to])
|
||||
r.off += n
|
||||
|
||||
if n < len(p) {
|
||||
return n, r.Err
|
||||
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
Generated
Vendored
-26
@@ -1,26 +0,0 @@
|
||||
// Package s3manageriface provides an interface for the s3manager package
|
||||
package s3manageriface
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
)
|
||||
|
||||
// DownloaderAPI is the interface type for s3manager.Downloader.
|
||||
type DownloaderAPI interface {
|
||||
Download(io.WriterAt, *s3.GetObjectInput, ...func(*s3manager.Downloader)) (int64, error)
|
||||
DownloadWithContext(aws.Context, io.WriterAt, *s3.GetObjectInput, ...func(*s3manager.Downloader)) (int64, error)
|
||||
}
|
||||
|
||||
var _ DownloaderAPI = (*s3manager.Downloader)(nil)
|
||||
|
||||
// UploaderAPI is the interface type for s3manager.Uploader.
|
||||
type UploaderAPI interface {
|
||||
Upload(*s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
|
||||
UploadWithContext(aws.Context, *s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
|
||||
}
|
||||
|
||||
var _ UploaderAPI = (*s3manager.Uploader)(nil)
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
package s3manager_test
|
||||
|
||||
var buf12MB = make([]byte, 1024*1024*12)
|
||||
var buf2MB = make([]byte, 1024*1024*2)
|
||||
-1003
File diff suppressed because it is too large
Load Diff
-111
@@ -1,111 +0,0 @@
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
func TestSSECustomerKeyOverHTTPError(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{DisableSSL: aws.Bool(true)})
|
||||
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
CopySource: aws.String("bucket/source"),
|
||||
Key: aws.String("dest"),
|
||||
SSECustomerKey: aws.String("key"),
|
||||
})
|
||||
err := req.Build()
|
||||
|
||||
if err == nil {
|
||||
t.Error("expected an error")
|
||||
}
|
||||
if e, a := "ConfigError", err.(awserr.Error).Code(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if !strings.Contains(err.(awserr.Error).Message(), "cannot send SSE keys over HTTP") {
|
||||
t.Errorf("expected error to contain 'cannot send SSE keys over HTTP', but received %s", err.(awserr.Error).Message())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopySourceSSECustomerKeyOverHTTPError(t *testing.T) {
|
||||
s := s3.New(unit.Session, &aws.Config{DisableSSL: aws.Bool(true)})
|
||||
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
CopySource: aws.String("bucket/source"),
|
||||
Key: aws.String("dest"),
|
||||
CopySourceSSECustomerKey: aws.String("key"),
|
||||
})
|
||||
err := req.Build()
|
||||
|
||||
if err == nil {
|
||||
t.Error("expected an error")
|
||||
}
|
||||
if e, a := "ConfigError", err.(awserr.Error).Code(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if !strings.Contains(err.(awserr.Error).Message(), "cannot send SSE keys over HTTP") {
|
||||
t.Errorf("expected error to contain 'cannot send SSE keys over HTTP', but received %s", err.(awserr.Error).Message())
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSSEKeys(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
CopySource: aws.String("bucket/source"),
|
||||
Key: aws.String("dest"),
|
||||
SSECustomerKey: aws.String("key"),
|
||||
CopySourceSSECustomerKey: aws.String("key"),
|
||||
})
|
||||
err := req.Build()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "PG4LipwVIkqCKLmpjKFTHQ==", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key-md5"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "PG4LipwVIkqCKLmpjKFTHQ==", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key-md5"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeSSEKeysShortcircuit(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
req, _ := s.CopyObjectRequest(&s3.CopyObjectInput{
|
||||
Bucket: aws.String("bucket"),
|
||||
CopySource: aws.String("bucket/source"),
|
||||
Key: aws.String("dest"),
|
||||
SSECustomerKey: aws.String("key"),
|
||||
CopySourceSSECustomerKey: aws.String("key"),
|
||||
SSECustomerKeyMD5: aws.String("MD5"),
|
||||
CopySourceSSECustomerKeyMD5: aws.String("MD5"),
|
||||
})
|
||||
err := req.Build()
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "a2V5", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "MD5", req.HTTPRequest.Header.Get("x-amz-server-side-encryption-customer-key-md5"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "MD5", req.HTTPRequest.Header.Get("x-amz-copy-source-server-side-encryption-customer-key-md5"); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
-166
@@ -1,166 +0,0 @@
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
const errMsg = `<Error><Code>ErrorCode</Code><Message>message body</Message><RequestId>requestID</RequestId><HostId>hostID=</HostId></Error>`
|
||||
|
||||
var lastModifiedTime = time.Date(2009, 11, 23, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestCopyObjectNoError(t *testing.T) {
|
||||
const successMsg = `
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2009-11-23T0:00:00Z</LastModified><ETag>"1da64c7f13d1e8dbeaea40b905fd586c"</ETag></CopyObjectResult>`
|
||||
|
||||
res, err := newCopyTestSvc(successMsg).CopyObject(&s3.CopyObjectInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
CopySource: aws.String("bucketname/exists.txt"),
|
||||
Key: aws.String("destination.txt"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
if e, a := fmt.Sprintf(`%q`, "1da64c7f13d1e8dbeaea40b905fd586c"), *res.CopyObjectResult.ETag; e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := lastModifiedTime, *res.CopyObjectResult.LastModified; !e.Equal(a) {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyObjectError(t *testing.T) {
|
||||
_, err := newCopyTestSvc(errMsg).CopyObject(&s3.CopyObjectInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
CopySource: aws.String("bucketname/doesnotexist.txt"),
|
||||
Key: aws.String("destination.txt"),
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Error("expected error, but received none")
|
||||
}
|
||||
e := err.(awserr.Error)
|
||||
|
||||
if e, a := "ErrorCode", e.Code(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "message body", e.Message(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPartCopySuccess(t *testing.T) {
|
||||
const successMsg = `
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<UploadPartCopyResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2009-11-23T0:00:00Z</LastModified><ETag>"1da64c7f13d1e8dbeaea40b905fd586c"</ETag></UploadPartCopyResult>`
|
||||
|
||||
res, err := newCopyTestSvc(successMsg).UploadPartCopy(&s3.UploadPartCopyInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
CopySource: aws.String("bucketname/doesnotexist.txt"),
|
||||
Key: aws.String("destination.txt"),
|
||||
PartNumber: aws.Int64(0),
|
||||
UploadId: aws.String("uploadID"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if e, a := fmt.Sprintf(`%q`, "1da64c7f13d1e8dbeaea40b905fd586c"), *res.CopyPartResult.ETag; e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := lastModifiedTime, *res.CopyPartResult.LastModified; !e.Equal(a) {
|
||||
t.Errorf("expected %v, but received %v", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadPartCopyError(t *testing.T) {
|
||||
_, err := newCopyTestSvc(errMsg).UploadPartCopy(&s3.UploadPartCopyInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
CopySource: aws.String("bucketname/doesnotexist.txt"),
|
||||
Key: aws.String("destination.txt"),
|
||||
PartNumber: aws.Int64(0),
|
||||
UploadId: aws.String("uploadID"),
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Error("expected an error")
|
||||
}
|
||||
e := err.(awserr.Error)
|
||||
|
||||
if e, a := "ErrorCode", e.Code(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "message body", e.Message(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteMultipartUploadSuccess(t *testing.T) {
|
||||
const successMsg = `
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Location>locationName</Location><Bucket>bucketName</Bucket><Key>keyName</Key><ETag>"etagVal"</ETag></CompleteMultipartUploadResult>`
|
||||
res, err := newCopyTestSvc(successMsg).CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
Key: aws.String("key"),
|
||||
UploadId: aws.String("uploadID"),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, but received %v", err)
|
||||
}
|
||||
|
||||
if e, a := `"etagVal"`, *res.ETag; e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "bucketName", *res.Bucket; e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "keyName", *res.Key; e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "locationName", *res.Location; e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteMultipartUploadError(t *testing.T) {
|
||||
_, err := newCopyTestSvc(errMsg).CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String("bucketname"),
|
||||
Key: aws.String("key"),
|
||||
UploadId: aws.String("uploadID"),
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Error("expected an error")
|
||||
}
|
||||
e := err.(awserr.Error)
|
||||
|
||||
if e, a := "ErrorCode", e.Code(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
if e, a := "message body", e.Message(); e != a {
|
||||
t.Errorf("expected %s, but received %s", e, a)
|
||||
}
|
||||
}
|
||||
|
||||
func newCopyTestSvc(errMsg string) *s3.S3 {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, errMsg, http.StatusOK)
|
||||
}))
|
||||
return s3.New(unit.Session, aws.NewConfig().
|
||||
WithEndpoint(server.URL).
|
||||
WithDisableSSL(true).
|
||||
WithMaxRetries(0).
|
||||
WithS3ForcePathStyle(true))
|
||||
}
|
||||
-178
@@ -1,178 +0,0 @@
|
||||
[
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3.amazonaws.com",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3.us-west-1.amazonaws.com",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-with-number-1",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-with-number-1.s3.us-west-1.amazonaws.com",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3.cn-north-1.amazonaws.com.cn",
|
||||
"Region": "cn-north-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "BucketName",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.amazonaws.com/BucketName",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "BucketName",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.amazonaws.com/BucketName",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket_name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket_name",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket.name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket.name",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "-bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/-bucket-name",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name-",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/bucket-name-",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "aa",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/aa",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.us-west-1.amazonaws.com/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": true
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
|
||||
"Region": "us-west-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": true
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3.dualstack.us-east-1.amazonaws.com",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": true,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3.dualstack.us-west-2.amazonaws.com",
|
||||
"Region": "us-west-2",
|
||||
"UseDualstack": true,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket.name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://s3.dualstack.us-west-2.amazonaws.com/bucket.name",
|
||||
"Region": "us-west-2",
|
||||
"UseDualstack": true,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "default",
|
||||
"ExpectedUri": "https://bucket-name.s3-accelerate.dualstack.amazonaws.com",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": true,
|
||||
"UseS3Accelerate": true
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "path",
|
||||
"ExpectedUri": "https://s3.amazonaws.com/bucket-name",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "path",
|
||||
"ExpectedUri": "https://bucket-name.s3-accelerate.amazonaws.com",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": false,
|
||||
"UseS3Accelerate": true
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "path",
|
||||
"ExpectedUri": "https://s3.dualstack.us-east-1.amazonaws.com/bucket-name",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": true,
|
||||
"UseS3Accelerate": false
|
||||
},
|
||||
{
|
||||
"Bucket": "bucket-name",
|
||||
"ConfiguredAddressingStyle": "path",
|
||||
"ExpectedUri": "https://bucket-name.s3-accelerate.dualstack.amazonaws.com",
|
||||
"Region": "us-east-1",
|
||||
"UseDualstack": true,
|
||||
"UseS3Accelerate": true
|
||||
}
|
||||
]
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/awstesting"
|
||||
)
|
||||
|
||||
func TestUnmarhsalErrorLeak(t *testing.T) {
|
||||
req := &request.Request{
|
||||
HTTPRequest: &http.Request{
|
||||
Header: make(http.Header),
|
||||
Body: &awstesting.ReadCloser{Size: 2048},
|
||||
},
|
||||
}
|
||||
req.HTTPResponse = &http.Response{
|
||||
Body: &awstesting.ReadCloser{Size: 2048},
|
||||
Header: http.Header{
|
||||
"X-Amzn-Requestid": []string{"1"},
|
||||
},
|
||||
StatusCode: http.StatusOK,
|
||||
}
|
||||
|
||||
reader := req.HTTPResponse.Body.(*awstesting.ReadCloser)
|
||||
unmarshalError(req)
|
||||
|
||||
if req.Error == nil {
|
||||
t.Error("expected an error, but received none")
|
||||
}
|
||||
|
||||
if !reader.Closed {
|
||||
t.Error("expected reader to be closed")
|
||||
}
|
||||
|
||||
if e, a := 0, reader.Size; e != a {
|
||||
t.Errorf("expected %d, but received %d", e, a)
|
||||
}
|
||||
}
|
||||
-253
@@ -1,253 +0,0 @@
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/awstesting/unit"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
type testErrorCase struct {
|
||||
RespFn func() *http.Response
|
||||
ReqID, HostID string
|
||||
Code, Msg string
|
||||
WithoutStatusMsg bool
|
||||
}
|
||||
|
||||
var testUnmarshalCases = []testErrorCase{
|
||||
{
|
||||
RespFn: func() *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: 301,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewReader(nil)),
|
||||
ContentLength: -1,
|
||||
}
|
||||
},
|
||||
ReqID: "abc123",
|
||||
HostID: "321cba",
|
||||
Code: "BucketRegionError", Msg: "incorrect region, the bucket is not in 'mock-region' region",
|
||||
},
|
||||
{
|
||||
RespFn: func() *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: 403,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewReader(nil)),
|
||||
ContentLength: 0,
|
||||
}
|
||||
},
|
||||
ReqID: "abc123",
|
||||
HostID: "321cba",
|
||||
Code: "Forbidden", Msg: "Forbidden",
|
||||
},
|
||||
{
|
||||
RespFn: func() *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: 400,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewReader(nil)),
|
||||
ContentLength: 0,
|
||||
}
|
||||
},
|
||||
ReqID: "abc123",
|
||||
HostID: "321cba",
|
||||
Code: "BadRequest", Msg: "Bad Request",
|
||||
},
|
||||
{
|
||||
RespFn: func() *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: 404,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewReader(nil)),
|
||||
ContentLength: 0,
|
||||
}
|
||||
},
|
||||
ReqID: "abc123",
|
||||
HostID: "321cba",
|
||||
Code: "NotFound", Msg: "Not Found",
|
||||
},
|
||||
{
|
||||
// SDK only reads request ID and host ID from the header. The values
|
||||
// in message body are ignored.
|
||||
RespFn: func() *http.Response {
|
||||
body := `<Error><Code>SomeException</Code><Message>Exception message</Message><RequestId>ignored-request-id</RequestId><HostId>ignored-host-id</HostId></Error>`
|
||||
return &http.Response{
|
||||
StatusCode: 500,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"taken-request-id"},
|
||||
"X-Amz-Id-2": []string{"taken-host-id"},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(body)),
|
||||
ContentLength: int64(len(body)),
|
||||
}
|
||||
},
|
||||
ReqID: "taken-request-id",
|
||||
HostID: "taken-host-id",
|
||||
Code: "SomeException", Msg: "Exception message",
|
||||
},
|
||||
{
|
||||
RespFn: func() *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: 404,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewReader(nil)),
|
||||
ContentLength: -1,
|
||||
}
|
||||
},
|
||||
ReqID: "abc123",
|
||||
HostID: "321cba",
|
||||
Code: "NotFound", Msg: "Not Found", WithoutStatusMsg: true,
|
||||
},
|
||||
{
|
||||
RespFn: func() *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: 404,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewReader(nil)),
|
||||
ContentLength: -1,
|
||||
}
|
||||
},
|
||||
ReqID: "abc123",
|
||||
HostID: "321cba",
|
||||
Code: "NotFound", Msg: "Not Found",
|
||||
},
|
||||
}
|
||||
|
||||
func TestUnmarshalError(t *testing.T) {
|
||||
for i, c := range testUnmarshalCases {
|
||||
s := s3.New(unit.Session)
|
||||
s.Handlers.Send.Clear()
|
||||
s.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
r.HTTPResponse = c.RespFn()
|
||||
if !c.WithoutStatusMsg {
|
||||
r.HTTPResponse.Status = fmt.Sprintf("%d%s",
|
||||
r.HTTPResponse.StatusCode,
|
||||
http.StatusText(r.HTTPResponse.StatusCode))
|
||||
}
|
||||
})
|
||||
_, err := s.PutBucketAcl(&s3.PutBucketAclInput{
|
||||
Bucket: aws.String("bucket"), ACL: aws.String("public-read"),
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatalf("%d, expected error, got nil", i)
|
||||
}
|
||||
if e, a := c.Code, err.(awserr.Error).Code(); e != a {
|
||||
t.Errorf("%d, Code: expect %s, got %s", i, e, a)
|
||||
}
|
||||
if e, a := c.Msg, err.(awserr.Error).Message(); e != a {
|
||||
t.Errorf("%d, Message: expect %s, got %s", i, e, a)
|
||||
}
|
||||
if e, a := c.ReqID, err.(awserr.RequestFailure).RequestID(); e != a {
|
||||
t.Errorf("%d, RequestID: expect %s, got %s", i, e, a)
|
||||
}
|
||||
if e, a := c.HostID, err.(s3.RequestFailure).HostID(); e != a {
|
||||
t.Errorf("%d, HostID: expect %s, got %s", i, e, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const completeMultiResp = `
|
||||
163
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Location>https://bucket.s3-us-west-2.amazonaws.com/key</Location><Bucket>bucket</Bucket><Key>key</Key><ETag>"a7d414b9133d6483d9a1c4e04e856e3b-2"</ETag></CompleteMultipartUploadResult>
|
||||
0
|
||||
`
|
||||
|
||||
func Test200NoErrorUnmarshalError(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
s.Handlers.Send.Clear()
|
||||
s.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(completeMultiResp)),
|
||||
ContentLength: -1,
|
||||
}
|
||||
r.HTTPResponse.Status = http.StatusText(r.HTTPResponse.StatusCode)
|
||||
})
|
||||
_, err := s.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String("bucket"), Key: aws.String("key"),
|
||||
UploadId: aws.String("id"),
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{Parts: []*s3.CompletedPart{
|
||||
{ETag: aws.String("etag"), PartNumber: aws.Int64(1)},
|
||||
}},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("expect no error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
const completeMultiErrResp = `<Error><Code>SomeException</Code><Message>Exception message</Message></Error>`
|
||||
|
||||
func Test200WithErrorUnmarshalError(t *testing.T) {
|
||||
s := s3.New(unit.Session)
|
||||
s.Handlers.Send.Clear()
|
||||
s.Handlers.Send.PushBack(func(r *request.Request) {
|
||||
r.HTTPResponse = &http.Response{
|
||||
StatusCode: 200,
|
||||
Header: http.Header{
|
||||
"X-Amz-Request-Id": []string{"abc123"},
|
||||
"X-Amz-Id-2": []string{"321cba"},
|
||||
},
|
||||
Body: ioutil.NopCloser(strings.NewReader(completeMultiErrResp)),
|
||||
ContentLength: -1,
|
||||
}
|
||||
r.HTTPResponse.Status = http.StatusText(r.HTTPResponse.StatusCode)
|
||||
})
|
||||
_, err := s.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String("bucket"), Key: aws.String("key"),
|
||||
UploadId: aws.String("id"),
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{Parts: []*s3.CompletedPart{
|
||||
{ETag: aws.String("etag"), PartNumber: aws.Int64(1)},
|
||||
}},
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if e, a := "SomeException", err.(awserr.Error).Code(); e != a {
|
||||
t.Errorf("Code: expect %s, got %s", e, a)
|
||||
}
|
||||
if e, a := "Exception message", err.(awserr.Error).Message(); e != a {
|
||||
t.Errorf("Message: expect %s, got %s", e, a)
|
||||
}
|
||||
if e, a := "abc123", err.(s3.RequestFailure).RequestID(); e != a {
|
||||
t.Errorf("RequestID: expect %s, got %s", e, a)
|
||||
}
|
||||
if e, a := "321cba", err.(s3.RequestFailure).HostID(); e != a {
|
||||
t.Errorf("HostID: expect %s, got %s", e, a)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user