vendor: add qingstor-sdk-go for QingStor

2017-08-04 17:09:28 +01:00
parent f682002b84
commit 466dd22b44
136 changed files with 15952 additions and 1 deletions
@@ -0,0 +1,303 @@
+// +-------------------------------------------------------------------------
+// | Copyright (C) 2016 Yunify, Inc.
+// +-------------------------------------------------------------------------
+// | Licensed under the Apache License, Version 2.0 (the "License");
+// | you may not use this work except in compliance with the License.
+// | You may obtain a copy of the License in the LICENSE file, or at:
+// |
+// | http://www.apache.org/licenses/LICENSE-2.0
+// |
+// | Unless required by applicable law or agreed to in writing, software
+// | distributed under the License is distributed on an "AS IS" BASIS,
+// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// | See the License for the specific language governing permissions and
+// | limitations under the License.
+// +-------------------------------------------------------------------------
+
+package builder
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/pengsrc/go-shared/convert"
+	"github.com/pengsrc/go-shared/json"
+
+	"github.com/yunify/qingstor-sdk-go/request/data"
+	"github.com/yunify/qingstor-sdk-go/utils"
+)
+
+// BaseBuilder is the base builder for all services.
+type BaseBuilder struct {
+	parsedURL        string
+	parsedProperties *map[string]string
+	parsedParams     *map[string]string
+	parsedHeaders    *map[string]string
+	parsedBodyString string
+	parsedBody       io.Reader
+
+	operation *data.Operation
+	input     *reflect.Value
+}
+
+// BuildHTTPRequest builds http request with an operation and an input.
+func (b *BaseBuilder) BuildHTTPRequest(o *data.Operation, i *reflect.Value) (*http.Request, error) {
+	b.operation = o
+	b.input = i
+
+	_, err := b.parse()
+	if err != nil {
+		return nil, err
+	}
+
+	return b.build()
+}
+
+func (b *BaseBuilder) build() (*http.Request, error) {
+	httpRequest, err := http.NewRequest(b.operation.RequestMethod, b.parsedURL, b.parsedBody)
+	if err != nil {
+		return nil, err
+	}
+
+	err = b.setupHeaders(httpRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	return httpRequest, nil
+}
+
+func (b *BaseBuilder) parse() (*BaseBuilder, error) {
+	err := b.parseRequestParamsAndHeaders()
+	if err != nil {
+		return b, err
+	}
+	err = b.parseRequestBody()
+	if err != nil {
+		return b, err
+	}
+	err = b.parseRequestProperties()
+	if err != nil {
+		return b, err
+	}
+	err = b.parseRequestURL()
+	if err != nil {
+		return b, err
+	}
+
+	return b, nil
+}
+
+func (b *BaseBuilder) parseRequestParamsAndHeaders() error {
+	requestParams := map[string]string{}
+	requestHeaders := map[string]string{}
+	maps := map[string](map[string]string){
+		"params":  requestParams,
+		"headers": requestHeaders,
+	}
+
+	b.parsedParams = &requestParams
+	b.parsedHeaders = &requestHeaders
+
+	if !b.input.IsValid() {
+		return nil
+	}
+
+	fields := b.input.Elem()
+	if !fields.IsValid() {
+		return nil
+	}
+
+	for i := 0; i < fields.NumField(); i++ {
+		tagName := fields.Type().Field(i).Tag.Get("name")
+		tagLocation := fields.Type().Field(i).Tag.Get("location")
+		if tagDefault := fields.Type().Field(i).Tag.Get("default"); tagDefault != "" {
+			maps[tagLocation][tagName] = tagDefault
+		}
+		if tagName != "" && tagLocation != "" && maps[tagLocation] != nil {
+			switch value := fields.Field(i).Interface().(type) {
+			case *string:
+				if value != nil {
+					maps[tagLocation][tagName] = *value
+				}
+			case *int:
+				if value != nil {
+					maps[tagLocation][tagName] = strconv.Itoa(int(*value))
+				}
+			case *int64:
+				if value != nil {
+					maps[tagLocation][tagName] = strconv.FormatInt(int64(*value), 10)
+				}
+			case *bool:
+			case *time.Time:
+				if value != nil {
+					formatString := fields.Type().Field(i).Tag.Get("format")
+					format := ""
+					switch formatString {
+					case "RFC 822":
+						format = convert.RFC822
+					case "ISO 8601":
+						format = convert.ISO8601
+					}
+					maps[tagLocation][tagName] = convert.TimeToString(*value, format)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *BaseBuilder) parseRequestBody() error {
+	requestData := map[string]interface{}{}
+
+	if !b.input.IsValid() {
+		return nil
+	}
+
+	fields := b.input.Elem()
+	if !fields.IsValid() {
+		return nil
+	}
+
+	for i := 0; i < fields.NumField(); i++ {
+		location := fields.Type().Field(i).Tag.Get("location")
+		if location == "elements" {
+			name := fields.Type().Field(i).Tag.Get("name")
+			requestData[name] = fields.Field(i).Interface()
+		}
+	}
+
+	if len(requestData) != 0 {
+		dataValue, err := json.Encode(requestData, true)
+		if err != nil {
+			return err
+		}
+
+		b.parsedBodyString = string(dataValue)
+		b.parsedBody = strings.NewReader(b.parsedBodyString)
+		(*b.parsedHeaders)["Content-Type"] = "application/json"
+	} else {
+		value := fields.FieldByName("Body")
+		if value.IsValid() {
+			switch value.Interface().(type) {
+			case string:
+				if value.String() != "" {
+					b.parsedBodyString = value.String()
+					b.parsedBody = strings.NewReader(value.String())
+				}
+			case io.Reader:
+				if value.Interface().(io.Reader) != nil {
+					b.parsedBody = value.Interface().(io.Reader)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *BaseBuilder) parseRequestProperties() error {
+	propertiesMap := map[string]string{}
+	b.parsedProperties = &propertiesMap
+
+	if b.operation.Properties != nil {
+		fields := reflect.ValueOf(b.operation.Properties).Elem()
+		if fields.IsValid() {
+			for i := 0; i < fields.NumField(); i++ {
+				switch value := fields.Field(i).Interface().(type) {
+				case *string:
+					if value != nil {
+						propertiesMap[fields.Type().Field(i).Tag.Get("name")] = *value
+					}
+				case *int:
+					if value != nil {
+						numberString := strconv.Itoa(int(*value))
+						propertiesMap[fields.Type().Field(i).Tag.Get("name")] = numberString
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *BaseBuilder) parseRequestURL() error {
+	return nil
+}
+
+func (b *BaseBuilder) setupHeaders(httpRequest *http.Request) error {
+	if b.parsedHeaders != nil {
+
+		for headerKey, headerValue := range *b.parsedHeaders {
+			if headerKey == "X-QS-Fetch-Source" {
+				// header X-QS-Fetch-Source is a URL to fetch.
+				// We should first parse this URL.
+				requestURL, err := url.Parse(headerValue)
+				if err != nil {
+					return fmt.Errorf("invalid HTTP header value: %s", headerValue)
+				}
+				headerValue = requestURL.String()
+			} else {
+				for _, r := range headerValue {
+					if r > unicode.MaxASCII {
+						headerValue = utils.URLQueryEscape(headerValue)
+						break
+					}
+				}
+			}
+
+			httpRequest.Header.Set(headerKey, headerValue)
+		}
+	}
+
+	if httpRequest.Header.Get("Content-Length") == "" {
+		var length int64
+		switch body := b.parsedBody.(type) {
+		case nil:
+			length = 0
+		case io.Seeker:
+			//start, err := body.Seek(0, io.SeekStart)
+			start, err := body.Seek(0, 0)
+			if err != nil {
+				return err
+			}
+			//end, err := body.Seek(0, io.SeekEnd)
+			end, err := body.Seek(0, 2)
+			if err != nil {
+				return err
+			}
+			//body.Seek(0, io.SeekStart)
+			body.Seek(0, 0)
+			length = end - start
+		default:
+			return errors.New("Can not get Content-Length")
+		}
+		if length > 0 {
+			httpRequest.ContentLength = length
+			httpRequest.Header.Set("Content-Length", strconv.Itoa(int(length)))
+		} else {
+			httpRequest.Header.Set("Content-Length", "0")
+		}
+	}
+	length, err := strconv.Atoi(httpRequest.Header.Get("Content-Length"))
+	if err != nil {
+		return err
+	}
+	httpRequest.ContentLength = int64(length)
+
+	if httpRequest.Header.Get("Date") == "" {
+		httpRequest.Header.Set("Date", convert.TimeToString(time.Now(), convert.RFC822))
+	}
+
+	return nil
+}
@@ -0,0 +1,125 @@
+// +-------------------------------------------------------------------------
+// | Copyright (C) 2016 Yunify, Inc.
+// +-------------------------------------------------------------------------
+// | Licensed under the Apache License, Version 2.0 (the "License");
+// | you may not use this work except in compliance with the License.
+// | You may obtain a copy of the License in the LICENSE file, or at:
+// |
+// | http://www.apache.org/licenses/LICENSE-2.0
+// |
+// | Unless required by applicable law or agreed to in writing, software
+// | distributed under the License is distributed on an "AS IS" BASIS,
+// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// | See the License for the specific language governing permissions and
+// | limitations under the License.
+// +-------------------------------------------------------------------------
+
+package builder
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/yunify/qingstor-sdk-go/config"
+	"github.com/yunify/qingstor-sdk-go/request/data"
+)
+
+type FakeProperties struct {
+	A  *string `name:"a"`
+	B  *string `name:"b"`
+	CD *int    `name:"c-d"`
+}
+type FakeInput struct {
+	ParamA    *string    `location:"params" name:"a"`
+	ParamB    *string    `location:"params" name:"b"`
+	ParamCD   *int       `location:"params" name:"c_d" default:"1024"`
+	HeaderA   *string    `location:"headers" name:"A"`
+	HeaderB   *time.Time `location:"headers" name:"B" format:"RFC 822"`
+	HeaderCD  *int       `location:"headers" name:"C-D"`
+	ElementA  *string    `location:"elements" name:"a"`
+	ElementB  *string    `location:"elements" name:"b"`
+	ElementCD *int64     `location:"elements" name:"cd"`
+	Body      *string    `localtion:"body"`
+}
+
+func (i *FakeInput) Validate() error {
+	return nil
+}
+
+func String(v string) *string {
+	return &v
+}
+
+func Int(v int) *int {
+	return &v
+}
+
+func Int64(v int64) *int64 {
+	return &v
+}
+
+func Time(v time.Time) *time.Time {
+	return &v
+}
+
+func TestBaseBuilder_BuildHTTPRequest(t *testing.T) {
+	conf, err := config.NewDefault()
+	assert.Nil(t, err)
+
+	tz, err := time.LoadLocation("Asia/Shanghai")
+	assert.Nil(t, err)
+
+	builder := BaseBuilder{}
+	operation := &data.Operation{
+		Config:      conf,
+		APIName:     "This is API name",
+		ServiceName: "Base",
+		Properties: &FakeProperties{
+			A:  String("property_a"),
+			B:  String("property_b"),
+			CD: Int(0),
+		},
+		RequestMethod: "GET",
+		RequestURI:    "/hello/<a>/<c-d>/<b>/world",
+		StatusCodes: []int{
+			200,
+			201,
+		},
+	}
+	inputValue := reflect.ValueOf(&FakeInput{
+		ParamA:    String("param_a"),
+		ParamCD:   Int(1024),
+		HeaderA:   String("header_a"),
+		HeaderB:   Time(time.Date(2016, 9, 1, 15, 30, 0, 0, tz)),
+		ElementA:  String("element_a"),
+		ElementB:  String("element_b"),
+		ElementCD: Int64(0),
+		Body:      String("This is body string"),
+	})
+	httpRequest, err := builder.BuildHTTPRequest(operation, &inputValue)
+	assert.Nil(t, err)
+	assert.Equal(t, &map[string]string{
+		"a":   "property_a",
+		"b":   "property_b",
+		"c-d": "0",
+	}, builder.parsedProperties)
+	assert.Equal(t, &map[string]string{
+		"a":   "param_a",
+		"c_d": "1024",
+	}, builder.parsedParams)
+	assert.Equal(t, &map[string]string{
+		"A":            "header_a",
+		"B":            "Thu, 01 Sep 2016 07:30:00 GMT",
+		"Content-Type": "application/json",
+	}, builder.parsedHeaders)
+	assert.NotNil(t, httpRequest.Header.Get("Date"))
+	assert.Equal(t, "40", httpRequest.Header.Get("Content-Length"))
+
+	buffer := &bytes.Buffer{}
+	buffer.ReadFrom(httpRequest.Body)
+	httpRequest.Body.Close()
+	assert.Equal(t, "{\"a\":\"element_a\",\"b\":\"element_b\",\"cd\":0}", buffer.String())
+}
@@ -0,0 +1,167 @@
+// +-------------------------------------------------------------------------
+// | Copyright (C) 2016 Yunify, Inc.
+// +-------------------------------------------------------------------------
+// | Licensed under the Apache License, Version 2.0 (the "License");
+// | you may not use this work except in compliance with the License.
+// | You may obtain a copy of the License in the LICENSE file, or at:
+// |
+// | http://www.apache.org/licenses/LICENSE-2.0
+// |
+// | Unless required by applicable law or agreed to in writing, software
+// | distributed under the License is distributed on an "AS IS" BASIS,
+// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// | See the License for the specific language governing permissions and
+// | limitations under the License.
+// +-------------------------------------------------------------------------
+
+package builder
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/url"
+	"path"
+	"reflect"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/pengsrc/go-shared/convert"
+
+	"github.com/yunify/qingstor-sdk-go"
+	"github.com/yunify/qingstor-sdk-go/logger"
+	"github.com/yunify/qingstor-sdk-go/request/data"
+	"github.com/yunify/qingstor-sdk-go/utils"
+)
+
+// QingStorBuilder is the request builder for QingStor service.
+type QingStorBuilder struct {
+	baseBuilder *BaseBuilder
+}
+
+// BuildHTTPRequest builds http request with an operation and an input.
+func (qb *QingStorBuilder) BuildHTTPRequest(o *data.Operation, i *reflect.Value) (*http.Request, error) {
+	qb.baseBuilder = &BaseBuilder{}
+	qb.baseBuilder.operation = o
+	qb.baseBuilder.input = i
+
+	_, err := qb.baseBuilder.parse()
+	if err != nil {
+		return nil, err
+	}
+	err = qb.parseURL()
+	if err != nil {
+		return nil, err
+	}
+
+	httpRequest, err := http.NewRequest(qb.baseBuilder.operation.RequestMethod,
+		qb.baseBuilder.parsedURL, qb.baseBuilder.parsedBody)
+	if err != nil {
+		return nil, err
+	}
+
+	err = qb.baseBuilder.setupHeaders(httpRequest)
+	if err != nil {
+		return nil, err
+	}
+	err = qb.setupHeaders(httpRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	logger.Info(fmt.Sprintf(
+		"Built QingStor request: [%d] %s",
+		convert.StringToUnixTimestamp(httpRequest.Header.Get("Date"), convert.RFC822),
+		httpRequest.URL.String()),
+	)
+
+	logger.Info(fmt.Sprintf(
+		"QingStor request headers: [%d] %s",
+		convert.StringToUnixTimestamp(httpRequest.Header.Get("Date"), convert.RFC822),
+		fmt.Sprint(httpRequest.Header)),
+	)
+
+	if qb.baseBuilder.parsedBodyString != "" {
+		logger.Info(fmt.Sprintf(
+			"QingStor request body string: [%d] %s",
+			convert.StringToUnixTimestamp(httpRequest.Header.Get("Date"), convert.RFC822),
+			qb.baseBuilder.parsedBodyString),
+		)
+	}
+
+	return httpRequest, nil
+}
+
+func (qb *QingStorBuilder) parseURL() error {
+	config := qb.baseBuilder.operation.Config
+
+	zone := (*qb.baseBuilder.parsedProperties)["zone"]
+	port := strconv.Itoa(config.Port)
+	endpoint := config.Protocol + "://" + config.Host + ":" + port
+	if zone != "" {
+		endpoint = config.Protocol + "://" + zone + "." + config.Host + ":" + port
+	}
+
+	requestURI := qb.baseBuilder.operation.RequestURI
+	for key, value := range *qb.baseBuilder.parsedProperties {
+		endpoint = strings.Replace(endpoint, "<"+key+">", utils.URLQueryEscape(value), -1)
+		requestURI = strings.Replace(requestURI, "<"+key+">", utils.URLQueryEscape(value), -1)
+	}
+	requestURI = regexp.MustCompile(`/+`).ReplaceAllString(requestURI, "/")
+
+	requestURL, err := url.Parse(endpoint + requestURI)
+	if err != nil {
+		return err
+	}
+
+	if qb.baseBuilder.parsedParams != nil {
+		queryValue := requestURL.Query()
+		for key, value := range *qb.baseBuilder.parsedParams {
+			queryValue.Set(key, value)
+		}
+		requestURL.RawQuery = queryValue.Encode()
+	}
+
+	qb.baseBuilder.parsedURL = requestURL.String()
+
+	return nil
+}
+
+func (qb *QingStorBuilder) setupHeaders(httpRequest *http.Request) error {
+	method := httpRequest.Method
+	if method == "POST" || method == "PUT" || method == "DELETE" {
+		if httpRequest.Header.Get("Content-Type") == "" {
+			mimeType := mime.TypeByExtension(path.Ext(httpRequest.URL.Path))
+			if mimeType != "" {
+				httpRequest.Header.Set("Content-Type", mimeType)
+			}
+		}
+	}
+
+	if httpRequest.Header.Get("User-Agent") == "" {
+		version := fmt.Sprintf(`Go v%s`, strings.Replace(runtime.Version(), "go", "", -1))
+		system := fmt.Sprintf(`%s_%s_%s`, runtime.GOOS, runtime.GOARCH, runtime.Compiler)
+		ua := fmt.Sprintf(`qingstor-sdk-go/%s (%s; %s)`, sdk.Version, version, system)
+		if qb.baseBuilder.operation.Config.AdditionalUserAgent != "" {
+			ua = fmt.Sprintf(`%s %s`, ua, qb.baseBuilder.operation.Config.AdditionalUserAgent)
+		}
+		httpRequest.Header.Set("User-Agent", ua)
+	}
+
+	if qb.baseBuilder.operation.APIName == "Delete Multiple Objects" {
+		buffer := &bytes.Buffer{}
+		buffer.ReadFrom(httpRequest.Body)
+		httpRequest.Body = ioutil.NopCloser(bytes.NewReader(buffer.Bytes()))
+
+		md5Value := md5.Sum(buffer.Bytes())
+		httpRequest.Header.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5Value[:]))
+	}
+
+	return nil
+}
@@ -0,0 +1,83 @@
+// +-------------------------------------------------------------------------
+// | Copyright (C) 2016 Yunify, Inc.
+// +-------------------------------------------------------------------------
+// | Licensed under the Apache License, Version 2.0 (the "License");
+// | you may not use this work except in compliance with the License.
+// | You may obtain a copy of the License in the LICENSE file, or at:
+// |
+// | http://www.apache.org/licenses/LICENSE-2.0
+// |
+// | Unless required by applicable law or agreed to in writing, software
+// | distributed under the License is distributed on an "AS IS" BASIS,
+// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// | See the License for the specific language governing permissions and
+// | limitations under the License.
+// +-------------------------------------------------------------------------
+
+package builder
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/yunify/qingstor-sdk-go/config"
+	"github.com/yunify/qingstor-sdk-go/request/data"
+)
+
+type ObjectSubServiceProperties struct {
+	BucketName *string `json:"bucket-name" name:"bucket-name"`
+	ObjectKey  *string `json:"object-key" name:"object-key"`
+	Zone       *string `json:"zone" name:"zone"`
+}
+type GetObjectInput struct {
+	IfMatch           *string    `json:"If-Match" name:"If-Match" location:"headers"`
+	IfModifiedSince   *time.Time `json:"If-Modified-Since" name:"If-Modified-Since" format:"RFC 822" location:"headers"`
+	IfNoneMatch       *string    `json:"If-None-Match" name:"If-None-Match" location:"headers"`
+	IfUnmodifiedSince time.Time  `json:"If-Unmodified-Since" name:"If-Unmodified-Since" format:"RFC 822" location:"headers"`
+	// Specified range of the Object
+	Range *string `json:"Range" name:"Range" location:"headers"`
+}
+
+func (i *GetObjectInput) Validate() error {
+	return nil
+}
+
+func TestQingStorBuilder_BuildHTTPRequest(t *testing.T) {
+	conf, err := config.NewDefault()
+	assert.Nil(t, err)
+	conf.Host = "qingstor.dev"
+
+	tz, err := time.LoadLocation("Asia/Shanghai")
+	assert.Nil(t, err)
+
+	qsBuilder := &QingStorBuilder{}
+	operation := &data.Operation{
+		Config:      conf,
+		APIName:     "GET Object",
+		ServiceName: "QingStor",
+		Properties: &ObjectSubServiceProperties{
+			BucketName: String("test"),
+			ObjectKey:  String("path/to/key.txt"),
+			Zone:       String("beta"),
+		},
+		RequestMethod: "GET",
+		RequestURI:    "/<bucket-name>/<object-key>",
+		StatusCodes: []int{
+			201,
+		},
+	}
+	inputValue := reflect.ValueOf(&GetObjectInput{
+		IfModifiedSince: Time(time.Date(2016, 9, 1, 15, 30, 0, 0, tz)),
+		Range:           String("100-"),
+	})
+	httpRequest, err := qsBuilder.BuildHTTPRequest(operation, &inputValue)
+	assert.Nil(t, err)
+	assert.NotNil(t, httpRequest.Header.Get("Date"))
+	assert.Equal(t, "0", httpRequest.Header.Get("Content-Length"))
+	assert.Equal(t, "", httpRequest.Header.Get("If-Match"))
+	assert.Equal(t, "Thu, 01 Sep 2016 07:30:00 GMT", httpRequest.Header.Get("If-Modified-Since"))
+	assert.Equal(t, "100-", httpRequest.Header.Get("Range"))
+	assert.Equal(t, "https://beta.qingstor.dev:443/test/path/to/key.txt", httpRequest.URL.String())
+}