vendor: update all dependencies

vendor/cloud.google.com/go/httpreplay/internal/proxy/record.go

@@ -0,0 +1,215 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+// The proxy package provides a record/replay HTTP proxy. It is designed to support
+// both an in-memory API (cloud.google.com/go/httpreplay) and a standalone server
+// (cloud.google.com/go/httpreplay/cmd/httpr).
+package proxy
+
+// See github.com/google/martian/cmd/proxy/main.go for the origin of much of this.
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/google/martian"
+	"github.com/google/martian/fifo"
+	"github.com/google/martian/har"
+	"github.com/google/martian/httpspec"
+	"github.com/google/martian/martianlog"
+	"github.com/google/martian/mitm"
+)
+
+// A Proxy is an HTTP proxy that supports recording or replaying requests.
+type Proxy struct {
+	// The certificate that the proxy uses to participate in TLS.
+	CACert *x509.Certificate
+
+	// The URL of the proxy.
+	URL *url.URL
+
+	// Initial state of the client. Must be serializable with json.Marshal.
+	Initial interface{}
+
+	mproxy   *martian.Proxy
+	filename string      // for log
+	logger   *har.Logger // for recording only
+}
+
+// ForRecording returns a Proxy configured to record.
+func ForRecording(filename string, port int) (*Proxy, error) {
+	p, err := newProxy(filename)
+	if err != nil {
+		return nil, err
+	}
+	// Configure the transport for the proxy's outgoing traffic.
+	p.mproxy.SetRoundTripper(&http.Transport{
+		Dial: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: time.Second,
+	})
+
+	// Construct a group that performs the standard proxy stack of request/response
+	// modifications.
+	stack, _ := httpspec.NewStack("httpr") // second arg is an internal group that we don't need
+	p.mproxy.SetRequestModifier(stack)
+	p.mproxy.SetResponseModifier(stack)
+
+	// Make a group for logging requests and responses.
+	logGroup := fifo.NewGroup()
+	skipAuth := skipLoggingByHost("accounts.google.com")
+	logGroup.AddRequestModifier(skipAuth)
+	logGroup.AddResponseModifier(skipAuth)
+	p.logger = har.NewLogger()
+	logGroup.AddRequestModifier(martian.RequestModifierFunc(
+		func(req *http.Request) error { return withRedactedHeaders(req, p.logger) }))
+	logGroup.AddResponseModifier(p.logger)
+
+	stack.AddRequestModifier(logGroup)
+	stack.AddResponseModifier(logGroup)
+
+	// Ordinary debug logging.
+	logger := martianlog.NewLogger()
+	logger.SetDecode(true)
+	stack.AddRequestModifier(logger)
+	stack.AddResponseModifier(logger)
+
+	if err := p.start(port); err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func newProxy(filename string) (*Proxy, error) {
+	mproxy := martian.NewProxy()
+	// Set up a man-in-the-middle configuration with a CA certificate so the proxy can
+	// participate in TLS.
+	x509c, priv, err := mitm.NewAuthority("cloud.google.com/go/httpreplay", "HTTPReplay Authority", time.Hour)
+	if err != nil {
+		return nil, err
+	}
+	mc, err := mitm.NewConfig(x509c, priv)
+	if err != nil {
+		return nil, err
+	}
+	mc.SetValidity(time.Hour)
+	mc.SetOrganization("cloud.google.com/go/httpreplay")
+	mc.SkipTLSVerify(false)
+	if err != nil {
+		return nil, err
+	}
+	mproxy.SetMITM(mc)
+	return &Proxy{
+		mproxy:   mproxy,
+		CACert:   x509c,
+		filename: filename,
+	}, nil
+}
+
+func (p *Proxy) start(port int) error {
+	l, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return err
+	}
+	p.URL = &url.URL{Scheme: "http", Host: l.Addr().String()}
+	go p.mproxy.Serve(l)
+	return nil
+}
+
+// Transport returns an http.Transport for clients who want to talk to the proxy.
+func (p *Proxy) Transport() *http.Transport {
+	caCertPool := x509.NewCertPool()
+	caCertPool.AddCert(p.CACert)
+	return &http.Transport{
+		TLSClientConfig: &tls.Config{RootCAs: caCertPool},
+		Proxy:           func(*http.Request) (*url.URL, error) { return p.URL, nil },
+	}
+}
+
+// Close closes the proxy. If the proxy is recording, it also writes the log.
+func (p *Proxy) Close() error {
+	p.mproxy.Close()
+	if p.logger != nil {
+		return p.writeLog()
+	}
+	return nil
+}
+
+type httprFile struct {
+	Initial interface{}
+	HAR     *har.HAR
+}
+
+func (p *Proxy) writeLog() error {
+	f := httprFile{
+		Initial: p.Initial,
+		HAR:     p.logger.ExportAndReset(),
+	}
+	bytes, err := json.Marshal(f)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(p.filename, bytes, 0600) // only accessible by owner
+}
+
+// Headers that may contain sensitive data (auth tokens, keys).
+var sensitiveHeaders = []string{
+	"Authorization",
+	"X-Goog-Encryption-Key",             // used by Cloud Storage for customer-supplied encryption
+	"X-Goog-Copy-Source-Encryption-Key", // ditto
+}
+
+// withRedactedHeaders removes sensitive header contents before calling mod.
+func withRedactedHeaders(req *http.Request, mod martian.RequestModifier) error {
+	// We have to change the headers, then log, then restore them.
+	replaced := map[string]string{}
+	for _, h := range sensitiveHeaders {
+		if v := req.Header.Get(h); v != "" {
+			replaced[h] = v
+			req.Header.Set(h, "REDACTED")
+		}
+	}
+	err := mod.ModifyRequest(req)
+	for h, v := range replaced {
+		req.Header.Set(h, v)
+	}
+	return err
+}
+
+// skipLoggingByHost disables logging for traffic to a particular host.
+type skipLoggingByHost string
+
+func (s skipLoggingByHost) ModifyRequest(req *http.Request) error {
+	if strings.HasPrefix(req.Host, string(s)) {
+		martian.NewContext(req).SkipLogging()
+	}
+	return nil
+}
+
+func (s skipLoggingByHost) ModifyResponse(res *http.Response) error {
+	return s.ModifyRequest(res.Request)
+}

vendor/cloud.google.com/go/httpreplay/internal/proxy/record_test.go

@@ -0,0 +1,65 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package proxy
+
+import (
+	"net/http"
+	"testing"
+
+	"cloud.google.com/go/internal/testutil"
+	"github.com/google/martian"
+)
+
+func TestWithRedactedHeaders(t *testing.T) {
+	clone := func(h http.Header) http.Header {
+		h2 := http.Header{}
+		for k, v := range h {
+			h2[k] = v
+		}
+		return h2
+	}
+
+	orig := http.Header{
+		"Content-Type":                      {"text/plain"},
+		"Authorization":                     {"oauth2-token"},
+		"X-Goog-Encryption-Key":             {"a-secret-key"},
+		"X-Goog-Copy-Source-Encryption-Key": {"another-secret-key"},
+	}
+	req := &http.Request{Header: clone(orig)}
+	var got http.Header
+	mod := martian.RequestModifierFunc(func(req *http.Request) error {
+		got = clone(req.Header)
+		return nil
+	})
+	if err := withRedactedHeaders(req, mod); err != nil {
+		t.Fatal(err)
+	}
+	// Logged headers should be redacted.
+	want := http.Header{
+		"Content-Type":                      {"text/plain"},
+		"Authorization":                     {"REDACTED"},
+		"X-Goog-Encryption-Key":             {"REDACTED"},
+		"X-Goog-Copy-Source-Encryption-Key": {"REDACTED"},
+	}
+	if !testutil.Equal(got, want) {
+		t.Errorf("got  %+v\nwant %+v", got, want)
+	}
+	// The request's headers should be the same.
+	if got, want := req.Header, orig; !testutil.Equal(got, want) {
+		t.Errorf("got  %+v\nwant %+v", got, want)
+	}
+}

vendor/cloud.google.com/go/httpreplay/internal/proxy/replay.go

@@ -0,0 +1,336 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package proxy
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"reflect"
+	"strings"
+
+	"github.com/google/martian/har"
+	"github.com/google/martian/martianlog"
+)
+
+// ForReplaying returns a Proxy configured to replay.
+func ForReplaying(filename string, port int) (*Proxy, error) {
+	p, err := newProxy(filename)
+	if err != nil {
+		return nil, err
+	}
+	calls, initial, err := readLog(filename)
+	if err != nil {
+		return nil, err
+	}
+	p.mproxy.SetRoundTripper(replayRoundTripper{calls: calls})
+	p.Initial = initial
+
+	// Debug logging.
+	// TODO(jba): factor out from here and ForRecording.
+	logger := martianlog.NewLogger()
+	logger.SetDecode(true)
+	p.mproxy.SetRequestModifier(logger)
+	p.mproxy.SetResponseModifier(logger)
+
+	if err := p.start(port); err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+// A call is an HTTP request and its matching response.
+type call struct {
+	req     *har.Request
+	reqBody *requestBody // parsed request body
+	res     *har.Response
+}
+
+func readLog(filename string) ([]*call, interface{}, error) {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, nil, err
+	}
+	var f httprFile
+	if err := json.Unmarshal(bytes, &f); err != nil {
+		return nil, nil, err
+	}
+	ignoreIDs := map[string]bool{} // IDs of requests to ignore
+	callsByID := map[string]*call{}
+	var calls []*call
+	for _, e := range f.HAR.Log.Entries {
+		if ignoreIDs[e.ID] {
+			continue
+		}
+		c, ok := callsByID[e.ID]
+		switch {
+		case !ok:
+			if e.Request == nil {
+				return nil, nil, fmt.Errorf("first entry for ID %s does not have a request", e.ID)
+			}
+			if e.Request.Method == "CONNECT" {
+				// Ignore CONNECT methods.
+				ignoreIDs[e.ID] = true
+			} else {
+				reqBody, err := newRequestBodyFromHAR(e.Request)
+				if err != nil {
+					return nil, nil, err
+				}
+				c := &call{e.Request, reqBody, e.Response}
+				calls = append(calls, c)
+				callsByID[e.ID] = c
+			}
+		case e.Request != nil:
+			if e.Response != nil {
+				return nil, nil, errors.New("HAR entry has both request and response")
+			}
+			c.req = e.Request
+		case e.Response != nil:
+			c.res = e.Response
+		default:
+			return nil, nil, errors.New("HAR entry has neither request nor response")
+		}
+	}
+	for _, c := range calls {
+		if c.req == nil || c.res == nil {
+			return nil, nil, fmt.Errorf("missing request or response: %+v", c)
+		}
+	}
+	return calls, f.Initial, nil
+}
+
+type replayRoundTripper struct {
+	calls []*call
+}
+
+func (r replayRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	reqBody, err := newRequestBodyFromHTTP(req)
+	if err != nil {
+		return nil, err
+	}
+	for i, call := range r.calls {
+		if call == nil {
+			continue
+		}
+		if requestsMatch(req, reqBody, call.req, call.reqBody) {
+			r.calls[i] = nil // nil out this call so we don't reuse it
+			res := harResponseToHTTPResponse(call.res)
+			res.Request = req
+			return res, nil
+		}
+	}
+	return nil, fmt.Errorf("no matching request for %+v", req)
+}
+
+// Headers that shouldn't be compared, becuase they may differ on different executions
+// of the same code, or may not be present during record or replay.
+var ignoreHeaders = map[string]bool{}
+
+func init() {
+	// Sensitive headers are redacted in the log, so they won't be equal to incoming values.
+	for _, h := range sensitiveHeaders {
+		ignoreHeaders[h] = true
+	}
+	for _, h := range []string{
+		"Content-Type", // handled by requestBody
+		"Date",
+		"Host",
+		"Transfer-Encoding",
+		"Via",
+		"X-Forwarded-For",
+		"X-Forwarded-Host",
+		"X-Forwarded-Proto",
+		"X-Forwarded-Url",
+	} {
+		ignoreHeaders[h] = true
+	}
+}
+
+// Report whether the incoming request in matches the candidate request cand.
+func requestsMatch(in *http.Request, inBody *requestBody, cand *har.Request, candBody *requestBody) bool {
+	// TODO(jba): compare headers?
+	if in.Method != cand.Method {
+		return false
+	}
+	if in.URL.String() != cand.URL {
+		return false
+	}
+	if !inBody.equal(candBody) {
+		return false
+	}
+	// Check headers last. See DebugHeaders.
+	return headersMatch(in.Header, harHeadersToHTTP(cand.Headers), ignoreHeaders)
+}
+
+func harHeadersToHTTP(hhs []har.Header) http.Header {
+	// Unfortunately, the har package joins multiple header values with ", ",
+	// which isn't reversible if any of the values contains a comma.
+	// We hope for the best.
+	res := http.Header{}
+	for _, hh := range hhs {
+		res[hh.Name] = strings.Split(hh.Value, ", ")
+	}
+	return res
+}
+
+// Convert a HAR response to a Go http.Response.
+// HAR (Http ARchive) is a standard for storing HTTP interactions.
+// See http://www.softwareishard.com/blog/har-12-spec.
+func harResponseToHTTPResponse(hr *har.Response) *http.Response {
+	return &http.Response{
+		StatusCode: hr.Status,
+		Status:     hr.StatusText,
+		Proto:      hr.HTTPVersion,
+		// TODO(jba): headers?
+		Body:          ioutil.NopCloser(bytes.NewReader(hr.Content.Text)),
+		ContentLength: int64(len(hr.Content.Text)),
+	}
+}
+
+// A requestBody represents the body of a request. If the content type is multipart, the
+// body is split into parts.
+//
+// The replaying proxy needs to understand multipart bodies because the boundaries are
+// generated randomly, so we can't just compare the entire bodies for equality.
+type requestBody struct {
+	mediaType string   // the media type part of the Content-Type header
+	parts     [][]byte // the parts of the body, or just a single []byte if not multipart
+}
+
+func newRequestBodyFromHTTP(req *http.Request) (*requestBody, error) {
+	defer req.Body.Close()
+	return newRequestBody(req.Header.Get("Content-Type"), req.Body)
+}
+
+func newRequestBodyFromHAR(req *har.Request) (*requestBody, error) {
+	if req.PostData == nil {
+		return nil, nil
+	}
+	var cth string
+	for _, h := range req.Headers {
+		if h.Name == "Content-Type" {
+			cth = h.Value
+			break
+		}
+	}
+	return newRequestBody(cth, strings.NewReader(req.PostData.Text))
+}
+
+// newRequestBody parses the Content-Type header, reads the body, and splits it into
+// parts if necessary.
+func newRequestBody(contentType string, body io.Reader) (*requestBody, error) {
+	if contentType == "" {
+		// No content-type header. There should not be a body.
+		if _, err := body.Read(make([]byte, 1)); err != io.EOF {
+			return nil, errors.New("no Content-Type, but body")
+		}
+		return nil, nil
+	}
+	mediaType, params, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return nil, err
+	}
+	rb := &requestBody{mediaType: mediaType}
+	if strings.HasPrefix(mediaType, "multipart/") {
+		mr := multipart.NewReader(body, params["boundary"])
+		for {
+			p, err := mr.NextPart()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return nil, err
+			}
+			part, err := ioutil.ReadAll(p)
+			if err != nil {
+				return nil, err
+			}
+			// TODO(jba): care about part headers?
+			rb.parts = append(rb.parts, part)
+		}
+	} else {
+		bytes, err := ioutil.ReadAll(body)
+		if err != nil {
+			return nil, err
+		}
+		rb.parts = [][]byte{bytes}
+	}
+	return rb, nil
+}
+
+func (r1 *requestBody) equal(r2 *requestBody) bool {
+	if r1 == nil || r2 == nil {
+		return r1 == r2
+	}
+	if r1.mediaType != r2.mediaType {
+		return false
+	}
+	if len(r1.parts) != len(r2.parts) {
+		return false
+	}
+	for i, p1 := range r1.parts {
+		if !bytes.Equal(p1, r2.parts[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// DebugHeaders helps to determine whether a header should be ignored.
+// When true, if requests have the same method, URL and body but differ
+// in a header, the first mismatched header is logged.
+var DebugHeaders = false
+
+func headersMatch(in, cand http.Header, ignores map[string]bool) bool {
+	for k1, v1 := range in {
+		if ignores[k1] {
+			continue
+		}
+		v2 := cand[k1]
+		if v2 == nil {
+			if DebugHeaders {
+				log.Printf("header %s: present in incoming request but not candidate", k1)
+			}
+			return false
+		}
+		if !reflect.DeepEqual(v1, v2) {
+			if DebugHeaders {
+				log.Printf("header %s: incoming %v, candidate %v", k1, v1, v2)
+			}
+			return false
+		}
+	}
+	for k2 := range cand {
+		if ignores[k2] {
+			continue
+		}
+		if in[k2] == nil {
+			if DebugHeaders {
+				log.Printf("header %s: not in incoming request but present in candidate", k2)
+			}
+			return false
+		}
+	}
+	return true
+}

vendor/cloud.google.com/go/httpreplay/internal/proxy/replay_test.go

@@ -0,0 +1,114 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package proxy
+
+import (
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"cloud.google.com/go/internal/testutil"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestRequestBody(t *testing.T) {
+	req1 := &http.Request{
+		Header: http.Header{"Content-Type": {"multipart/mixed; boundary=foo"}},
+		Body: ioutil.NopCloser(strings.NewReader(
+			"--foo\r\nFoo: one\r\n\r\nA section\r\n" +
+				"--foo\r\nFoo: two\r\n\r\nAnd another\r\n" +
+				"--foo--\r\n")),
+	}
+	rb1, err := newRequestBodyFromHTTP(req1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := &requestBody{
+		mediaType: "multipart/mixed",
+		parts: [][]byte{
+			[]byte("A section"),
+			[]byte("And another"),
+		},
+	}
+	if diff := testutil.Diff(rb1, want, cmp.AllowUnexported(requestBody{})); diff != "" {
+		t.Error(diff)
+	}
+
+	// Same contents, different boundary.
+	req2 := &http.Request{
+		Header: http.Header{"Content-Type": {"multipart/mixed; boundary=bar"}},
+		Body: ioutil.NopCloser(strings.NewReader(
+			"--bar\r\nFoo: one\r\n\r\nA section\r\n" +
+				"--bar\r\nFoo: two\r\n\r\nAnd another\r\n" +
+				"--bar--\r\n")),
+	}
+	rb2, err := newRequestBodyFromHTTP(req2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := testutil.Diff(rb1, want, cmp.AllowUnexported(requestBody{})); diff != "" {
+		t.Error(diff)
+	}
+
+	if !rb1.equal(rb2) {
+		t.Error("equal returned false, want true")
+	}
+}
+
+func TestHeadersMatch(t *testing.T) {
+	for _, test := range []struct {
+		h1, h2 http.Header
+		want   bool
+	}{
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			true,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			http.Header{"A": {"x"}, "B": {"w"}},
+			false,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"foo"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"bar"}},
+			true,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"bar"}},
+			true,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"foo"}},
+			http.Header{"A": {"x"}, "I": {"bar"}},
+			false,
+		},
+		{
+			http.Header{"A": {"x"}, "I": {"foo"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"bar"}},
+			false,
+		},
+	} {
+		got := headersMatch(test.h1, test.h2, map[string]bool{"I": true})
+		if got != test.want {
+			t.Errorf("%v, %v: got %t, want %t", test.h1, test.h2, got, test.want)
+		}
+	}
+}