vendor: update all dependencies

vendor/cloud.google.com/go/httpreplay/cmd/httpr/httpr.go

@@ -0,0 +1,128 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// httpr is a proxy that can record or replay HTTP requests.
+// Start httpr with either the -record or -replay flags, providing a filename.
+// Terminate the process with an interrupt (kill -2) to write the log file when recording.
+// To get the CA certificate of the proxy, issue a GET to http://localhost:CP/authority.cer, where
+// CP is the control port.
+
+// +build go1.8
+
+package main
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+
+	"cloud.google.com/go/httpreplay/internal/proxy"
+	"github.com/google/martian/martianhttp"
+)
+
+var (
+	port         = flag.Int("port", 8080, "port of the proxy")
+	controlPort  = flag.Int("control-port", 8181, "port for controlling the proxy")
+	record       = flag.String("record", "", "record traffic and save to filename")
+	replay       = flag.String("replay", "", "read filename and replay traffic")
+	debugHeaders = flag.Bool("debug-headers", false, "log header mismatches")
+)
+
+func main() {
+	flag.Parse()
+	if *record == "" && *replay == "" {
+		log.Fatal("provide either -record or -replay")
+	}
+	if *record != "" && *replay != "" {
+		log.Fatal("provide only one of -record and -replay")
+	}
+	log.Printf("httpr: starting proxy on port %d and control on port %d", *port, *controlPort)
+
+	var pr *proxy.Proxy
+	var err error
+	if *record != "" {
+		pr, err = proxy.ForRecording(*record, *port)
+	} else {
+		pr, err = proxy.ForReplaying(*replay, *port)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+	proxy.DebugHeaders = *debugHeaders
+
+	// Expose handlers on the control port.
+	mux := http.NewServeMux()
+	mux.Handle("/authority.cer", martianhttp.NewAuthorityHandler(pr.CACert))
+	mux.HandleFunc("/initial", handleInitial(pr))
+	lControl, err := net.Listen("tcp", fmt.Sprintf(":%d", *controlPort))
+	if err != nil {
+		log.Fatal(err)
+	}
+	go http.Serve(lControl, mux)
+
+	sigc := make(chan os.Signal, 1)
+	signal.Notify(sigc, os.Interrupt)
+
+	<-sigc
+
+	log.Println("httpr: shutting down")
+	if err := pr.Close(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func handleInitial(pr *proxy.Proxy) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		switch req.Method {
+		case "GET":
+			if pr.Initial != nil {
+				switch x := pr.Initial.(type) {
+				case []byte:
+					w.Write(x)
+				case string:
+					// If it's base64, then it's most likely from the JSON in the saved file
+					// (json.Marshal encodes []byte as a base64 string).  Decode it.
+					if bytes, err := base64.StdEncoding.DecodeString(x); err == nil {
+						w.Write(bytes)
+					} else {
+						// If it's not base64, write the string out directly.
+						w.Write([]byte(x))
+					}
+				default:
+					// We don't know what it is, so just print it.
+					fmt.Fprint(w, x)
+				}
+			}
+
+		case "POST":
+			bytes, err := ioutil.ReadAll(req.Body)
+			req.Body.Close()
+			if err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				fmt.Fprintf(w, "reading body: %v", err)
+			}
+			pr.Initial = bytes
+
+		default:
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprint(w, "use GET to retrieve initial or POST to set it")
+		}
+	}
+}

vendor/cloud.google.com/go/httpreplay/cmd/httpr/integration_test.go

@@ -0,0 +1,230 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package main_test
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"cloud.google.com/go/internal/testutil"
+	"cloud.google.com/go/storage"
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
+	"google.golang.org/api/option"
+)
+
+const initial = "initial state"
+
+func TestIntegration_HTTPR(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Integration tests skipped in short mode")
+	}
+	if testutil.ProjID() == "" {
+		t.Fatal("set GCLOUD_TESTS_GOLANG_PROJECT_ID and GCLOUD_TESTS_GOLANG_KEY")
+	}
+	// Get a unique temporary filename.
+	f, err := ioutil.TempFile("", "httpreplay")
+	if err != nil {
+		t.Fatal(err)
+	}
+	replayFilename := f.Name()
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(replayFilename)
+
+	if err := exec.Command("go", "build").Run(); err != nil {
+		t.Fatalf("running 'go build': %v", err)
+	}
+	defer os.Remove("./httpr")
+	want := runRecord(t, replayFilename)
+	got := runReplay(t, replayFilename)
+	if got != want {
+		t.Fatalf("got %q, want %q", got, want)
+	}
+}
+
+func runRecord(t *testing.T, filename string) string {
+	cmd, tr, cport, err := start("-record", filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stop(t, cmd)
+
+	ctx := context.Background()
+	hc := &http.Client{
+		Transport: &oauth2.Transport{
+			Base:   tr,
+			Source: testutil.TokenSource(ctx, storage.ScopeFullControl),
+		},
+	}
+	res, err := http.Post(
+		fmt.Sprintf("http://localhost:%s/initial", cport),
+		"text/plain",
+		strings.NewReader(initial))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.StatusCode != 200 {
+		t.Fatalf("from POST: %s", res.Status)
+	}
+	info, err := getBucketInfo(ctx, hc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return info
+}
+
+func runReplay(t *testing.T, filename string) string {
+	cmd, tr, cport, err := start("-replay", filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer stop(t, cmd)
+
+	hc := &http.Client{Transport: tr}
+	res, err := http.Get(fmt.Sprintf("http://localhost:%s/initial", cport))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.StatusCode != 200 {
+		t.Fatalf("from GET: %s", res.Status)
+	}
+	bytes, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := string(bytes), initial; got != want {
+		t.Errorf("initial: got %q, want %q", got, want)
+	}
+	info, err := getBucketInfo(context.Background(), hc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return info
+}
+
+// Start the proxy binary and wait for it to come up.
+// Return a transport that talks to the proxy, as well as the control port.
+// modeFlag must be either "-record" or "-replay".
+func start(modeFlag, filename string) (*exec.Cmd, *http.Transport, string, error) {
+	pport, err := pickPort()
+	if err != nil {
+		return nil, nil, "", err
+	}
+	cport, err := pickPort()
+	if err != nil {
+		return nil, nil, "", err
+	}
+	cmd := exec.Command("./httpr", "-port", pport, "-control-port", cport, modeFlag, filename, "-debug-headers")
+	if err := cmd.Start(); err != nil {
+		return nil, nil, "", err
+	}
+	// Wait for the server to come up.
+	serverUp := false
+	for i := 0; i < 10; i++ {
+		if conn, err := net.Dial("tcp", "localhost:"+cport); err == nil {
+			conn.Close()
+			serverUp = true
+			break
+		}
+		time.Sleep(time.Second)
+	}
+	if !serverUp {
+		return nil, nil, "", errors.New("server never came up")
+	}
+	tr, err := proxyTransport(pport, cport)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	return cmd, tr, cport, nil
+}
+
+func stop(t *testing.T, cmd *exec.Cmd) {
+	if err := cmd.Process.Signal(os.Interrupt); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// pickPort picks an unused port.
+func pickPort() (string, error) {
+	l, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return "", err
+	}
+	addr := l.Addr().String()
+	_, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return "", err
+	}
+	l.Close()
+	return port, nil
+}
+
+func proxyTransport(pport, cport string) (*http.Transport, error) {
+	caCert, err := getBody(fmt.Sprintf("http://localhost:%s/authority.cer", cport))
+	if err != nil {
+		return nil, err
+	}
+	caCertPool := x509.NewCertPool()
+	if !caCertPool.AppendCertsFromPEM([]byte(caCert)) {
+		return nil, errors.New("bad CA Cert")
+	}
+	return &http.Transport{
+		Proxy:           http.ProxyURL(&url.URL{Host: "localhost:" + pport}),
+		TLSClientConfig: &tls.Config{RootCAs: caCertPool},
+	}, nil
+}
+
+func getBucketInfo(ctx context.Context, hc *http.Client) (string, error) {
+	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
+	if err != nil {
+		return "", err
+	}
+	defer client.Close()
+	b := client.Bucket(testutil.ProjID())
+	attrs, err := b.Attrs(ctx)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("name:%s reqpays:%v location:%s sclass:%s",
+		attrs.Name, attrs.RequesterPays, attrs.Location, attrs.StorageClass), nil
+}
+
+func getBody(url string) ([]byte, error) {
+	res, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("response: %s", res.Status)
+	}
+	defer res.Body.Close()
+	return ioutil.ReadAll(res.Body)
+}

vendor/cloud.google.com/go/httpreplay/httpreplay.go

@@ -0,0 +1,148 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+// Package httpreplay provides an API for recording and replaying traffic
+// from HTTP-based Google API clients.
+//
+// To record:
+// 1.  Call NewRecorder to get a Recorder.
+// 2.  Use its Client method to obtain an HTTP client to use when making API calls.
+// 3.  Close the Recorder when you're done. That will save the
+//     log of interactions to the file you provided to NewRecorder.
+//
+// To replay:
+// 1.  Call NewReplayer with the same filename you used to record to get a Replayer.
+// 2.  Call its Client method and use the client to make the same API calls.
+//     You will get back the recorded responses.
+// 3.  Close the Replayer when you're done.
+//
+// This package is EXPERIMENTAL and is subject to change or removal without notice.
+// It requires Go version 1.8 or higher.
+package httpreplay
+
+// TODO(jba): add examples.
+
+import (
+	"fmt"
+	"net/http"
+
+	"cloud.google.com/go/httpreplay/internal/proxy"
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
+	"google.golang.org/api/option"
+	htransport "google.golang.org/api/transport/http"
+)
+
+// A Recorder records HTTP interactions.
+type Recorder struct {
+	filename string
+	proxy    *proxy.Proxy
+}
+
+// NewRecorder creates a recorder that writes to filename. The file will
+// also store initial state that can be retrieved to configure replay. The "initial"
+// argument must work with json.Marshal.
+//
+// You must call Close on the Recorder to ensure that all data is written.
+func NewRecorder(filename string, initial interface{}) (*Recorder, error) {
+	p, err := proxy.ForRecording(filename, 0)
+	if err != nil {
+		return nil, err
+	}
+	p.Initial = initial
+	return &Recorder{proxy: p}, nil
+}
+
+// Client returns an http.Client to be used for recording. Provide authentication options
+// like option.WithTokenSource as you normally would, or omit them to use Application Default
+// Credentials.
+func (r *Recorder) Client(ctx context.Context, opts ...option.ClientOption) (*http.Client, error) {
+	hc, _, err := htransport.NewClient(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+	// The http.Client returned by htransport.NewClient contains an
+	// http.RoundTripper. We want to somehow plug in a Transport that calls the proxy
+	// (returned by r.proxy.Transport).
+	//
+	// htransport.NewClient constructs its RoundTripper via the decorator pattern, by
+	// nesting several implementations of RoundTripper inside each other, ending with
+	// http.DefaultTransport. For example, one of the decorators is oauth2.Transport,
+	// which inserts an Authorization header and then calls the next RoundTripper in
+	// the sequence (stored in a field called Base).
+	//
+	// The problem is that we need to insert the proxy Transport at the end of this
+	// sequence, where http.DefaultTransport currently lives. But we can't traverse
+	// that sequence of RoundTrippers in general, because we don't know their types.
+	//
+	// For now, we only handle the special (but common) case where the first
+	// RoundTripper in the sequence is an oauth2.Transport. We can replace its Base
+	// field with the proxy transport. This causes us to lose the other RoundTrippers
+	// in the sequence, but those aren't essential for testing.
+	//
+	// A better solution would be to add option.WithBaseTransport, which would allow
+	// us to replace the http.DefaultTransport at the end of the sequence with the
+	// transport of our choice.
+	otrans, ok := hc.Transport.(*oauth2.Transport)
+	if !ok {
+		return nil, fmt.Errorf("can't handle Transport of type %T", hc.Transport)
+	}
+	otrans.Base = r.proxy.Transport()
+	return hc, nil
+}
+
+// Close closes the Recorder and saves the log file.
+func (r *Recorder) Close() error {
+	return r.proxy.Close()
+}
+
+// A Replayer replays previously recorded HTTP interactions.
+type Replayer struct {
+	proxy *proxy.Proxy
+}
+
+// NewReplayer creates a replayer that reads from filename.
+func NewReplayer(filename string) (*Replayer, error) {
+	p, err := proxy.ForReplaying(filename, 0)
+	if err != nil {
+		return nil, err
+	}
+	return &Replayer{proxy: p}, nil
+}
+
+// Client returns an HTTP client for replaying. The client does not need to be
+// configured with credentials for authenticating to a server, since it never
+// contacts a real backend.
+func (r *Replayer) Client(ctx context.Context) (*http.Client, error) {
+	return &http.Client{Transport: r.proxy.Transport()}, nil
+}
+
+// Initial returns the initial state saved by the Recorder.
+func (r *Replayer) Initial() interface{} {
+	return r.proxy.Initial
+}
+
+// Close closes the replayer.
+func (r *Replayer) Close() error {
+	return r.proxy.Close()
+}
+
+// DebugHeaders helps to determine whether a header should be ignored.
+// When true, if requests have the same method, URL and body but differ
+// in a header, the first mismatched header is logged.
+func DebugHeaders() {
+	proxy.DebugHeaders = true
+}

vendor/cloud.google.com/go/httpreplay/httpreplay_test.go

@@ -0,0 +1,123 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package httpreplay_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+
+	"cloud.google.com/go/httpreplay"
+	"cloud.google.com/go/internal/testutil"
+	"cloud.google.com/go/storage"
+	"golang.org/x/net/context"
+	"google.golang.org/api/option"
+)
+
+func TestIntegration_RecordAndReplay(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Integration tests skipped in short mode")
+	}
+	f, err := ioutil.TempFile("", "httpreplay")
+	if err != nil {
+		t.Fatal(err)
+	}
+	replayFilename := f.Name()
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(replayFilename)
+	projectID := testutil.ProjID()
+	if projectID == "" {
+		t.Skip("Need project ID. See CONTRIBUTING.md for details.")
+	}
+	ctx := context.Background()
+
+	// Record.
+	rec, err := httpreplay.NewRecorder(replayFilename, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hc, err := rec.Client(ctx, option.WithTokenSource(
+		testutil.TokenSource(ctx, storage.ScopeFullControl)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	wanta, wantc := run(t, hc)
+	if err := rec.Close(); err != nil {
+		t.Fatalf("rec.Close: %v", err)
+	}
+
+	// Replay.
+	rep, err := httpreplay.NewReplayer(replayFilename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rep.Close()
+	hc, err = rep.Client(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	gota, gotc := run(t, hc)
+
+	if diff := testutil.Diff(gota, wanta); diff != "" {
+		t.Error(diff)
+	}
+	if !bytes.Equal(gotc, wantc) {
+		t.Errorf("got %q, want %q", gotc, wantc)
+	}
+	if got, want := rep.Initial(), "initial state"; got != want {
+		t.Errorf("initial: got %v, want %q", got, want)
+	}
+}
+
+// TODO(jba): test errors
+
+func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+	b := client.Bucket(testutil.ProjID())
+	attrs, err := b.Attrs(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	obj := b.Object("replay-test")
+	w := obj.NewWriter(ctx)
+	if _, err := w.Write([]byte("hello")); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	r, err := obj.NewReader(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+	contents, err := ioutil.ReadAll(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return attrs, contents
+}

vendor/cloud.google.com/go/httpreplay/internal/proxy/record.go

@@ -0,0 +1,215 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+// The proxy package provides a record/replay HTTP proxy. It is designed to support
+// both an in-memory API (cloud.google.com/go/httpreplay) and a standalone server
+// (cloud.google.com/go/httpreplay/cmd/httpr).
+package proxy
+
+// See github.com/google/martian/cmd/proxy/main.go for the origin of much of this.
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/google/martian"
+	"github.com/google/martian/fifo"
+	"github.com/google/martian/har"
+	"github.com/google/martian/httpspec"
+	"github.com/google/martian/martianlog"
+	"github.com/google/martian/mitm"
+)
+
+// A Proxy is an HTTP proxy that supports recording or replaying requests.
+type Proxy struct {
+	// The certificate that the proxy uses to participate in TLS.
+	CACert *x509.Certificate
+
+	// The URL of the proxy.
+	URL *url.URL
+
+	// Initial state of the client. Must be serializable with json.Marshal.
+	Initial interface{}
+
+	mproxy   *martian.Proxy
+	filename string      // for log
+	logger   *har.Logger // for recording only
+}
+
+// ForRecording returns a Proxy configured to record.
+func ForRecording(filename string, port int) (*Proxy, error) {
+	p, err := newProxy(filename)
+	if err != nil {
+		return nil, err
+	}
+	// Configure the transport for the proxy's outgoing traffic.
+	p.mproxy.SetRoundTripper(&http.Transport{
+		Dial: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: time.Second,
+	})
+
+	// Construct a group that performs the standard proxy stack of request/response
+	// modifications.
+	stack, _ := httpspec.NewStack("httpr") // second arg is an internal group that we don't need
+	p.mproxy.SetRequestModifier(stack)
+	p.mproxy.SetResponseModifier(stack)
+
+	// Make a group for logging requests and responses.
+	logGroup := fifo.NewGroup()
+	skipAuth := skipLoggingByHost("accounts.google.com")
+	logGroup.AddRequestModifier(skipAuth)
+	logGroup.AddResponseModifier(skipAuth)
+	p.logger = har.NewLogger()
+	logGroup.AddRequestModifier(martian.RequestModifierFunc(
+		func(req *http.Request) error { return withRedactedHeaders(req, p.logger) }))
+	logGroup.AddResponseModifier(p.logger)
+
+	stack.AddRequestModifier(logGroup)
+	stack.AddResponseModifier(logGroup)
+
+	// Ordinary debug logging.
+	logger := martianlog.NewLogger()
+	logger.SetDecode(true)
+	stack.AddRequestModifier(logger)
+	stack.AddResponseModifier(logger)
+
+	if err := p.start(port); err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func newProxy(filename string) (*Proxy, error) {
+	mproxy := martian.NewProxy()
+	// Set up a man-in-the-middle configuration with a CA certificate so the proxy can
+	// participate in TLS.
+	x509c, priv, err := mitm.NewAuthority("cloud.google.com/go/httpreplay", "HTTPReplay Authority", time.Hour)
+	if err != nil {
+		return nil, err
+	}
+	mc, err := mitm.NewConfig(x509c, priv)
+	if err != nil {
+		return nil, err
+	}
+	mc.SetValidity(time.Hour)
+	mc.SetOrganization("cloud.google.com/go/httpreplay")
+	mc.SkipTLSVerify(false)
+	if err != nil {
+		return nil, err
+	}
+	mproxy.SetMITM(mc)
+	return &Proxy{
+		mproxy:   mproxy,
+		CACert:   x509c,
+		filename: filename,
+	}, nil
+}
+
+func (p *Proxy) start(port int) error {
+	l, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return err
+	}
+	p.URL = &url.URL{Scheme: "http", Host: l.Addr().String()}
+	go p.mproxy.Serve(l)
+	return nil
+}
+
+// Transport returns an http.Transport for clients who want to talk to the proxy.
+func (p *Proxy) Transport() *http.Transport {
+	caCertPool := x509.NewCertPool()
+	caCertPool.AddCert(p.CACert)
+	return &http.Transport{
+		TLSClientConfig: &tls.Config{RootCAs: caCertPool},
+		Proxy:           func(*http.Request) (*url.URL, error) { return p.URL, nil },
+	}
+}
+
+// Close closes the proxy. If the proxy is recording, it also writes the log.
+func (p *Proxy) Close() error {
+	p.mproxy.Close()
+	if p.logger != nil {
+		return p.writeLog()
+	}
+	return nil
+}
+
+type httprFile struct {
+	Initial interface{}
+	HAR     *har.HAR
+}
+
+func (p *Proxy) writeLog() error {
+	f := httprFile{
+		Initial: p.Initial,
+		HAR:     p.logger.ExportAndReset(),
+	}
+	bytes, err := json.Marshal(f)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(p.filename, bytes, 0600) // only accessible by owner
+}
+
+// Headers that may contain sensitive data (auth tokens, keys).
+var sensitiveHeaders = []string{
+	"Authorization",
+	"X-Goog-Encryption-Key",             // used by Cloud Storage for customer-supplied encryption
+	"X-Goog-Copy-Source-Encryption-Key", // ditto
+}
+
+// withRedactedHeaders removes sensitive header contents before calling mod.
+func withRedactedHeaders(req *http.Request, mod martian.RequestModifier) error {
+	// We have to change the headers, then log, then restore them.
+	replaced := map[string]string{}
+	for _, h := range sensitiveHeaders {
+		if v := req.Header.Get(h); v != "" {
+			replaced[h] = v
+			req.Header.Set(h, "REDACTED")
+		}
+	}
+	err := mod.ModifyRequest(req)
+	for h, v := range replaced {
+		req.Header.Set(h, v)
+	}
+	return err
+}
+
+// skipLoggingByHost disables logging for traffic to a particular host.
+type skipLoggingByHost string
+
+func (s skipLoggingByHost) ModifyRequest(req *http.Request) error {
+	if strings.HasPrefix(req.Host, string(s)) {
+		martian.NewContext(req).SkipLogging()
+	}
+	return nil
+}
+
+func (s skipLoggingByHost) ModifyResponse(res *http.Response) error {
+	return s.ModifyRequest(res.Request)
+}

vendor/cloud.google.com/go/httpreplay/internal/proxy/record_test.go

@@ -0,0 +1,65 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package proxy
+
+import (
+	"net/http"
+	"testing"
+
+	"cloud.google.com/go/internal/testutil"
+	"github.com/google/martian"
+)
+
+func TestWithRedactedHeaders(t *testing.T) {
+	clone := func(h http.Header) http.Header {
+		h2 := http.Header{}
+		for k, v := range h {
+			h2[k] = v
+		}
+		return h2
+	}
+
+	orig := http.Header{
+		"Content-Type":                      {"text/plain"},
+		"Authorization":                     {"oauth2-token"},
+		"X-Goog-Encryption-Key":             {"a-secret-key"},
+		"X-Goog-Copy-Source-Encryption-Key": {"another-secret-key"},
+	}
+	req := &http.Request{Header: clone(orig)}
+	var got http.Header
+	mod := martian.RequestModifierFunc(func(req *http.Request) error {
+		got = clone(req.Header)
+		return nil
+	})
+	if err := withRedactedHeaders(req, mod); err != nil {
+		t.Fatal(err)
+	}
+	// Logged headers should be redacted.
+	want := http.Header{
+		"Content-Type":                      {"text/plain"},
+		"Authorization":                     {"REDACTED"},
+		"X-Goog-Encryption-Key":             {"REDACTED"},
+		"X-Goog-Copy-Source-Encryption-Key": {"REDACTED"},
+	}
+	if !testutil.Equal(got, want) {
+		t.Errorf("got  %+v\nwant %+v", got, want)
+	}
+	// The request's headers should be the same.
+	if got, want := req.Header, orig; !testutil.Equal(got, want) {
+		t.Errorf("got  %+v\nwant %+v", got, want)
+	}
+}

vendor/cloud.google.com/go/httpreplay/internal/proxy/replay.go

@@ -0,0 +1,336 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package proxy
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"reflect"
+	"strings"
+
+	"github.com/google/martian/har"
+	"github.com/google/martian/martianlog"
+)
+
+// ForReplaying returns a Proxy configured to replay.
+func ForReplaying(filename string, port int) (*Proxy, error) {
+	p, err := newProxy(filename)
+	if err != nil {
+		return nil, err
+	}
+	calls, initial, err := readLog(filename)
+	if err != nil {
+		return nil, err
+	}
+	p.mproxy.SetRoundTripper(replayRoundTripper{calls: calls})
+	p.Initial = initial
+
+	// Debug logging.
+	// TODO(jba): factor out from here and ForRecording.
+	logger := martianlog.NewLogger()
+	logger.SetDecode(true)
+	p.mproxy.SetRequestModifier(logger)
+	p.mproxy.SetResponseModifier(logger)
+
+	if err := p.start(port); err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+// A call is an HTTP request and its matching response.
+type call struct {
+	req     *har.Request
+	reqBody *requestBody // parsed request body
+	res     *har.Response
+}
+
+func readLog(filename string) ([]*call, interface{}, error) {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, nil, err
+	}
+	var f httprFile
+	if err := json.Unmarshal(bytes, &f); err != nil {
+		return nil, nil, err
+	}
+	ignoreIDs := map[string]bool{} // IDs of requests to ignore
+	callsByID := map[string]*call{}
+	var calls []*call
+	for _, e := range f.HAR.Log.Entries {
+		if ignoreIDs[e.ID] {
+			continue
+		}
+		c, ok := callsByID[e.ID]
+		switch {
+		case !ok:
+			if e.Request == nil {
+				return nil, nil, fmt.Errorf("first entry for ID %s does not have a request", e.ID)
+			}
+			if e.Request.Method == "CONNECT" {
+				// Ignore CONNECT methods.
+				ignoreIDs[e.ID] = true
+			} else {
+				reqBody, err := newRequestBodyFromHAR(e.Request)
+				if err != nil {
+					return nil, nil, err
+				}
+				c := &call{e.Request, reqBody, e.Response}
+				calls = append(calls, c)
+				callsByID[e.ID] = c
+			}
+		case e.Request != nil:
+			if e.Response != nil {
+				return nil, nil, errors.New("HAR entry has both request and response")
+			}
+			c.req = e.Request
+		case e.Response != nil:
+			c.res = e.Response
+		default:
+			return nil, nil, errors.New("HAR entry has neither request nor response")
+		}
+	}
+	for _, c := range calls {
+		if c.req == nil || c.res == nil {
+			return nil, nil, fmt.Errorf("missing request or response: %+v", c)
+		}
+	}
+	return calls, f.Initial, nil
+}
+
+type replayRoundTripper struct {
+	calls []*call
+}
+
+func (r replayRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	reqBody, err := newRequestBodyFromHTTP(req)
+	if err != nil {
+		return nil, err
+	}
+	for i, call := range r.calls {
+		if call == nil {
+			continue
+		}
+		if requestsMatch(req, reqBody, call.req, call.reqBody) {
+			r.calls[i] = nil // nil out this call so we don't reuse it
+			res := harResponseToHTTPResponse(call.res)
+			res.Request = req
+			return res, nil
+		}
+	}
+	return nil, fmt.Errorf("no matching request for %+v", req)
+}
+
+// Headers that shouldn't be compared, becuase they may differ on different executions
+// of the same code, or may not be present during record or replay.
+var ignoreHeaders = map[string]bool{}
+
+func init() {
+	// Sensitive headers are redacted in the log, so they won't be equal to incoming values.
+	for _, h := range sensitiveHeaders {
+		ignoreHeaders[h] = true
+	}
+	for _, h := range []string{
+		"Content-Type", // handled by requestBody
+		"Date",
+		"Host",
+		"Transfer-Encoding",
+		"Via",
+		"X-Forwarded-For",
+		"X-Forwarded-Host",
+		"X-Forwarded-Proto",
+		"X-Forwarded-Url",
+	} {
+		ignoreHeaders[h] = true
+	}
+}
+
+// Report whether the incoming request in matches the candidate request cand.
+func requestsMatch(in *http.Request, inBody *requestBody, cand *har.Request, candBody *requestBody) bool {
+	// TODO(jba): compare headers?
+	if in.Method != cand.Method {
+		return false
+	}
+	if in.URL.String() != cand.URL {
+		return false
+	}
+	if !inBody.equal(candBody) {
+		return false
+	}
+	// Check headers last. See DebugHeaders.
+	return headersMatch(in.Header, harHeadersToHTTP(cand.Headers), ignoreHeaders)
+}
+
+func harHeadersToHTTP(hhs []har.Header) http.Header {
+	// Unfortunately, the har package joins multiple header values with ", ",
+	// which isn't reversible if any of the values contains a comma.
+	// We hope for the best.
+	res := http.Header{}
+	for _, hh := range hhs {
+		res[hh.Name] = strings.Split(hh.Value, ", ")
+	}
+	return res
+}
+
+// Convert a HAR response to a Go http.Response.
+// HAR (Http ARchive) is a standard for storing HTTP interactions.
+// See http://www.softwareishard.com/blog/har-12-spec.
+func harResponseToHTTPResponse(hr *har.Response) *http.Response {
+	return &http.Response{
+		StatusCode: hr.Status,
+		Status:     hr.StatusText,
+		Proto:      hr.HTTPVersion,
+		// TODO(jba): headers?
+		Body:          ioutil.NopCloser(bytes.NewReader(hr.Content.Text)),
+		ContentLength: int64(len(hr.Content.Text)),
+	}
+}
+
+// A requestBody represents the body of a request. If the content type is multipart, the
+// body is split into parts.
+//
+// The replaying proxy needs to understand multipart bodies because the boundaries are
+// generated randomly, so we can't just compare the entire bodies for equality.
+type requestBody struct {
+	mediaType string   // the media type part of the Content-Type header
+	parts     [][]byte // the parts of the body, or just a single []byte if not multipart
+}
+
+func newRequestBodyFromHTTP(req *http.Request) (*requestBody, error) {
+	defer req.Body.Close()
+	return newRequestBody(req.Header.Get("Content-Type"), req.Body)
+}
+
+func newRequestBodyFromHAR(req *har.Request) (*requestBody, error) {
+	if req.PostData == nil {
+		return nil, nil
+	}
+	var cth string
+	for _, h := range req.Headers {
+		if h.Name == "Content-Type" {
+			cth = h.Value
+			break
+		}
+	}
+	return newRequestBody(cth, strings.NewReader(req.PostData.Text))
+}
+
+// newRequestBody parses the Content-Type header, reads the body, and splits it into
+// parts if necessary.
+func newRequestBody(contentType string, body io.Reader) (*requestBody, error) {
+	if contentType == "" {
+		// No content-type header. There should not be a body.
+		if _, err := body.Read(make([]byte, 1)); err != io.EOF {
+			return nil, errors.New("no Content-Type, but body")
+		}
+		return nil, nil
+	}
+	mediaType, params, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return nil, err
+	}
+	rb := &requestBody{mediaType: mediaType}
+	if strings.HasPrefix(mediaType, "multipart/") {
+		mr := multipart.NewReader(body, params["boundary"])
+		for {
+			p, err := mr.NextPart()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return nil, err
+			}
+			part, err := ioutil.ReadAll(p)
+			if err != nil {
+				return nil, err
+			}
+			// TODO(jba): care about part headers?
+			rb.parts = append(rb.parts, part)
+		}
+	} else {
+		bytes, err := ioutil.ReadAll(body)
+		if err != nil {
+			return nil, err
+		}
+		rb.parts = [][]byte{bytes}
+	}
+	return rb, nil
+}
+
+func (r1 *requestBody) equal(r2 *requestBody) bool {
+	if r1 == nil || r2 == nil {
+		return r1 == r2
+	}
+	if r1.mediaType != r2.mediaType {
+		return false
+	}
+	if len(r1.parts) != len(r2.parts) {
+		return false
+	}
+	for i, p1 := range r1.parts {
+		if !bytes.Equal(p1, r2.parts[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// DebugHeaders helps to determine whether a header should be ignored.
+// When true, if requests have the same method, URL and body but differ
+// in a header, the first mismatched header is logged.
+var DebugHeaders = false
+
+func headersMatch(in, cand http.Header, ignores map[string]bool) bool {
+	for k1, v1 := range in {
+		if ignores[k1] {
+			continue
+		}
+		v2 := cand[k1]
+		if v2 == nil {
+			if DebugHeaders {
+				log.Printf("header %s: present in incoming request but not candidate", k1)
+			}
+			return false
+		}
+		if !reflect.DeepEqual(v1, v2) {
+			if DebugHeaders {
+				log.Printf("header %s: incoming %v, candidate %v", k1, v1, v2)
+			}
+			return false
+		}
+	}
+	for k2 := range cand {
+		if ignores[k2] {
+			continue
+		}
+		if in[k2] == nil {
+			if DebugHeaders {
+				log.Printf("header %s: not in incoming request but present in candidate", k2)
+			}
+			return false
+		}
+	}
+	return true
+}

vendor/cloud.google.com/go/httpreplay/internal/proxy/replay_test.go

@@ -0,0 +1,114 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package proxy
+
+import (
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"cloud.google.com/go/internal/testutil"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestRequestBody(t *testing.T) {
+	req1 := &http.Request{
+		Header: http.Header{"Content-Type": {"multipart/mixed; boundary=foo"}},
+		Body: ioutil.NopCloser(strings.NewReader(
+			"--foo\r\nFoo: one\r\n\r\nA section\r\n" +
+				"--foo\r\nFoo: two\r\n\r\nAnd another\r\n" +
+				"--foo--\r\n")),
+	}
+	rb1, err := newRequestBodyFromHTTP(req1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := &requestBody{
+		mediaType: "multipart/mixed",
+		parts: [][]byte{
+			[]byte("A section"),
+			[]byte("And another"),
+		},
+	}
+	if diff := testutil.Diff(rb1, want, cmp.AllowUnexported(requestBody{})); diff != "" {
+		t.Error(diff)
+	}
+
+	// Same contents, different boundary.
+	req2 := &http.Request{
+		Header: http.Header{"Content-Type": {"multipart/mixed; boundary=bar"}},
+		Body: ioutil.NopCloser(strings.NewReader(
+			"--bar\r\nFoo: one\r\n\r\nA section\r\n" +
+				"--bar\r\nFoo: two\r\n\r\nAnd another\r\n" +
+				"--bar--\r\n")),
+	}
+	rb2, err := newRequestBodyFromHTTP(req2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := testutil.Diff(rb1, want, cmp.AllowUnexported(requestBody{})); diff != "" {
+		t.Error(diff)
+	}
+
+	if !rb1.equal(rb2) {
+		t.Error("equal returned false, want true")
+	}
+}
+
+func TestHeadersMatch(t *testing.T) {
+	for _, test := range []struct {
+		h1, h2 http.Header
+		want   bool
+	}{
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			true,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			http.Header{"A": {"x"}, "B": {"w"}},
+			false,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"foo"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"bar"}},
+			true,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"bar"}},
+			true,
+		},
+		{
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"foo"}},
+			http.Header{"A": {"x"}, "I": {"bar"}},
+			false,
+		},
+		{
+			http.Header{"A": {"x"}, "I": {"foo"}},
+			http.Header{"A": {"x"}, "B": {"y", "z"}, "I": {"bar"}},
+			false,
+		},
+	} {
+		got := headersMatch(test.h1, test.h2, map[string]bool{"I": true})
+		if got != test.want {
+			t.Errorf("%v, %v: got %t, want %t", test.h1, test.h2, got, test.want)
+		}
+	}
+}