vendor: update all dependencies to latest versions

vendor/cloud.google.com/go/storage/acl.go

@@ -15,7 +15,6 @@
 package storage
 
 import (
-	"fmt"
 	"net/http"
 	"reflect"
 
@@ -106,21 +105,17 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
 		return err
 	})
 	if err != nil {
-		return nil, fmt.Errorf("storage: error listing default object ACL for bucket %q: %v", a.bucket, err)
+		return nil, err
 	}
 	return toACLRules(acls.Items), nil
 }
 
 func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error {
-	err := runWithRetry(ctx, func() error {
+	return runWithRetry(ctx, func() error {
 		req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity))
 		a.configureCall(req, ctx)
 		return req.Do()
 	})
-	if err != nil {
-		return fmt.Errorf("storage: error deleting default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
-	}
-	return nil
 }
 
 func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
@@ -133,7 +128,7 @@ func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
 		return err
 	})
 	if err != nil {
-		return nil, fmt.Errorf("storage: error listing bucket ACL for bucket %q: %v", a.bucket, err)
+		return nil, err
 	}
 	r := make([]ACLRule, len(acls.Items))
 	for i, v := range acls.Items {
@@ -156,7 +151,7 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol
 		return err
 	})
 	if err != nil {
-		return fmt.Errorf("storage: error updating bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
+		return err
 	}
 	return nil
 }
@@ -168,7 +163,7 @@ func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
 		return req.Do()
 	})
 	if err != nil {
-		return fmt.Errorf("storage: error deleting bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
+		return err
 	}
 	return nil
 }
@@ -183,7 +178,7 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
 		return err
 	})
 	if err != nil {
-		return nil, fmt.Errorf("storage: error listing object ACL for bucket %q, file %q: %v", a.bucket, a.object, err)
+		return nil, err
 	}
 	return toACLRules(acls.Items), nil
 }
@@ -206,30 +201,18 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol
 		req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl)
 	}
 	a.configureCall(req, ctx)
-	err := runWithRetry(ctx, func() error {
+	return runWithRetry(ctx, func() error {
 		_, err := req.Do()
 		return err
 	})
-	if err != nil {
-		if isBucketDefault {
-			return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
-		} else {
-			return fmt.Errorf("storage: error updating object ACL entry for bucket %q, object %q, entity %q: %v", a.bucket, a.object, entity, err)
-		}
-	}
-	return nil
 }
 
 func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
-	err := runWithRetry(ctx, func() error {
+	return runWithRetry(ctx, func() error {
 		req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
 		a.configureCall(req, ctx)
 		return req.Do()
 	})
-	if err != nil {
-		return fmt.Errorf("storage: error deleting object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err)
-	}
-	return nil
 }
 
 func (a *ACLHandle) configureCall(call interface {

vendor/cloud.google.com/go/storage/bucket.go

@@ -35,7 +35,7 @@ type BucketHandle struct {
 	acl              ACLHandle
 	defaultObjectACL ACLHandle
 	conds            *BucketConditions
-	userProject      string // project for requester-pays buckets
+	userProject      string // project for Requester Pays buckets
 }
 
 // Bucket returns a BucketHandle, which provides operations on the named bucket.
@@ -237,6 +237,9 @@ type BucketAttrs struct {
 	Labels map[string]string
 
 	// RequesterPays reports whether the bucket is a Requester Pays bucket.
+	// Clients performing operations on Requester Pays buckets must provide
+	// a user project (see BucketHandle.UserProject), which will be billed
+	// for the operations.
 	RequesterPays bool
 	// Lifecycle is the lifecycle configuration for objects in the bucket.
 	Lifecycle Lifecycle
@@ -506,8 +509,10 @@ func (c *BucketConditions) validate(method string) error {
 }
 
 // UserProject returns a new BucketHandle that passes the project ID as the user
-// project for all subsequent calls. A user project is required for all operations
-// on requester-pays buckets.
+// project for all subsequent calls. Calls with a user project will be billed to that
+// project rather than to the bucket's owning project.
+//
+// A user project is required for all operations on Requester Pays buckets.
 func (b *BucketHandle) UserProject(projectID string) *BucketHandle {
 	b2 := *b
 	b2.userProject = projectID
@@ -604,6 +609,7 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle {
 		if rr.Condition.CreatedBefore != "" {
 			r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore)
 		}
+		l.Rules = append(l.Rules, r)
 	}
 	return l
 }

vendor/cloud.google.com/go/storage/bucket_test.go

@@ -24,6 +24,7 @@ import (
 
 	"cloud.google.com/go/internal/pretty"
 	"cloud.google.com/go/internal/testutil"
+	"google.golang.org/api/googleapi"
 	raw "google.golang.org/api/storage/v1"
 )
 
@@ -52,7 +53,7 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
 		},
 		Location:     "loc",
 		StorageClass: "class",
-		Versioning:   nil, // ignore VersioningEnabled if flase
+		Versioning:   nil, // ignore VersioningEnabled if false
 		Labels:       map[string]string{"label": "value"},
 	}
 	msg, ok, err := pretty.Diff(want, got)
@@ -171,10 +172,20 @@ func TestCallBuilders(t *testing.T) {
 		},
 		{
 			func(b *BucketHandle) (interface{}, error) {
-				return b.newPatchCall(&BucketAttrsToUpdate{VersioningEnabled: false})
+				return b.newPatchCall(&BucketAttrsToUpdate{
+					VersioningEnabled: false,
+					RequesterPays:     false,
+				})
 			},
 			rc.Buckets.Patch("name", &raw.Bucket{
-				Versioning: &raw.BucketVersioning{Enabled: false, ForceSendFields: []string{"Enabled"}},
+				Versioning: &raw.BucketVersioning{
+					Enabled:         false,
+					ForceSendFields: []string{"Enabled"},
+				},
+				Billing: &raw.BucketBilling{
+					RequesterPays:   false,
+					ForceSendFields: []string{"RequesterPays"},
+				},
 			}).Projection("full"),
 			func(req interface{}) { req.(*raw.BucketsPatchCall).IfMetagenerationMatch(metagen).UserProject("p") },
 		},
@@ -209,3 +220,69 @@ func TestCallBuilders(t *testing.T) {
 		t.Errorf("got nil, want error")
 	}
 }
+
+func TestNewBucket(t *testing.T) {
+	labels := map[string]string{"a": "b"}
+	matchClasses := []string{"MULTI_REGIONAL", "REGIONAL", "STANDARD"}
+	rb := &raw.Bucket{
+		Name:           "name",
+		Location:       "loc",
+		Metageneration: 3,
+		StorageClass:   "sc",
+		TimeCreated:    "2017-10-23T04:05:06Z",
+		Versioning:     &raw.BucketVersioning{Enabled: true},
+		Labels:         labels,
+		Billing:        &raw.BucketBilling{RequesterPays: true},
+		Lifecycle: &raw.BucketLifecycle{
+			Rule: []*raw.BucketLifecycleRule{{
+				Action: &raw.BucketLifecycleRuleAction{
+					Type:         "SetStorageClass",
+					StorageClass: "NEARLINE",
+				},
+				Condition: &raw.BucketLifecycleRuleCondition{
+					Age:                 10,
+					IsLive:              googleapi.Bool(true),
+					CreatedBefore:       "2017-01-02",
+					MatchesStorageClass: matchClasses,
+					NumNewerVersions:    3,
+				},
+			}},
+		},
+		Acl: []*raw.BucketAccessControl{
+			{Bucket: "name", Role: "READER", Email: "joe@example.com", Entity: "allUsers"},
+		},
+	}
+	want := &BucketAttrs{
+		Name:              "name",
+		Location:          "loc",
+		MetaGeneration:    3,
+		StorageClass:      "sc",
+		Created:           time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
+		VersioningEnabled: true,
+		Labels:            labels,
+		RequesterPays:     true,
+		Lifecycle: Lifecycle{
+			Rules: []LifecycleRule{
+				{
+					Action: LifecycleAction{
+						Type:         SetStorageClassAction,
+						StorageClass: "NEARLINE",
+					},
+					Condition: LifecycleCondition{
+						AgeInDays:             10,
+						Liveness:              Live,
+						CreatedBefore:         time.Date(2017, 1, 2, 0, 0, 0, 0, time.UTC),
+						MatchesStorageClasses: matchClasses,
+						NumNewerVersions:      3,
+					},
+				},
+			},
+		},
+		ACL:              []ACLRule{{Entity: "allUsers", Role: RoleReader}},
+		DefaultObjectACL: []ACLRule{},
+	}
+	got := newBucket(rb)
+	if diff := testutil.Diff(got, want); diff != "" {
+		t.Errorf("got=-, want=+:\n%s", diff)
+	}
+}

vendor/cloud.google.com/go/storage/doc.go

@@ -36,6 +36,13 @@ To start working with this package, create a client:
         // TODO: Handle error.
     }
 
+The client will use your default application credentials.
+
+If you only wish to access public data, you can create
+an unauthenticated client with
+
+    client, err := storage.NewClient(ctx, option.WithoutAuthentication())
+
 Buckets
 
 A Google Cloud Storage bucket is a collection of objects. To work with a

vendor/cloud.google.com/go/storage/example_test.go

@@ -26,10 +26,14 @@ import (
 	"cloud.google.com/go/storage"
 	"golang.org/x/net/context"
 	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
 )
 
 func ExampleNewClient() {
 	ctx := context.Background()
+	// Use Google Application Default Credentials to authorize and authenticate the client.
+	// More information about Application Default Credentials and how to enable is at
+	// https://developers.google.com/identity/protocols/application-default-credentials.
 	client, err := storage.NewClient(ctx)
 	if err != nil {
 		// TODO: handle error.
@@ -42,21 +46,19 @@ func ExampleNewClient() {
 	}
 }
 
-func ExampleNewClient_auth() {
+// This example shows how to create an unauthenticated client, which
+// can be used to access public data.
+func ExampleNewClient_unauthenticated() {
 	ctx := context.Background()
-	// Use Google Application Default Credentials to authorize and authenticate the client.
-	// More information about Application Default Credentials and how to enable is at
-	// https://developers.google.com/identity/protocols/application-default-credentials.
-	client, err := storage.NewClient(ctx)
+	client, err := storage.NewClient(ctx, option.WithoutAuthentication())
 	if err != nil {
-		log.Fatal(err)
+		// TODO: handle error.
 	}
-
 	// Use the client.
 
 	// Close the client when finished.
 	if err := client.Close(); err != nil {
-		log.Fatal(err)
+		// TODO: handle error.
 	}
 }
 
@@ -176,6 +178,57 @@ func ExampleBucketHandle_Objects() {
 	_ = it // TODO: iterate using Next or iterator.Pager.
 }
 
+func ExampleBucketHandle_AddNotification() {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		// TODO: handle error.
+	}
+	b := client.Bucket("my-bucket")
+	n, err := b.AddNotification(ctx, &storage.Notification{
+		TopicProjectID: "my-project",
+		TopicID:        "my-topic",
+		PayloadFormat:  storage.JSONPayload,
+	})
+	if err != nil {
+		// TODO: handle error.
+	}
+	fmt.Println(n.ID)
+}
+
+func ExampleBucketHandle_Notifications() {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		// TODO: handle error.
+	}
+	b := client.Bucket("my-bucket")
+	ns, err := b.Notifications(ctx)
+	if err != nil {
+		// TODO: handle error.
+	}
+	for id, n := range ns {
+		fmt.Printf("%s: %+v\n", id, n)
+	}
+}
+
+var notificationID string
+
+func ExampleBucketHandle_DeleteNotification() {
+	ctx := context.Background()
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		// TODO: handle error.
+	}
+	b := client.Bucket("my-bucket")
+	// TODO: Obtain notificationID from BucketHandle.AddNotification
+	// or BucketHandle.Notifications.
+	err = b.DeleteNotification(ctx, notificationID)
+	if err != nil {
+		// TODO: handle error.
+	}
+}
+
 func ExampleObjectIterator_Next() {
 	ctx := context.Background()
 	client, err := storage.NewClient(ctx)

vendor/cloud.google.com/go/storage/iam.go

@@ -23,21 +23,28 @@ import (
 
 // IAM provides access to IAM access control for the bucket.
 func (b *BucketHandle) IAM() *iam.Handle {
-	return iam.InternalNewHandleClient(&iamClient{raw: b.c.raw}, b.name)
+	return iam.InternalNewHandleClient(&iamClient{
+		raw:         b.c.raw,
+		userProject: b.userProject,
+	}, b.name)
 }
 
 // iamClient implements the iam.client interface.
 type iamClient struct {
-	raw *raw.Service
+	raw         *raw.Service
+	userProject string
 }
 
 func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, error) {
-	req := c.raw.Buckets.GetIamPolicy(resource)
-	setClientHeader(req.Header())
+	call := c.raw.Buckets.GetIamPolicy(resource)
+	setClientHeader(call.Header())
+	if c.userProject != "" {
+		call.UserProject(c.userProject)
+	}
 	var rp *raw.Policy
 	var err error
 	err = runWithRetry(ctx, func() error {
-		rp, err = req.Context(ctx).Do()
+		rp, err = call.Context(ctx).Do()
 		return err
 	})
 	if err != nil {
@@ -48,21 +55,27 @@ func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, er
 
 func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) error {
 	rp := iamToStoragePolicy(p)
-	req := c.raw.Buckets.SetIamPolicy(resource, rp)
-	setClientHeader(req.Header())
+	call := c.raw.Buckets.SetIamPolicy(resource, rp)
+	setClientHeader(call.Header())
+	if c.userProject != "" {
+		call.UserProject(c.userProject)
+	}
 	return runWithRetry(ctx, func() error {
-		_, err := req.Context(ctx).Do()
+		_, err := call.Context(ctx).Do()
 		return err
 	})
 }
 
 func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) {
-	req := c.raw.Buckets.TestIamPermissions(resource, perms)
-	setClientHeader(req.Header())
+	call := c.raw.Buckets.TestIamPermissions(resource, perms)
+	setClientHeader(call.Header())
+	if c.userProject != "" {
+		call.UserProject(c.userProject)
+	}
 	var res *raw.TestIamPermissionsResponse
 	var err error
 	err = runWithRetry(ctx, func() error {
-		res, err = req.Context(ctx).Do()
+		res, err = call.Context(ctx).Do()
 		return err
 	})
 	if err != nil {

vendor/cloud.google.com/go/storage/integration_test.go

@@ -20,6 +20,7 @@ import (
 	"crypto/md5"
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"hash/crc32"
@@ -114,7 +115,7 @@ func config(ctx context.Context) (*Client, string) {
 	return client, p + suffix
 }
 
-func TestBucketMethods(t *testing.T) {
+func TestIntegration_BucketMethods(t *testing.T) {
 	ctx := context.Background()
 	client, bucket := testConfig(ctx, t)
 	defer client.Close()
@@ -308,12 +309,8 @@ func TestIntegration_ConditionalDelete(t *testing.T) {
 	}
 }
 
-func TestObjects(t *testing.T) {
-	// TODO(djd): there are a lot of closely-related tests here which share
-	// a common setup. Once we can depend on Go 1.7 features, we should refactor
-	// this test to use the sub-test feature. This will increase the readability
-	// of this test, and should also reduce the time it takes to execute.
-	// https://golang.org/pkg/testing/#hdr-Subtests_and_Sub_benchmarks
+func TestIntegration_Objects(t *testing.T) {
+	// TODO(jba): Use subtests (Go 1.7).
 	ctx := context.Background()
 	client, bucket := testConfig(ctx, t)
 	defer client.Close()
@@ -385,33 +382,6 @@ func TestObjects(t *testing.T) {
 		if err := rc.Close(); err != nil {
 			t.Errorf("%v Close: %v", obj, err)
 		}
-
-		// Test SignedURL
-		opts := &SignedURLOptions{
-			GoogleAccessID: "xxx@clientid",
-			PrivateKey:     dummyKey("rsa"),
-			Method:         "GET",
-			MD5:            "ICy5YqxZB1uWSwcVLSNLcA==",
-			Expires:        time.Date(2020, time.October, 2, 10, 0, 0, 0, time.UTC),
-			ContentType:    "application/json",
-			Headers:        []string{"x-header1", "x-header2"},
-		}
-		u, err := SignedURL(bucket, obj, opts)
-		if err != nil {
-			t.Fatalf("SignedURL(%q, %q) errored with %v", bucket, obj, err)
-		}
-		res, err := client.hc.Get(u)
-		if err != nil {
-			t.Fatalf("Can't get URL %q: %v", u, err)
-		}
-		slurp, err = ioutil.ReadAll(res.Body)
-		if err != nil {
-			t.Fatalf("Can't ReadAll signed object %v, errored with %v", obj, err)
-		}
-		if got, want := slurp, contents[obj]; !bytes.Equal(got, want) {
-			t.Errorf("Contents (%v) = %q; want %q", obj, got, want)
-		}
-		res.Body.Close()
 	}
 
 	obj := objects[0]
@@ -762,7 +732,116 @@ func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) {
 	// TODO(jba): test query.Delimiter != ""
 }
 
-func TestACL(t *testing.T) {
+func TestIntegration_SignedURL(t *testing.T) {
+	// To test SignedURL, we need a real user email and private key. Extract them
+	// from the JSON key file.
+	jwtConf, err := testutil.JWTConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if jwtConf == nil {
+		t.Skip("JSON key file is not present")
+	}
+
+	ctx := context.Background()
+	client, bucket := testConfig(ctx, t)
+	defer client.Close()
+
+	bkt := client.Bucket(bucket)
+	obj := "signedURL"
+	contents := []byte("This is a test of SignedURL.\n")
+	md5 := "Jyxvgwm9n2MsrGTMPbMeYA==" // base64-encoded MD5 of contents
+	if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil {
+		t.Fatalf("writing: %v", err)
+	}
+	for _, test := range []struct {
+		desc    string
+		opts    SignedURLOptions
+		headers map[string][]string
+		fail    bool
+	}{
+		{
+			desc: "basic",
+		},
+		{
+			desc:    "MD5 sent and matches",
+			opts:    SignedURLOptions{MD5: md5},
+			headers: map[string][]string{"Content-MD5": {md5}},
+		},
+		{
+			desc: "MD5 not sent",
+			opts: SignedURLOptions{MD5: md5},
+			fail: true,
+		},
+		{
+			desc:    "Content-Type sent and matches",
+			opts:    SignedURLOptions{ContentType: "text/plain"},
+			headers: map[string][]string{"Content-Type": {"text/plain"}},
+		},
+		{
+			desc:    "Content-Type sent but does not match",
+			opts:    SignedURLOptions{ContentType: "text/plain"},
+			headers: map[string][]string{"Content-Type": {"application/json"}},
+			fail:    true,
+		},
+		{
+			desc: "Canonical headers sent and match",
+			opts: SignedURLOptions{Headers: []string{
+				" X-Goog-Foo: Bar baz ",
+				"X-Goog-Novalue", // ignored: no value
+				"X-Google-Foo",   // ignored: wrong prefix
+			}},
+			headers: map[string][]string{"X-Goog-foo": {"Bar baz  "}},
+		},
+		{
+			desc:    "Canonical headers sent but don't match",
+			opts:    SignedURLOptions{Headers: []string{" X-Goog-Foo: Bar baz"}},
+			headers: map[string][]string{"X-Goog-Foo": {"bar baz"}},
+			fail:    true,
+		},
+	} {
+		opts := test.opts
+		opts.GoogleAccessID = jwtConf.Email
+		opts.PrivateKey = jwtConf.PrivateKey
+		opts.Method = "GET"
+		opts.Expires = time.Now().Add(time.Hour)
+		u, err := SignedURL(bucket, obj, &opts)
+		if err != nil {
+			t.Errorf("%s: SignedURL: %v", test.desc, err)
+			continue
+		}
+		got, err := getURL(u, test.headers)
+		if err != nil && !test.fail {
+			t.Errorf("%s: getURL %q: %v", test.desc, u, err)
+		} else if err == nil && !bytes.Equal(got, contents) {
+			t.Errorf("%s: got %q, want %q", test.desc, got, contents)
+		}
+	}
+}
+
+// Make a GET request to a URL using an unauthenticated client, and return its contents.
+func getURL(url string, headers map[string][]string) ([]byte, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = headers
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	bytes, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes))
+	}
+	return bytes, nil
+}
+
+func TestIntegration_ACL(t *testing.T) {
 	ctx := context.Background()
 	client, bucket := testConfig(ctx, t)
 	defer client.Close()
@@ -831,7 +910,7 @@ func hasRule(acl []ACLRule, rule ACLRule) bool {
 	return false
 }
 
-func TestValidObjectNames(t *testing.T) {
+func TestIntegration_ValidObjectNames(t *testing.T) {
 	ctx := context.Background()
 	client, bucket := testConfig(ctx, t)
 	defer client.Close()
@@ -868,7 +947,7 @@ func TestValidObjectNames(t *testing.T) {
 	}
 }
 
-func TestWriterContentType(t *testing.T) {
+func TestIntegration_WriterContentType(t *testing.T) {
 	ctx := context.Background()
 	client, bucket := testConfig(ctx, t)
 	defer client.Close()
@@ -912,7 +991,7 @@ func TestWriterContentType(t *testing.T) {
 	}
 }
 
-func TestZeroSizedObject(t *testing.T) {
+func TestIntegration_ZeroSizedObject(t *testing.T) {
 	t.Parallel()
 	ctx := context.Background()
 	client, bucket := testConfig(ctx, t)
@@ -1011,7 +1090,7 @@ func TestIntegration_Encryption(t *testing.T) {
 		}
 		gotContents := string(got)
 		if gotContents != wantContents {
-			t.Errorf("%s: got %q, want %q", gotContents, wantContents)
+			t.Errorf("%s: got %q, want %q", msg, gotContents, wantContents)
 		}
 	}
 
@@ -1333,11 +1412,69 @@ func TestIntegration_BucketIAM(t *testing.T) {
 }
 
 func TestIntegration_RequesterPays(t *testing.T) {
+	// This test needs a second project and user (token source) to test
+	// all possibilities. Since we need these things for Firestore already,
+	// we use them here.
+	//
+	// There are up to three entities involved in a requester-pays call:
+	//
+	// 1. The user making the request. Here, we use
+	//    a. The account used to create the token source used for all our
+	//       integration tests (see testutil.TokenSource).
+	//    b. The account used for the Firestore tests.
+	// 2. The project that owns the requester-pays bucket. Here, that
+	//    is the test project ID (see testutil.ProjID).
+	// 3. The project provided as the userProject parameter of the request;
+	//    the project to be billed. This test uses:
+	//    a. The project that owns the requester-pays bucket (same as (2))
+	//    b. Another project (the Firestore project).
+	//
+	// The following must hold for this test to work:
+	// - (1a) must have resourcemanager.projects.createBillingAssignment permission
+	//       (Owner role) on (2) (the project, not the bucket).
+	// - (1b) must NOT have that permission on (2).
+	// - (1b) must have serviceusage.services.use permission (Editor role) on (3b).
+	// - (1b) must NOT have that permission on (3a).
+	// - (1a) must NOT have that permission on (3b).
+	const wantErrorCode = 400
+
 	ctx := context.Background()
 	client, bucketName := testConfig(ctx, t)
 	defer client.Close()
-	b := client.Bucket(bucketName + "-rp")
+	bucketName += "-rp"
+	b := client.Bucket(bucketName)
 	projID := testutil.ProjID()
+	// Use Firestore project as a project that does not contain the bucket.
+	otherProjID := os.Getenv(envFirestoreProjID)
+	if otherProjID == "" {
+		t.Fatalf("need a second project (env var %s)", envFirestoreProjID)
+	}
+	ts := testutil.TokenSourceEnv(ctx, envFirestorePrivateKey, ScopeFullControl)
+	if ts == nil {
+		t.Fatalf("need a second account (env var %s)", envFirestorePrivateKey)
+	}
+	otherClient, err := NewClient(ctx, option.WithTokenSource(ts))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer otherClient.Close()
+	ob := otherClient.Bucket(bucketName)
+	user, err := keyFileEmail(os.Getenv("GCLOUD_TESTS_GOLANG_KEY"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	otherUser, err := keyFileEmail(os.Getenv(envFirestorePrivateKey))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a requester-pays bucket. The bucket is contained in the project projID.
+	if err := b.Create(ctx, projID, &BucketAttrs{RequesterPays: true}); err != nil {
+		t.Fatal(err)
+	}
+	if err := b.ACL().Set(ctx, ACLEntity("user-"+otherUser), RoleOwner); err != nil {
+		t.Fatal(err)
+	}
 
 	// Extract the error code from err if it's a googleapi.Error.
 	errCode := func(err error) int {
@@ -1350,42 +1487,80 @@ func TestIntegration_RequesterPays(t *testing.T) {
 		return -1
 	}
 
-	// Call f twice on b, first without and then with a user project.
-	call := func(msg string, f func(b *BucketHandle) error) {
-		if err := f(b); err == nil {
-			if got, want := errCode(err), 400; got != want {
-				t.Errorf("%s: got error code %d, want %d", msg, got, want)
-			}
+	// Call f under various conditions.
+	// Here b and ob refer to the same bucket, but b is bound to client,
+	// while ob is bound to otherClient. The clients differ in their credentials,
+	// i.e. the identity of the user making the RPC: b's user is an Owner on the
+	// bucket's containing project, ob's is not.
+	call := func(msg string, f func(*BucketHandle) error) {
+		// user: an Owner on the containing project
+		// userProject: absent
+		// result: success, by the rule permitting access by owners of the containing bucket.
+		if err := f(b); err != nil {
+			t.Errorf("%s: %v, want nil\n"+
+				"confirm that %s is an Owner on %s",
+				msg, err, user, projID)
 		}
+		// user: an Owner on the containing project
+		// userProject: containing project
+		// result: success, by the same rule as above; userProject is unnecessary but allowed.
 		if err := f(b.UserProject(projID)); err != nil {
 			t.Errorf("%s: got %v, want nil", msg, err)
 		}
+		// user: not an Owner on the containing project
+		// userProject: absent
+		// result: failure, by the standard requester-pays rule
+		err := f(ob)
+		if got, want := errCode(err), wantErrorCode; got != want {
+			t.Errorf("%s: got error %s, want code %d\n"+
+				"confirm that %s is NOT an Owner on %s",
+				msg, err, want, otherUser, projID)
+		}
+		// user: not an Owner on the containing project
+		// userProject: not the containing one, but user has Editor role on it
+		// result: success, by the standard requester-pays rule
+		if err := f(ob.UserProject(otherProjID)); err != nil {
+			t.Errorf("%s: got %v, want nil\n"+
+				"confirm that %s is an Editor on %s and that that project has billing enabled",
+				msg, err, otherUser, otherProjID)
+		}
+		// user: not an Owner on the containing project
+		// userProject: the containing one, on which the user does NOT have Editor permission.
+		// result: failure
+		err = f(ob.UserProject("veener-jba"))
+		if got, want := errCode(err), 403; got != want {
+			t.Errorf("%s: got error %s, want code %d\n"+
+				"confirm that %s is NOT an Editor on %s",
+				msg, err, want, otherUser, "veener-jba")
+		}
 	}
 
-	// Create a requester-pays bucket.
-	err := b.Create(ctx, projID, &BucketAttrs{RequesterPays: true})
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Getting its attributes requires a user project.
 	var attrs *BucketAttrs
-	call("Bucket attrs", func(b *BucketHandle) (err error) {
-		attrs, err = b.Attrs(ctx)
+	call("Bucket attrs", func(b *BucketHandle) error {
+		a, err := b.Attrs(ctx)
+		if a != nil {
+			attrs = a
+		}
 		return err
 	})
-	if got, want := attrs.RequesterPays, true; got != want {
-		t.Fatalf("attr.RequesterPays = %b, want %b", got, want)
+	if attrs != nil {
+		if got, want := attrs.RequesterPays, true; got != want {
+			t.Fatalf("attr.RequesterPays = %t, want %t", got, want)
+		}
 	}
-
 	// Object operations.
 	call("write object", func(b *BucketHandle) error {
 		return writeObject(ctx, b.Object("foo"), "text/plain", []byte("hello"))
 	})
-	// // TODO(jba): add read test when XML API has requester-pays support.
-	// callObject("object attrs", o, func(o *ObjectHandle) error {
-	// 	_, err := o.Attrs(ctx)
-	// 	return err
-	// })
+	call("read object", func(b *BucketHandle) error {
+		_, err := readObject(ctx, b.Object("foo"))
+		return err
+	})
+	call("object attrs", func(b *BucketHandle) error {
+		_, err := b.Object("foo").Attrs(ctx)
+		return err
+	})
 	call("update object", func(b *BucketHandle) error {
 		_, err := b.Object("foo").Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"})
 		return err
@@ -1401,7 +1576,13 @@ func TestIntegration_RequesterPays(t *testing.T) {
 		return err
 	})
 	call("bucket acl delete", func(b *BucketHandle) error {
-		return b.ACL().Delete(ctx, entity)
+		err := b.ACL().Delete(ctx, entity)
+		if errCode(err) == 404 {
+			// Since we call the function multiple times, it will
+			// fail with NotFound for all but the first.
+			return nil
+		}
+		return err
 	})
 	call("default object acl set", func(b *BucketHandle) error {
 		return b.DefaultObjectACL().Set(ctx, entity, RoleReader)
@@ -1411,7 +1592,11 @@ func TestIntegration_RequesterPays(t *testing.T) {
 		return err
 	})
 	call("default object acl delete", func(b *BucketHandle) error {
-		return b.DefaultObjectACL().Delete(ctx, entity)
+		err := b.DefaultObjectACL().Delete(ctx, entity)
+		if errCode(err) == 404 {
+			return nil
+		}
+		return err
 	})
 	call("object acl set", func(b *BucketHandle) error {
 		return b.Object("foo").ACL().Set(ctx, entity, RoleReader)
@@ -1421,7 +1606,11 @@ func TestIntegration_RequesterPays(t *testing.T) {
 		return err
 	})
 	call("object acl delete", func(b *BucketHandle) error {
-		return b.Object("foo").ACL().Delete(ctx, entity)
+		err := b.Object("foo").ACL().Delete(ctx, entity)
+		if errCode(err) == 404 {
+			return nil
+		}
+		return err
 	})
 
 	// Copy and compose.
@@ -1436,16 +1625,145 @@ func TestIntegration_RequesterPays(t *testing.T) {
 
 	// Deletion.
 	call("delete object", func(b *BucketHandle) error {
-		return b.Object("foo").Delete(ctx)
+		err := b.Object("foo").Delete(ctx)
+		if err == ErrObjectNotExist {
+			return nil
+		}
+		return err
 	})
 	for _, obj := range []string{"copy", "compose"} {
 		if err := b.UserProject(projID).Object(obj).Delete(ctx); err != nil {
 			t.Fatalf("could not delete %q: %v", obj, err)
 		}
 	}
-	call("delete bucket", func(b *BucketHandle) error {
-		return b.Delete(ctx)
-	})
+	if err := b.Delete(ctx); err != nil {
+		t.Fatalf("deleting bucket: %v", err)
+	}
+}
+
+// TODO(jba): move to testutil, factor out from firestore/integration_test.go.
+const (
+	envFirestoreProjID     = "GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID"
+	envFirestorePrivateKey = "GCLOUD_TESTS_GOLANG_FIRESTORE_KEY"
+)
+
+func keyFileEmail(filename string) (string, error) {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return "", err
+	}
+	var v struct {
+		ClientEmail string `json:"client_email"`
+	}
+	if err := json.Unmarshal(bytes, &v); err != nil {
+		return "", err
+	}
+	return v.ClientEmail, nil
+}
+
+func TestNotifications(t *testing.T) {
+	ctx := context.Background()
+	client, bucket := testConfig(ctx, t)
+	defer client.Close()
+	bkt := client.Bucket(bucket)
+
+	checkNotifications := func(msg string, want map[string]*Notification) {
+		got, err := bkt.Notifications(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if diff := testutil.Diff(got, want); diff != "" {
+			t.Errorf("%s: got=-, want=+:\n%s", msg, diff)
+		}
+	}
+	checkNotifications("initial", map[string]*Notification{})
+
+	nArg := &Notification{
+		TopicProjectID: testutil.ProjID(),
+		TopicID:        "go-storage-notification-test",
+		PayloadFormat:  NoPayload,
+	}
+	n, err := bkt.AddNotification(ctx, nArg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	nArg.ID = n.ID
+	if !testutil.Equal(n, nArg) {
+		t.Errorf("got %+v, want %+v", n, nArg)
+	}
+	checkNotifications("after add", map[string]*Notification{n.ID: n})
+
+	if err := bkt.DeleteNotification(ctx, n.ID); err != nil {
+		t.Fatal(err)
+	}
+	checkNotifications("after delete", map[string]*Notification{})
+}
+
+func TestIntegration_Public(t *testing.T) {
+	// Confirm that an unauthenticated client can access a public bucket.
+
+	// See https://cloud.google.com/storage/docs/public-datasets/landsat
+	const landsatBucket = "gcp-public-data-landsat"
+	const landsatPrefix = "LC08/PRE/044/034/LC80440342016259LGN00/"
+	const landsatObject = landsatPrefix + "LC80440342016259LGN00_MTL.txt"
+
+	// Create an unauthenticated client.
+	ctx := context.Background()
+	client, err := NewClient(ctx, option.WithoutAuthentication())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+	bkt := client.Bucket(landsatBucket)
+	obj := bkt.Object(landsatObject)
+
+	// Read a public object.
+	bytes, err := readObject(ctx, obj)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, want := len(bytes), 7903; got != want {
+		t.Errorf("len(bytes) = %d, want %d", got, want)
+	}
+
+	// List objects in a public bucket.
+	iter := bkt.Objects(ctx, &Query{Prefix: landsatPrefix})
+	gotCount := 0
+	for {
+		_, err := iter.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		gotCount++
+	}
+	if wantCount := 13; gotCount != wantCount {
+		t.Errorf("object count: got %d, want %d", gotCount, wantCount)
+	}
+
+	errCode := func(err error) int {
+		if err, ok := err.(*googleapi.Error); !ok {
+			return -1
+		} else {
+			return err.Code
+		}
+	}
+
+	// Reading from or writing to a non-public bucket fails.
+	c, bucketName := testConfig(ctx, t)
+	defer c.Close()
+	nonPublicObj := client.Bucket(bucketName).Object("noauth")
+	// Oddly, reading returns 403 but writing returns 401.
+	_, err = readObject(ctx, nonPublicObj)
+	if got, want := errCode(err), 403; got != want {
+		t.Errorf("got code %d; want %d\nerror: %v", got, want, err)
+	}
+	err = writeObject(ctx, nonPublicObj, "text/plain", []byte("b"))
+	if got, want := errCode(err), 401; got != want {
+		t.Errorf("got code %d; want %d\nerror: %v", got, want, err)
+	}
 }
 
 func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error {

vendor/cloud.google.com/go/storage/notifications.go

@@ -0,0 +1,179 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+
+	"golang.org/x/net/context"
+	raw "google.golang.org/api/storage/v1"
+)
+
+// A Notification describes how to send Cloud PubSub messages when certain
+// events occur in a bucket.
+type Notification struct {
+	//The ID of the notification.
+	ID string
+
+	// The ID of the topic to which this subscription publishes.
+	TopicID string
+
+	// The ID of the project to which the topic belongs.
+	TopicProjectID string
+
+	// Only send notifications about listed event types. If empty, send notifications
+	// for all event types.
+	// See https://cloud.google.com/storage/docs/pubsub-notifications#events.
+	EventTypes []string
+
+	// If present, only apply this notification configuration to object names that
+	// begin with this prefix.
+	ObjectNamePrefix string
+
+	// An optional list of additional attributes to attach to each Cloud PubSub
+	// message published for this notification subscription.
+	CustomAttributes map[string]string
+
+	// The contents of the message payload.
+	// See https://cloud.google.com/storage/docs/pubsub-notifications#payload.
+	PayloadFormat string
+}
+
+// Values for Notification.PayloadFormat.
+const (
+	// Send no payload with notification messages.
+	NoPayload = "NONE"
+
+	// Send object metadata as JSON with notification messages.
+	JSONPayload = "JSON_API_V1"
+)
+
+// Values for Notification.EventTypes.
+const (
+	// Event that occurs when an object is successfully created.
+	ObjectFinalizeEvent = "OBJECT_FINALIZE"
+
+	// Event that occurs when the metadata of an existing object changes.
+	ObjectMetadataUpdateEvent = "OBJECT_METADATA_UPDATE"
+
+	// Event that occurs when an object is permanently deleted.
+	ObjectDeleteEvent = "OBJECT_DELETE"
+
+	// Event that occurs when the live version of an object becomes an
+	// archived version.
+	ObjectArchiveEvent = "OBJECT_ARCHIVE"
+)
+
+func toNotification(rn *raw.Notification) *Notification {
+	n := &Notification{
+		ID:               rn.Id,
+		EventTypes:       rn.EventTypes,
+		ObjectNamePrefix: rn.ObjectNamePrefix,
+		CustomAttributes: rn.CustomAttributes,
+		PayloadFormat:    rn.PayloadFormat,
+	}
+	n.TopicProjectID, n.TopicID = parseNotificationTopic(rn.Topic)
+	return n
+}
+
+var topicRE = regexp.MustCompile("^//pubsub.googleapis.com/projects/([^/]+)/topics/([^/]+)")
+
+// parseNotificationTopic extracts the project and topic IDs from from the full
+// resource name returned by the service. If the name is malformed, it returns
+// "?" for both IDs.
+func parseNotificationTopic(nt string) (projectID, topicID string) {
+	matches := topicRE.FindStringSubmatch(nt)
+	if matches == nil {
+		return "?", "?"
+	}
+	return matches[1], matches[2]
+}
+
+func toRawNotification(n *Notification) *raw.Notification {
+	return &raw.Notification{
+		Id: n.ID,
+		Topic: fmt.Sprintf("//pubsub.googleapis.com/projects/%s/topics/%s",
+			n.TopicProjectID, n.TopicID),
+		EventTypes:       n.EventTypes,
+		ObjectNamePrefix: n.ObjectNamePrefix,
+		CustomAttributes: n.CustomAttributes,
+		PayloadFormat:    string(n.PayloadFormat),
+	}
+}
+
+// AddNotification adds a notification to b. You must set n's TopicProjectID, TopicID
+// and PayloadFormat, and must not set its ID. The other fields are all optional. The
+// returned Notification's ID can be used to refer to it.
+func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (*Notification, error) {
+	if n.ID != "" {
+		return nil, errors.New("storage: AddNotification: ID must not be set")
+	}
+	if n.TopicProjectID == "" {
+		return nil, errors.New("storage: AddNotification: missing TopicProjectID")
+	}
+	if n.TopicID == "" {
+		return nil, errors.New("storage: AddNotification: missing TopicID")
+	}
+	call := b.c.raw.Notifications.Insert(b.name, toRawNotification(n))
+	setClientHeader(call.Header())
+	if b.userProject != "" {
+		call.UserProject(b.userProject)
+	}
+	rn, err := call.Context(ctx).Do()
+	if err != nil {
+		return nil, err
+	}
+	return toNotification(rn), nil
+}
+
+// Notifications returns all the Notifications configured for this bucket, as a map
+// indexed by notification ID.
+func (b *BucketHandle) Notifications(ctx context.Context) (map[string]*Notification, error) {
+	call := b.c.raw.Notifications.List(b.name)
+	setClientHeader(call.Header())
+	if b.userProject != "" {
+		call.UserProject(b.userProject)
+	}
+	var res *raw.Notifications
+	var err error
+	err = runWithRetry(ctx, func() error {
+		res, err = call.Context(ctx).Do()
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+	return notificationsToMap(res.Items), nil
+}
+
+func notificationsToMap(rns []*raw.Notification) map[string]*Notification {
+	m := map[string]*Notification{}
+	for _, rn := range rns {
+		m[rn.Id] = toNotification(rn)
+	}
+	return m
+}
+
+// DeleteNotification deletes the notification with the given ID.
+func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) error {
+	call := b.c.raw.Notifications.Delete(b.name, id)
+	setClientHeader(call.Header())
+	if b.userProject != "" {
+		call.UserProject(b.userProject)
+	}
+	return call.Context(ctx).Do()
+}

vendor/cloud.google.com/go/storage/notifications_test.go

@@ -0,0 +1,97 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"testing"
+
+	"cloud.google.com/go/internal/testutil"
+	raw "google.golang.org/api/storage/v1"
+)
+
+func TestParseNotificationTopic(t *testing.T) {
+	for _, test := range []struct {
+		in            string
+		wantProjectID string
+		wantTopicID   string
+	}{
+		{"", "?", "?"},
+		{"foobar", "?", "?"},
+		{"//pubsub.googleapis.com/projects/foo", "?", "?"},
+		{"//pubsub.googleapis.com/projects/my-project/topics/my-topic",
+			"my-project", "my-topic"},
+	} {
+		gotProjectID, gotTopicID := parseNotificationTopic(test.in)
+		if gotProjectID != test.wantProjectID || gotTopicID != test.wantTopicID {
+			t.Errorf("%q: got (%q, %q), want (%q, %q)",
+				test.in, gotProjectID, gotTopicID, test.wantProjectID, test.wantTopicID)
+		}
+	}
+
+}
+
+func TestConvertNotification(t *testing.T) {
+	want := &Notification{
+		ID:               "id",
+		TopicProjectID:   "my-project",
+		TopicID:          "my-topic",
+		EventTypes:       []string{ObjectFinalizeEvent},
+		ObjectNamePrefix: "prefix",
+		CustomAttributes: map[string]string{"a": "b"},
+		PayloadFormat:    JSONPayload,
+	}
+	got := toNotification(toRawNotification(want))
+	if diff := testutil.Diff(got, want); diff != "" {
+		t.Errorf("got=-, want=+:\n%s", diff)
+	}
+}
+
+func TestNotificationsToMap(t *testing.T) {
+	got := notificationsToMap(nil)
+	want := map[string]*Notification{}
+	if !testutil.Equal(got, want) {
+		t.Errorf("got %+v, want %+v", got, want)
+	}
+
+	in := []*raw.Notification{
+		{Id: "a", Topic: "//pubsub.googleapis.com/projects/P1/topics/T1"},
+		{Id: "b", Topic: "//pubsub.googleapis.com/projects/P2/topics/T2"},
+		{Id: "c", Topic: "//pubsub.googleapis.com/projects/P3/topics/T3"},
+	}
+	got = notificationsToMap(in)
+	want = map[string]*Notification{
+		"a": &Notification{ID: "a", TopicProjectID: "P1", TopicID: "T1"},
+		"b": &Notification{ID: "b", TopicProjectID: "P2", TopicID: "T2"},
+		"c": &Notification{ID: "c", TopicProjectID: "P3", TopicID: "T3"},
+	}
+	if diff := testutil.Diff(got, want); diff != "" {
+		t.Errorf("got=-, want=+:\n%s", diff)
+	}
+}
+
+func TestAddNotificationsErrors(t *testing.T) {
+	c := &Client{}
+	b := c.Bucket("b")
+	for _, n := range []*Notification{
+		{ID: "foo", TopicProjectID: "p", TopicID: "t"}, // has ID
+		{TopicProjectID: "p"},                          // missing TopicID
+		{TopicID: "t"},                                 // missing TopicProjectID
+	} {
+		_, err := b.AddNotification(nil, n)
+		if err == nil {
+			t.Errorf("%+v: got nil, want error", n)
+		}
+	}
+}

vendor/cloud.google.com/go/storage/reader.go

@@ -25,13 +25,14 @@ var crc32cTable = crc32.MakeTable(crc32.Castagnoli)
 // Reader reads a Cloud Storage object.
 // It implements io.Reader.
 type Reader struct {
-	body         io.ReadCloser
-	remain, size int64
-	contentType  string
-	cacheControl string
-	checkCRC     bool   // should we check the CRC?
-	wantCRC      uint32 // the CRC32c value the server sent in the header
-	gotCRC       uint32 // running crc
+	body            io.ReadCloser
+	remain, size    int64
+	contentType     string
+	contentEncoding string
+	cacheControl    string
+	checkCRC        bool   // should we check the CRC?
+	wantCRC         uint32 // the CRC32c value the server sent in the header
+	gotCRC          uint32 // running crc
 }
 
 // Close closes the Reader. It must be called when done reading.
@@ -74,6 +75,11 @@ func (r *Reader) ContentType() string {
 	return r.contentType
 }
 
+// ContentEncoding returns the content encoding of the object.
+func (r *Reader) ContentEncoding() string {
+	return r.contentEncoding
+}
+
 // CacheControl returns the cache control of the object.
 func (r *Reader) CacheControl() string {
 	return r.cacheControl

vendor/cloud.google.com/go/storage/storage.go

@@ -30,6 +30,8 @@ import (
 	"net/http"
 	"net/url"
 	"reflect"
+	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -170,7 +172,7 @@ type SignedURLOptions struct {
 	// Optional.
 	ContentType string
 
-	// Headers is a list of extention headers the client must provide
+	// Headers is a list of extension headers the client must provide
 	// in order to use the generated signed URL.
 	// Optional.
 	Headers []string
@@ -182,6 +184,60 @@ type SignedURLOptions struct {
 	MD5 string
 }
 
+var (
+	canonicalHeaderRegexp    = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`)
+	excludedCanonicalHeaders = map[string]bool{
+		"x-goog-encryption-key":        true,
+		"x-goog-encryption-key-sha256": true,
+	}
+)
+
+// sanitizeHeaders applies the specifications for canonical extension headers at
+// https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers.
+func sanitizeHeaders(hdrs []string) []string {
+	headerMap := map[string][]string{}
+	for _, hdr := range hdrs {
+		// No leading or trailing whitespaces.
+		sanitizedHeader := strings.TrimSpace(hdr)
+
+		// Only keep canonical headers, discard any others.
+		headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader)
+		if len(headerMatches) == 0 {
+			continue
+		}
+
+		header := strings.ToLower(strings.TrimSpace(headerMatches[1]))
+		if excludedCanonicalHeaders[headerMatches[1]] {
+			// Do not keep any deliberately excluded canonical headers when signing.
+			continue
+		}
+		value := strings.TrimSpace(headerMatches[2])
+		if len(value) > 0 {
+			// Remove duplicate headers by appending the values of duplicates
+			// in their order of appearance.
+			headerMap[header] = append(headerMap[header], value)
+		}
+	}
+
+	var sanitizedHeaders []string
+	for header, values := range headerMap {
+		// There should be no spaces around the colon separating the
+		// header name from the header value or around the values
+		// themselves. The values should be separated by commas.
+		// NOTE: The semantics for headers without a value are not clear.
+		//       However from specifications these should be edge-cases
+		//       anyway and we should assume that there will be no
+		//       canonical headers using empty values. Any such headers
+		//       are discarded at the regexp stage above.
+		sanitizedHeaders = append(
+			sanitizedHeaders,
+			fmt.Sprintf("%s:%s", header, strings.Join(values, ",")),
+		)
+	}
+	sort.Strings(sanitizedHeaders)
+	return sanitizedHeaders
+}
+
 // SignedURL returns a URL for the specified object. Signed URLs allow
 // the users access to a restricted resource for a limited time without having a
 // Google account or signing in. For more information about the signed
@@ -208,6 +264,7 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) {
 			return "", errors.New("storage: invalid MD5 checksum")
 		}
 	}
+	opts.Headers = sanitizeHeaders(opts.Headers)
 
 	signBytes := opts.SignBytes
 	if opts.PrivateKey != nil {
@@ -258,14 +315,15 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) {
 // ObjectHandle provides operations on an object in a Google Cloud Storage bucket.
 // Use BucketHandle.Object to get a handle.
 type ObjectHandle struct {
-	c             *Client
-	bucket        string
-	object        string
-	acl           ACLHandle
-	gen           int64 // a negative value indicates latest
-	conds         *Conditions
-	encryptionKey []byte // AES-256 key
-	userProject   string // for requester-pays buckets
+	c              *Client
+	bucket         string
+	object         string
+	acl            ACLHandle
+	gen            int64 // a negative value indicates latest
+	conds          *Conditions
+	encryptionKey  []byte // AES-256 key
+	userProject    string // for requester-pays buckets
+	readCompressed bool   // Accept-Encoding: gzip
 }
 
 // ACL provides access to the object's access control list.
@@ -467,6 +525,13 @@ func (o *ObjectHandle) Delete(ctx context.Context) error {
 	return err
 }
 
+// ReadCompressed when true causes the read to happen without decompressing.
+func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle {
+	o2 := *o
+	o2.readCompressed = compressed
+	return &o2
+}
+
 // NewReader creates a new Reader to read the contents of the
 // object.
 // ErrObjectNotExist will be returned if the object is not found.
@@ -514,6 +579,9 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
 	if o.userProject != "" {
 		req.Header.Set("X-Goog-User-Project", o.userProject)
 	}
+	if o.readCompressed {
+		req.Header.Set("Accept-Encoding", "gzip")
+	}
 	if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil {
 		return nil, err
 	}
@@ -576,13 +644,14 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
 		crc, checkCRC = parseCRC32c(res)
 	}
 	return &Reader{
-		body:         body,
-		size:         size,
-		remain:       remain,
-		contentType:  res.Header.Get("Content-Type"),
-		cacheControl: res.Header.Get("Cache-Control"),
-		wantCRC:      crc,
-		checkCRC:     checkCRC,
+		body:            body,
+		size:            size,
+		remain:          remain,
+		contentType:     res.Header.Get("Content-Type"),
+		contentEncoding: res.Header.Get("Content-Encoding"),
+		cacheControl:    res.Header.Get("Cache-Control"),
+		wantCRC:         crc,
+		checkCRC:        checkCRC,
 	}, nil
 }
 
@@ -638,11 +707,10 @@ func (o *ObjectHandle) validate() error {
 	return nil
 }
 
-// parseKey converts the binary contents of a private key file
-// to an *rsa.PrivateKey. It detects whether the private key is in a
-// PEM container or not. If so, it extracts the the private key
-// from PEM container before conversion. It only supports PEM
-// containers with no passphrase.
+// parseKey converts the binary contents of a private key file to an
+// *rsa.PrivateKey. It detects whether the private key is in a PEM container or
+// not. If so, it extracts the private key from PEM container before
+// conversion. It only supports PEM containers with no passphrase.
 func parseKey(key []byte) (*rsa.PrivateKey, error) {
 	if block, _ := pem.Decode(key); block != nil {
 		key = block.Bytes

vendor/cloud.google.com/go/storage/storage_test.go

@@ -24,10 +24,13 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
 
+	"cloud.google.com/go/iam"
 	"cloud.google.com/go/internal/testutil"
 
 	"golang.org/x/net/context"
@@ -37,6 +40,52 @@ import (
 	raw "google.golang.org/api/storage/v1"
 )
 
+func TestHeaderSanitization(t *testing.T) {
+	t.Parallel()
+	var tests = []struct {
+		desc string
+		in   []string
+		want []string
+	}{
+		{
+			desc: "already sanitized headers should not be modified",
+			in:   []string{"x-goog-header1:true", "x-goog-header2:0"},
+			want: []string{"x-goog-header1:true", "x-goog-header2:0"},
+		},
+		{
+			desc: "sanitized headers should be sorted",
+			in:   []string{"x-goog-header2:0", "x-goog-header1:true"},
+			want: []string{"x-goog-header1:true", "x-goog-header2:0"},
+		},
+		{
+			desc: "non-canonical headers should be removed",
+			in:   []string{"x-goog-header1:true", "x-goog-no-value", "non-canonical-header:not-of-use"},
+			want: []string{"x-goog-header1:true"},
+		},
+		{
+			desc: "excluded canonical headers should be removed",
+			in:   []string{"x-goog-header1:true", "x-goog-encryption-key:my_key", "x-goog-encryption-key-sha256:my_sha256"},
+			want: []string{"x-goog-header1:true"},
+		},
+		{
+			desc: "dirty headers should be formatted correctly",
+			in:   []string{" x-goog-header1 : \textra-spaces ", "X-Goog-Header2:CamelCaseValue"},
+			want: []string{"x-goog-header1:extra-spaces", "x-goog-header2:CamelCaseValue"},
+		},
+		{
+			desc: "duplicate headers should be merged",
+			in:   []string{"x-goog-header1:value1", "X-Goog-Header1:value2"},
+			want: []string{"x-goog-header1:value1,value2"},
+		},
+	}
+	for _, test := range tests {
+		got := sanitizeHeaders(test.in)
+		if !testutil.Equal(got, test.want) {
+			t.Errorf("%s: got %v, want %v", test.desc, got, test.want)
+		}
+	}
+}
+
 func TestSignedURL(t *testing.T) {
 	t.Parallel()
 	expires, _ := time.Parse(time.RFC3339, "2002-10-02T10:00:00-05:00")
@@ -47,20 +96,20 @@ func TestSignedURL(t *testing.T) {
 		MD5:            "ICy5YqxZB1uWSwcVLSNLcA==",
 		Expires:        expires,
 		ContentType:    "application/json",
-		Headers:        []string{"x-header1", "x-header2"},
+		Headers:        []string{"x-goog-header1:true", "x-goog-header2:false"},
 	})
 	if err != nil {
 		t.Error(err)
 	}
 	want := "https://storage.googleapis.com/bucket-name/object-name?" +
 		"Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" +
-		"ZMw18bZVhySNYAMEX87RMyuZCUMtGLVi%2B2zU2ByiQ0Rxgij%2BhFZ5LsT" +
-		"5ZPIH5h3QXB%2BiSb1URJnZo3aF0exVP%2FYR1hpg2e65w9HHt7yYjIqcg" +
-		"%2FfAOIyxriFtgRYk3oAv%2FFLF62fI8iF%2BCp0fWSm%2FHggz22blVnQz" +
-		"EtSP%2BuRhFle4172L%2B710sfMDtyQLKTz6W4TmRjC9ymTi8mVj95dZgyF" +
-		"RXbibTdtw0JzndE0Ig4c6pU4xDPPiyaziUSVDMIpzZDJH1GYOGHxbFasba4" +
-		"1rRoWWkdBnsMtHm2ck%2FsFD2leL6u8q0OpVAc4ZdxseucL4OpCy%2BCLhQ" +
-		"JFQT5bqSljP0g%3D%3D"
+		"RfsHlPtbB2JUYjzCgNr2Mi%2BjggdEuL1V7E6N9o6aaqwVLBDuTv3I0%2B9" +
+		"x94E6rmmr%2FVgnmZigkIUxX%2Blfl7LgKf30uPGLt0mjKGH2p7r9ey1ONJ" +
+		"%2BhVec23FnTRcSgopglvHPuCMWU2oNJE%2F1y8EwWE27baHrG1RhRHbLVF" +
+		"bPpLZ9xTRFK20pluIkfHV00JGljB1imqQHXM%2B2XPWqBngLr%2FwqxLN7i" +
+		"FcUiqR8xQEOHF%2F2e7fbkTHPNq4TazaLZ8X0eZ3eFdJ55A5QmNi8atlN4W" +
+		"5q7Hvs0jcxElG3yqIbx439A995BkspLiAcA%2Fo4%2BxAwEMkGLICdbvakq" +
+		"3eEprNCojw%3D%3D"
 	if url != want {
 		t.Fatalf("Unexpected signed URL; found %v", url)
 	}
@@ -76,16 +125,17 @@ func TestSignedURL_PEMPrivateKey(t *testing.T) {
 		MD5:            "ICy5YqxZB1uWSwcVLSNLcA==",
 		Expires:        expires,
 		ContentType:    "application/json",
-		Headers:        []string{"x-header1", "x-header2"},
+		Headers:        []string{"x-goog-header1:true", "x-goog-header2:false"},
 	})
 	if err != nil {
 		t.Error(err)
 	}
 	want := "https://storage.googleapis.com/bucket-name/object-name?" +
 		"Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" +
-		"gHlh63sOxJnNj22X%2B%2F4kwOSNMeqwXWr4udEfrzJPQcq1xzxA8ovMM5SOrOc%" +
-		"2FuE%2Ftc9%2Bq7a42CDBwZff1PsvuJMBDaPbluU257h%2Bvxx8lHMnb%2Bg1wD1" +
-		"99FiCE014MRH9TlIg%2FdXRkErosVWTy4GqAgZemmKHo0HwDGT6IovB9mdg%3D"
+		"TiyKD%2FgGb6Kh0kkb2iF%2FfF%2BnTx7L0J4YiZua8AcTmnidutePEGIU5" +
+		"NULYlrGl6l52gz4zqFb3VFfIRTcPXMdXnnFdMCDhz2QuJBUpsU1Ai9zlyTQ" +
+		"dkb6ShG03xz9%2BEXWAUQO4GBybJw%2FULASuv37xA00SwLdkqj8YdyS5II" +
+		"1lro%3D"
 	if url != want {
 		t.Fatalf("Unexpected signed URL; found %v", url)
 	}
@@ -103,7 +153,7 @@ func TestSignedURL_SignBytes(t *testing.T) {
 		MD5:         "ICy5YqxZB1uWSwcVLSNLcA==",
 		Expires:     expires,
 		ContentType: "application/json",
-		Headers:     []string{"x-header1", "x-header2"},
+		Headers:     []string{"x-goog-header1:true", "x-goog-header2:false"},
 	})
 	if err != nil {
 		t.Error(err)
@@ -126,16 +176,16 @@ func TestSignedURL_URLUnsafeObjectName(t *testing.T) {
 		MD5:            "ICy5YqxZB1uWSwcVLSNLcA==",
 		Expires:        expires,
 		ContentType:    "application/json",
-		Headers:        []string{"x-header1", "x-header2"},
+		Headers:        []string{"x-goog-header1:true", "x-goog-header2:false"},
 	})
 	if err != nil {
 		t.Error(err)
 	}
 	want := "https://storage.googleapis.com/bucket-name/object%20name%E7%95%8C?" +
-		"Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=" +
-		"LSxs1YwXNKOa7mQv1ZAI2ao0Fuv6yXLLU7%2BQ97z2B7hYZ57OiFwQ72EdGXSiIM" +
-		"JwLisEKkwoSlYCMm3uuTdgJtXXVi7SYXMfdeKaonyQwMv531KETCBTSewt8CW%2B" +
-		"FaUJ5SEYG44SeJCiqeIr3GF7t90UNWs6TdFXDaKShpQzBGg%3D"
+		"Expires=1033570800&GoogleAccessId=xxx%40clientid&Signature=bxVH1%2Bl%2" +
+		"BSxpnj3XuqKz6mOFk6M94Y%2B4w85J6FCmJan%2FNhGSpndP6fAw1uLHlOn%2F8xUaY%2F" +
+		"SfZ5GzcQ%2BbxOL1WA37yIwZ7xgLYlO%2ByAi3GuqMUmHZiNCai28emODXQ8RtWHvgv6dE" +
+		"SQ%2F0KpDMIWW7rYCaUa63UkUyeSQsKhrVqkIA%3D"
 	if url != want {
 		t.Fatalf("Unexpected signed URL; found %v", url)
 	}
@@ -322,7 +372,7 @@ func TestObjectNames(t *testing.T) {
 		MD5:            "ICy5YqxZB1uWSwcVLSNLcA==",
 		Expires:        time.Date(2002, time.October, 2, 10, 0, 0, 0, time.UTC),
 		ContentType:    "application/json",
-		Headers:        []string{"x-header1", "x-header2"},
+		Headers:        []string{"x-goog-header1", "x-goog-header2"},
 	}
 
 	for _, test := range tests {
@@ -769,6 +819,83 @@ func TestBucketAttrs(t *testing.T) {
 	}
 }
 
+func TestUserProject(t *testing.T) {
+	// Verify that the userProject query param is sent.
+	t.Parallel()
+	ctx := context.Background()
+	gotURL := make(chan *url.URL, 1)
+	hClient, close := newTestServer(func(w http.ResponseWriter, r *http.Request) {
+		io.Copy(ioutil.Discard, r.Body)
+		gotURL <- r.URL
+		if strings.Contains(r.URL.String(), "/rewriteTo/") {
+			res := &raw.RewriteResponse{Done: true}
+			bytes, err := res.MarshalJSON()
+			if err != nil {
+				t.Fatal(err)
+			}
+			w.Write(bytes)
+		} else {
+			fmt.Fprintf(w, "{}")
+		}
+	})
+	defer close()
+	client, err := NewClient(ctx, option.WithHTTPClient(hClient))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	re := regexp.MustCompile(`\buserProject=p\b`)
+	b := client.Bucket("b").UserProject("p")
+	o := b.Object("o")
+
+	check := func(msg string, f func()) {
+		f()
+		select {
+		case u := <-gotURL:
+			if !re.MatchString(u.RawQuery) {
+				t.Errorf("%s: query string %q does not contain userProject", msg, u.RawQuery)
+			}
+		case <-time.After(2 * time.Second):
+			t.Errorf("%s: timed out", msg)
+		}
+	}
+
+	check("buckets.delete", func() { b.Delete(ctx) })
+	check("buckets.get", func() { b.Attrs(ctx) })
+	check("buckets.patch", func() { b.Update(ctx, BucketAttrsToUpdate{}) })
+	check("storage.objects.compose", func() { o.ComposerFrom(b.Object("x")).Run(ctx) })
+	check("storage.objects.delete", func() { o.Delete(ctx) })
+	check("storage.objects.get", func() { o.Attrs(ctx) })
+	check("storage.objects.insert", func() { o.NewWriter(ctx).Close() })
+	check("storage.objects.list", func() { b.Objects(ctx, nil).Next() })
+	check("storage.objects.patch", func() { o.Update(ctx, ObjectAttrsToUpdate{}) })
+	check("storage.objects.rewrite", func() { o.CopierFrom(b.Object("x")).Run(ctx) })
+	check("storage.objectAccessControls.list", func() { o.ACL().List(ctx) })
+	check("storage.objectAccessControls.update", func() { o.ACL().Set(ctx, "", "") })
+	check("storage.objectAccessControls.delete", func() { o.ACL().Delete(ctx, "") })
+	check("storage.bucketAccessControls.list", func() { b.ACL().List(ctx) })
+	check("storage.bucketAccessControls.update", func() { b.ACL().Set(ctx, "", "") })
+	check("storage.bucketAccessControls.delete", func() { b.ACL().Delete(ctx, "") })
+	check("storage.defaultObjectAccessControls.list",
+		func() { b.DefaultObjectACL().List(ctx) })
+	check("storage.defaultObjectAccessControls.update",
+		func() { b.DefaultObjectACL().Set(ctx, "", "") })
+	check("storage.defaultObjectAccessControls.delete",
+		func() { b.DefaultObjectACL().Delete(ctx, "") })
+	check("buckets.getIamPolicy", func() { b.IAM().Policy(ctx) })
+	check("buckets.setIamPolicy", func() {
+		p := &iam.Policy{}
+		p.Add("m", iam.Owner)
+		b.IAM().SetPolicy(ctx, p)
+	})
+	check("buckets.testIamPermissions", func() { b.IAM().TestPermissions(ctx, nil) })
+	check("storage.notifications.insert", func() {
+		b.AddNotification(ctx, &Notification{TopicProjectID: "p", TopicID: "t"})
+	})
+	check("storage.notifications.delete", func() { b.DeleteNotification(ctx, "n") })
+	check("storage.notifications.list", func() { b.Notifications(ctx) })
+}
+
 func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) {
 	ts := httptest.NewTLSServer(http.HandlerFunc(handler))
 	tlsConf := &tls.Config{InsecureSkipVerify: true}