vendor: update all dependencies

2018-06-17 17:59:12 +01:00
parent 3f0789e2db
commit 08021c4636
2474 changed files with 435818 additions and 282709 deletions
@@ -147,7 +147,10 @@ func runWithRetry(ctx context.Context, call func() error) error {
 	})
 }

-// This is the correct definition of retryable according to the BigQuery team.
+// This is the correct definition of retryable according to the BigQuery team. It
+// also considers 502 ("Bad Gateway") and 503 ("Service Unavailable") errors
+// retryable; these are returned by systems between the client and the BigQuery
+// service.
 func retryableError(err error) bool {
 	e, ok := err.(*googleapi.Error)
 	if !ok {
@@ -157,5 +160,5 @@ func retryableError(err error) bool {
 	if len(e.Errors) > 0 {
 		reason = e.Errors[0].Reason
 	}
-	return e.Code == http.StatusBadGateway || reason == "backendError" || reason == "rateLimitExceeded"
+	return e.Code == http.StatusServiceUnavailable || e.Code == http.StatusBadGateway || reason == "backendError" || reason == "rateLimitExceeded"
 }
@@ -147,12 +147,21 @@ func accessListToBQ(a []*AccessEntry) ([]*bq.DatasetAccess, error) {
 	return q, nil
 }

-// Delete deletes the dataset.
+// Delete deletes the dataset.  Delete will fail if the dataset is not empty.
 func (d *Dataset) Delete(ctx context.Context) (err error) {
+	return d.deleteInternal(ctx, false)
+}
+
+// DeleteWithContents deletes the dataset, as well as contained resources.
+func (d *Dataset) DeleteWithContents(ctx context.Context) (err error) {
+	return d.deleteInternal(ctx, true)
+}
+
+func (d *Dataset) deleteInternal(ctx context.Context, deleteContents bool) (err error) {
 	ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Dataset.Delete")
 	defer func() { trace.EndSpan(ctx, err) }()

-	call := d.c.bqs.Datasets.Delete(d.ProjectID, d.DatasetID).Context(ctx)
+	call := d.c.bqs.Datasets.Delete(d.ProjectID, d.DatasetID).Context(ctx).DeleteContents(deleteContents)
 	setClientHeader(call.Header())
 	return call.Do()
 }
@@ -336,6 +345,9 @@ func (it *TableIterator) fetch(pageSize int, pageToken string) (string, error) {
 }

 func bqToTable(tr *bq.TableReference, c *Client) *Table {
+	if tr == nil {
+		return nil
+	}
 	return &Table{
 		ProjectID: tr.ProjectId,
 		DatasetID: tr.DatasetId,
@@ -20,6 +20,9 @@ Note: This package is in beta.  Some backwards-incompatible changes may occur.
 The following assumes a basic familiarity with BigQuery concepts.
 See https://cloud.google.com/bigquery/docs.

+See https://godoc.org/cloud.google.com/go for authentication, timeouts,
+connection pooling and similar aspects of this package.
+

 Creating a Client

@@ -294,10 +297,5 @@ Extractor, then optionally configure it, and lastly call its Run method.
    extractor.DisableHeader = true
    job, err = extractor.Run(ctx)
    // Poll the job for completion if desired, as above.
-
-Authentication
-
-See examples of authorization and authentication at
-https://godoc.org/cloud.google.com/go#pkg-examples.
 */
 package bigquery // import "cloud.google.com/go/bigquery"
@@ -32,6 +32,7 @@ const (
 	DatastoreBackup DataFormat = "DATASTORE_BACKUP"
 	GoogleSheets    DataFormat = "GOOGLE_SHEETS"
 	Bigtable        DataFormat = "BIGTABLE"
+	Parquet         DataFormat = "PARQUET"
 )

 // ExternalData is a table which is stored outside of BigQuery. It is implemented by
@@ -48,8 +48,8 @@ func (r *ReaderSource) populateLoadConfig(lc *bq.JobConfigurationLoad) io.Reader
 // file may live in Google Cloud Storage (see GCSReference), or it may be
 // loaded into a table via the Table.LoaderFromReader.
 type FileConfig struct {
-	// SourceFormat is the format of the GCS data to be read.
-	// Allowed values are: CSV, Avro, JSON, DatastoreBackup.  The default is CSV.
+	// SourceFormat is the format of the data to be read.
+	// Allowed values are: CSV, Avro, Parquet, JSON, DatastoreBackup.  The default is CSV.
 	SourceFormat DataFormat

 	// Indicates if we should automatically infer the options and
@@ -34,6 +34,7 @@ import (
 	"cloud.google.com/go/internal"
 	"cloud.google.com/go/internal/pretty"
 	"cloud.google.com/go/internal/testutil"
+	"cloud.google.com/go/internal/uid"
 	"cloud.google.com/go/storage"
 	"golang.org/x/net/context"
 	"google.golang.org/api/googleapi"
@@ -55,8 +56,8 @@ var (
 	testTableExpiration time.Time
 	// BigQuery does not accept hyphens in dataset or table IDs, so we create IDs
 	// with underscores.
-	datasetIDs = testutil.NewUIDSpaceSep("dataset", '_')
-	tableIDs   = testutil.NewUIDSpaceSep("table", '_')
+	datasetIDs = uid.NewSpace("dataset", &uid.Options{Sep: '_'})
+	tableIDs   = uid.NewSpace("table", &uid.Options{Sep: '_'})
 )

 // Note: integration tests cannot be run in parallel, because TestIntegration_Location
@@ -105,28 +106,12 @@ func initIntegrationTest() func() {
 	}
 	testTableExpiration = time.Now().Add(10 * time.Minute).Round(time.Second)
 	return func() {
-		if err := deleteDataset(ctx, dataset); err != nil {
+		if err := dataset.DeleteWithContents(ctx); err != nil {
 			log.Printf("could not delete %s", dataset.DatasetID)
 		}
 	}
 }

-func deleteDataset(ctx context.Context, ds *Dataset) error {
-	it := ds.Tables(ctx)
-	for {
-		tbl, err := it.Next()
-		if err == iterator.Done {
-			break
-		}
-		if err != nil {
-			return err
-		}
-		if err := tbl.Delete(ctx); err != nil {
-			return err
-		}
-	}
-	return ds.Delete(ctx)
-}
 func TestIntegration_TableCreate(t *testing.T) {
 	// Check that creating a record field with an empty schema is an error.
 	if client == nil {
@@ -167,7 +152,9 @@ func TestIntegration_TableCreateView(t *testing.T) {
 	if err != nil {
 		t.Fatalf("table.create: Did not expect an error, got: %v", err)
 	}
-	view.Delete(ctx)
+	if err := view.Delete(ctx); err != nil {
+		t.Fatal(err)
+	}
 }

 func TestIntegration_TableMetadata(t *testing.T) {
@@ -312,6 +299,28 @@ func TestIntegration_DatasetDelete(t *testing.T) {
 	}
 }

+func TestIntegration_DatasetDeleteWithContents(t *testing.T) {
+	if client == nil {
+		t.Skip("Integration tests skipped")
+	}
+	ctx := context.Background()
+	ds := client.Dataset(datasetIDs.New())
+	if err := ds.Create(ctx, nil); err != nil {
+		t.Fatalf("creating dataset %s: %v", ds.DatasetID, err)
+	}
+	table := ds.Table(tableIDs.New())
+	if err := table.Create(ctx, nil); err != nil {
+		t.Fatalf("creating table %s in dataset %s: %v", table.TableID, table.DatasetID, err)
+	}
+	// We expect failure here
+	if err := ds.Delete(ctx); err == nil {
+		t.Fatalf("non-recursive delete of dataset %s succeeded unexpectedly.", ds.DatasetID)
+	}
+	if err := ds.DeleteWithContents(ctx); err != nil {
+		t.Fatalf("deleting recursively dataset %s: %v", ds.DatasetID, err)
+	}
+}
+
 func TestIntegration_DatasetUpdateETags(t *testing.T) {
 	if client == nil {
 		t.Skip("Integration tests skipped")
@@ -1570,7 +1579,7 @@ func TestIntegration_TableUseLegacySQL(t *testing.T) {
 		} else if !gotErr && test.err {
 			t.Errorf("%+v:\nsucceeded, but want error", test)
 		}
-		view.Delete(ctx)
+		_ = view.Delete(ctx)
 	}
 }

@@ -404,6 +404,9 @@ type QueryStatistics struct {
 	// statements INSERT, UPDATE or DELETE.
 	NumDMLAffectedRows int64

+	// Describes a timeline of job execution.
+	Timeline []*QueryTimelineSample
+
 	// ReferencedTables: [Output-only, Experimental] Referenced tables for
 	// the job. Queries that reference more than 50 tables will not have a
 	// complete list.
@@ -413,25 +416,59 @@ type QueryStatistics struct {
 	// non-legacy SQL queries.
 	Schema Schema

+	// Slot-milliseconds consumed by this query job.
+	SlotMillis int64
+
 	// Standard SQL: list of undeclared query parameter names detected during a
 	// dry run validation.
 	UndeclaredQueryParameterNames []string
+
+	// DDL target table.
+	DDLTargetTable *Table
+
+	// DDL Operation performed on the target table.  Used to report how the
+	// query impacted the DDL target table.
+	DDLOperationPerformed string
 }

 // ExplainQueryStage describes one stage of a query.
 type ExplainQueryStage struct {
+	// CompletedParallelInputs: Number of parallel input segments completed.
+	CompletedParallelInputs int64
+
+	// ComputeAvg: Duration the average shard spent on CPU-bound tasks.
+	ComputeAvg time.Duration
+
+	// ComputeMax: Duration the slowest shard spent on CPU-bound tasks.
+	ComputeMax time.Duration
+
 	// Relative amount of the total time the average shard spent on CPU-bound tasks.
 	ComputeRatioAvg float64

 	// Relative amount of the total time the slowest shard spent on CPU-bound tasks.
 	ComputeRatioMax float64

+	// EndTime: Stage end time.
+	EndTime time.Time
+
 	// Unique ID for stage within plan.
 	ID int64

+	// InputStages: IDs for stages that are inputs to this stage.
+	InputStages []int64
+
 	// Human-readable name for stage.
 	Name string

+	// ParallelInputs: Number of parallel input segments to be processed.
+	ParallelInputs int64
+
+	// ReadAvg: Duration the average shard spent reading input.
+	ReadAvg time.Duration
+
+	// ReadMax: Duration the slowest shard spent reading input.
+	ReadMax time.Duration
+
 	// Relative amount of the total time the average shard spent reading input.
 	ReadRatioAvg float64

@@ -444,6 +481,16 @@ type ExplainQueryStage struct {
 	// Number of records written by the stage.
 	RecordsWritten int64

+	// ShuffleOutputBytes: Total number of bytes written to shuffle.
+	ShuffleOutputBytes int64
+
+	// ShuffleOutputBytesSpilled: Total number of bytes written to shuffle
+	// and spilled to disk.
+	ShuffleOutputBytesSpilled int64
+
+	// StartTime: Stage start time.
+	StartTime time.Time
+
 	// Current status for the stage.
 	Status string

@@ -451,12 +498,24 @@ type ExplainQueryStage struct {
 	// chronological).
 	Steps []*ExplainQueryStep

+	// WaitAvg: Duration the average shard spent waiting to be scheduled.
+	WaitAvg time.Duration
+
+	// WaitMax: Duration the slowest shard spent waiting to be scheduled.
+	WaitMax time.Duration
+
 	// Relative amount of the total time the average shard spent waiting to be scheduled.
 	WaitRatioAvg float64

 	// Relative amount of the total time the slowest shard spent waiting to be scheduled.
 	WaitRatioMax float64

+	// WriteAvg: Duration the average shard spent on writing output.
+	WriteAvg time.Duration
+
+	// WriteMax: Duration the slowest shard spent on writing output.
+	WriteMax time.Duration
+
 	// Relative amount of the total time the average shard spent on writing output.
 	WriteRatioAvg float64

@@ -473,6 +532,25 @@ type ExplainQueryStep struct {
 	Substeps []string
 }

+// QueryTimelineSample represents a sample of execution statistics at a point in time.
+type QueryTimelineSample struct {
+
+	// Total number of units currently being processed by workers, represented as largest value since last sample.
+	ActiveUnits int64
+
+	// Total parallel units of work completed by this query.
+	CompletedUnits int64
+
+	// Time elapsed since start of query execution.
+	Elapsed time.Duration
+
+	// Total parallel units of work remaining for the active stages.
+	PendingUnits int64
+
+	// Cumulative slot-milliseconds consumed by the query.
+	SlotMillis int64
+}
+
 func (*ExtractStatistics) implementsStatistics() {}
 func (*LoadStatistics) implementsStatistics()    {}
 func (*QueryStatistics) implementsStatistics()   {}
@@ -667,12 +745,16 @@ func (j *Job) setStatistics(s *bq.JobStatistics, c *Client) {
 		js.Details = &QueryStatistics{
 			BillingTier:                   s.Query.BillingTier,
 			CacheHit:                      s.Query.CacheHit,
+			DDLTargetTable:                bqToTable(s.Query.DdlTargetTable, c),
+			DDLOperationPerformed:         s.Query.DdlOperationPerformed,
 			StatementType:                 s.Query.StatementType,
 			TotalBytesBilled:              s.Query.TotalBytesBilled,
 			TotalBytesProcessed:           s.Query.TotalBytesProcessed,
 			NumDMLAffectedRows:            s.Query.NumDmlAffectedRows,
 			QueryPlan:                     queryPlanFromProto(s.Query.QueryPlan),
 			Schema:                        bqToSchema(s.Query.Schema),
+			SlotMillis:                    s.Query.TotalSlotMs,
+			Timeline:                      timelineFromProto(s.Query.Timeline),
 			ReferencedTables:              tables,
 			UndeclaredQueryParameterNames: names,
 		}
@@ -691,20 +773,49 @@ func queryPlanFromProto(stages []*bq.ExplainQueryStage) []*ExplainQueryStage {
 			})
 		}
 		res = append(res, &ExplainQueryStage{
-			ComputeRatioAvg: s.ComputeRatioAvg,
-			ComputeRatioMax: s.ComputeRatioMax,
-			ID:              s.Id,
-			Name:            s.Name,
-			ReadRatioAvg:    s.ReadRatioAvg,
-			ReadRatioMax:    s.ReadRatioMax,
-			RecordsRead:     s.RecordsRead,
-			RecordsWritten:  s.RecordsWritten,
-			Status:          s.Status,
-			Steps:           steps,
-			WaitRatioAvg:    s.WaitRatioAvg,
-			WaitRatioMax:    s.WaitRatioMax,
-			WriteRatioAvg:   s.WriteRatioAvg,
-			WriteRatioMax:   s.WriteRatioMax,
+			CompletedParallelInputs:   s.CompletedParallelInputs,
+			ComputeAvg:                time.Duration(s.ComputeMsAvg) * time.Millisecond,
+			ComputeMax:                time.Duration(s.ComputeMsMax) * time.Millisecond,
+			ComputeRatioAvg:           s.ComputeRatioAvg,
+			ComputeRatioMax:           s.ComputeRatioMax,
+			EndTime:                   time.Unix(0, s.EndMs*1e6),
+			ID:                        s.Id,
+			InputStages:               s.InputStages,
+			Name:                      s.Name,
+			ParallelInputs:            s.ParallelInputs,
+			ReadAvg:                   time.Duration(s.ReadMsAvg) * time.Millisecond,
+			ReadMax:                   time.Duration(s.ReadMsMax) * time.Millisecond,
+			ReadRatioAvg:              s.ReadRatioAvg,
+			ReadRatioMax:              s.ReadRatioMax,
+			RecordsRead:               s.RecordsRead,
+			RecordsWritten:            s.RecordsWritten,
+			ShuffleOutputBytes:        s.ShuffleOutputBytes,
+			ShuffleOutputBytesSpilled: s.ShuffleOutputBytesSpilled,
+			StartTime:                 time.Unix(0, s.StartMs*1e6),
+			Status:                    s.Status,
+			Steps:                     steps,
+			WaitAvg:                   time.Duration(s.WaitMsAvg) * time.Millisecond,
+			WaitMax:                   time.Duration(s.WaitMsMax) * time.Millisecond,
+			WaitRatioAvg:              s.WaitRatioAvg,
+			WaitRatioMax:              s.WaitRatioMax,
+			WriteAvg:                  time.Duration(s.WriteMsAvg) * time.Millisecond,
+			WriteMax:                  time.Duration(s.WriteMsMax) * time.Millisecond,
+			WriteRatioAvg:             s.WriteRatioAvg,
+			WriteRatioMax:             s.WriteRatioMax,
+		})
+	}
+	return res
+}
+
+func timelineFromProto(timeline []*bq.QueryTimelineSample) []*QueryTimelineSample {
+	var res []*QueryTimelineSample
+	for _, s := range timeline {
+		res = append(res, &QueryTimelineSample{
+			ActiveUnits:    s.ActiveUnits,
+			CompletedUnits: s.CompletedUnits,
+			Elapsed:        time.Duration(s.ElapsedMs) * time.Millisecond,
+			PendingUnits:   s.PendingUnits,
+			SlotMillis:     s.TotalSlotMs,
 		})
 	}
 	return res
@@ -47,8 +47,8 @@ type LoadConfig struct {
 	// Custom encryption configuration (e.g., Cloud KMS keys).
 	DestinationEncryptionConfig *EncryptionConfig

-	// SchemaUpdateOptions allows the schema of the destination table to be
-	// updated as a side effect of the load job.
+	// Allows the schema of the destination table to be updated as a side effect of
+	// the load job.
 	SchemaUpdateOptions []string
 }

@@ -119,6 +119,10 @@ type QueryConfig struct {

 	// Custom encryption configuration (e.g., Cloud KMS keys).
 	DestinationEncryptionConfig *EncryptionConfig
+
+	// Allows the schema of the destination table to be updated as a side effect of
+	// the query job.
+	SchemaUpdateOptions []string
 }

 func (qc *QueryConfig) toBQ() (*bq.JobConfiguration, error) {
@@ -131,6 +135,7 @@ func (qc *QueryConfig) toBQ() (*bq.JobConfiguration, error) {
 		MaximumBytesBilled:                 qc.MaxBytesBilled,
 		TimePartitioning:                   qc.TimePartitioning.toBQ(),
 		DestinationEncryptionConfiguration: qc.DestinationEncryptionConfig.toBQ(),
+		SchemaUpdateOptions:                qc.SchemaUpdateOptions,
 	}
 	if len(qc.TableDefinitions) > 0 {
 		qconf.TableDefinitions = make(map[string]bq.ExternalDataConfiguration)
@@ -162,11 +167,12 @@ func (qc *QueryConfig) toBQ() (*bq.JobConfiguration, error) {
 	if len(qc.Parameters) > 0 && qc.UseLegacySQL {
 		return nil, errors.New("bigquery: cannot provide both Parameters (implying standard SQL) and UseLegacySQL")
 	}
+	ptrue := true
+	pfalse := false
 	if qc.UseLegacySQL {
-		qconf.UseLegacySql = true
+		qconf.UseLegacySql = &ptrue
 	} else {
-		qconf.UseLegacySql = false
-		qconf.ForceSendFields = append(qconf.ForceSendFields, "UseLegacySql")
+		qconf.UseLegacySql = &pfalse
 	}
 	if qc.Dst != nil && !qc.Dst.implicitTable() {
 		qconf.DestinationTable = qc.Dst.toBQ()
@@ -188,18 +194,21 @@ func (qc *QueryConfig) toBQ() (*bq.JobConfiguration, error) {
 func bqToQueryConfig(q *bq.JobConfiguration, c *Client) (*QueryConfig, error) {
 	qq := q.Query
 	qc := &QueryConfig{
-		Labels:            q.Labels,
-		DryRun:            q.DryRun,
-		Q:                 qq.Query,
-		CreateDisposition: TableCreateDisposition(qq.CreateDisposition),
-		WriteDisposition:  TableWriteDisposition(qq.WriteDisposition),
-		AllowLargeResults: qq.AllowLargeResults,
-		Priority:          QueryPriority(qq.Priority),
-		MaxBytesBilled:    qq.MaximumBytesBilled,
-		UseLegacySQL:      qq.UseLegacySql,
-		UseStandardSQL:    !qq.UseLegacySql,
-		TimePartitioning:  bqToTimePartitioning(qq.TimePartitioning),
+		Labels:                      q.Labels,
+		DryRun:                      q.DryRun,
+		Q:                           qq.Query,
+		CreateDisposition:           TableCreateDisposition(qq.CreateDisposition),
+		WriteDisposition:            TableWriteDisposition(qq.WriteDisposition),
+		AllowLargeResults:           qq.AllowLargeResults,
+		Priority:                    QueryPriority(qq.Priority),
+		MaxBytesBilled:              qq.MaximumBytesBilled,
+		UseLegacySQL:                qq.UseLegacySql == nil || *qq.UseLegacySql,
+		TimePartitioning:            bqToTimePartitioning(qq.TimePartitioning),
+		DestinationEncryptionConfig: bqToEncryptionConfig(qq.DestinationEncryptionConfiguration),
+		SchemaUpdateOptions:         qq.SchemaUpdateOptions,
 	}
+	qc.UseStandardSQL = !qc.UseLegacySQL
+
 	if len(qq.TableDefinitions) > 0 {
 		qc.TableDefinitions = make(map[string]ExternalData)
 	}
@@ -26,6 +26,7 @@ import (
 )

 func defaultQueryJob() *bq.Job {
+	pfalse := false
 	return &bq.Job{
 		JobReference: &bq.JobReference{JobId: "RANDOM", ProjectId: "client-project-id"},
 		Configuration: &bq.JobConfiguration{
@@ -40,8 +41,7 @@ func defaultQueryJob() *bq.Job {
 					ProjectId: "def-project-id",
 					DatasetId: "def-dataset-id",
 				},
-				UseLegacySql:    false,
-				ForceSendFields: []string{"UseLegacySql"},
+				UseLegacySql: &pfalse,
 			},
 		},
 	}
@@ -271,7 +271,8 @@ func TestQuery(t *testing.T) {
 			},
 			want: func() *bq.Job {
 				j := defaultQueryJob()
-				j.Configuration.Query.UseLegacySql = true
+				ptrue := true
+				j.Configuration.Query.UseLegacySql = &ptrue
 				j.Configuration.Query.ForceSendFields = nil
 				return j
 			}(),
@@ -351,9 +352,12 @@ func TestConfiguringQuery(t *testing.T) {
 	query.DefaultDatasetID = "def-dataset-id"
 	query.TimePartitioning = &TimePartitioning{Expiration: 1234 * time.Second, Field: "f"}
 	query.DestinationEncryptionConfig = &EncryptionConfig{KMSKeyName: "keyName"}
+	query.SchemaUpdateOptions = []string{"ALLOW_FIELD_ADDITION"}
+
 	// Note: Other configuration fields are tested in other tests above.
 	// A lot of that can be consolidated once Client.Copy is gone.

+	pfalse := false
 	want := &bq.Job{
 		Configuration: &bq.JobConfiguration{
 			Query: &bq.JobConfigurationQuery{
@@ -362,10 +366,10 @@ func TestConfiguringQuery(t *testing.T) {
 					ProjectId: "def-project-id",
 					DatasetId: "def-dataset-id",
 				},
-				UseLegacySql:                       false,
+				UseLegacySql:                       &pfalse,
 				TimePartitioning:                   &bq.TimePartitioning{ExpirationMs: 1234000, Field: "f", Type: "DAY"},
 				DestinationEncryptionConfiguration: &bq.EncryptionConfiguration{KmsKeyName: "keyName"},
-				ForceSendFields:                    []string{"UseLegacySql"},
+				SchemaUpdateOptions:                []string{"ALLOW_FIELD_ADDITION"},
 			},
 		},
 		JobReference: &bq.JobReference{