f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vendor: update all dependencies

Browse Source
This commit is contained in:
Nick Craig-Wood
2017-07-23 08:51:42 +01:00
parent 0b6fba34a3
commit eb87cf6f12
2008 changed files with 352617 additions and 1004734 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/cloud.google.com/go/bigquery/bigquery.go
Generated Vendored
+9
View File
@@ -74,3 +74,12 @@ func NewClient(ctx context.Context, projectID string, opts ...option.ClientOptio
func (c *Client) Close() error {
return nil
}
func (c *Client) insertJob(ctx context.Context, conf *insertJobConf) (*Job, error) {
job, err := c.service.insertJob(ctx, c.projectID, conf)
if err != nil {
return nil, err
}
job.c = c
return job, nil
}
vendor/cloud.google.com/go/bigquery/copy.go
Generated Vendored
+1 -1
View File
@@ -70,5 +70,5 @@ func (c *Copier) Run(ctx context.Context) (*Job, error) {
}
job := &bq.Job{Configuration: &bq.JobConfiguration{Copy: conf}}
setJobRef(job, c.JobID, c.c.projectID)
return c.c.service.insertJob(ctx, c.c.projectID, &insertJobConf{job: job})
return c.c.insertJob(ctx, &insertJobConf{job: job})
}
vendor/cloud.google.com/go/bigquery/extract.go
Generated Vendored
+1 -1
View File
@@ -72,5 +72,5 @@ func (e *Extractor) Run(ctx context.Context) (*Job, error) {
conf.PrintHeader = &f
}
return e.c.service.insertJob(ctx, e.c.projectID, &insertJobConf{job: job})
return e.c.insertJob(ctx, &insertJobConf{job: job})
}
vendor/cloud.google.com/go/bigquery/integration_test.go
Generated Vendored
+349 -65
View File
@@ -15,6 +15,7 @@
package bigquery
import (
"errors"
"flag"
"fmt"
"log"
@@ -26,7 +27,10 @@ import (
"testing"
"time"
gax "github.com/googleapis/gax-go"
"cloud.google.com/go/civil"
"cloud.google.com/go/internal"
"cloud.google.com/go/internal/pretty"
"cloud.google.com/go/internal/testutil"
"golang.org/x/net/context"
@@ -42,7 +46,7 @@ var (
{Name: "name", Type: StringFieldType},
{Name: "num", Type: IntegerFieldType},
}
fiveMinutesFromNow time.Time
testTableExpiration time.Time
)
func TestMain(m *testing.M) {
@@ -79,6 +83,7 @@ func initIntegrationTest() {
if err := dataset.Create(ctx); err != nil && !hasStatusCode(err, http.StatusConflict) { // AlreadyExists is 409
log.Fatalf("creating dataset: %v", err)
}
testTableExpiration = time.Now().Add(10 * time.Minute).Round(time.Second)
}
func TestIntegration_Create(t *testing.T) {
@@ -136,7 +141,7 @@ func TestIntegration_TableMetadata(t *testing.T) {
if got, want := md.Type, RegularTable; got != want {
t.Errorf("metadata.Type: got %v, want %v", got, want)
}
if got, want := md.ExpirationTime, fiveMinutesFromNow; !got.Equal(want) {
if got, want := md.ExpirationTime, testTableExpiration; !got.Equal(want) {
t.Errorf("metadata.Type: got %v, want %v", got, want)
}
@@ -221,32 +226,37 @@ func TestIntegration_Tables(t *testing.T) {
ctx := context.Background()
table := newTable(t, schema)
defer table.Delete(ctx)
wantName := table.FullyQualifiedName()
// Iterate over tables in the dataset.
it := dataset.Tables(ctx)
var tables []*Table
for {
tbl, err := it.Next()
if err == iterator.Done {
break
// This test is flaky due to eventual consistency.
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
err := internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
// Iterate over tables in the dataset.
it := dataset.Tables(ctx)
var tableNames []string
for {
tbl, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
return false, err
}
tableNames = append(tableNames, tbl.FullyQualifiedName())
}
if err != nil {
t.Fatal(err)
// Other tests may be running with this dataset, so there might be more
// than just our table in the list. So don't try for an exact match; just
// make sure that our table is there somewhere.
for _, tn := range tableNames {
if tn == wantName {
return true, nil
}
}
tables = append(tables, tbl)
}
// Other tests may be running with this dataset, so there might be more
// than just our table in the list. So don't try for an exact match; just
// make sure that our table is there somewhere.
found := false
for _, tbl := range tables {
if reflect.DeepEqual(tbl, table) {
found = true
break
}
}
if !found {
t.Errorf("Tables: got %v\nshould see %v in the list", pretty.Value(tables), pretty.Value(table))
return false, fmt.Errorf("got %v\nwant %s in the list", tableNames, wantName)
})
if err != nil {
t.Fatal(err)
}
}
@@ -302,6 +312,7 @@ func TestIntegration_UploadAndRead(t *testing.T) {
if err != nil {
t.Fatal(err)
}
job2, err := client.JobFromID(ctx, job1.ID())
if err != nil {
t.Fatal(err)
@@ -312,6 +323,18 @@ func TestIntegration_UploadAndRead(t *testing.T) {
}
checkRead(t, "job.Read", rit, wantRows)
// Get statistics.
jobStatus, err := job2.Status(ctx)
if err != nil {
t.Fatal(err)
}
if jobStatus.Statistics == nil {
t.Fatal("jobStatus missing statistics")
}
if _, ok := jobStatus.Statistics.Details.(*QueryStatistics); !ok {
t.Errorf("expected QueryStatistics, got %T", jobStatus.Statistics.Details)
}
// Test reading directly into a []Value.
valueLists, err := readAll(table.Read(ctx))
if err != nil {
@@ -348,20 +371,39 @@ func TestIntegration_UploadAndRead(t *testing.T) {
}
}
type SubSubTestStruct struct {
Integer int64
}
type SubTestStruct struct {
String string
Record SubSubTestStruct
RecordArray []SubSubTestStruct
}
type TestStruct struct {
Name string
Nums []int
Sub Sub
Subs []*Sub
}
Name string
Bytes []byte
Integer int64
Float float64
Boolean bool
Timestamp time.Time
Date civil.Date
Time civil.Time
DateTime civil.DateTime
type Sub struct {
B bool
SubSub SubSub
SubSubs []*SubSub
}
StringArray []string
IntegerArray []int64
FloatArray []float64
BooleanArray []bool
TimestampArray []time.Time
DateArray []civil.Date
TimeArray []civil.Time
DateTimeArray []civil.DateTime
type SubSub struct{ Count int }
Record SubTestStruct
RecordArray []SubTestStruct
}
func TestIntegration_UploadAndReadStructs(t *testing.T) {
if client == nil {
@@ -376,16 +418,61 @@ func TestIntegration_UploadAndReadStructs(t *testing.T) {
table := newTable(t, schema)
defer table.Delete(ctx)
d := civil.Date{2016, 3, 20}
tm := civil.Time{15, 4, 5, 0}
ts := time.Date(2016, 3, 20, 15, 4, 5, 0, time.UTC)
dtm := civil.DateTime{d, tm}
d2 := civil.Date{1994, 5, 15}
tm2 := civil.Time{1, 2, 4, 0}
ts2 := time.Date(1994, 5, 15, 1, 2, 4, 0, time.UTC)
dtm2 := civil.DateTime{d2, tm2}
// Populate the table.
upl := table.Uploader()
want := []*TestStruct{
{Name: "a", Nums: []int{1, 2}, Sub: Sub{B: true}, Subs: []*Sub{{B: false}, {B: true}}},
{Name: "b", Nums: []int{1}, Subs: []*Sub{{B: false}, {B: false}, {B: true}}},
{Name: "c", Sub: Sub{B: true}},
{
Name: "d",
Sub: Sub{SubSub: SubSub{12}, SubSubs: []*SubSub{{1}, {2}, {3}}},
Subs: []*Sub{{B: false, SubSub: SubSub{4}}, {B: true, SubSubs: []*SubSub{{5}, {6}}}},
"a",
[]byte("byte"),
42,
3.14,
true,
ts,
d,
tm,
dtm,
[]string{"a", "b"},
[]int64{1, 2},
[]float64{1, 1.41},
[]bool{true, false},
[]time.Time{ts, ts2},
[]civil.Date{d, d2},
[]civil.Time{tm, tm2},
[]civil.DateTime{dtm, dtm2},
SubTestStruct{
"string",
SubSubTestStruct{24},
[]SubSubTestStruct{{1}, {2}},
},
[]SubTestStruct{
{String: "empty"},
{
"full",
SubSubTestStruct{1},
[]SubSubTestStruct{{1}, {2}},
},
},
},
{
Name: "b",
Bytes: []byte("byte2"),
Integer: 24,
Float: 4.13,
Boolean: false,
Timestamp: ts,
Date: d,
Time: tm,
DateTime: dtm,
},
}
var savers []*StructSaver
@@ -573,28 +660,38 @@ func TestIntegration_DML(t *testing.T) {
t.Skip("Integration tests skipped")
}
ctx := context.Background()
table := newTable(t, schema)
defer table.Delete(ctx)
// Retry insert; sometimes it fails with INTERNAL.
err := internal.Retry(ctx, gax.Backoff{}, func() (bool, error) {
table := newTable(t, schema)
defer table.Delete(ctx)
// Use DML to insert.
wantRows := [][]Value{
[]Value{"a", int64(0)},
[]Value{"b", int64(1)},
[]Value{"c", int64(2)},
}
query := fmt.Sprintf("INSERT bigquery_integration_test.%s (name, num) "+
"VALUES ('a', 0), ('b', 1), ('c', 2)",
table.TableID)
q := client.Query(query)
q.UseStandardSQL = true // necessary for DML
job, err := q.Run(ctx)
// Use DML to insert.
wantRows := [][]Value{
[]Value{"a", int64(0)},
[]Value{"b", int64(1)},
[]Value{"c", int64(2)},
}
query := fmt.Sprintf("INSERT bigquery_integration_test.%s (name, num) "+
"VALUES ('a', 0), ('b', 1), ('c', 2)",
table.TableID)
q := client.Query(query)
q.UseStandardSQL = true // necessary for DML
job, err := q.Run(ctx)
if err != nil {
return false, err
}
if err := wait(ctx, job); err != nil {
return false, err
}
if msg, ok := compareRead(table.Read(ctx), wantRows); !ok {
// Stop on read error, because that has never been flaky.
return true, errors.New(msg)
}
return true, nil
})
if err != nil {
t.Fatal(err)
}
if err := wait(ctx, job); err != nil {
t.Fatal(err)
}
checkRead(t, "INSERT", table.Read(ctx), wantRows)
}
func TestIntegration_TimeTypes(t *testing.T) {
@@ -645,12 +742,183 @@ func TestIntegration_TimeTypes(t *testing.T) {
checkRead(t, "TimeTypes", table.Read(ctx), wantRows)
}
func TestIntegration_StandardQuery(t *testing.T) {
if client == nil {
t.Skip("Integration tests skipped")
}
ctx := context.Background()
d := civil.Date{2016, 3, 20}
tm := civil.Time{15, 04, 05, 0}
ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
dtm := ts.Format("2006-01-02 15:04:05")
// Constructs Value slices made up of int64s.
ints := func(args ...int) []Value {
vals := make([]Value, len(args))
for i, arg := range args {
vals[i] = int64(arg)
}
return vals
}
testCases := []struct {
query string
wantRow []Value
}{
{"SELECT 1", ints(1)},
{"SELECT 1.3", []Value{1.3}},
{"SELECT TRUE", []Value{true}},
{"SELECT 'ABC'", []Value{"ABC"}},
{"SELECT CAST('foo' AS BYTES)", []Value{[]byte("foo")}},
{fmt.Sprintf("SELECT TIMESTAMP '%s'", dtm), []Value{ts}},
{fmt.Sprintf("SELECT [TIMESTAMP '%s', TIMESTAMP '%s']", dtm, dtm), []Value{[]Value{ts, ts}}},
{fmt.Sprintf("SELECT ('hello', TIMESTAMP '%s')", dtm), []Value{[]Value{"hello", ts}}},
{fmt.Sprintf("SELECT DATETIME(TIMESTAMP '%s')", dtm), []Value{civil.DateTime{d, tm}}},
{fmt.Sprintf("SELECT DATE(TIMESTAMP '%s')", dtm), []Value{d}},
{fmt.Sprintf("SELECT TIME(TIMESTAMP '%s')", dtm), []Value{tm}},
{"SELECT (1, 2)", []Value{ints(1, 2)}},
{"SELECT [1, 2, 3]", []Value{ints(1, 2, 3)}},
{"SELECT ([1, 2], 3, [4, 5])", []Value{[]Value{ints(1, 2), int64(3), ints(4, 5)}}},
{"SELECT [(1, 2, 3), (4, 5, 6)]", []Value{[]Value{ints(1, 2, 3), ints(4, 5, 6)}}},
{"SELECT [([1, 2, 3], 4), ([5, 6], 7)]", []Value{[]Value{[]Value{ints(1, 2, 3), int64(4)}, []Value{ints(5, 6), int64(7)}}}},
{"SELECT ARRAY(SELECT STRUCT([1, 2]))", []Value{[]Value{[]Value{ints(1, 2)}}}},
}
for _, c := range testCases {
q := client.Query(c.query)
q.UseStandardSQL = true
it, err := q.Read(ctx)
if err != nil {
t.Fatal(err)
}
checkRead(t, "StandardQuery", it, [][]Value{c.wantRow})
}
}
func TestIntegration_LegacyQuery(t *testing.T) {
if client == nil {
t.Skip("Integration tests skipped")
}
ctx := context.Background()
ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
dtm := ts.Format("2006-01-02 15:04:05")
testCases := []struct {
query string
wantRow []Value
}{
{"SELECT 1", []Value{int64(1)}},
{"SELECT 1.3", []Value{1.3}},
{"SELECT TRUE", []Value{true}},
{"SELECT 'ABC'", []Value{"ABC"}},
{"SELECT CAST('foo' AS BYTES)", []Value{[]byte("foo")}},
{fmt.Sprintf("SELECT TIMESTAMP('%s')", dtm), []Value{ts}},
{fmt.Sprintf("SELECT DATE(TIMESTAMP('%s'))", dtm), []Value{"2016-03-20"}},
{fmt.Sprintf("SELECT TIME(TIMESTAMP('%s'))", dtm), []Value{"15:04:05"}},
}
for _, c := range testCases {
q := client.Query(c.query)
it, err := q.Read(ctx)
if err != nil {
t.Fatal(err)
}
checkRead(t, "LegacyQuery", it, [][]Value{c.wantRow})
}
}
func TestIntegration_QueryParameters(t *testing.T) {
if client == nil {
t.Skip("Integration tests skipped")
}
ctx := context.Background()
d := civil.Date{2016, 3, 20}
tm := civil.Time{15, 04, 05, 0}
dtm := civil.DateTime{d, tm}
ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
type ss struct {
String string
}
type s struct {
Timestamp time.Time
StringArray []string
SubStruct ss
SubStructArray []ss
}
testCases := []struct {
query string
parameters []QueryParameter
wantRow []Value
}{
{"SELECT @val", []QueryParameter{{"val", 1}}, []Value{int64(1)}},
{"SELECT @val", []QueryParameter{{"val", 1.3}}, []Value{1.3}},
{"SELECT @val", []QueryParameter{{"val", true}}, []Value{true}},
{"SELECT @val", []QueryParameter{{"val", "ABC"}}, []Value{"ABC"}},
{"SELECT @val", []QueryParameter{{"val", []byte("foo")}}, []Value{[]byte("foo")}},
{"SELECT @val", []QueryParameter{{"val", ts}}, []Value{ts}},
{"SELECT @val", []QueryParameter{{"val", []time.Time{ts, ts}}}, []Value{[]Value{ts, ts}}},
{"SELECT @val", []QueryParameter{{"val", dtm}}, []Value{dtm}},
{"SELECT @val", []QueryParameter{{"val", d}}, []Value{d}},
{"SELECT @val", []QueryParameter{{"val", tm}}, []Value{tm}},
{"SELECT @val", []QueryParameter{{"val", s{ts, []string{"a", "b"}, ss{"c"}, []ss{{"d"}, {"e"}}}}},
[]Value{[]Value{ts, []Value{"a", "b"}, []Value{"c"}, []Value{[]Value{"d"}, []Value{"e"}}}}},
{"SELECT @val.Timestamp, @val.SubStruct.String", []QueryParameter{{"val", s{Timestamp: ts, SubStruct: ss{"a"}}}}, []Value{ts, "a"}},
}
for _, c := range testCases {
q := client.Query(c.query)
q.Parameters = c.parameters
it, err := q.Read(ctx)
if err != nil {
t.Fatal(err)
}
checkRead(t, "QueryParameters", it, [][]Value{c.wantRow})
}
}
func TestIntegration_ReadNullIntoStruct(t *testing.T) {
// Reading a null into a struct field should return an error (not panic).
if client == nil {
t.Skip("Integration tests skipped")
}
ctx := context.Background()
table := newTable(t, schema)
defer table.Delete(ctx)
upl := table.Uploader()
row := &ValuesSaver{
Schema: schema,
Row: []Value{"name", nil},
}
if err := upl.Put(ctx, []*ValuesSaver{row}); err != nil {
t.Fatal(putError(err))
}
if err := waitForRow(ctx, table); err != nil {
t.Fatal(err)
}
q := client.Query(fmt.Sprintf("select name, num from %s", table.TableID))
q.DefaultProjectID = dataset.ProjectID
q.DefaultDatasetID = dataset.DatasetID
it, err := q.Read(ctx)
if err != nil {
t.Fatal(err)
}
type S struct{ Num int64 }
var s S
if err := it.Next(&s); err == nil {
t.Fatal("got nil, want error")
}
}
// Creates a new, temporary table with a unique name and the given schema.
func newTable(t *testing.T, s Schema) *Table {
fiveMinutesFromNow = time.Now().Add(5 * time.Minute).Round(time.Second)
name := fmt.Sprintf("t%d", time.Now().UnixNano())
table := dataset.Table(name)
err := table.Create(context.Background(), s, TableExpiration(fiveMinutesFromNow))
err := table.Create(context.Background(), s, TableExpiration(testTableExpiration))
if err != nil {
t.Fatal(err)
}
@@ -658,21 +926,28 @@ func newTable(t *testing.T, s Schema) *Table {
}
func checkRead(t *testing.T, msg string, it *RowIterator, want [][]Value) {
if msg2, ok := compareRead(it, want); !ok {
t.Errorf("%s: %s", msg, msg2)
}
}
func compareRead(it *RowIterator, want [][]Value) (msg string, ok bool) {
got, err := readAll(it)
if err != nil {
t.Fatalf("%s: %v", msg, err)
return err.Error(), false
}
if len(got) != len(want) {
t.Errorf("%s: got %d rows, want %d", msg, len(got), len(want))
return fmt.Sprintf("got %d rows, want %d", len(got), len(want)), false
}
sort.Sort(byCol0(got))
for i, r := range got {
gotRow := []Value(r)
wantRow := want[i]
if !reflect.DeepEqual(gotRow, wantRow) {
t.Errorf("%s #%d: got %v, want %v", msg, i, gotRow, wantRow)
return fmt.Sprintf("#%d: got %v, want %v", i, gotRow, wantRow), false
}
}
return "", true
}
func readAll(it *RowIterator) ([][]Value, error) {
@@ -721,6 +996,15 @@ func wait(ctx context.Context, job *Job) error {
if status.Err() != nil {
return fmt.Errorf("job status error: %#v", status.Err())
}
if status.Statistics == nil {
return errors.New("nil Statistics")
}
if status.Statistics.EndTime.IsZero() {
return errors.New("EndTime is zero")
}
if status.Statistics.Details == nil {
return errors.New("nil Statistics.Details")
}
return nil
}
vendor/cloud.google.com/go/bigquery/iterator.go
Generated Vendored
+4 -8
View File
@@ -92,6 +92,9 @@ type RowIterator struct {
// A repeated field corresponds to a slice or array of the element type.
// A RECORD type (nested schema) corresponds to a nested struct or struct pointer.
// All calls to Next on the same iterator must use the same struct type.
//
// It is an error to attempt to read a BigQuery NULL value into a struct field.
// If your table contains NULLs, use a *[]Value or *map[string]Value.
func (it *RowIterator) Next(dst interface{}) error {
var vl ValueLoader
switch dst := dst.(type) {
@@ -141,14 +144,7 @@ func (it *RowIterator) fetch(pageSize int, pageToken string) (string, error) {
pc.startIndex = it.StartIndex
}
it.pf.setPaging(pc)
var res *readDataResult
var err error
for {
res, err = it.pf.fetch(it.ctx, it.service, pageToken)
if err != errIncompleteJob {
break
}
}
res, err := it.pf.fetch(it.ctx, it.service, pageToken)
if err != nil {
return "", err
}
vendor/cloud.google.com/go/bigquery/iterator_test.go
Generated Vendored
-50
View File
@@ -284,56 +284,6 @@ func consumeRowIterator(it *RowIterator) ([][]Value, Schema, error) {
}
}
type delayedPageFetcher struct {
pageFetcherStub
delayCount int
}
func (pf *delayedPageFetcher) fetch(ctx context.Context, s service, token string) (*readDataResult, error) {
if pf.delayCount > 0 {
pf.delayCount--
return nil, errIncompleteJob
}
return pf.pageFetcherStub.fetch(ctx, s, token)
}
func TestIterateIncompleteJob(t *testing.T) {
want := [][]Value{{1, 2}, {11, 12}, {101, 102}, {111, 112}}
pf := pageFetcherStub{
fetchResponses: map[string]fetchResponse{
"": {
result: &readDataResult{
pageToken: "a",
rows: [][]Value{{1, 2}, {11, 12}},
},
},
"a": {
result: &readDataResult{
pageToken: "",
rows: [][]Value{{101, 102}, {111, 112}},
},
},
},
}
dpf := &delayedPageFetcher{
pageFetcherStub: pf,
delayCount: 1,
}
it := newRowIterator(context.Background(), nil, dpf)
values, _, err := consumeRowIterator(it)
if err != nil {
t.Fatal(err)
}
if (len(values) != 0 || len(want) != 0) && !reflect.DeepEqual(values, want) {
t.Errorf("values: got:\n%v\nwant:\n%v", values, want)
}
if dpf.delayCount != 0 {
t.Errorf("delayCount: got: %v, want: 0", dpf.delayCount)
}
}
func TestNextDuringErrorState(t *testing.T) {
pf := &pageFetcherStub{
fetchResponses: map[string]fetchResponse{
vendor/cloud.google.com/go/bigquery/job.go
Generated Vendored
+206 -13
View File
@@ -15,6 +15,9 @@
package bigquery
import (
"errors"
"time"
"cloud.google.com/go/internal"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
@@ -23,28 +26,24 @@ import (
// A Job represents an operation which has been submitted to BigQuery for processing.
type Job struct {
service service
c *Client
projectID string
jobID string
isQuery bool
isQuery bool
destinationTable *bq.TableReference // table to read query results from
}
// JobFromID creates a Job which refers to an existing BigQuery job. The job
// need not have been created by this package. For example, the job may have
// been created in the BigQuery console.
func (c *Client) JobFromID(ctx context.Context, id string) (*Job, error) {
jobType, err := c.service.getJobType(ctx, c.projectID, id)
job, err := c.service.getJob(ctx, c.projectID, id)
if err != nil {
return nil, err
}
return &Job{
service: c.service,
projectID: c.projectID,
jobID: id,
isQuery: jobType == queryJobType,
}, nil
job.c = c
return job, nil
}
func (j *Job) ID() string {
@@ -69,6 +68,9 @@ type JobStatus struct {
// All errors encountered during the running of the job.
// Not all Errors are fatal, so errors here do not necessarily mean that the job has completed or was unsuccessful.
Errors []*Error
// Statistics about the job.
Statistics *JobStatistics
}
// setJobRef initializes job's JobReference if given a non-empty jobID.
@@ -99,22 +101,47 @@ func (s *JobStatus) Err() error {
// Status returns the current status of the job. It fails if the Status could not be determined.
func (j *Job) Status(ctx context.Context) (*JobStatus, error) {
return j.service.jobStatus(ctx, j.projectID, j.jobID)
js, err := j.c.service.jobStatus(ctx, j.projectID, j.jobID)
if err != nil {
return nil, err
}
// Fill in the client field of Tables in the statistics.
if js.Statistics != nil {
if qs, ok := js.Statistics.Details.(*QueryStatistics); ok {
for _, t := range qs.ReferencedTables {
t.c = j.c
}
}
}
return js, nil
}
// Cancel requests that a job be cancelled. This method returns without waiting for
// cancellation to take effect. To check whether the job has terminated, use Job.Status.
// Cancelled jobs may still incur costs.
func (j *Job) Cancel(ctx context.Context) error {
return j.service.jobCancel(ctx, j.projectID, j.jobID)
return j.c.service.jobCancel(ctx, j.projectID, j.jobID)
}
// Wait blocks until the job or th context is done. It returns the final status
// Wait blocks until the job or the context is done. It returns the final status
// of the job.
// If an error occurs while retrieving the status, Wait returns that error. But
// Wait returns nil if the status was retrieved successfully, even if
// status.Err() != nil. So callers must check both errors. See the example.
func (j *Job) Wait(ctx context.Context) (*JobStatus, error) {
if j.isQuery {
// We can avoid polling for query jobs.
if _, err := j.c.service.waitForQuery(ctx, j.projectID, j.jobID); err != nil {
return nil, err
}
// Note: extra RPC even if you just want to wait for the query to finish.
js, err := j.Status(ctx)
if err != nil {
return nil, err
}
return js, nil
}
// Non-query jobs must poll.
var js *JobStatus
err := internal.Retry(ctx, gax.Backoff{}, func() (stop bool, err error) {
js, err = j.Status(ctx)
@@ -131,3 +158,169 @@ func (j *Job) Wait(ctx context.Context) (*JobStatus, error) {
}
return js, nil
}
// Read fetches the results of a query job.
// If j is not a query job, Read returns an error.
func (j *Job) Read(ctx context.Context) (*RowIterator, error) {
if !j.isQuery {
return nil, errors.New("bigquery: cannot read from a non-query job")
}
var projectID string
if j.destinationTable != nil {
projectID = j.destinationTable.ProjectId
} else {
projectID = j.c.projectID
}
schema, err := j.c.service.waitForQuery(ctx, projectID, j.jobID)
if err != nil {
return nil, err
}
// The destination table should only be nil if there was a query error.
if j.destinationTable == nil {
return nil, errors.New("bigquery: query job missing destination table")
}
return newRowIterator(ctx, j.c.service, &readTableConf{
projectID: j.destinationTable.ProjectId,
datasetID: j.destinationTable.DatasetId,
tableID: j.destinationTable.TableId,
schema: schema,
}), nil
}
// JobStatistics contains statistics about a job.
type JobStatistics struct {
CreationTime time.Time
StartTime time.Time
EndTime time.Time
TotalBytesProcessed int64
Details Statistics
}
// Statistics is one of ExtractStatistics, LoadStatistics or QueryStatistics.
type Statistics interface {
implementsStatistics()
}
// ExtractStatistics contains statistics about an extract job.
type ExtractStatistics struct {
// The number of files per destination URI or URI pattern specified in the
// extract configuration. These values will be in the same order as the
// URIs specified in the 'destinationUris' field.
DestinationURIFileCounts []int64
}
// LoadStatistics contains statistics about a load job.
type LoadStatistics struct {
// The number of bytes of source data in a load job.
InputFileBytes int64
// The number of source files in a load job.
InputFiles int64
// Size of the loaded data in bytes. Note that while a load job is in the
// running state, this value may change.
OutputBytes int64
// The number of rows imported in a load job. Note that while an import job is
// in the running state, this value may change.
OutputRows int64
}
// QueryStatistics contains statistics about a query job.
type QueryStatistics struct {
// Billing tier for the job.
BillingTier int64
// Whether the query result was fetched from the query cache.
CacheHit bool
// The type of query statement, if valid.
StatementType string
// Total bytes billed for the job.
TotalBytesBilled int64
// Total bytes processed for the job.
TotalBytesProcessed int64
// Describes execution plan for the query.
QueryPlan []*ExplainQueryStage
// The number of rows affected by a DML statement. Present only for DML
// statements INSERT, UPDATE or DELETE.
NumDMLAffectedRows int64
// ReferencedTables: [Output-only, Experimental] Referenced tables for
// the job. Queries that reference more than 50 tables will not have a
// complete list.
ReferencedTables []*Table
// The schema of the results. Present only for successful dry run of
// non-legacy SQL queries.
Schema Schema
// Standard SQL: list of undeclared query parameter names detected during a
// dry run validation.
UndeclaredQueryParameterNames []string
}
// ExplainQueryStage describes one stage of a query.
type ExplainQueryStage struct {
// Relative amount of the total time the average shard spent on CPU-bound tasks.
ComputeRatioAvg float64
// Relative amount of the total time the slowest shard spent on CPU-bound tasks.
ComputeRatioMax float64
// Unique ID for stage within plan.
ID int64
// Human-readable name for stage.
Name string
// Relative amount of the total time the average shard spent reading input.
ReadRatioAvg float64
// Relative amount of the total time the slowest shard spent reading input.
ReadRatioMax float64
// Number of records read into the stage.
RecordsRead int64
// Number of records written by the stage.
RecordsWritten int64
// Current status for the stage.
Status string
// List of operations within the stage in dependency order (approximately
// chronological).
Steps []*ExplainQueryStep
// Relative amount of the total time the average shard spent waiting to be scheduled.
WaitRatioAvg float64
// Relative amount of the total time the slowest shard spent waiting to be scheduled.
WaitRatioMax float64
// Relative amount of the total time the average shard spent on writing output.
WriteRatioAvg float64
// Relative amount of the total time the slowest shard spent on writing output.
WriteRatioMax float64
}
// ExplainQueryStep describes one step of a query stage.
type ExplainQueryStep struct {
// Machine-readable operation type.
Kind string
// Human-readable stage descriptions.
Substeps []string
}
func (*ExtractStatistics) implementsStatistics() {}
func (*LoadStatistics) implementsStatistics() {}
func (*QueryStatistics) implementsStatistics() {}
vendor/cloud.google.com/go/bigquery/load.go
Generated Vendored
+1 -1
View File
@@ -82,5 +82,5 @@ func (l *Loader) Run(ctx context.Context) (*Job, error) {
job.Configuration.Load.DestinationTable = l.Dst.tableRefProto()
return l.c.service.insertJob(ctx, l.c.projectID, conf)
return l.c.insertJob(ctx, conf)
}
vendor/cloud.google.com/go/bigquery/query.go
Generated Vendored
+12 -2
View File
@@ -46,7 +46,7 @@ type QueryConfig struct {
CreateDisposition TableCreateDisposition
// WriteDisposition specifies how existing data in the destination table is treated.
// The default is WriteAppend.
// The default is WriteEmpty.
WriteDisposition TableWriteDisposition
// DisableQueryCache prevents results being fetched from the query cache.
@@ -132,7 +132,7 @@ func (q *Query) Run(ctx context.Context) (*Job, error) {
if err := q.QueryConfig.populateJobQueryConfig(job.Configuration.Query); err != nil {
return nil, err
}
j, err := q.client.service.insertJob(ctx, q.client.projectID, &insertJobConf{job: job})
j, err := q.client.insertJob(ctx, &insertJobConf{job: job})
if err != nil {
return nil, err
}
@@ -194,3 +194,13 @@ func (q *QueryConfig) populateJobQueryConfig(conf *bq.JobConfigurationQuery) err
}
return nil
}
// Read submits a query for execution and returns the results via a RowIterator.
// It is a shorthand for Query.Run followed by Job.Read.
func (q *Query) Read(ctx context.Context) (*RowIterator, error) {
job, err := q.Run(ctx)
if err != nil {
return nil, err
}
return job.Read(ctx)
}
vendor/cloud.google.com/go/bigquery/read.go
Generated Vendored
-64
View File
@@ -1,64 +0,0 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bigquery
import (
"errors"
"golang.org/x/net/context"
)
func (conf *readTableConf) fetch(ctx context.Context, s service, token string) (*readDataResult, error) {
return s.readTabledata(ctx, conf, token)
}
func (conf *readTableConf) setPaging(pc *pagingConf) { conf.paging = *pc }
// Read fetches the contents of the table.
func (t *Table) Read(ctx context.Context) *RowIterator {
return newRowIterator(ctx, t.c.service, &readTableConf{
projectID: t.ProjectID,
datasetID: t.DatasetID,
tableID: t.TableID,
})
}
func (conf *readQueryConf) fetch(ctx context.Context, s service, token string) (*readDataResult, error) {
return s.readQuery(ctx, conf, token)
}
func (conf *readQueryConf) setPaging(pc *pagingConf) { conf.paging = *pc }
// Read fetches the results of a query job.
// If j is not a query job, Read returns an error.
func (j *Job) Read(ctx context.Context) (*RowIterator, error) {
if !j.isQuery {
return nil, errors.New("Cannot read from a non-query job")
}
return newRowIterator(ctx, j.service, &readQueryConf{
projectID: j.projectID,
jobID: j.jobID,
}), nil
}
// Read submits a query for execution and returns the results via a RowIterator.
// It is a shorthand for Query.Run followed by Job.Read.
func (q *Query) Read(ctx context.Context) (*RowIterator, error) {
job, err := q.Run(ctx)
if err != nil {
return nil, err
}
return job.Read(ctx)
}
vendor/cloud.google.com/go/bigquery/read_test.go
Generated Vendored
+24 -67
View File
@@ -20,6 +20,7 @@ import (
"testing"
"golang.org/x/net/context"
bq "google.golang.org/api/bigquery/v2"
"google.golang.org/api/iterator"
)
@@ -28,11 +29,6 @@ type readTabledataArgs struct {
tok string
}
type readQueryArgs struct {
conf *readQueryConf
tok string
}
// readServiceStub services read requests by returning data from an in-memory list of values.
type readServiceStub struct {
// values and pageTokens are used as sources of data to return in response to calls to readTabledata or readQuery.
@@ -41,7 +37,6 @@ type readServiceStub struct {
// arguments are recorded for later inspection.
readTabledataCalls []readTabledataArgs
readQueryCalls []readQueryArgs
service
}
@@ -55,13 +50,13 @@ func (s *readServiceStub) readValues(tok string) *readDataResult {
return result
}
func (s *readServiceStub) readTabledata(ctx context.Context, conf *readTableConf, token string) (*readDataResult, error) {
s.readTabledataCalls = append(s.readTabledataCalls, readTabledataArgs{conf, token})
return s.readValues(token), nil
func (s *readServiceStub) waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error) {
return nil, nil
}
func (s *readServiceStub) readQuery(ctx context.Context, conf *readQueryConf, token string) (*readDataResult, error) {
s.readQueryCalls = append(s.readQueryCalls, readQueryArgs{conf, token})
func (s *readServiceStub) readTabledata(ctx context.Context, conf *readTableConf, token string) (*readDataResult, error) {
s.readTabledataCalls = append(s.readTabledataCalls, readTabledataArgs{conf, token})
return s.readValues(token), nil
}
@@ -77,8 +72,13 @@ func TestRead(t *testing.T) {
queryJob := &Job{
projectID: "project-id",
jobID: "job-id",
service: service,
c: c,
isQuery: true,
destinationTable: &bq.TableReference{
ProjectId: "project-id",
DatasetId: "dataset-id",
TableId: "table-id",
},
}
for _, readFunc := range []func() *RowIterator{
@@ -159,55 +159,6 @@ func TestNoMoreValues(t *testing.T) {
}
}
// delayedReadStub simulates reading results from a query that has not yet
// completed. Its readQuery method initially reports that the query job is not
// yet complete. Subsequently, it proxies the request through to another
// service stub.
type delayedReadStub struct {
numDelays int
readServiceStub
}
func (s *delayedReadStub) readQuery(ctx context.Context, conf *readQueryConf, token string) (*readDataResult, error) {
if s.numDelays > 0 {
s.numDelays--
return nil, errIncompleteJob
}
return s.readServiceStub.readQuery(ctx, conf, token)
}
// TestIncompleteJob tests that an Iterator which reads from a query job will block until the job is complete.
func TestIncompleteJob(t *testing.T) {
service := &delayedReadStub{
numDelays: 2,
readServiceStub: readServiceStub{
values: [][][]Value{{{1, 2}}},
},
}
queryJob := &Job{
projectID: "project-id",
jobID: "job-id",
service: service,
isQuery: true,
}
it, err := queryJob.Read(context.Background())
if err != nil {
t.Fatalf("err calling Read: %v", err)
}
var got []Value
want := []Value{1, 2}
if err := it.Next(&got); err != nil {
t.Fatalf("Next: got: %v: want: nil", err)
}
if service.numDelays != 0 {
t.Errorf("remaining numDelays : got: %v want:0", service.numDelays)
}
if !reflect.DeepEqual(got, want) {
t.Errorf("reading: got:\n%v\nwant:\n%v", got, want)
}
}
type errorReadService struct {
service
}
@@ -272,8 +223,13 @@ func TestReadQueryOptions(t *testing.T) {
queryJob := &Job{
projectID: "project-id",
jobID: "job-id",
service: s,
c: &Client{service: s},
isQuery: true,
destinationTable: &bq.TableReference{
ProjectId: "project-id",
DatasetId: "dataset-id",
TableId: "table-id",
},
}
it, err := queryJob.Read(context.Background())
if err != nil {
@@ -285,10 +241,11 @@ func TestReadQueryOptions(t *testing.T) {
t.Fatalf("Next: got: %v: want: nil", err)
}
want := []readQueryArgs{{
conf: &readQueryConf{
want := []readTabledataArgs{{
conf: &readTableConf{
projectID: "project-id",
jobID: "job-id",
datasetID: "dataset-id",
tableID: "table-id",
paging: pagingConf{
recordsPerRequest: 5,
setRecordsPerRequest: true,
@@ -297,7 +254,7 @@ func TestReadQueryOptions(t *testing.T) {
tok: "",
}}
if !reflect.DeepEqual(s.readQueryCalls, want) {
t.Errorf("reading: got:\n%v\nwant:\n%v", s.readQueryCalls, want)
if !reflect.DeepEqual(s.readTabledataCalls, want) {
t.Errorf("reading: got:\n%v\nwant:\n%v", s.readTabledataCalls, want)
}
}
vendor/cloud.google.com/go/bigquery/schema.go
Generated Vendored
+4 -1
View File
@@ -98,6 +98,9 @@ func convertTableFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema {
}
func convertTableSchema(ts *bq.TableSchema) Schema {
if ts == nil {
return nil
}
var s Schema
for _, f := range ts.Fields {
s = append(s, convertTableFieldSchema(f))
@@ -283,7 +286,7 @@ func (l *typeList) has(t reflect.Type) bool {
// hasRecursiveType reports whether t or any type inside t refers to itself, directly or indirectly,
// via exported fields. (Schema inference ignores unexported fields.)
func hasRecursiveType(t reflect.Type, seen *typeList) (bool, error) {
if t.Kind() == reflect.Ptr {
for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice || t.Kind() == reflect.Array {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
vendor/cloud.google.com/go/bigquery/schema_test.go
Generated Vendored
+4
View File
@@ -769,6 +769,9 @@ func TestHasRecursiveType(t *testing.T) {
A int
R *rec
}
recSlicePointer struct {
A []*recSlicePointer
}
)
for _, test := range []struct {
in interface{}
@@ -780,6 +783,7 @@ func TestHasRecursiveType(t *testing.T) {
{rec{}, true},
{recUnexported{}, false},
{hasRec{}, true},
{&recSlicePointer{}, true},
} {
got, err := hasRecursiveType(reflect.TypeOf(test.in), nil)
if err != nil {
vendor/cloud.google.com/go/bigquery/service.go
Generated Vendored
+212 -104
View File
@@ -15,7 +15,6 @@
package bigquery
import (
"errors"
"fmt"
"io"
"net/http"
@@ -23,6 +22,7 @@ import (
"time"
"cloud.google.com/go/internal"
"cloud.google.com/go/internal/version"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
@@ -37,7 +37,7 @@ import (
type service interface {
// Jobs
insertJob(ctx context.Context, projectId string, conf *insertJobConf) (*Job, error)
getJobType(ctx context.Context, projectId, jobID string) (jobType, error)
getJob(ctx context.Context, projectId, jobID string) (*Job, error)
jobCancel(ctx context.Context, projectId, jobID string) error
jobStatus(ctx context.Context, projectId, jobID string) (*JobStatus, error)
@@ -61,15 +61,19 @@ type service interface {
// Misc
// readQuery reads data resulting from a query job. If the job is
// incomplete, an errIncompleteJob is returned. readQuery may be called
// repeatedly to poll for job completion.
readQuery(ctx context.Context, conf *readQueryConf, pageToken string) (*readDataResult, error)
// Waits for a query to complete.
waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error)
// listDatasets returns a page of Datasets and a next page token. Note: the Datasets do not have their c field populated.
listDatasets(ctx context.Context, projectID string, maxResults int, pageToken string, all bool, filter string) ([]*Dataset, string, error)
}
var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo)
func setClientHeader(headers http.Header) {
headers.Set("x-goog-api-client", xGoogHeader)
}
type bigqueryService struct {
s *bq.Service
}
@@ -104,16 +108,43 @@ type insertJobConf struct {
media io.Reader
}
// Calls the Jobs.Insert RPC and returns a Job. Callers must set the returned Job's
// client.
func (s *bigqueryService) insertJob(ctx context.Context, projectID string, conf *insertJobConf) (*Job, error) {
call := s.s.Jobs.Insert(projectID, conf.job).Context(ctx)
setClientHeader(call.Header())
if conf.media != nil {
call.Media(conf.media)
}
res, err := call.Do()
var res *bq.Job
var err error
invoke := func() error {
res, err = call.Do()
return err
}
// A job with a client-generated ID can be retried; the presence of the
// ID makes the insert operation idempotent.
// We don't retry if there is media, because it is an io.Reader. We'd
// have to read the contents and keep it in memory, and that could be expensive.
// TODO(jba): Look into retrying if media != nil.
if conf.job.JobReference != nil && conf.media == nil {
err = runWithRetry(ctx, invoke)
} else {
err = invoke()
}
if err != nil {
return nil, err
}
return &Job{service: s, projectID: projectID, jobID: res.JobReference.JobId}, nil
var dt *bq.TableReference
if qc := res.Configuration.Query; qc != nil {
dt = qc.DestinationTable
}
return &Job{
projectID: projectID,
jobID: res.JobReference.JobId,
destinationTable: dt,
}, nil
}
type pagingConf struct {
@@ -129,6 +160,12 @@ type readTableConf struct {
schema Schema // lazily initialized when the first page of data is fetched.
}
func (conf *readTableConf) fetch(ctx context.Context, s service, token string) (*readDataResult, error) {
return s.readTabledata(ctx, conf, token)
}
func (conf *readTableConf) setPaging(pc *pagingConf) { conf.paging = *pc }
type readDataResult struct {
pageToken string
rows [][]Value
@@ -136,14 +173,10 @@ type readDataResult struct {
schema Schema
}
type readQueryConf struct {
projectID, jobID string
paging pagingConf
}
func (s *bigqueryService) readTabledata(ctx context.Context, conf *readTableConf, pageToken string) (*readDataResult, error) {
// Prepare request to fetch one page of table data.
req := s.s.Tabledata.List(conf.projectID, conf.datasetID, conf.tableID)
setClientHeader(req.Header())
if pageToken != "" {
req.PageToken(pageToken)
@@ -195,47 +228,30 @@ func (s *bigqueryService) readTabledata(ctx context.Context, conf *readTableConf
return result, nil
}
var errIncompleteJob = errors.New("internal error: query results not available because job is not complete")
// getQueryResultsTimeout controls the maximum duration of a request to the
// BigQuery GetQueryResults endpoint. Setting a long timeout here does not
// cause increased overall latency, as results are returned as soon as they are
// available.
const getQueryResultsTimeout = time.Minute
func (s *bigqueryService) readQuery(ctx context.Context, conf *readQueryConf, pageToken string) (*readDataResult, error) {
req := s.s.Jobs.GetQueryResults(conf.projectID, conf.jobID).
TimeoutMs(getQueryResultsTimeout.Nanoseconds() / 1e6)
if pageToken != "" {
req.PageToken(pageToken)
} else {
req.StartIndex(conf.paging.startIndex)
func (s *bigqueryService) waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error) {
// Use GetQueryResults only to wait for completion, not to read results.
req := s.s.Jobs.GetQueryResults(projectID, jobID).Context(ctx).MaxResults(0)
setClientHeader(req.Header())
backoff := gax.Backoff{
Initial: 1 * time.Second,
Multiplier: 2,
Max: 60 * time.Second,
}
if conf.paging.setRecordsPerRequest {
req.MaxResults(conf.paging.recordsPerRequest)
}
res, err := req.Context(ctx).Do()
var res *bq.GetQueryResultsResponse
err := internal.Retry(ctx, backoff, func() (stop bool, err error) {
res, err = req.Do()
if err != nil {
return !retryableError(err), err
}
if !res.JobComplete { // GetQueryResults may return early without error; retry.
return false, nil
}
return true, nil
})
if err != nil {
return nil, err
}
if !res.JobComplete {
return nil, errIncompleteJob
}
schema := convertTableSchema(res.Schema)
result := &readDataResult{
pageToken: res.PageToken,
totalRows: res.TotalRows,
schema: schema,
}
result.rows, err = convertRows(res.Rows, schema)
if err != nil {
return nil, err
}
return result, nil
return convertTableSchema(res.Schema), nil
}
type insertRowsConf struct {
@@ -263,7 +279,9 @@ func (s *bigqueryService) insertRows(ctx context.Context, projectID, datasetID,
var res *bq.TableDataInsertAllResponse
err := runWithRetry(ctx, func() error {
var err error
res, err = s.s.Tabledata.InsertAll(projectID, datasetID, tableID, req).Context(ctx).Do()
req := s.s.Tabledata.InsertAll(projectID, datasetID, tableID, req).Context(ctx)
setClientHeader(req.Header())
res, err = req.Do()
return err
})
if err != nil {
@@ -290,37 +308,26 @@ func (s *bigqueryService) insertRows(ctx context.Context, projectID, datasetID,
return errs
}
type jobType int
const (
copyJobType jobType = iota
extractJobType
loadJobType
queryJobType
)
func (s *bigqueryService) getJobType(ctx context.Context, projectID, jobID string) (jobType, error) {
func (s *bigqueryService) getJob(ctx context.Context, projectID, jobID string) (*Job, error) {
res, err := s.s.Jobs.Get(projectID, jobID).
Fields("configuration").
Context(ctx).
Do()
if err != nil {
return 0, err
return nil, err
}
switch {
case res.Configuration.Copy != nil:
return copyJobType, nil
case res.Configuration.Extract != nil:
return extractJobType, nil
case res.Configuration.Load != nil:
return loadJobType, nil
case res.Configuration.Query != nil:
return queryJobType, nil
default:
return 0, errors.New("unknown job type")
var isQuery bool
var dest *bq.TableReference
if res.Configuration.Query != nil {
isQuery = true
dest = res.Configuration.Query.DestinationTable
}
return &Job{
projectID: projectID,
jobID: jobID,
isQuery: isQuery,
destinationTable: dest,
}, nil
}
func (s *bigqueryService) jobCancel(ctx context.Context, projectID, jobID string) error {
@@ -338,13 +345,18 @@ func (s *bigqueryService) jobCancel(ctx context.Context, projectID, jobID string
func (s *bigqueryService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
res, err := s.s.Jobs.Get(projectID, jobID).
Fields("status"). // Only fetch what we need.
Fields("status", "statistics"). // Only fetch what we need.
Context(ctx).
Do()
if err != nil {
return nil, err
}
return jobStatusFromProto(res.Status)
st, err := jobStatusFromProto(res.Status)
if err != nil {
return nil, err
}
st.Statistics = jobStatisticsFromProto(res.Statistics)
return st, nil
}
var stateMap = map[string]State{"PENDING": Pending, "RUNNING": Running, "DONE": Done}
@@ -369,12 +381,87 @@ func jobStatusFromProto(status *bq.JobStatus) (*JobStatus, error) {
return newStatus, nil
}
func jobStatisticsFromProto(s *bq.JobStatistics) *JobStatistics {
js := &JobStatistics{
CreationTime: unixMillisToTime(s.CreationTime),
StartTime: unixMillisToTime(s.StartTime),
EndTime: unixMillisToTime(s.EndTime),
TotalBytesProcessed: s.TotalBytesProcessed,
}
switch {
case s.Extract != nil:
js.Details = &ExtractStatistics{
DestinationURIFileCounts: []int64(s.Extract.DestinationUriFileCounts),
}
case s.Load != nil:
js.Details = &LoadStatistics{
InputFileBytes: s.Load.InputFileBytes,
InputFiles: s.Load.InputFiles,
OutputBytes: s.Load.OutputBytes,
OutputRows: s.Load.OutputRows,
}
case s.Query != nil:
var names []string
for _, qp := range s.Query.UndeclaredQueryParameters {
names = append(names, qp.Name)
}
var tables []*Table
for _, tr := range s.Query.ReferencedTables {
tables = append(tables, convertTableReference(tr))
}
js.Details = &QueryStatistics{
BillingTier: s.Query.BillingTier,
CacheHit: s.Query.CacheHit,
StatementType: s.Query.StatementType,
TotalBytesBilled: s.Query.TotalBytesBilled,
TotalBytesProcessed: s.Query.TotalBytesProcessed,
NumDMLAffectedRows: s.Query.NumDmlAffectedRows,
QueryPlan: queryPlanFromProto(s.Query.QueryPlan),
Schema: convertTableSchema(s.Query.Schema),
ReferencedTables: tables,
UndeclaredQueryParameterNames: names,
}
}
return js
}
func queryPlanFromProto(stages []*bq.ExplainQueryStage) []*ExplainQueryStage {
var res []*ExplainQueryStage
for _, s := range stages {
var steps []*ExplainQueryStep
for _, p := range s.Steps {
steps = append(steps, &ExplainQueryStep{
Kind: p.Kind,
Substeps: p.Substeps,
})
}
res = append(res, &ExplainQueryStage{
ComputeRatioAvg: s.ComputeRatioAvg,
ComputeRatioMax: s.ComputeRatioMax,
ID: s.Id,
Name: s.Name,
ReadRatioAvg: s.ReadRatioAvg,
ReadRatioMax: s.ReadRatioMax,
RecordsRead: s.RecordsRead,
RecordsWritten: s.RecordsWritten,
Status: s.Status,
Steps: steps,
WaitRatioAvg: s.WaitRatioAvg,
WaitRatioMax: s.WaitRatioMax,
WriteRatioAvg: s.WriteRatioAvg,
WriteRatioMax: s.WriteRatioMax,
})
}
return res
}
// listTables returns a subset of tables that belong to a dataset, and a token for fetching the next subset.
func (s *bigqueryService) listTables(ctx context.Context, projectID, datasetID string, pageSize int, pageToken string) ([]*Table, string, error) {
var tables []*Table
req := s.s.Tables.List(projectID, datasetID).
PageToken(pageToken).
Context(ctx)
setClientHeader(req.Header())
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
@@ -383,7 +470,7 @@ func (s *bigqueryService) listTables(ctx context.Context, projectID, datasetID s
return nil, "", err
}
for _, t := range res.Tables {
tables = append(tables, s.convertListedTable(t))
tables = append(tables, convertTableReference(t.TableReference))
}
return tables, res.NextPageToken, nil
}
@@ -433,12 +520,16 @@ func (s *bigqueryService) createTable(ctx context.Context, conf *createTableConf
}
}
_, err := s.s.Tables.Insert(conf.projectID, conf.datasetID, table).Context(ctx).Do()
req := s.s.Tables.Insert(conf.projectID, conf.datasetID, table).Context(ctx)
setClientHeader(req.Header())
_, err := req.Do()
return err
}
func (s *bigqueryService) getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error) {
table, err := s.s.Tables.Get(projectID, datasetID, tableID).Context(ctx).Do()
req := s.s.Tables.Get(projectID, datasetID, tableID).Context(ctx)
setClientHeader(req.Header())
table, err := req.Do()
if err != nil {
return nil, err
}
@@ -446,7 +537,9 @@ func (s *bigqueryService) getTableMetadata(ctx context.Context, projectID, datas
}
func (s *bigqueryService) deleteTable(ctx context.Context, projectID, datasetID, tableID string) error {
return s.s.Tables.Delete(projectID, datasetID, tableID).Context(ctx).Do()
req := s.s.Tables.Delete(projectID, datasetID, tableID).Context(ctx)
setClientHeader(req.Header())
return req.Do()
}
func bqTableToMetadata(t *bq.Table) *TableMetadata {
@@ -468,9 +561,17 @@ func bqTableToMetadata(t *bq.Table) *TableMetadata {
md.View = t.View.Query
}
if t.TimePartitioning != nil {
md.TimePartitioning = &TimePartitioning{time.Duration(t.TimePartitioning.ExpirationMs) * time.Millisecond}
md.TimePartitioning = &TimePartitioning{
Expiration: time.Duration(t.TimePartitioning.ExpirationMs) * time.Millisecond,
}
}
if t.StreamingBuffer != nil {
md.StreamingBuffer = &StreamingBuffer{
EstimatedBytes: t.StreamingBuffer.EstimatedBytes,
EstimatedRows: t.StreamingBuffer.EstimatedRows,
OldestEntryTime: unixMillisToTime(int64(t.StreamingBuffer.OldestEntryTime)),
}
}
return md
}
@@ -498,11 +599,11 @@ func unixMillisToTime(m int64) time.Time {
return time.Unix(0, m*1e6)
}
func (s *bigqueryService) convertListedTable(t *bq.TableListTables) *Table {
func convertTableReference(tr *bq.TableReference) *Table {
return &Table{
ProjectID: t.TableReference.ProjectId,
DatasetID: t.TableReference.DatasetId,
TableID: t.TableReference.TableId,
ProjectID: tr.ProjectId,
DatasetID: tr.DatasetId,
TableID: tr.TableId,
}
}
@@ -545,16 +646,22 @@ func (s *bigqueryService) insertDataset(ctx context.Context, datasetID, projectI
ds := &bq.Dataset{
DatasetReference: &bq.DatasetReference{DatasetId: datasetID},
}
_, err := s.s.Datasets.Insert(projectID, ds).Context(ctx).Do()
req := s.s.Datasets.Insert(projectID, ds).Context(ctx)
setClientHeader(req.Header())
_, err := req.Do()
return err
}
func (s *bigqueryService) deleteDataset(ctx context.Context, datasetID, projectID string) error {
return s.s.Datasets.Delete(projectID, datasetID).Context(ctx).Do()
req := s.s.Datasets.Delete(projectID, datasetID).Context(ctx)
setClientHeader(req.Header())
return req.Do()
}
func (s *bigqueryService) getDatasetMetadata(ctx context.Context, projectID, datasetID string) (*DatasetMetadata, error) {
table, err := s.s.Datasets.Get(projectID, datasetID).Context(ctx).Do()
req := s.s.Datasets.Get(projectID, datasetID).Context(ctx)
setClientHeader(req.Header())
table, err := req.Do()
if err != nil {
return nil, err
}
@@ -566,6 +673,7 @@ func (s *bigqueryService) listDatasets(ctx context.Context, projectID string, ma
Context(ctx).
PageToken(pageToken).
All(all)
setClientHeader(req.Header())
if maxResults > 0 {
req.MaxResults(int64(maxResults))
}
@@ -605,19 +713,19 @@ func runWithRetry(ctx context.Context, call func() error) error {
if err == nil {
return true, nil
}
e, ok := err.(*googleapi.Error)
if !ok {
return true, err
}
var reason string
if len(e.Errors) > 0 {
reason = e.Errors[0].Reason
}
// Retry using the criteria in
// https://cloud.google.com/bigquery/troubleshooting-errors
if reason == "backendError" && (e.Code == 500 || e.Code == 503) {
return false, nil
}
return true, err
return !retryableError(err), err
})
}
// Use the criteria in https://cloud.google.com/bigquery/troubleshooting-errors.
func retryableError(err error) bool {
e, ok := err.(*googleapi.Error)
if !ok {
return false
}
var reason string
if len(e.Errors) > 0 {
reason = e.Errors[0].Reason
}
return reason == "backendError" && (e.Code == 500 || e.Code == 503)
}
vendor/cloud.google.com/go/bigquery/service_test.go
Generated Vendored
+83
View File
@@ -0,0 +1,83 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bigquery
import (
"reflect"
"testing"
"time"
bq "google.golang.org/api/bigquery/v2"
)
func TestBQTableToMetadata(t *testing.T) {
aTime := time.Date(2017, 1, 26, 0, 0, 0, 0, time.Local)
aTimeMillis := aTime.UnixNano() / 1e6
for _, test := range []struct {
in *bq.Table
want *TableMetadata
}{
{&bq.Table{}, &TableMetadata{}}, // test minimal case
{
&bq.Table{
CreationTime: aTimeMillis,
Description: "desc",
Etag: "etag",
ExpirationTime: aTimeMillis,
FriendlyName: "fname",
Id: "id",
LastModifiedTime: uint64(aTimeMillis),
Location: "loc",
NumBytes: 123,
NumLongTermBytes: 23,
NumRows: 7,
StreamingBuffer: &bq.Streamingbuffer{
EstimatedBytes: 11,
EstimatedRows: 3,
OldestEntryTime: uint64(aTimeMillis),
},
TimePartitioning: &bq.TimePartitioning{
ExpirationMs: 7890,
Type: "DAY",
},
Type: "EXTERNAL",
View: &bq.ViewDefinition{Query: "view-query"},
},
&TableMetadata{
Description: "desc",
Name: "fname",
View: "view-query",
ID: "id",
Type: ExternalTable,
ExpirationTime: aTime.Truncate(time.Millisecond),
CreationTime: aTime.Truncate(time.Millisecond),
LastModifiedTime: aTime.Truncate(time.Millisecond),
NumBytes: 123,
NumRows: 7,
TimePartitioning: &TimePartitioning{Expiration: time.Duration(7890) * time.Millisecond},
StreamingBuffer: &StreamingBuffer{
EstimatedBytes: 11,
EstimatedRows: 3,
OldestEntryTime: aTime,
},
},
},
} {
got := bqTableToMetadata(test.in)
if !reflect.DeepEqual(got, test.want) {
t.Errorf("%v:\ngot %+v\nwant %+v", test.in, got, test.want)
}
}
}
vendor/cloud.google.com/go/bigquery/table.go
Generated Vendored
+31 -2
View File
@@ -64,6 +64,11 @@ type TableMetadata struct {
// The time-based partitioning settings for this table.
TimePartitioning *TimePartitioning
// Contains information regarding this table's streaming buffer, if one is
// present. This field will be nil if the table is not being streamed to or if
// there is no data in the streaming buffer.
StreamingBuffer *StreamingBuffer
}
// TableCreateDisposition specifies the circumstances under which destination table will be created.
@@ -101,10 +106,25 @@ const (
type TableType string
const (
RegularTable TableType = "TABLE"
ViewTable TableType = "VIEW"
RegularTable TableType = "TABLE"
ViewTable TableType = "VIEW"
ExternalTable TableType = "EXTERNAL"
)
// StreamingBuffer holds information about the streaming buffer.
type StreamingBuffer struct {
// A lower-bound estimate of the number of bytes currently in the streaming
// buffer.
EstimatedBytes uint64
// A lower-bound estimate of the number of rows currently in the streaming
// buffer.
EstimatedRows uint64
// The time of the oldest entry in the streaming buffer.
OldestEntryTime time.Time
}
func (t *Table) tableRefProto() *bq.TableReference {
return &bq.TableReference{
ProjectId: t.ProjectID,
@@ -193,6 +213,15 @@ func (opt TimePartitioning) customizeCreateTable(conf *createTableConf) {
conf.timePartitioning = &opt
}
// Read fetches the contents of the table.
func (t *Table) Read(ctx context.Context) *RowIterator {
return newRowIterator(ctx, t.c.service, &readTableConf{
projectID: t.ProjectID,
datasetID: t.DatasetID,
tableID: t.TableID,
})
}
// Update modifies specific Table metadata fields.
func (t *Table) Update(ctx context.Context, tm TableMetadataToUpdate) (*TableMetadata, error) {
var conf patchTableConf
vendor/cloud.google.com/go/bigquery/uploader.go
Generated Vendored
+17 -2
View File
@@ -15,6 +15,7 @@
package bigquery
import (
"errors"
"fmt"
"reflect"
@@ -114,6 +115,16 @@ func valueSavers(src interface{}) ([]ValueSaver, error) {
// Make a ValueSaver from x, which must implement ValueSaver already
// or be a struct or pointer to struct.
func toValueSaver(x interface{}) (ValueSaver, bool, error) {
if _, ok := x.(StructSaver); ok {
return nil, false, errors.New("bigquery: use &StructSaver, not StructSaver")
}
var insertID string
// Handle StructSavers specially so we can infer the schema if necessary.
if ss, ok := x.(*StructSaver); ok && ss.Schema == nil {
x = ss.Struct
insertID = ss.InsertID
// Fall through so we can infer the schema.
}
if saver, ok := x.(ValueSaver); ok {
return saver, ok, nil
}
@@ -128,11 +139,15 @@ func toValueSaver(x interface{}) (ValueSaver, bool, error) {
if v.Kind() != reflect.Struct {
return nil, false, nil
}
schema, err := inferSchemaReflect(v.Type())
schema, err := inferSchemaReflectCached(v.Type())
if err != nil {
return nil, false, err
}
return &StructSaver{Struct: x, Schema: schema}, true, nil
return &StructSaver{
Struct: x,
InsertID: insertID,
Schema: schema,
}, true, nil
}
func (u *Uploader) putMulti(ctx context.Context, src []ValueSaver) error {
vendor/cloud.google.com/go/bigquery/uploader_test.go
Generated Vendored
+14 -20
View File
@@ -35,27 +35,18 @@ func (ts testSaver) Save() (map[string]Value, string, error) {
func TestRejectsNonValueSavers(t *testing.T) {
client := &Client{projectID: "project-id"}
u := Uploader{t: client.Dataset("dataset-id").Table("table-id")}
testCases := []struct {
src interface{}
}{
{
src: 1,
},
{
src: []int{1, 2},
},
{
src: []interface{}{
testSaver{ir: &insertionRow{"a", map[string]Value{"one": 1}}},
1,
},
inputs := []interface{}{
1,
[]int{1, 2},
[]interface{}{
testSaver{ir: &insertionRow{"a", map[string]Value{"one": 1}}},
1,
},
StructSaver{},
}
for _, tc := range testCases {
if err := u.Put(context.Background(), tc.src); err == nil {
t.Errorf("put value: %v; got nil, want error", tc.src)
for _, in := range inputs {
if err := u.Put(context.Background(), in); err == nil {
t.Errorf("put value: %v; got nil, want error", in)
}
}
}
@@ -265,13 +256,16 @@ func TestValueSavers(t *testing.T) {
&StructSaver{Schema: schema, Struct: T{I: 1}},
&StructSaver{Schema: schema, Struct: &T{I: 2}},
}},
{&StructSaver{Struct: T{I: 3}, InsertID: "foo"},
[]ValueSaver{
&StructSaver{Schema: schema, Struct: T{I: 3}, InsertID: "foo"},
}},
} {
got, err := valueSavers(test.in)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got, test.want) {
t.Errorf("%+v: got %v, want %v", test.in, pretty.Value(got), pretty.Value(test.want))
}
// Make sure Save is successful.
vendor/cloud.google.com/go/bigquery/value.go
Generated Vendored
+20
View File
@@ -103,12 +103,20 @@ type structLoaderOp struct {
repeated bool
}
var errNoNulls = errors.New("bigquery: NULL values cannot be read into structs")
func setAny(v reflect.Value, x interface{}) error {
if x == nil {
return errNoNulls
}
v.Set(reflect.ValueOf(x))
return nil
}
func setInt(v reflect.Value, x interface{}) error {
if x == nil {
return errNoNulls
}
xx := x.(int64)
if v.OverflowInt(xx) {
return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
@@ -118,6 +126,9 @@ func setInt(v reflect.Value, x interface{}) error {
}
func setFloat(v reflect.Value, x interface{}) error {
if x == nil {
return errNoNulls
}
xx := x.(float64)
if v.OverflowFloat(xx) {
return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
@@ -127,16 +138,25 @@ func setFloat(v reflect.Value, x interface{}) error {
}
func setBool(v reflect.Value, x interface{}) error {
if x == nil {
return errNoNulls
}
v.SetBool(x.(bool))
return nil
}
func setString(v reflect.Value, x interface{}) error {
if x == nil {
return errNoNulls
}
v.SetString(x.(string))
return nil
}
func setBytes(v reflect.Value, x interface{}) error {
if x == nil {
return errNoNulls
}
v.SetBytes(x.([]byte))
return nil
}