f-denkena/rclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vendor: update all dependencies

Browse Source
This commit is contained in:
Nick Craig-Wood
2017-07-23 08:51:42 +01:00
parent 0b6fba34a3
commit eb87cf6f12
2008 changed files with 352617 additions and 1004734 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/cloud.google.com/go/bigquery/service.go
Generated Vendored
+212 -104
View File
@@ -15,7 +15,6 @@
package bigquery
import (
"errors"
"fmt"
"io"
"net/http"
@@ -23,6 +22,7 @@ import (
"time"
"cloud.google.com/go/internal"
"cloud.google.com/go/internal/version"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
@@ -37,7 +37,7 @@ import (
type service interface {
// Jobs
insertJob(ctx context.Context, projectId string, conf *insertJobConf) (*Job, error)
getJobType(ctx context.Context, projectId, jobID string) (jobType, error)
getJob(ctx context.Context, projectId, jobID string) (*Job, error)
jobCancel(ctx context.Context, projectId, jobID string) error
jobStatus(ctx context.Context, projectId, jobID string) (*JobStatus, error)
@@ -61,15 +61,19 @@ type service interface {
// Misc
// readQuery reads data resulting from a query job. If the job is
// incomplete, an errIncompleteJob is returned. readQuery may be called
// repeatedly to poll for job completion.
readQuery(ctx context.Context, conf *readQueryConf, pageToken string) (*readDataResult, error)
// Waits for a query to complete.
waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error)
// listDatasets returns a page of Datasets and a next page token. Note: the Datasets do not have their c field populated.
listDatasets(ctx context.Context, projectID string, maxResults int, pageToken string, all bool, filter string) ([]*Dataset, string, error)
}
var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo)
func setClientHeader(headers http.Header) {
headers.Set("x-goog-api-client", xGoogHeader)
}
type bigqueryService struct {
s *bq.Service
}
@@ -104,16 +108,43 @@ type insertJobConf struct {
media io.Reader
}
// Calls the Jobs.Insert RPC and returns a Job. Callers must set the returned Job's
// client.
func (s *bigqueryService) insertJob(ctx context.Context, projectID string, conf *insertJobConf) (*Job, error) {
call := s.s.Jobs.Insert(projectID, conf.job).Context(ctx)
setClientHeader(call.Header())
if conf.media != nil {
call.Media(conf.media)
}
res, err := call.Do()
var res *bq.Job
var err error
invoke := func() error {
res, err = call.Do()
return err
}
// A job with a client-generated ID can be retried; the presence of the
// ID makes the insert operation idempotent.
// We don't retry if there is media, because it is an io.Reader. We'd
// have to read the contents and keep it in memory, and that could be expensive.
// TODO(jba): Look into retrying if media != nil.
if conf.job.JobReference != nil && conf.media == nil {
err = runWithRetry(ctx, invoke)
} else {
err = invoke()
}
if err != nil {
return nil, err
}
return &Job{service: s, projectID: projectID, jobID: res.JobReference.JobId}, nil
var dt *bq.TableReference
if qc := res.Configuration.Query; qc != nil {
dt = qc.DestinationTable
}
return &Job{
projectID: projectID,
jobID: res.JobReference.JobId,
destinationTable: dt,
}, nil
}
type pagingConf struct {
@@ -129,6 +160,12 @@ type readTableConf struct {
schema Schema // lazily initialized when the first page of data is fetched.
}
func (conf *readTableConf) fetch(ctx context.Context, s service, token string) (*readDataResult, error) {
return s.readTabledata(ctx, conf, token)
}
func (conf *readTableConf) setPaging(pc *pagingConf) { conf.paging = *pc }
type readDataResult struct {
pageToken string
rows [][]Value
@@ -136,14 +173,10 @@ type readDataResult struct {
schema Schema
}
type readQueryConf struct {
projectID, jobID string
paging pagingConf
}
func (s *bigqueryService) readTabledata(ctx context.Context, conf *readTableConf, pageToken string) (*readDataResult, error) {
// Prepare request to fetch one page of table data.
req := s.s.Tabledata.List(conf.projectID, conf.datasetID, conf.tableID)
setClientHeader(req.Header())
if pageToken != "" {
req.PageToken(pageToken)
@@ -195,47 +228,30 @@ func (s *bigqueryService) readTabledata(ctx context.Context, conf *readTableConf
return result, nil
}
var errIncompleteJob = errors.New("internal error: query results not available because job is not complete")
// getQueryResultsTimeout controls the maximum duration of a request to the
// BigQuery GetQueryResults endpoint. Setting a long timeout here does not
// cause increased overall latency, as results are returned as soon as they are
// available.
const getQueryResultsTimeout = time.Minute
func (s *bigqueryService) readQuery(ctx context.Context, conf *readQueryConf, pageToken string) (*readDataResult, error) {
req := s.s.Jobs.GetQueryResults(conf.projectID, conf.jobID).
TimeoutMs(getQueryResultsTimeout.Nanoseconds() / 1e6)
if pageToken != "" {
req.PageToken(pageToken)
} else {
req.StartIndex(conf.paging.startIndex)
func (s *bigqueryService) waitForQuery(ctx context.Context, projectID, jobID string) (Schema, error) {
// Use GetQueryResults only to wait for completion, not to read results.
req := s.s.Jobs.GetQueryResults(projectID, jobID).Context(ctx).MaxResults(0)
setClientHeader(req.Header())
backoff := gax.Backoff{
Initial: 1 * time.Second,
Multiplier: 2,
Max: 60 * time.Second,
}
if conf.paging.setRecordsPerRequest {
req.MaxResults(conf.paging.recordsPerRequest)
}
res, err := req.Context(ctx).Do()
var res *bq.GetQueryResultsResponse
err := internal.Retry(ctx, backoff, func() (stop bool, err error) {
res, err = req.Do()
if err != nil {
return !retryableError(err), err
}
if !res.JobComplete { // GetQueryResults may return early without error; retry.
return false, nil
}
return true, nil
})
if err != nil {
return nil, err
}
if !res.JobComplete {
return nil, errIncompleteJob
}
schema := convertTableSchema(res.Schema)
result := &readDataResult{
pageToken: res.PageToken,
totalRows: res.TotalRows,
schema: schema,
}
result.rows, err = convertRows(res.Rows, schema)
if err != nil {
return nil, err
}
return result, nil
return convertTableSchema(res.Schema), nil
}
type insertRowsConf struct {
@@ -263,7 +279,9 @@ func (s *bigqueryService) insertRows(ctx context.Context, projectID, datasetID,
var res *bq.TableDataInsertAllResponse
err := runWithRetry(ctx, func() error {
var err error
res, err = s.s.Tabledata.InsertAll(projectID, datasetID, tableID, req).Context(ctx).Do()
req := s.s.Tabledata.InsertAll(projectID, datasetID, tableID, req).Context(ctx)
setClientHeader(req.Header())
res, err = req.Do()
return err
})
if err != nil {
@@ -290,37 +308,26 @@ func (s *bigqueryService) insertRows(ctx context.Context, projectID, datasetID,
return errs
}
type jobType int
const (
copyJobType jobType = iota
extractJobType
loadJobType
queryJobType
)
func (s *bigqueryService) getJobType(ctx context.Context, projectID, jobID string) (jobType, error) {
func (s *bigqueryService) getJob(ctx context.Context, projectID, jobID string) (*Job, error) {
res, err := s.s.Jobs.Get(projectID, jobID).
Fields("configuration").
Context(ctx).
Do()
if err != nil {
return 0, err
return nil, err
}
switch {
case res.Configuration.Copy != nil:
return copyJobType, nil
case res.Configuration.Extract != nil:
return extractJobType, nil
case res.Configuration.Load != nil:
return loadJobType, nil
case res.Configuration.Query != nil:
return queryJobType, nil
default:
return 0, errors.New("unknown job type")
var isQuery bool
var dest *bq.TableReference
if res.Configuration.Query != nil {
isQuery = true
dest = res.Configuration.Query.DestinationTable
}
return &Job{
projectID: projectID,
jobID: jobID,
isQuery: isQuery,
destinationTable: dest,
}, nil
}
func (s *bigqueryService) jobCancel(ctx context.Context, projectID, jobID string) error {
@@ -338,13 +345,18 @@ func (s *bigqueryService) jobCancel(ctx context.Context, projectID, jobID string
func (s *bigqueryService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
res, err := s.s.Jobs.Get(projectID, jobID).
Fields("status"). // Only fetch what we need.
Fields("status", "statistics"). // Only fetch what we need.
Context(ctx).
Do()
if err != nil {
return nil, err
}
return jobStatusFromProto(res.Status)
st, err := jobStatusFromProto(res.Status)
if err != nil {
return nil, err
}
st.Statistics = jobStatisticsFromProto(res.Statistics)
return st, nil
}
var stateMap = map[string]State{"PENDING": Pending, "RUNNING": Running, "DONE": Done}
@@ -369,12 +381,87 @@ func jobStatusFromProto(status *bq.JobStatus) (*JobStatus, error) {
return newStatus, nil
}
func jobStatisticsFromProto(s *bq.JobStatistics) *JobStatistics {
js := &JobStatistics{
CreationTime: unixMillisToTime(s.CreationTime),
StartTime: unixMillisToTime(s.StartTime),
EndTime: unixMillisToTime(s.EndTime),
TotalBytesProcessed: s.TotalBytesProcessed,
}
switch {
case s.Extract != nil:
js.Details = &ExtractStatistics{
DestinationURIFileCounts: []int64(s.Extract.DestinationUriFileCounts),
}
case s.Load != nil:
js.Details = &LoadStatistics{
InputFileBytes: s.Load.InputFileBytes,
InputFiles: s.Load.InputFiles,
OutputBytes: s.Load.OutputBytes,
OutputRows: s.Load.OutputRows,
}
case s.Query != nil:
var names []string
for _, qp := range s.Query.UndeclaredQueryParameters {
names = append(names, qp.Name)
}
var tables []*Table
for _, tr := range s.Query.ReferencedTables {
tables = append(tables, convertTableReference(tr))
}
js.Details = &QueryStatistics{
BillingTier: s.Query.BillingTier,
CacheHit: s.Query.CacheHit,
StatementType: s.Query.StatementType,
TotalBytesBilled: s.Query.TotalBytesBilled,
TotalBytesProcessed: s.Query.TotalBytesProcessed,
NumDMLAffectedRows: s.Query.NumDmlAffectedRows,
QueryPlan: queryPlanFromProto(s.Query.QueryPlan),
Schema: convertTableSchema(s.Query.Schema),
ReferencedTables: tables,
UndeclaredQueryParameterNames: names,
}
}
return js
}
func queryPlanFromProto(stages []*bq.ExplainQueryStage) []*ExplainQueryStage {
var res []*ExplainQueryStage
for _, s := range stages {
var steps []*ExplainQueryStep
for _, p := range s.Steps {
steps = append(steps, &ExplainQueryStep{
Kind: p.Kind,
Substeps: p.Substeps,
})
}
res = append(res, &ExplainQueryStage{
ComputeRatioAvg: s.ComputeRatioAvg,
ComputeRatioMax: s.ComputeRatioMax,
ID: s.Id,
Name: s.Name,
ReadRatioAvg: s.ReadRatioAvg,
ReadRatioMax: s.ReadRatioMax,
RecordsRead: s.RecordsRead,
RecordsWritten: s.RecordsWritten,
Status: s.Status,
Steps: steps,
WaitRatioAvg: s.WaitRatioAvg,
WaitRatioMax: s.WaitRatioMax,
WriteRatioAvg: s.WriteRatioAvg,
WriteRatioMax: s.WriteRatioMax,
})
}
return res
}
// listTables returns a subset of tables that belong to a dataset, and a token for fetching the next subset.
func (s *bigqueryService) listTables(ctx context.Context, projectID, datasetID string, pageSize int, pageToken string) ([]*Table, string, error) {
var tables []*Table
req := s.s.Tables.List(projectID, datasetID).
PageToken(pageToken).
Context(ctx)
setClientHeader(req.Header())
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
@@ -383,7 +470,7 @@ func (s *bigqueryService) listTables(ctx context.Context, projectID, datasetID s
return nil, "", err
}
for _, t := range res.Tables {
tables = append(tables, s.convertListedTable(t))
tables = append(tables, convertTableReference(t.TableReference))
}
return tables, res.NextPageToken, nil
}
@@ -433,12 +520,16 @@ func (s *bigqueryService) createTable(ctx context.Context, conf *createTableConf
}
}
_, err := s.s.Tables.Insert(conf.projectID, conf.datasetID, table).Context(ctx).Do()
req := s.s.Tables.Insert(conf.projectID, conf.datasetID, table).Context(ctx)
setClientHeader(req.Header())
_, err := req.Do()
return err
}
func (s *bigqueryService) getTableMetadata(ctx context.Context, projectID, datasetID, tableID string) (*TableMetadata, error) {
table, err := s.s.Tables.Get(projectID, datasetID, tableID).Context(ctx).Do()
req := s.s.Tables.Get(projectID, datasetID, tableID).Context(ctx)
setClientHeader(req.Header())
table, err := req.Do()
if err != nil {
return nil, err
}
@@ -446,7 +537,9 @@ func (s *bigqueryService) getTableMetadata(ctx context.Context, projectID, datas
}
func (s *bigqueryService) deleteTable(ctx context.Context, projectID, datasetID, tableID string) error {
return s.s.Tables.Delete(projectID, datasetID, tableID).Context(ctx).Do()
req := s.s.Tables.Delete(projectID, datasetID, tableID).Context(ctx)
setClientHeader(req.Header())
return req.Do()
}
func bqTableToMetadata(t *bq.Table) *TableMetadata {
@@ -468,9 +561,17 @@ func bqTableToMetadata(t *bq.Table) *TableMetadata {
md.View = t.View.Query
}
if t.TimePartitioning != nil {
md.TimePartitioning = &TimePartitioning{time.Duration(t.TimePartitioning.ExpirationMs) * time.Millisecond}
md.TimePartitioning = &TimePartitioning{
Expiration: time.Duration(t.TimePartitioning.ExpirationMs) * time.Millisecond,
}
}
if t.StreamingBuffer != nil {
md.StreamingBuffer = &StreamingBuffer{
EstimatedBytes: t.StreamingBuffer.EstimatedBytes,
EstimatedRows: t.StreamingBuffer.EstimatedRows,
OldestEntryTime: unixMillisToTime(int64(t.StreamingBuffer.OldestEntryTime)),
}
}
return md
}
@@ -498,11 +599,11 @@ func unixMillisToTime(m int64) time.Time {
return time.Unix(0, m*1e6)
}
func (s *bigqueryService) convertListedTable(t *bq.TableListTables) *Table {
func convertTableReference(tr *bq.TableReference) *Table {
return &Table{
ProjectID: t.TableReference.ProjectId,
DatasetID: t.TableReference.DatasetId,
TableID: t.TableReference.TableId,
ProjectID: tr.ProjectId,
DatasetID: tr.DatasetId,
TableID: tr.TableId,
}
}
@@ -545,16 +646,22 @@ func (s *bigqueryService) insertDataset(ctx context.Context, datasetID, projectI
ds := &bq.Dataset{
DatasetReference: &bq.DatasetReference{DatasetId: datasetID},
}
_, err := s.s.Datasets.Insert(projectID, ds).Context(ctx).Do()
req := s.s.Datasets.Insert(projectID, ds).Context(ctx)
setClientHeader(req.Header())
_, err := req.Do()
return err
}
func (s *bigqueryService) deleteDataset(ctx context.Context, datasetID, projectID string) error {
return s.s.Datasets.Delete(projectID, datasetID).Context(ctx).Do()
req := s.s.Datasets.Delete(projectID, datasetID).Context(ctx)
setClientHeader(req.Header())
return req.Do()
}
func (s *bigqueryService) getDatasetMetadata(ctx context.Context, projectID, datasetID string) (*DatasetMetadata, error) {
table, err := s.s.Datasets.Get(projectID, datasetID).Context(ctx).Do()
req := s.s.Datasets.Get(projectID, datasetID).Context(ctx)
setClientHeader(req.Header())
table, err := req.Do()
if err != nil {
return nil, err
}
@@ -566,6 +673,7 @@ func (s *bigqueryService) listDatasets(ctx context.Context, projectID string, ma
Context(ctx).
PageToken(pageToken).
All(all)
setClientHeader(req.Header())
if maxResults > 0 {
req.MaxResults(int64(maxResults))
}
@@ -605,19 +713,19 @@ func runWithRetry(ctx context.Context, call func() error) error {
if err == nil {
return true, nil
}
e, ok := err.(*googleapi.Error)
if !ok {
return true, err
}
var reason string
if len(e.Errors) > 0 {
reason = e.Errors[0].Reason
}
// Retry using the criteria in
// https://cloud.google.com/bigquery/troubleshooting-errors
if reason == "backendError" && (e.Code == 500 || e.Code == 503) {
return false, nil
}
return true, err
return !retryableError(err), err
})
}
// Use the criteria in https://cloud.google.com/bigquery/troubleshooting-errors.
func retryableError(err error) bool {
e, ok := err.(*googleapi.Error)
if !ok {
return false
}
var reason string
if len(e.Errors) > 0 {
reason = e.Errors[0].Reason
}
return reason == "backendError" && (e.Code == 500 || e.Code == 503)
}