Switch to using the dep tool and update all the dependencies
This commit is contained in:
+93
@@ -0,0 +1,93 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// concat_table is an example client of the bigquery client library.
|
||||
// It concatenates two BigQuery tables and writes the result to another table.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/bigquery"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var (
|
||||
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
|
||||
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
|
||||
src1 = flag.String("src1", "", "The ID of the first BigQuery table to concatenate")
|
||||
src2 = flag.String("src2", "", "The ID of the second BigQuery table to concatenate")
|
||||
dest = flag.String("dest", "", "The ID of the BigQuery table to write the result to")
|
||||
pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
flagsOk := true
|
||||
for _, f := range []string{"project", "dataset", "src1", "src2", "dest"} {
|
||||
if flag.Lookup(f).Value.String() == "" {
|
||||
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
|
||||
flagsOk = false
|
||||
}
|
||||
}
|
||||
if !flagsOk {
|
||||
os.Exit(1)
|
||||
}
|
||||
if *src1 == *src2 || *src1 == *dest || *src2 == *dest {
|
||||
log.Fatalf("Different values must be supplied for each of --src1, --src2 and --dest")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := bigquery.NewClient(ctx, *project)
|
||||
if err != nil {
|
||||
log.Fatalf("Creating bigquery client: %v", err)
|
||||
}
|
||||
|
||||
s1 := client.Dataset(*dataset).Table(*src1)
|
||||
s2 := client.Dataset(*dataset).Table(*src2)
|
||||
d := client.Dataset(*dataset).Table(*dest)
|
||||
|
||||
// Concatenate data.
|
||||
copier := d.CopierFrom(s1, s2)
|
||||
copier.WriteDisposition = bigquery.WriteTruncate
|
||||
job, err := copier.Run(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("Concatenating: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Job for concatenation operation: %+v\n", job)
|
||||
fmt.Printf("Waiting for job to complete.\n")
|
||||
|
||||
for range time.Tick(*pollint) {
|
||||
status, err := job.Status(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("Failure determining status: %v", err)
|
||||
break
|
||||
}
|
||||
if !status.Done() {
|
||||
continue
|
||||
}
|
||||
if err := status.Err(); err == nil {
|
||||
fmt.Printf("Success\n")
|
||||
} else {
|
||||
fmt.Printf("Failure: %+v\n", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// load is an example client of the bigquery client library.
|
||||
// It loads a file from Google Cloud Storage into a BigQuery table.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/bigquery"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var (
|
||||
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
|
||||
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
|
||||
table = flag.String("table", "", "The ID of a BigQuery table to load data into")
|
||||
bucket = flag.String("bucket", "", "The name of a Google Cloud Storage bucket to load data from")
|
||||
object = flag.String("object", "", "The name of a Google Cloud Storage object to load data from. Must exist within the bucket specified by --bucket")
|
||||
skiprows = flag.Int64("skiprows", 0, "The number of rows of the source data to skip when loading")
|
||||
pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
flagsOk := true
|
||||
for _, f := range []string{"project", "dataset", "table", "bucket", "object"} {
|
||||
if flag.Lookup(f).Value.String() == "" {
|
||||
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
|
||||
flagsOk = false
|
||||
}
|
||||
}
|
||||
if !flagsOk {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := bigquery.NewClient(ctx, *project)
|
||||
if err != nil {
|
||||
log.Fatalf("Creating bigquery client: %v", err)
|
||||
}
|
||||
|
||||
table := client.Dataset(*dataset).Table(*table)
|
||||
|
||||
gcs := bigquery.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object))
|
||||
gcs.SkipLeadingRows = *skiprows
|
||||
gcs.MaxBadRecords = 1
|
||||
gcs.AllowQuotedNewlines = true
|
||||
|
||||
// Load data from Google Cloud Storage into a BigQuery table.
|
||||
loader := table.LoaderFrom(gcs)
|
||||
loader.WriteDisposition = bigquery.WriteTruncate
|
||||
job, err := loader.Run(ctx)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Loading data: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Job for data load operation: %+v\n", job)
|
||||
fmt.Printf("Waiting for job to complete.\n")
|
||||
|
||||
for range time.Tick(*pollint) {
|
||||
status, err := job.Status(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("Failure determining status: %v", err)
|
||||
break
|
||||
}
|
||||
if !status.Done() {
|
||||
continue
|
||||
}
|
||||
if err := status.Err(); err == nil {
|
||||
fmt.Printf("Success\n")
|
||||
} else {
|
||||
fmt.Printf("Failure: %+v\n", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// query is an example client of the bigquery client library.
|
||||
// It submits a query and writes the result to a table.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/bigquery"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var (
|
||||
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
|
||||
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
|
||||
q = flag.String("q", "", "The query string")
|
||||
dest = flag.String("dest", "", "The ID of the BigQuery table to write the result to. If unset, an ephemeral table ID will be generated.")
|
||||
pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
|
||||
wait = flag.Bool("wait", false, "Whether to wait for the query job to complete.")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
flagsOk := true
|
||||
for _, f := range []string{"project", "dataset", "q"} {
|
||||
if flag.Lookup(f).Value.String() == "" {
|
||||
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
|
||||
flagsOk = false
|
||||
}
|
||||
}
|
||||
if !flagsOk {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := bigquery.NewClient(ctx, *project)
|
||||
if err != nil {
|
||||
log.Fatalf("Creating bigquery client: %v", err)
|
||||
}
|
||||
|
||||
query := client.Query(*q)
|
||||
query.DefaultProjectID = *project
|
||||
query.DefaultDatasetID = *dataset
|
||||
query.WriteDisposition = bigquery.WriteTruncate
|
||||
|
||||
if *dest != "" {
|
||||
query.Dst = client.Dataset(*dataset).Table(*dest)
|
||||
}
|
||||
|
||||
// Query data.
|
||||
job, err := query.Run(ctx)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Querying: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Submitted query. Job ID: %s\n", job.ID())
|
||||
if !*wait {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("Waiting for job to complete.\n")
|
||||
|
||||
for range time.Tick(*pollint) {
|
||||
status, err := job.Status(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("Failure determining status: %v", err)
|
||||
break
|
||||
}
|
||||
if !status.Done() {
|
||||
continue
|
||||
}
|
||||
if err := status.Err(); err == nil {
|
||||
fmt.Printf("Success\n")
|
||||
} else {
|
||||
fmt.Printf("Failure: %+v\n", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// read is an example client of the bigquery client library.
|
||||
// It reads from a table, returning the data via an Iterator.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"cloud.google.com/go/bigquery"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/iterator"
|
||||
)
|
||||
|
||||
var (
|
||||
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
|
||||
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
|
||||
table = flag.String("table", ".*", "A regular expression to match the IDs of tables to read.")
|
||||
jobID = flag.String("jobid", "", "The ID of a query job that has already been submitted."+
|
||||
" If set, --dataset, --table will be ignored, and results will be read from the specified job.")
|
||||
)
|
||||
|
||||
func printValues(ctx context.Context, it *bigquery.RowIterator) {
|
||||
// one-space padding.
|
||||
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
|
||||
|
||||
for {
|
||||
var vals []bigquery.Value
|
||||
err := it.Next(&vals)
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("err calling get: %v\n", err)
|
||||
}
|
||||
sep := ""
|
||||
for _, v := range vals {
|
||||
fmt.Fprintf(tw, "%s%v", sep, v)
|
||||
sep = "\t"
|
||||
}
|
||||
fmt.Fprintf(tw, "\n")
|
||||
}
|
||||
tw.Flush()
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
func printTable(ctx context.Context, client *bigquery.Client, t *bigquery.Table) {
|
||||
it := t.Read(ctx)
|
||||
id := t.FullyQualifiedName()
|
||||
fmt.Printf("%s\n%s\n", id, strings.Repeat("-", len(id)))
|
||||
printValues(ctx, it)
|
||||
}
|
||||
|
||||
func printQueryResults(ctx context.Context, client *bigquery.Client, queryJobID string) {
|
||||
job, err := client.JobFromID(ctx, queryJobID)
|
||||
if err != nil {
|
||||
log.Fatalf("Loading job: %v", err)
|
||||
}
|
||||
|
||||
it, err := job.Read(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("Reading: %v", err)
|
||||
}
|
||||
|
||||
// TODO: print schema.
|
||||
printValues(ctx, it)
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
flagsOk := true
|
||||
if flag.Lookup("project").Value.String() == "" {
|
||||
fmt.Fprintf(os.Stderr, "Flag --project is required\n")
|
||||
flagsOk = false
|
||||
}
|
||||
|
||||
var sourceFlagCount int
|
||||
if flag.Lookup("dataset").Value.String() != "" {
|
||||
sourceFlagCount++
|
||||
}
|
||||
if flag.Lookup("jobid").Value.String() != "" {
|
||||
sourceFlagCount++
|
||||
}
|
||||
if sourceFlagCount != 1 {
|
||||
fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n")
|
||||
flagsOk = false
|
||||
}
|
||||
|
||||
if !flagsOk {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
tableRE, err := regexp.Compile(*table)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
client, err := bigquery.NewClient(ctx, *project)
|
||||
if err != nil {
|
||||
log.Fatalf("Creating bigquery client: %v", err)
|
||||
}
|
||||
|
||||
if *jobID != "" {
|
||||
printQueryResults(ctx, client, *jobID)
|
||||
return
|
||||
}
|
||||
ds := client.Dataset(*dataset)
|
||||
tableIter := ds.Tables(context.Background())
|
||||
for {
|
||||
t, err := tableIter.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("Listing tables: %v", err)
|
||||
}
|
||||
if tableRE.MatchString(t.TableID) {
|
||||
printTable(ctx, client, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
# Cloud Bigtable Hello World in Go
|
||||
|
||||
This is a simple application that demonstrates using the [Google Cloud APIs Go
|
||||
Client Library](https://github.com/GoogleCloudPlatform/google-cloud-go) to connect
|
||||
to and interact with Cloud Bigtable.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. Set up Cloud Console.
|
||||
1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project.
|
||||
You will need the project ID later.
|
||||
1. Go to **Settings > Project Billing Settings** and enable billing.
|
||||
1. Select **APIs & Auth > APIs**.
|
||||
1. Enable the **Cloud Bigtable API** and the **Cloud Bigtable Admin API**.
|
||||
(You may need to search for the API).
|
||||
1. Set up gcloud.
|
||||
1. `gcloud components update`
|
||||
1. `gcloud auth login`
|
||||
1. `gcloud config set project PROJECT_ID`
|
||||
1. Provision a Cloud Bigtable instance
|
||||
1. Follow the instructions in the [user
|
||||
documentation](https://cloud.google.com/bigtable/docs/creating-instance) to
|
||||
create a Google Cloud Platform project and Cloud Bigtable instance if necessary.
|
||||
1. You'll need to reference your project id and instance id to run the application.
|
||||
|
||||
## Running
|
||||
|
||||
1. From the hello_world example folder, `go run main.go -project PROJECT_ID -instance INSTANCE_ID`, substituting your project id and instance id.
|
||||
|
||||
## Cleaning up
|
||||
|
||||
To avoid incurring extra charges to your Google Cloud Platform account, remove
|
||||
the resources created for this sample.
|
||||
|
||||
1. Go to the Clusters page in the [Cloud
|
||||
Console](https://console.cloud.google.com).
|
||||
|
||||
[Go to the Clusters page](https://console.cloud.google.com/project/_/bigtable/clusters)
|
||||
|
||||
1. Click the cluster name.
|
||||
|
||||
1. Click **Delete**.
|
||||
|
||||

|
||||
|
||||
1. Type the cluster ID, then click **Delete** to delete the cluster.
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
// Copyright 2016 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Hello world is a sample program demonstrating use of the Bigtable client
|
||||
// library to perform basic CRUD operations
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// User-provided constants.
|
||||
const (
|
||||
tableName = "Hello-Bigtable"
|
||||
columnFamilyName = "cf1"
|
||||
columnName = "greeting"
|
||||
)
|
||||
|
||||
var greetings = []string{"Hello World!", "Hello Cloud Bigtable!", "Hello golang!"}
|
||||
|
||||
// sliceContains reports whether the provided string is present in the given slice of strings.
|
||||
func sliceContains(list []string, target string) bool {
|
||||
for _, s := range list {
|
||||
if s == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
project := flag.String("project", "", "The Google Cloud Platform project ID. Required.")
|
||||
instance := flag.String("instance", "", "The Google Cloud Bigtable instance ID. Required.")
|
||||
flag.Parse()
|
||||
|
||||
for _, f := range []string{"project", "instance"} {
|
||||
if flag.Lookup(f).Value.String() == "" {
|
||||
log.Fatalf("The %s flag is required.", f)
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Set up admin client, tables, and column families.
|
||||
// NewAdminClient uses Application Default Credentials to authenticate.
|
||||
adminClient, err := bigtable.NewAdminClient(ctx, *project, *instance)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create admin client: %v", err)
|
||||
}
|
||||
|
||||
tables, err := adminClient.Tables(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not fetch table list: %v", err)
|
||||
}
|
||||
|
||||
if !sliceContains(tables, tableName) {
|
||||
log.Printf("Creating table %s", tableName)
|
||||
if err := adminClient.CreateTable(ctx, tableName); err != nil {
|
||||
log.Fatalf("Could not create table %s: %v", tableName, err)
|
||||
}
|
||||
}
|
||||
|
||||
tblInfo, err := adminClient.TableInfo(ctx, tableName)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not read info for table %s: %v", tableName, err)
|
||||
}
|
||||
|
||||
if !sliceContains(tblInfo.Families, columnFamilyName) {
|
||||
if err := adminClient.CreateColumnFamily(ctx, tableName, columnFamilyName); err != nil {
|
||||
log.Fatalf("Could not create column family %s: %v", columnFamilyName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set up Bigtable data operations client.
|
||||
// NewClient uses Application Default Credentials to authenticate.
|
||||
client, err := bigtable.NewClient(ctx, *project, *instance)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create data operations client: %v", err)
|
||||
}
|
||||
|
||||
tbl := client.Open(tableName)
|
||||
muts := make([]*bigtable.Mutation, len(greetings))
|
||||
rowKeys := make([]string, len(greetings))
|
||||
|
||||
log.Printf("Writing greeting rows to table")
|
||||
for i, greeting := range greetings {
|
||||
muts[i] = bigtable.NewMutation()
|
||||
muts[i].Set(columnFamilyName, columnName, bigtable.Now(), []byte(greeting))
|
||||
|
||||
// Each row has a unique row key.
|
||||
//
|
||||
// Note: This example uses sequential numeric IDs for simplicity, but
|
||||
// this can result in poor performance in a production application.
|
||||
// Since rows are stored in sorted order by key, sequential keys can
|
||||
// result in poor distribution of operations across nodes.
|
||||
//
|
||||
// For more information about how to design a Bigtable schema for the
|
||||
// best performance, see the documentation:
|
||||
//
|
||||
// https://cloud.google.com/bigtable/docs/schema-design
|
||||
rowKeys[i] = fmt.Sprintf("%s%d", columnName, i)
|
||||
}
|
||||
|
||||
rowErrs, err := tbl.ApplyBulk(ctx, rowKeys, muts)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not apply bulk row mutation: %v", err)
|
||||
}
|
||||
if rowErrs != nil {
|
||||
for _, rowErr := range rowErrs {
|
||||
log.Printf("Error writing row: %v", rowErr)
|
||||
}
|
||||
log.Fatalf("Could not write some rows")
|
||||
}
|
||||
|
||||
log.Printf("Getting a single greeting by row key:")
|
||||
row, err := tbl.ReadRow(ctx, rowKeys[0], bigtable.RowFilter(bigtable.ColumnFilter(columnName)))
|
||||
if err != nil {
|
||||
log.Fatalf("Could not read row with key %s: %v", rowKeys[0], err)
|
||||
}
|
||||
log.Printf("\t%s = %s\n", rowKeys[0], string(row[columnFamilyName][0].Value))
|
||||
|
||||
log.Printf("Reading all greeting rows:")
|
||||
err = tbl.ReadRows(ctx, bigtable.PrefixRange(columnName), func(row bigtable.Row) bool {
|
||||
item := row[columnFamilyName][0]
|
||||
log.Printf("\t%s = %s\n", item.Row, string(item.Value))
|
||||
return true
|
||||
}, bigtable.RowFilter(bigtable.ColumnFilter(columnName)))
|
||||
|
||||
if err = client.Close(); err != nil {
|
||||
log.Fatalf("Could not close data operations client: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("Deleting the table")
|
||||
if err = adminClient.DeleteTable(ctx, tableName); err != nil {
|
||||
log.Fatalf("Could not delete table %s: %v", tableName, err)
|
||||
}
|
||||
|
||||
if err = adminClient.Close(); err != nil {
|
||||
log.Fatalf("Could not close admin client: %v", err)
|
||||
}
|
||||
}
|
||||
+453
@@ -0,0 +1,453 @@
|
||||
/*
|
||||
Copyright 2015 Google Inc. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Search is a sample web server that uses Cloud Bigtable as the storage layer
|
||||
// for a simple document-storage and full-text-search service.
|
||||
// It has four functions:
|
||||
// - Initialize and clear the table.
|
||||
// - Add a document. This adds the content of a user-supplied document to the
|
||||
// Bigtable, and adds references to the document to an index in the Bigtable.
|
||||
// The document is indexed under each unique word in the document.
|
||||
// - Search the index. This returns documents containing each word in a user
|
||||
// query, with snippets and links to view the whole document.
|
||||
// - Copy table. This copies the documents and index from another table and
|
||||
// adds them to the current one.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var (
|
||||
addTemplate = template.Must(template.New("").Parse(`<html><body>
|
||||
Added {{.Title}}
|
||||
</body></html>`))
|
||||
|
||||
contentTemplate = template.Must(template.New("").Parse(`<html><body>
|
||||
<b>{{.Title}}</b><br><br>
|
||||
{{.Content}}
|
||||
</body></html>`))
|
||||
|
||||
searchTemplate = template.Must(template.New("").Parse(`<html><body>
|
||||
Results for <b>{{.Query}}</b>:<br><br>
|
||||
{{range .Results}}
|
||||
<a href="/content?name={{.Title}}">{{.Title}}</a><br>
|
||||
<i>{{.Snippet}}</i><br><br>
|
||||
{{end}}
|
||||
</body></html>`))
|
||||
)
|
||||
|
||||
const (
|
||||
indexColumnFamily = "i"
|
||||
contentColumnFamily = "c"
|
||||
mainPage = `
|
||||
<html>
|
||||
<head>
|
||||
<title>Document Search</title>
|
||||
</head>
|
||||
<body>
|
||||
Initialize and clear table:
|
||||
<form action="/reset" method="post">
|
||||
<div><input type="submit" value="Init"></div>
|
||||
</form>
|
||||
|
||||
Search for documents:
|
||||
<form action="/search" method="post">
|
||||
<div><input type="text" name="q" size=80></div>
|
||||
<div><input type="submit" value="Search"></div>
|
||||
</form>
|
||||
|
||||
Add a document:
|
||||
<form action="/add" method="post">
|
||||
Document name:
|
||||
<div><textarea name="name" rows="1" cols="80"></textarea></div>
|
||||
Document text:
|
||||
<div><textarea name="content" rows="20" cols="80"></textarea></div>
|
||||
<div><input type="submit" value="Submit"></div>
|
||||
</form>
|
||||
|
||||
Copy data from another table:
|
||||
<form action="/copy" method="post">
|
||||
Source table name:
|
||||
<div><input type="text" name="name" size=80></div>
|
||||
<div><input type="submit" value="Copy"></div>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
)
|
||||
|
||||
func main() {
|
||||
var (
|
||||
project = flag.String("project", "", "The name of the project.")
|
||||
instance = flag.String("instance", "", "The name of the Cloud Bigtable instance.")
|
||||
tableName = flag.String("table", "docindex", "The name of the table containing the documents and index.")
|
||||
port = flag.Int("port", 8080, "TCP port for server.")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
// Make an admin client.
|
||||
adminClient, err := bigtable.NewAdminClient(context.Background(), *project, *instance)
|
||||
if err != nil {
|
||||
log.Fatal("Bigtable NewAdminClient:", err)
|
||||
}
|
||||
|
||||
// Make a regular client.
|
||||
client, err := bigtable.NewClient(context.Background(), *project, *instance)
|
||||
if err != nil {
|
||||
log.Fatal("Bigtable NewClient:", err)
|
||||
}
|
||||
|
||||
// Open the table.
|
||||
table := client.Open(*tableName)
|
||||
|
||||
// Set up HTML handlers, and start the web server.
|
||||
http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) { handleSearch(w, r, table) })
|
||||
http.HandleFunc("/content", func(w http.ResponseWriter, r *http.Request) { handleContent(w, r, table) })
|
||||
http.HandleFunc("/add", func(w http.ResponseWriter, r *http.Request) { handleAddDoc(w, r, table) })
|
||||
http.HandleFunc("/reset", func(w http.ResponseWriter, r *http.Request) { handleReset(w, r, *tableName, adminClient) })
|
||||
http.HandleFunc("/copy", func(w http.ResponseWriter, r *http.Request) { handleCopy(w, r, *tableName, client, adminClient) })
|
||||
http.HandleFunc("/", handleMain)
|
||||
log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
|
||||
}
|
||||
|
||||
// handleMain outputs the home page.
|
||||
func handleMain(w http.ResponseWriter, r *http.Request) {
|
||||
io.WriteString(w, mainPage)
|
||||
}
|
||||
|
||||
// tokenize splits a string into tokens.
|
||||
// This is very simple, it's not a good tokenization function.
|
||||
func tokenize(s string) []string {
|
||||
wordMap := make(map[string]bool)
|
||||
f := strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) })
|
||||
for _, word := range f {
|
||||
word = strings.ToLower(word)
|
||||
wordMap[word] = true
|
||||
}
|
||||
words := make([]string, 0, len(wordMap))
|
||||
for word := range wordMap {
|
||||
words = append(words, word)
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
// handleContent fetches the content of a document from the Bigtable and returns it.
|
||||
func handleContent(w http.ResponseWriter, r *http.Request, table *bigtable.Table) {
|
||||
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
name := r.FormValue("name")
|
||||
if len(name) == 0 {
|
||||
http.Error(w, "No document name supplied.", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
row, err := table.ReadRow(ctx, name)
|
||||
if err != nil {
|
||||
http.Error(w, "Error reading content: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
content := row[contentColumnFamily]
|
||||
if len(content) == 0 {
|
||||
http.Error(w, "Document not found.", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := contentTemplate.ExecuteTemplate(&buf, "", struct{ Title, Content string }{name, string(content[0].Value)}); err != nil {
|
||||
http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
io.Copy(w, &buf)
|
||||
}
|
||||
|
||||
// handleSearch responds to search queries, returning links and snippets for matching documents.
|
||||
func handleSearch(w http.ResponseWriter, r *http.Request, table *bigtable.Table) {
|
||||
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
query := r.FormValue("q")
|
||||
// Split the query into words.
|
||||
words := tokenize(query)
|
||||
if len(words) == 0 {
|
||||
http.Error(w, "Empty query.", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// readRows reads from many rows concurrently.
|
||||
readRows := func(rows []string) ([]bigtable.Row, error) {
|
||||
results := make([]bigtable.Row, len(rows))
|
||||
errors := make([]error, len(rows))
|
||||
var wg sync.WaitGroup
|
||||
for i, row := range rows {
|
||||
wg.Add(1)
|
||||
go func(i int, row string) {
|
||||
defer wg.Done()
|
||||
results[i], errors[i] = table.ReadRow(ctx, row, bigtable.RowFilter(bigtable.LatestNFilter(1)))
|
||||
}(i, row)
|
||||
}
|
||||
wg.Wait()
|
||||
for _, err := range errors {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// For each query word, get the list of documents containing it.
|
||||
results, err := readRows(words)
|
||||
if err != nil {
|
||||
http.Error(w, "Error reading index: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Count how many of the query words each result contained.
|
||||
hits := make(map[string]int)
|
||||
for _, r := range results {
|
||||
for _, r := range r[indexColumnFamily] {
|
||||
hits[r.Column]++
|
||||
}
|
||||
}
|
||||
|
||||
// Build a slice of all the documents that matched every query word.
|
||||
var matches []string
|
||||
for doc, count := range hits {
|
||||
if count == len(words) {
|
||||
matches = append(matches, doc[len(indexColumnFamily+":"):])
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch the content of those documents from the Bigtable.
|
||||
content, err := readRows(matches)
|
||||
if err != nil {
|
||||
http.Error(w, "Error reading results: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
type result struct{ Title, Snippet string }
|
||||
data := struct {
|
||||
Query string
|
||||
Results []result
|
||||
}{query, nil}
|
||||
|
||||
// Output links and snippets.
|
||||
for i, doc := range matches {
|
||||
var text string
|
||||
c := content[i][contentColumnFamily]
|
||||
if len(c) > 0 {
|
||||
text = string(c[0].Value)
|
||||
}
|
||||
if len(text) > 100 {
|
||||
text = text[:100] + "..."
|
||||
}
|
||||
data.Results = append(data.Results, result{doc, text})
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := searchTemplate.ExecuteTemplate(&buf, "", data); err != nil {
|
||||
http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
io.Copy(w, &buf)
|
||||
}
|
||||
|
||||
// handleAddDoc adds a document to the index.
|
||||
func handleAddDoc(w http.ResponseWriter, r *http.Request, table *bigtable.Table) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "POST requests only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
ctx, _ := context.WithTimeout(context.Background(), time.Minute)
|
||||
|
||||
name := r.FormValue("name")
|
||||
if len(name) == 0 {
|
||||
http.Error(w, "Empty document name!", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
content := r.FormValue("content")
|
||||
if len(content) == 0 {
|
||||
http.Error(w, "Empty document content!", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
writeErr error // Set if any write fails.
|
||||
mu sync.Mutex // Protects writeErr
|
||||
wg sync.WaitGroup // Used to wait for all writes to finish.
|
||||
)
|
||||
|
||||
// writeOneColumn writes one column in one row, updates err if there is an error,
|
||||
// and signals wg that one operation has finished.
|
||||
writeOneColumn := func(row, family, column, value string, ts bigtable.Timestamp) {
|
||||
mut := bigtable.NewMutation()
|
||||
mut.Set(family, column, ts, []byte(value))
|
||||
err := table.Apply(ctx, row, mut)
|
||||
if err != nil {
|
||||
mu.Lock()
|
||||
writeErr = err
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Start a write to store the document content.
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
writeOneColumn(name, contentColumnFamily, "", content, bigtable.Now())
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// Start writes to store the document name in the index for each word in the document.
|
||||
words := tokenize(content)
|
||||
for _, word := range words {
|
||||
var (
|
||||
row = word
|
||||
family = indexColumnFamily
|
||||
column = name
|
||||
value = ""
|
||||
ts = bigtable.Now()
|
||||
)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
// TODO: should use a semaphore to limit the number of concurrent writes.
|
||||
writeOneColumn(row, family, column, value, ts)
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if writeErr != nil {
|
||||
http.Error(w, "Error writing to Bigtable: "+writeErr.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := addTemplate.ExecuteTemplate(&buf, "", struct{ Title string }{name}); err != nil {
|
||||
http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
io.Copy(w, &buf)
|
||||
}
|
||||
|
||||
// handleReset deletes the table if it exists, creates it again, and creates its column families.
|
||||
func handleReset(w http.ResponseWriter, r *http.Request, table string, adminClient *bigtable.AdminClient) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "POST requests only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
adminClient.DeleteTable(ctx, table)
|
||||
if err := adminClient.CreateTable(ctx, table); err != nil {
|
||||
http.Error(w, "Error creating Bigtable: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Second)
|
||||
// Create two column families, and set the GC policy for each one to keep one version.
|
||||
for _, family := range []string{indexColumnFamily, contentColumnFamily} {
|
||||
if err := adminClient.CreateColumnFamily(ctx, table, family); err != nil {
|
||||
http.Error(w, "Error creating column family: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := adminClient.SetGCPolicy(ctx, table, family, bigtable.MaxVersionsPolicy(1)); err != nil {
|
||||
http.Error(w, "Error setting GC policy: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
w.Write([]byte("<html><body>Done.</body></html>"))
|
||||
return
|
||||
}
|
||||
|
||||
// copyTable copies data from one table to another.
|
||||
func copyTable(src, dst string, client *bigtable.Client, adminClient *bigtable.AdminClient) error {
|
||||
if src == "" || src == dst {
|
||||
return nil
|
||||
}
|
||||
ctx, _ := context.WithTimeout(context.Background(), time.Minute)
|
||||
|
||||
// Open the source and destination tables.
|
||||
srcTable := client.Open(src)
|
||||
dstTable := client.Open(dst)
|
||||
|
||||
var (
|
||||
writeErr error // Set if any write fails.
|
||||
mu sync.Mutex // Protects writeErr
|
||||
wg sync.WaitGroup // Used to wait for all writes to finish.
|
||||
)
|
||||
copyRowToTable := func(row bigtable.Row) bool {
|
||||
mu.Lock()
|
||||
failed := writeErr != nil
|
||||
mu.Unlock()
|
||||
if failed {
|
||||
return false
|
||||
}
|
||||
mut := bigtable.NewMutation()
|
||||
for family, items := range row {
|
||||
for _, item := range items {
|
||||
// Get the column name, excluding the column family name and ':' character.
|
||||
columnWithoutFamily := item.Column[len(family)+1:]
|
||||
mut.Set(family, columnWithoutFamily, bigtable.Now(), item.Value)
|
||||
}
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
// TODO: should use a semaphore to limit the number of concurrent writes.
|
||||
if err := dstTable.Apply(ctx, row.Key(), mut); err != nil {
|
||||
mu.Lock()
|
||||
writeErr = err
|
||||
mu.Unlock()
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// Create a filter that only accepts the column families we're interested in.
|
||||
filter := bigtable.FamilyFilter(indexColumnFamily + "|" + contentColumnFamily)
|
||||
// Read every row from srcTable, and call copyRowToTable to copy it to our table.
|
||||
err := srcTable.ReadRows(ctx, bigtable.InfiniteRange(""), copyRowToTable, bigtable.RowFilter(filter))
|
||||
wg.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeErr
|
||||
}
|
||||
|
||||
// handleCopy copies data from one table to another.
|
||||
func handleCopy(w http.ResponseWriter, r *http.Request, dst string, client *bigtable.Client, adminClient *bigtable.AdminClient) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "POST requests only", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
src := r.FormValue("name")
|
||||
if src == "" {
|
||||
http.Error(w, "No source table specified.", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := copyTable(src, dst, client, adminClient); err != nil {
|
||||
http.Error(w, "Failed to rebuild index: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, "Copied table.\n")
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
# User Counter
|
||||
# (Cloud Bigtable on Managed VMs using Go)
|
||||
|
||||
This app counts how often each user visits. The app uses Cloud Bigtable to store the visit counts for each user.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. Set up Cloud Console.
|
||||
1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project.
|
||||
You will need the project ID later.
|
||||
1. Go to **Settings > Project Billing Settings** and enable billing.
|
||||
1. Select **APIs & Auth > APIs**.
|
||||
1. Enable the **Cloud Bigtable API** and the **Cloud Bigtable Admin API**.
|
||||
(You may need to search for the API).
|
||||
1. Set up gcloud.
|
||||
1. `gcloud components update`
|
||||
1. `gcloud auth login`
|
||||
1. `gcloud config set project PROJECT_ID`
|
||||
1. Download App Engine SDK for Go.
|
||||
1. `go get -u google.golang.org/appengine/...`
|
||||
1. In main.go, change the `project` and `instance` constants.
|
||||
|
||||
## Running locally
|
||||
|
||||
1. From the sample project folder, `dev_appserver.py app.yaml`.
|
||||
|
||||
## Deploying on Google App Engine flexible environment
|
||||
|
||||
Follow the [deployment instructions](https://cloud.google.com/appengine/docs/flexible/go/testing-and-deploying-your-app).
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
runtime: go
|
||||
api_version: go1
|
||||
vm: true
|
||||
|
||||
manual_scaling:
|
||||
instances: 1
|
||||
|
||||
handlers:
|
||||
# Serve only the web root.
|
||||
- url: /
|
||||
script: _go_app
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
// Copyright 2015 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/*
|
||||
User counter is a program that tracks how often a user has visited the index page.
|
||||
|
||||
This program demonstrates usage of the Cloud Bigtable API for App Engine flexible environment and Go.
|
||||
Instructions for running this program are in the README.md.
|
||||
*/
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"html/template"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"cloud.google.com/go/bigtable"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/appengine"
|
||||
aelog "google.golang.org/appengine/log"
|
||||
"google.golang.org/appengine/user"
|
||||
)
|
||||
|
||||
// User-provided constants.
|
||||
const (
|
||||
project = "PROJECT_ID"
|
||||
instance = "INSTANCE"
|
||||
)
|
||||
|
||||
var (
|
||||
tableName = "user-visit-counter"
|
||||
familyName = "emails"
|
||||
|
||||
// Client is initialized by main.
|
||||
client *bigtable.Client
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Set up admin client, tables, and column families.
|
||||
// NewAdminClient uses Application Default Credentials to authenticate.
|
||||
adminClient, err := bigtable.NewAdminClient(ctx, project, instance)
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to create a table admin client. %v", err)
|
||||
}
|
||||
tables, err := adminClient.Tables(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to fetch table list. %v", err)
|
||||
}
|
||||
if !sliceContains(tables, tableName) {
|
||||
if err := adminClient.CreateTable(ctx, tableName); err != nil {
|
||||
log.Fatalf("Unable to create table: %v. %v", tableName, err)
|
||||
}
|
||||
}
|
||||
tblInfo, err := adminClient.TableInfo(ctx, tableName)
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to read info for table: %v. %v", tableName, err)
|
||||
}
|
||||
if !sliceContains(tblInfo.Families, familyName) {
|
||||
if err := adminClient.CreateColumnFamily(ctx, tableName, familyName); err != nil {
|
||||
log.Fatalf("Unable to create column family: %v. %v", familyName, err)
|
||||
}
|
||||
}
|
||||
adminClient.Close()
|
||||
|
||||
// Set up Bigtable data operations client.
|
||||
// NewClient uses Application Default Credentials to authenticate.
|
||||
client, err = bigtable.NewClient(ctx, project, instance)
|
||||
if err != nil {
|
||||
log.Fatalf("Unable to create data operations client. %v", err)
|
||||
}
|
||||
|
||||
http.Handle("/", appHandler(mainHandler))
|
||||
appengine.Main() // Never returns.
|
||||
}
|
||||
|
||||
// mainHandler tracks how many times each user has visited this page.
|
||||
func mainHandler(w http.ResponseWriter, r *http.Request) *appError {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx := appengine.NewContext(r)
|
||||
u := user.Current(ctx)
|
||||
if u == nil {
|
||||
login, err := user.LoginURL(ctx, r.URL.String())
|
||||
if err != nil {
|
||||
return &appError{err, "Error finding login URL", http.StatusInternalServerError}
|
||||
}
|
||||
http.Redirect(w, r, login, http.StatusFound)
|
||||
return nil
|
||||
}
|
||||
logoutURL, err := user.LogoutURL(ctx, "/")
|
||||
if err != nil {
|
||||
return &appError{err, "Error finding logout URL", http.StatusInternalServerError}
|
||||
}
|
||||
|
||||
// Increment visit count for user.
|
||||
tbl := client.Open(tableName)
|
||||
rmw := bigtable.NewReadModifyWrite()
|
||||
rmw.Increment(familyName, u.Email, 1)
|
||||
row, err := tbl.ApplyReadModifyWrite(ctx, u.Email, rmw)
|
||||
if err != nil {
|
||||
return &appError{err, "Error applying ReadModifyWrite to row: " + u.Email, http.StatusInternalServerError}
|
||||
}
|
||||
data := struct {
|
||||
Username, Logout string
|
||||
Visits uint64
|
||||
}{
|
||||
Username: u.Email,
|
||||
// Retrieve the most recently edited column.
|
||||
Visits: binary.BigEndian.Uint64(row[familyName][0].Value),
|
||||
Logout: logoutURL,
|
||||
}
|
||||
|
||||
// Display hello page.
|
||||
var buf bytes.Buffer
|
||||
if err := tmpl.Execute(&buf, data); err != nil {
|
||||
return &appError{err, "Error writing template", http.StatusInternalServerError}
|
||||
}
|
||||
buf.WriteTo(w)
|
||||
return nil
|
||||
}
|
||||
|
||||
var tmpl = template.Must(template.New("").Parse(`
|
||||
<html><body>
|
||||
|
||||
<p>
|
||||
{{with .Username}} Hello {{.}}{{end}}
|
||||
{{with .Logout}}<a href="{{.}}">Sign out</a>{{end}}
|
||||
|
||||
</p>
|
||||
|
||||
<p>
|
||||
You have visited {{.Visits}}
|
||||
</p>
|
||||
|
||||
</body></html>`))
|
||||
|
||||
// sliceContains reports whether the provided string is present in the given slice of strings.
|
||||
func sliceContains(list []string, target string) bool {
|
||||
for _, s := range list {
|
||||
if s == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// More info about this method of error handling can be found at: http://blog.golang.org/error-handling-and-go
|
||||
type appHandler func(http.ResponseWriter, *http.Request) *appError
|
||||
|
||||
type appError struct {
|
||||
Error error
|
||||
Message string
|
||||
Code int
|
||||
}
|
||||
|
||||
func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if e := fn(w, r); e != nil {
|
||||
ctx := appengine.NewContext(r)
|
||||
aelog.Errorf(ctx, "%v", e.Error)
|
||||
http.Error(w, e.Message, e.Code)
|
||||
}
|
||||
}
|
||||
+428
@@ -0,0 +1,428 @@
|
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//[START sample]
|
||||
// Package gcsdemo is an example App Engine app using the Google Cloud Storage API.
|
||||
package gcsdemo
|
||||
|
||||
//[START imports]
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/appengine"
|
||||
"google.golang.org/appengine/file"
|
||||
"google.golang.org/appengine/log"
|
||||
)
|
||||
|
||||
//[END imports]
|
||||
|
||||
func init() {
|
||||
http.HandleFunc("/", handler)
|
||||
}
|
||||
|
||||
// demo struct holds information needed to run the various demo functions.
|
||||
type demo struct {
|
||||
client *storage.Client
|
||||
bucketName string
|
||||
bucket *storage.BucketHandle
|
||||
|
||||
w io.Writer
|
||||
ctx context.Context
|
||||
// cleanUp is a list of filenames that need cleaning up at the end of the demo.
|
||||
cleanUp []string
|
||||
// failed indicates that one or more of the demo steps failed.
|
||||
failed bool
|
||||
}
|
||||
|
||||
func (d *demo) errorf(format string, args ...interface{}) {
|
||||
d.failed = true
|
||||
fmt.Fprintln(d.w, fmt.Sprintf(format, args...))
|
||||
log.Errorf(d.ctx, format, args...)
|
||||
}
|
||||
|
||||
// handler is the main demo entry point that calls the GCS operations.
|
||||
func handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := appengine.NewContext(r)
|
||||
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
//[START get_default_bucket]
|
||||
// Use `dev_appserver.py --default_gcs_bucket_name GCS_BUCKET_NAME`
|
||||
// when running locally.
|
||||
bucket, err := file.DefaultBucketName(ctx)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "failed to get default GCS bucket name: %v", err)
|
||||
}
|
||||
//[END get_default_bucket]
|
||||
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "failed to create client: %v", err)
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(ctx))
|
||||
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
d := &demo{
|
||||
w: buf,
|
||||
ctx: ctx,
|
||||
client: client,
|
||||
bucket: client.Bucket(bucket),
|
||||
bucketName: bucket,
|
||||
}
|
||||
|
||||
n := "demo-testfile-go"
|
||||
d.createFile(n)
|
||||
d.readFile(n)
|
||||
d.copyFile(n)
|
||||
d.statFile(n)
|
||||
d.createListFiles()
|
||||
d.listBucket()
|
||||
d.listBucketDirMode()
|
||||
d.defaultACL()
|
||||
d.putDefaultACLRule()
|
||||
d.deleteDefaultACLRule()
|
||||
d.bucketACL()
|
||||
d.putBucketACLRule()
|
||||
d.deleteBucketACLRule()
|
||||
d.acl(n)
|
||||
d.putACLRule(n)
|
||||
d.deleteACLRule(n)
|
||||
d.deleteFiles()
|
||||
|
||||
if d.failed {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
buf.WriteTo(w)
|
||||
fmt.Fprintf(w, "\nDemo failed.\n")
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
buf.WriteTo(w)
|
||||
fmt.Fprintf(w, "\nDemo succeeded.\n")
|
||||
}
|
||||
}
|
||||
|
||||
//[START write]
|
||||
// createFile creates a file in Google Cloud Storage.
|
||||
func (d *demo) createFile(fileName string) {
|
||||
fmt.Fprintf(d.w, "Creating file /%v/%v\n", d.bucketName, fileName)
|
||||
|
||||
wc := d.bucket.Object(fileName).NewWriter(d.ctx)
|
||||
wc.ContentType = "text/plain"
|
||||
wc.Metadata = map[string]string{
|
||||
"x-goog-meta-foo": "foo",
|
||||
"x-goog-meta-bar": "bar",
|
||||
}
|
||||
d.cleanUp = append(d.cleanUp, fileName)
|
||||
|
||||
if _, err := wc.Write([]byte("abcde\n")); err != nil {
|
||||
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil {
|
||||
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
if err := wc.Close(); err != nil {
|
||||
d.errorf("createFile: unable to close bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
//[END write]
|
||||
|
||||
//[START read]
|
||||
// readFile reads the named file in Google Cloud Storage.
|
||||
func (d *demo) readFile(fileName string) {
|
||||
io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n")
|
||||
|
||||
rc, err := d.bucket.Object(fileName).NewReader(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("readFile: unable to open file from bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
slurp, err := ioutil.ReadAll(rc)
|
||||
if err != nil {
|
||||
d.errorf("readFile: unable to read data from bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0])
|
||||
if len(slurp) > 1024 {
|
||||
fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:])
|
||||
} else {
|
||||
fmt.Fprintf(d.w, "%s\n", slurp)
|
||||
}
|
||||
}
|
||||
|
||||
//[END read]
|
||||
|
||||
//[START copy]
|
||||
// copyFile copies a file in Google Cloud Storage.
|
||||
func (d *demo) copyFile(fileName string) {
|
||||
copyName := fileName + "-copy"
|
||||
fmt.Fprintf(d.w, "Copying file /%v/%v to /%v/%v:\n", d.bucketName, fileName, d.bucketName, copyName)
|
||||
|
||||
obj, err := d.bucket.Object(copyName).CopierFrom(d.bucket.Object(fileName)).Run(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("copyFile: unable to copy /%v/%v to bucket %q, file %q: %v", d.bucketName, fileName, d.bucketName, copyName, err)
|
||||
return
|
||||
}
|
||||
d.cleanUp = append(d.cleanUp, copyName)
|
||||
|
||||
d.dumpStats(obj)
|
||||
}
|
||||
|
||||
//[END copy]
|
||||
|
||||
func (d *demo) dumpStats(obj *storage.ObjectAttrs) {
|
||||
fmt.Fprintf(d.w, "(filename: /%v/%v, ", obj.Bucket, obj.Name)
|
||||
fmt.Fprintf(d.w, "ContentType: %q, ", obj.ContentType)
|
||||
fmt.Fprintf(d.w, "ACL: %#v, ", obj.ACL)
|
||||
fmt.Fprintf(d.w, "Owner: %v, ", obj.Owner)
|
||||
fmt.Fprintf(d.w, "ContentEncoding: %q, ", obj.ContentEncoding)
|
||||
fmt.Fprintf(d.w, "Size: %v, ", obj.Size)
|
||||
fmt.Fprintf(d.w, "MD5: %q, ", obj.MD5)
|
||||
fmt.Fprintf(d.w, "CRC32C: %q, ", obj.CRC32C)
|
||||
fmt.Fprintf(d.w, "Metadata: %#v, ", obj.Metadata)
|
||||
fmt.Fprintf(d.w, "MediaLink: %q, ", obj.MediaLink)
|
||||
fmt.Fprintf(d.w, "StorageClass: %q, ", obj.StorageClass)
|
||||
if !obj.Deleted.IsZero() {
|
||||
fmt.Fprintf(d.w, "Deleted: %v, ", obj.Deleted)
|
||||
}
|
||||
fmt.Fprintf(d.w, "Updated: %v)\n", obj.Updated)
|
||||
}
|
||||
|
||||
//[START file_metadata]
|
||||
// statFile reads the stats of the named file in Google Cloud Storage.
|
||||
func (d *demo) statFile(fileName string) {
|
||||
io.WriteString(d.w, "\nFile stat:\n")
|
||||
|
||||
obj, err := d.bucket.Object(fileName).Attrs(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("statFile: unable to stat file from bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
|
||||
d.dumpStats(obj)
|
||||
}
|
||||
|
||||
//[END file_metadata]
|
||||
|
||||
// createListFiles creates files that will be used by listBucket.
|
||||
func (d *demo) createListFiles() {
|
||||
io.WriteString(d.w, "\nCreating more files for listbucket...\n")
|
||||
for _, n := range []string{"foo1", "foo2", "bar", "bar/1", "bar/2", "boo/"} {
|
||||
d.createFile(n)
|
||||
}
|
||||
}
|
||||
|
||||
//[START list_bucket]
|
||||
// listBucket lists the contents of a bucket in Google Cloud Storage.
|
||||
func (d *demo) listBucket() {
|
||||
io.WriteString(d.w, "\nListbucket result:\n")
|
||||
|
||||
query := &storage.Query{Prefix: "foo"}
|
||||
it := d.bucket.Objects(d.ctx, query)
|
||||
for {
|
||||
obj, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
d.errorf("listBucket: unable to list bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpStats(obj)
|
||||
}
|
||||
}
|
||||
|
||||
//[END list_bucket]
|
||||
|
||||
func (d *demo) listDir(name, indent string) {
|
||||
query := &storage.Query{Prefix: name, Delimiter: "/"}
|
||||
it := d.bucket.Objects(d.ctx, query)
|
||||
for {
|
||||
obj, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
d.errorf("listBucketDirMode: unable to list bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
if obj.Prefix == "" {
|
||||
fmt.Fprint(d.w, indent)
|
||||
d.dumpStats(obj)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(d.w, "%v(directory: /%v/%v)\n", indent, d.bucketName, obj.Prefix)
|
||||
d.listDir(obj.Prefix, indent+" ")
|
||||
}
|
||||
}
|
||||
|
||||
// listBucketDirMode lists the contents of a bucket in dir mode in Google Cloud Storage.
|
||||
func (d *demo) listBucketDirMode() {
|
||||
io.WriteString(d.w, "\nListbucket directory mode result:\n")
|
||||
d.listDir("b", "")
|
||||
}
|
||||
|
||||
// dumpDefaultACL prints out the default object ACL for this bucket.
|
||||
func (d *demo) dumpDefaultACL() {
|
||||
acl, err := d.bucket.ACL().List(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("defaultACL: unable to list default object ACL for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
for _, v := range acl {
|
||||
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
|
||||
}
|
||||
}
|
||||
|
||||
// defaultACL displays the default object ACL for this bucket.
|
||||
func (d *demo) defaultACL() {
|
||||
io.WriteString(d.w, "\nDefault object ACL:\n")
|
||||
d.dumpDefaultACL()
|
||||
}
|
||||
|
||||
// putDefaultACLRule adds the "allUsers" default object ACL rule for this bucket.
|
||||
func (d *demo) putDefaultACLRule() {
|
||||
io.WriteString(d.w, "\nPut Default object ACL Rule:\n")
|
||||
err := d.bucket.DefaultObjectACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
|
||||
if err != nil {
|
||||
d.errorf("putDefaultACLRule: unable to save default object ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpDefaultACL()
|
||||
}
|
||||
|
||||
// deleteDefaultACLRule deleted the "allUsers" default object ACL rule for this bucket.
|
||||
func (d *demo) deleteDefaultACLRule() {
|
||||
io.WriteString(d.w, "\nDelete Default object ACL Rule:\n")
|
||||
err := d.bucket.DefaultObjectACL().Delete(d.ctx, storage.AllUsers)
|
||||
if err != nil {
|
||||
d.errorf("deleteDefaultACLRule: unable to delete default object ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpDefaultACL()
|
||||
}
|
||||
|
||||
// dumpBucketACL prints out the bucket ACL.
|
||||
func (d *demo) dumpBucketACL() {
|
||||
acl, err := d.bucket.ACL().List(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("dumpBucketACL: unable to list bucket ACL for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
for _, v := range acl {
|
||||
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
|
||||
}
|
||||
}
|
||||
|
||||
// bucketACL displays the bucket ACL for this bucket.
|
||||
func (d *demo) bucketACL() {
|
||||
io.WriteString(d.w, "\nBucket ACL:\n")
|
||||
d.dumpBucketACL()
|
||||
}
|
||||
|
||||
// putBucketACLRule adds the "allUsers" bucket ACL rule for this bucket.
|
||||
func (d *demo) putBucketACLRule() {
|
||||
io.WriteString(d.w, "\nPut Bucket ACL Rule:\n")
|
||||
err := d.bucket.ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
|
||||
if err != nil {
|
||||
d.errorf("putBucketACLRule: unable to save bucket ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpBucketACL()
|
||||
}
|
||||
|
||||
// deleteBucketACLRule deleted the "allUsers" bucket ACL rule for this bucket.
|
||||
func (d *demo) deleteBucketACLRule() {
|
||||
io.WriteString(d.w, "\nDelete Bucket ACL Rule:\n")
|
||||
err := d.bucket.ACL().Delete(d.ctx, storage.AllUsers)
|
||||
if err != nil {
|
||||
d.errorf("deleteBucketACLRule: unable to delete bucket ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpBucketACL()
|
||||
}
|
||||
|
||||
// dumpACL prints out the ACL of the named file.
|
||||
func (d *demo) dumpACL(fileName string) {
|
||||
acl, err := d.bucket.Object(fileName).ACL().List(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("dumpACL: unable to list file ACL for bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
for _, v := range acl {
|
||||
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
|
||||
}
|
||||
}
|
||||
|
||||
// acl displays the ACL for the named file.
|
||||
func (d *demo) acl(fileName string) {
|
||||
fmt.Fprintf(d.w, "\nACL for file %v:\n", fileName)
|
||||
d.dumpACL(fileName)
|
||||
}
|
||||
|
||||
// putACLRule adds the "allUsers" ACL rule for the named file.
|
||||
func (d *demo) putACLRule(fileName string) {
|
||||
fmt.Fprintf(d.w, "\nPut ACL rule for file %v:\n", fileName)
|
||||
err := d.bucket.Object(fileName).ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
|
||||
if err != nil {
|
||||
d.errorf("putACLRule: unable to save ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
d.dumpACL(fileName)
|
||||
}
|
||||
|
||||
// deleteACLRule deleted the "allUsers" ACL rule for the named file.
|
||||
func (d *demo) deleteACLRule(fileName string) {
|
||||
fmt.Fprintf(d.w, "\nDelete ACL rule for file %v:\n", fileName)
|
||||
err := d.bucket.Object(fileName).ACL().Delete(d.ctx, storage.AllUsers)
|
||||
if err != nil {
|
||||
d.errorf("deleteACLRule: unable to delete ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
d.dumpACL(fileName)
|
||||
}
|
||||
|
||||
// deleteFiles deletes all the temporary files from a bucket created by this demo.
|
||||
func (d *demo) deleteFiles() {
|
||||
io.WriteString(d.w, "\nDeleting files...\n")
|
||||
for _, v := range d.cleanUp {
|
||||
fmt.Fprintf(d.w, "Deleting file %v\n", v)
|
||||
if err := d.bucket.Object(v).Delete(d.ctx); err != nil {
|
||||
d.errorf("deleteFiles: unable to delete bucket %q, file %q: %v", d.bucketName, v, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//[END sample]
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
application: your-app-id
|
||||
version: v1
|
||||
runtime: go
|
||||
api_version: go1
|
||||
|
||||
handlers:
|
||||
- url: /.*
|
||||
script: _go_app
|
||||
+432
@@ -0,0 +1,432 @@
|
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package main is an example Mananged VM app using the Google Cloud Storage API.
|
||||
package main
|
||||
|
||||
//[START imports]
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/appengine"
|
||||
"google.golang.org/appengine/file"
|
||||
"google.golang.org/appengine/log"
|
||||
)
|
||||
|
||||
//[END imports]
|
||||
|
||||
func main() {
|
||||
http.HandleFunc("/", handler)
|
||||
appengine.Main()
|
||||
}
|
||||
|
||||
//[START bucket_struct]
|
||||
// demo struct holds information needed to run the various demo functions.
|
||||
type demo struct {
|
||||
client *storage.Client
|
||||
bucketName string
|
||||
bucket *storage.BucketHandle
|
||||
|
||||
w io.Writer
|
||||
ctx context.Context
|
||||
// cleanUp is a list of filenames that need cleaning up at the end of the demo.
|
||||
cleanUp []string
|
||||
// failed indicates that one or more of the demo steps failed.
|
||||
failed bool
|
||||
}
|
||||
|
||||
//[END bucket_struct]
|
||||
|
||||
func (d *demo) errorf(format string, args ...interface{}) {
|
||||
d.failed = true
|
||||
fmt.Fprintln(d.w, fmt.Sprintf(format, args...))
|
||||
log.Errorf(d.ctx, format, args...)
|
||||
}
|
||||
|
||||
// handler is the main demo entry point that calls the GCS operations.
|
||||
func handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := appengine.NewContext(r)
|
||||
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
//[START get_default_bucket]
|
||||
// Use `dev_appserver.py --default_gcs_bucket_name GCS_BUCKET_NAME`
|
||||
// when running locally.
|
||||
bucket, err := file.DefaultBucketName(ctx)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "failed to get default GCS bucket name: %v", err)
|
||||
}
|
||||
//[END get_default_bucket]
|
||||
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "failed to create client: %v", err)
|
||||
return
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(ctx))
|
||||
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
d := &demo{
|
||||
w: buf,
|
||||
ctx: ctx,
|
||||
client: client,
|
||||
bucket: client.Bucket(bucket),
|
||||
bucketName: bucket,
|
||||
}
|
||||
|
||||
n := "demo-testfile-go"
|
||||
d.createFile(n)
|
||||
d.readFile(n)
|
||||
d.copyFile(n)
|
||||
d.statFile(n)
|
||||
d.createListFiles()
|
||||
d.listBucket()
|
||||
d.listBucketDirMode()
|
||||
d.defaultACL()
|
||||
d.putDefaultACLRule()
|
||||
d.deleteDefaultACLRule()
|
||||
d.bucketACL()
|
||||
d.putBucketACLRule()
|
||||
d.deleteBucketACLRule()
|
||||
d.acl(n)
|
||||
d.putACLRule(n)
|
||||
d.deleteACLRule(n)
|
||||
d.deleteFiles()
|
||||
|
||||
if d.failed {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
buf.WriteTo(w)
|
||||
fmt.Fprintf(w, "\nDemo failed.\n")
|
||||
} else {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
buf.WriteTo(w)
|
||||
fmt.Fprintf(w, "\nDemo succeeded.\n")
|
||||
}
|
||||
}
|
||||
|
||||
//[START write]
|
||||
// createFile creates a file in Google Cloud Storage.
|
||||
func (d *demo) createFile(fileName string) {
|
||||
fmt.Fprintf(d.w, "Creating file /%v/%v\n", d.bucket, fileName)
|
||||
|
||||
wc := d.bucket.Object(fileName).NewWriter(d.ctx)
|
||||
wc.ContentType = "text/plain"
|
||||
wc.Metadata = map[string]string{
|
||||
"x-goog-meta-foo": "foo",
|
||||
"x-goog-meta-bar": "bar",
|
||||
}
|
||||
d.cleanUp = append(d.cleanUp, fileName)
|
||||
|
||||
if _, err := wc.Write([]byte("abcde\n")); err != nil {
|
||||
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil {
|
||||
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
if err := wc.Close(); err != nil {
|
||||
d.errorf("createFile: unable to close bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
//[END write]
|
||||
|
||||
//[START read]
|
||||
// readFile reads the named file in Google Cloud Storage.
|
||||
func (d *demo) readFile(fileName string) {
|
||||
io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n")
|
||||
|
||||
rc, err := d.bucket.Object(fileName).NewReader(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("readFile: unable to open file from bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
slurp, err := ioutil.ReadAll(rc)
|
||||
if err != nil {
|
||||
d.errorf("readFile: unable to read data from bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0])
|
||||
if len(slurp) > 1024 {
|
||||
fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:])
|
||||
} else {
|
||||
fmt.Fprintf(d.w, "%s\n", slurp)
|
||||
}
|
||||
}
|
||||
|
||||
//[END read]
|
||||
|
||||
//[START copy]
|
||||
// copyFile copies a file in Google Cloud Storage.
|
||||
func (d *demo) copyFile(fileName string) {
|
||||
copyName := fileName + "-copy"
|
||||
fmt.Fprintf(d.w, "Copying file /%v/%v to /%v/%v:\n", d.bucket, fileName, d.bucketName, copyName)
|
||||
|
||||
obj, err := d.bucket.Object(copyName).CopierFrom(d.bucket.Object(fileName)).Run(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("copyFile: unable to copy /%v/%v to bucket %q, file %q: %v", d.bucketName, fileName, d.bucketName, copyName, err)
|
||||
return
|
||||
}
|
||||
d.cleanUp = append(d.cleanUp, copyName)
|
||||
|
||||
d.dumpStats(obj)
|
||||
}
|
||||
|
||||
//[END copy]
|
||||
|
||||
func (d *demo) dumpStats(obj *storage.ObjectAttrs) {
|
||||
fmt.Fprintf(d.w, "(filename: /%v/%v, ", obj.Bucket, obj.Name)
|
||||
fmt.Fprintf(d.w, "ContentType: %q, ", obj.ContentType)
|
||||
fmt.Fprintf(d.w, "ACL: %#v, ", obj.ACL)
|
||||
fmt.Fprintf(d.w, "Owner: %v, ", obj.Owner)
|
||||
fmt.Fprintf(d.w, "ContentEncoding: %q, ", obj.ContentEncoding)
|
||||
fmt.Fprintf(d.w, "Size: %v, ", obj.Size)
|
||||
fmt.Fprintf(d.w, "MD5: %q, ", obj.MD5)
|
||||
fmt.Fprintf(d.w, "CRC32C: %q, ", obj.CRC32C)
|
||||
fmt.Fprintf(d.w, "Metadata: %#v, ", obj.Metadata)
|
||||
fmt.Fprintf(d.w, "MediaLink: %q, ", obj.MediaLink)
|
||||
fmt.Fprintf(d.w, "StorageClass: %q, ", obj.StorageClass)
|
||||
if !obj.Deleted.IsZero() {
|
||||
fmt.Fprintf(d.w, "Deleted: %v, ", obj.Deleted)
|
||||
}
|
||||
fmt.Fprintf(d.w, "Updated: %v)\n", obj.Updated)
|
||||
}
|
||||
|
||||
//[START file_metadata]
|
||||
// statFile reads the stats of the named file in Google Cloud Storage.
|
||||
func (d *demo) statFile(fileName string) {
|
||||
io.WriteString(d.w, "\nFile stat:\n")
|
||||
|
||||
obj, err := d.bucket.Object(fileName).Attrs(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("statFile: unable to stat file from bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
|
||||
d.dumpStats(obj)
|
||||
}
|
||||
|
||||
//[END file_metadata]
|
||||
|
||||
// createListFiles creates files that will be used by listBucket.
|
||||
func (d *demo) createListFiles() {
|
||||
io.WriteString(d.w, "\nCreating more files for listbucket...\n")
|
||||
for _, n := range []string{"foo1", "foo2", "bar", "bar/1", "bar/2", "boo/"} {
|
||||
d.createFile(n)
|
||||
}
|
||||
}
|
||||
|
||||
//[START list_bucket]
|
||||
// listBucket lists the contents of a bucket in Google Cloud Storage.
|
||||
func (d *demo) listBucket() {
|
||||
io.WriteString(d.w, "\nListbucket result:\n")
|
||||
|
||||
query := &storage.Query{Prefix: "foo"}
|
||||
it := d.bucket.Objects(d.ctx, query)
|
||||
for {
|
||||
obj, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
d.errorf("listBucket: unable to list bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpStats(obj)
|
||||
}
|
||||
}
|
||||
|
||||
//[END list_bucket]
|
||||
|
||||
func (d *demo) listDir(name, indent string) {
|
||||
query := &storage.Query{Prefix: name, Delimiter: "/"}
|
||||
it := d.bucket.Objects(d.ctx, query)
|
||||
for {
|
||||
obj, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
d.errorf("listBucketDirMode: unable to list bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
if obj.Prefix == "" {
|
||||
fmt.Fprint(d.w, indent)
|
||||
d.dumpStats(obj)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(d.w, "%v(directory: /%v/%v)\n", indent, d.bucketName, obj.Prefix)
|
||||
d.listDir(obj.Prefix, indent+" ")
|
||||
}
|
||||
}
|
||||
|
||||
// listBucketDirMode lists the contents of a bucket in dir mode in Google Cloud Storage.
|
||||
func (d *demo) listBucketDirMode() {
|
||||
io.WriteString(d.w, "\nListbucket directory mode result:\n")
|
||||
d.listDir("b", "")
|
||||
}
|
||||
|
||||
// dumpDefaultACL prints out the default object ACL for this bucket.
|
||||
func (d *demo) dumpDefaultACL() {
|
||||
acl, err := d.bucket.ACL().List(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("defaultACL: unable to list default object ACL for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
for _, v := range acl {
|
||||
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
|
||||
}
|
||||
}
|
||||
|
||||
// defaultACL displays the default object ACL for this bucket.
|
||||
func (d *demo) defaultACL() {
|
||||
io.WriteString(d.w, "\nDefault object ACL:\n")
|
||||
d.dumpDefaultACL()
|
||||
}
|
||||
|
||||
// putDefaultACLRule adds the "allUsers" default object ACL rule for this bucket.
|
||||
func (d *demo) putDefaultACLRule() {
|
||||
io.WriteString(d.w, "\nPut Default object ACL Rule:\n")
|
||||
err := d.bucket.DefaultObjectACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
|
||||
if err != nil {
|
||||
d.errorf("putDefaultACLRule: unable to save default object ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpDefaultACL()
|
||||
}
|
||||
|
||||
// deleteDefaultACLRule deleted the "allUsers" default object ACL rule for this bucket.
|
||||
func (d *demo) deleteDefaultACLRule() {
|
||||
io.WriteString(d.w, "\nDelete Default object ACL Rule:\n")
|
||||
err := d.bucket.DefaultObjectACL().Delete(d.ctx, storage.AllUsers)
|
||||
if err != nil {
|
||||
d.errorf("deleteDefaultACLRule: unable to delete default object ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpDefaultACL()
|
||||
}
|
||||
|
||||
// dumpBucketACL prints out the bucket ACL.
|
||||
func (d *demo) dumpBucketACL() {
|
||||
acl, err := d.bucket.ACL().List(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("dumpBucketACL: unable to list bucket ACL for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
for _, v := range acl {
|
||||
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
|
||||
}
|
||||
}
|
||||
|
||||
// bucketACL displays the bucket ACL for this bucket.
|
||||
func (d *demo) bucketACL() {
|
||||
io.WriteString(d.w, "\nBucket ACL:\n")
|
||||
d.dumpBucketACL()
|
||||
}
|
||||
|
||||
// putBucketACLRule adds the "allUsers" bucket ACL rule for this bucket.
|
||||
func (d *demo) putBucketACLRule() {
|
||||
io.WriteString(d.w, "\nPut Bucket ACL Rule:\n")
|
||||
err := d.bucket.ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
|
||||
if err != nil {
|
||||
d.errorf("putBucketACLRule: unable to save bucket ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpBucketACL()
|
||||
}
|
||||
|
||||
// deleteBucketACLRule deleted the "allUsers" bucket ACL rule for this bucket.
|
||||
func (d *demo) deleteBucketACLRule() {
|
||||
io.WriteString(d.w, "\nDelete Bucket ACL Rule:\n")
|
||||
err := d.bucket.ACL().Delete(d.ctx, storage.AllUsers)
|
||||
if err != nil {
|
||||
d.errorf("deleteBucketACLRule: unable to delete bucket ACL rule for bucket %q: %v", d.bucketName, err)
|
||||
return
|
||||
}
|
||||
d.dumpBucketACL()
|
||||
}
|
||||
|
||||
// dumpACL prints out the ACL of the named file.
|
||||
func (d *demo) dumpACL(fileName string) {
|
||||
acl, err := d.bucket.Object(fileName).ACL().List(d.ctx)
|
||||
if err != nil {
|
||||
d.errorf("dumpACL: unable to list file ACL for bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
for _, v := range acl {
|
||||
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
|
||||
}
|
||||
}
|
||||
|
||||
// acl displays the ACL for the named file.
|
||||
func (d *demo) acl(fileName string) {
|
||||
fmt.Fprintf(d.w, "\nACL for file %v:\n", fileName)
|
||||
d.dumpACL(fileName)
|
||||
}
|
||||
|
||||
// putACLRule adds the "allUsers" ACL rule for the named file.
|
||||
func (d *demo) putACLRule(fileName string) {
|
||||
fmt.Fprintf(d.w, "\nPut ACL rule for file %v:\n", fileName)
|
||||
err := d.bucket.Object(fileName).ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
|
||||
if err != nil {
|
||||
d.errorf("putACLRule: unable to save ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
d.dumpACL(fileName)
|
||||
}
|
||||
|
||||
// deleteACLRule deleted the "allUsers" ACL rule for the named file.
|
||||
func (d *demo) deleteACLRule(fileName string) {
|
||||
fmt.Fprintf(d.w, "\nDelete ACL rule for file %v:\n", fileName)
|
||||
err := d.bucket.Object(fileName).ACL().Delete(d.ctx, storage.AllUsers)
|
||||
if err != nil {
|
||||
d.errorf("deleteACLRule: unable to delete ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
|
||||
return
|
||||
}
|
||||
d.dumpACL(fileName)
|
||||
}
|
||||
|
||||
//[START delete]
|
||||
// deleteFiles deletes all the temporary files from a bucket created by this demo.
|
||||
func (d *demo) deleteFiles() {
|
||||
io.WriteString(d.w, "\nDeleting files...\n")
|
||||
for _, v := range d.cleanUp {
|
||||
fmt.Fprintf(d.w, "Deleting file %v\n", v)
|
||||
if err := d.bucket.Object(v).Delete(d.ctx); err != nil {
|
||||
d.errorf("deleteFiles: unable to delete bucket %q, file %q: %v", d.bucketName, v, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//[END delete]
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
runtime: go
|
||||
api_version: 1
|
||||
vm: true
|
||||
|
||||
manual_scaling:
|
||||
instances: 1
|
||||
|
||||
handlers:
|
||||
- url: /.*
|
||||
script: _go_app
|
||||
Reference in New Issue
Block a user