Switch to using the dep tool and update all the dependencies

This commit is contained in:
Nick Craig-Wood
2017-05-11 15:39:54 +01:00
parent 5135ff73cb
commit 98c2d2c41b
5321 changed files with 4483197 additions and 5918 deletions
+93
View File
@@ -0,0 +1,93 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// concat_table is an example client of the bigquery client library.
// It concatenates two BigQuery tables and writes the result to another table.
package main
import (
"flag"
"fmt"
"log"
"os"
"time"
"cloud.google.com/go/bigquery"
"golang.org/x/net/context"
)
var (
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
src1 = flag.String("src1", "", "The ID of the first BigQuery table to concatenate")
src2 = flag.String("src2", "", "The ID of the second BigQuery table to concatenate")
dest = flag.String("dest", "", "The ID of the BigQuery table to write the result to")
pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
)
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "src1", "src2", "dest"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
if *src1 == *src2 || *src1 == *dest || *src2 == *dest {
log.Fatalf("Different values must be supplied for each of --src1, --src2 and --dest")
}
ctx := context.Background()
client, err := bigquery.NewClient(ctx, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
s1 := client.Dataset(*dataset).Table(*src1)
s2 := client.Dataset(*dataset).Table(*src2)
d := client.Dataset(*dataset).Table(*dest)
// Concatenate data.
copier := d.CopierFrom(s1, s2)
copier.WriteDisposition = bigquery.WriteTruncate
job, err := copier.Run(ctx)
if err != nil {
log.Fatalf("Concatenating: %v", err)
}
fmt.Printf("Job for concatenation operation: %+v\n", job)
fmt.Printf("Waiting for job to complete.\n")
for range time.Tick(*pollint) {
status, err := job.Status(ctx)
if err != nil {
fmt.Printf("Failure determining status: %v", err)
break
}
if !status.Done() {
continue
}
if err := status.Err(); err == nil {
fmt.Printf("Success\n")
} else {
fmt.Printf("Failure: %+v\n", err)
}
break
}
}
+95
View File
@@ -0,0 +1,95 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// load is an example client of the bigquery client library.
// It loads a file from Google Cloud Storage into a BigQuery table.
package main
import (
"flag"
"fmt"
"log"
"os"
"time"
"cloud.google.com/go/bigquery"
"golang.org/x/net/context"
)
var (
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
table = flag.String("table", "", "The ID of a BigQuery table to load data into")
bucket = flag.String("bucket", "", "The name of a Google Cloud Storage bucket to load data from")
object = flag.String("object", "", "The name of a Google Cloud Storage object to load data from. Must exist within the bucket specified by --bucket")
skiprows = flag.Int64("skiprows", 0, "The number of rows of the source data to skip when loading")
pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
)
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "table", "bucket", "object"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
ctx := context.Background()
client, err := bigquery.NewClient(ctx, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
table := client.Dataset(*dataset).Table(*table)
gcs := bigquery.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object))
gcs.SkipLeadingRows = *skiprows
gcs.MaxBadRecords = 1
gcs.AllowQuotedNewlines = true
// Load data from Google Cloud Storage into a BigQuery table.
loader := table.LoaderFrom(gcs)
loader.WriteDisposition = bigquery.WriteTruncate
job, err := loader.Run(ctx)
if err != nil {
log.Fatalf("Loading data: %v", err)
}
fmt.Printf("Job for data load operation: %+v\n", job)
fmt.Printf("Waiting for job to complete.\n")
for range time.Tick(*pollint) {
status, err := job.Status(ctx)
if err != nil {
fmt.Printf("Failure determining status: %v", err)
break
}
if !status.Done() {
continue
}
if err := status.Err(); err == nil {
fmt.Printf("Success\n")
} else {
fmt.Printf("Failure: %+v\n", err)
}
break
}
}
+98
View File
@@ -0,0 +1,98 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// query is an example client of the bigquery client library.
// It submits a query and writes the result to a table.
package main
import (
"flag"
"fmt"
"log"
"os"
"time"
"cloud.google.com/go/bigquery"
"golang.org/x/net/context"
)
var (
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
q = flag.String("q", "", "The query string")
dest = flag.String("dest", "", "The ID of the BigQuery table to write the result to. If unset, an ephemeral table ID will be generated.")
pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
wait = flag.Bool("wait", false, "Whether to wait for the query job to complete.")
)
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "q"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
ctx := context.Background()
client, err := bigquery.NewClient(ctx, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
query := client.Query(*q)
query.DefaultProjectID = *project
query.DefaultDatasetID = *dataset
query.WriteDisposition = bigquery.WriteTruncate
if *dest != "" {
query.Dst = client.Dataset(*dataset).Table(*dest)
}
// Query data.
job, err := query.Run(ctx)
if err != nil {
log.Fatalf("Querying: %v", err)
}
fmt.Printf("Submitted query. Job ID: %s\n", job.ID())
if !*wait {
return
}
fmt.Printf("Waiting for job to complete.\n")
for range time.Tick(*pollint) {
status, err := job.Status(ctx)
if err != nil {
fmt.Printf("Failure determining status: %v", err)
break
}
if !status.Done() {
continue
}
if err := status.Err(); err == nil {
fmt.Printf("Success\n")
} else {
fmt.Printf("Failure: %+v\n", err)
}
break
}
}
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// read is an example client of the bigquery client library.
// It reads from a table, returning the data via an Iterator.
package main
import (
"flag"
"fmt"
"log"
"os"
"regexp"
"strings"
"text/tabwriter"
"cloud.google.com/go/bigquery"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
)
var (
project = flag.String("project", "", "The ID of a Google Cloud Platform project")
dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
table = flag.String("table", ".*", "A regular expression to match the IDs of tables to read.")
jobID = flag.String("jobid", "", "The ID of a query job that has already been submitted."+
" If set, --dataset, --table will be ignored, and results will be read from the specified job.")
)
func printValues(ctx context.Context, it *bigquery.RowIterator) {
// one-space padding.
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
for {
var vals []bigquery.Value
err := it.Next(&vals)
if err == iterator.Done {
break
}
if err != nil {
fmt.Printf("err calling get: %v\n", err)
}
sep := ""
for _, v := range vals {
fmt.Fprintf(tw, "%s%v", sep, v)
sep = "\t"
}
fmt.Fprintf(tw, "\n")
}
tw.Flush()
fmt.Println()
}
func printTable(ctx context.Context, client *bigquery.Client, t *bigquery.Table) {
it := t.Read(ctx)
id := t.FullyQualifiedName()
fmt.Printf("%s\n%s\n", id, strings.Repeat("-", len(id)))
printValues(ctx, it)
}
func printQueryResults(ctx context.Context, client *bigquery.Client, queryJobID string) {
job, err := client.JobFromID(ctx, queryJobID)
if err != nil {
log.Fatalf("Loading job: %v", err)
}
it, err := job.Read(ctx)
if err != nil {
log.Fatalf("Reading: %v", err)
}
// TODO: print schema.
printValues(ctx, it)
}
func main() {
flag.Parse()
flagsOk := true
if flag.Lookup("project").Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --project is required\n")
flagsOk = false
}
var sourceFlagCount int
if flag.Lookup("dataset").Value.String() != "" {
sourceFlagCount++
}
if flag.Lookup("jobid").Value.String() != "" {
sourceFlagCount++
}
if sourceFlagCount != 1 {
fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n")
flagsOk = false
}
if !flagsOk {
os.Exit(1)
}
ctx := context.Background()
tableRE, err := regexp.Compile(*table)
if err != nil {
fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
os.Exit(1)
}
client, err := bigquery.NewClient(ctx, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
if *jobID != "" {
printQueryResults(ctx, client, *jobID)
return
}
ds := client.Dataset(*dataset)
tableIter := ds.Tables(context.Background())
for {
t, err := tableIter.Next()
if err == iterator.Done {
break
}
if err != nil {
log.Fatalf("Listing tables: %v", err)
}
if tableRE.MatchString(t.TableID) {
printTable(ctx, client, t)
}
}
}
+46
View File
@@ -0,0 +1,46 @@
# Cloud Bigtable Hello World in Go
This is a simple application that demonstrates using the [Google Cloud APIs Go
Client Library](https://github.com/GoogleCloudPlatform/google-cloud-go) to connect
to and interact with Cloud Bigtable.
## Prerequisites
1. Set up Cloud Console.
1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project.
You will need the project ID later.
1. Go to **Settings > Project Billing Settings** and enable billing.
1. Select **APIs & Auth > APIs**.
1. Enable the **Cloud Bigtable API** and the **Cloud Bigtable Admin API**.
(You may need to search for the API).
1. Set up gcloud.
1. `gcloud components update`
1. `gcloud auth login`
1. `gcloud config set project PROJECT_ID`
1. Provision a Cloud Bigtable instance
1. Follow the instructions in the [user
documentation](https://cloud.google.com/bigtable/docs/creating-instance) to
create a Google Cloud Platform project and Cloud Bigtable instance if necessary.
1. You'll need to reference your project id and instance id to run the application.
## Running
1. From the hello_world example folder, `go run main.go -project PROJECT_ID -instance INSTANCE_ID`, substituting your project id and instance id.
## Cleaning up
To avoid incurring extra charges to your Google Cloud Platform account, remove
the resources created for this sample.
1. Go to the Clusters page in the [Cloud
Console](https://console.cloud.google.com).
[Go to the Clusters page](https://console.cloud.google.com/project/_/bigtable/clusters)
1. Click the cluster name.
1. Click **Delete**.
![Delete](https://cloud.google.com/bigtable/img/delete-quickstart-cluster.png)
1. Type the cluster ID, then click **Delete** to delete the cluster.
+157
View File
@@ -0,0 +1,157 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Hello world is a sample program demonstrating use of the Bigtable client
// library to perform basic CRUD operations
package main
import (
"flag"
"fmt"
"log"
"cloud.google.com/go/bigtable"
"golang.org/x/net/context"
)
// User-provided constants.
const (
tableName = "Hello-Bigtable"
columnFamilyName = "cf1"
columnName = "greeting"
)
var greetings = []string{"Hello World!", "Hello Cloud Bigtable!", "Hello golang!"}
// sliceContains reports whether the provided string is present in the given slice of strings.
func sliceContains(list []string, target string) bool {
for _, s := range list {
if s == target {
return true
}
}
return false
}
func main() {
project := flag.String("project", "", "The Google Cloud Platform project ID. Required.")
instance := flag.String("instance", "", "The Google Cloud Bigtable instance ID. Required.")
flag.Parse()
for _, f := range []string{"project", "instance"} {
if flag.Lookup(f).Value.String() == "" {
log.Fatalf("The %s flag is required.", f)
}
}
ctx := context.Background()
// Set up admin client, tables, and column families.
// NewAdminClient uses Application Default Credentials to authenticate.
adminClient, err := bigtable.NewAdminClient(ctx, *project, *instance)
if err != nil {
log.Fatalf("Could not create admin client: %v", err)
}
tables, err := adminClient.Tables(ctx)
if err != nil {
log.Fatalf("Could not fetch table list: %v", err)
}
if !sliceContains(tables, tableName) {
log.Printf("Creating table %s", tableName)
if err := adminClient.CreateTable(ctx, tableName); err != nil {
log.Fatalf("Could not create table %s: %v", tableName, err)
}
}
tblInfo, err := adminClient.TableInfo(ctx, tableName)
if err != nil {
log.Fatalf("Could not read info for table %s: %v", tableName, err)
}
if !sliceContains(tblInfo.Families, columnFamilyName) {
if err := adminClient.CreateColumnFamily(ctx, tableName, columnFamilyName); err != nil {
log.Fatalf("Could not create column family %s: %v", columnFamilyName, err)
}
}
// Set up Bigtable data operations client.
// NewClient uses Application Default Credentials to authenticate.
client, err := bigtable.NewClient(ctx, *project, *instance)
if err != nil {
log.Fatalf("Could not create data operations client: %v", err)
}
tbl := client.Open(tableName)
muts := make([]*bigtable.Mutation, len(greetings))
rowKeys := make([]string, len(greetings))
log.Printf("Writing greeting rows to table")
for i, greeting := range greetings {
muts[i] = bigtable.NewMutation()
muts[i].Set(columnFamilyName, columnName, bigtable.Now(), []byte(greeting))
// Each row has a unique row key.
//
// Note: This example uses sequential numeric IDs for simplicity, but
// this can result in poor performance in a production application.
// Since rows are stored in sorted order by key, sequential keys can
// result in poor distribution of operations across nodes.
//
// For more information about how to design a Bigtable schema for the
// best performance, see the documentation:
//
// https://cloud.google.com/bigtable/docs/schema-design
rowKeys[i] = fmt.Sprintf("%s%d", columnName, i)
}
rowErrs, err := tbl.ApplyBulk(ctx, rowKeys, muts)
if err != nil {
log.Fatalf("Could not apply bulk row mutation: %v", err)
}
if rowErrs != nil {
for _, rowErr := range rowErrs {
log.Printf("Error writing row: %v", rowErr)
}
log.Fatalf("Could not write some rows")
}
log.Printf("Getting a single greeting by row key:")
row, err := tbl.ReadRow(ctx, rowKeys[0], bigtable.RowFilter(bigtable.ColumnFilter(columnName)))
if err != nil {
log.Fatalf("Could not read row with key %s: %v", rowKeys[0], err)
}
log.Printf("\t%s = %s\n", rowKeys[0], string(row[columnFamilyName][0].Value))
log.Printf("Reading all greeting rows:")
err = tbl.ReadRows(ctx, bigtable.PrefixRange(columnName), func(row bigtable.Row) bool {
item := row[columnFamilyName][0]
log.Printf("\t%s = %s\n", item.Row, string(item.Value))
return true
}, bigtable.RowFilter(bigtable.ColumnFilter(columnName)))
if err = client.Close(); err != nil {
log.Fatalf("Could not close data operations client: %v", err)
}
log.Printf("Deleting the table")
if err = adminClient.DeleteTable(ctx, tableName); err != nil {
log.Fatalf("Could not delete table %s: %v", tableName, err)
}
if err = adminClient.Close(); err != nil {
log.Fatalf("Could not close admin client: %v", err)
}
}
+453
View File
@@ -0,0 +1,453 @@
/*
Copyright 2015 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Search is a sample web server that uses Cloud Bigtable as the storage layer
// for a simple document-storage and full-text-search service.
// It has four functions:
// - Initialize and clear the table.
// - Add a document. This adds the content of a user-supplied document to the
// Bigtable, and adds references to the document to an index in the Bigtable.
// The document is indexed under each unique word in the document.
// - Search the index. This returns documents containing each word in a user
// query, with snippets and links to view the whole document.
// - Copy table. This copies the documents and index from another table and
// adds them to the current one.
package main
import (
"bytes"
"flag"
"fmt"
"html/template"
"io"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
"unicode"
"cloud.google.com/go/bigtable"
"golang.org/x/net/context"
)
var (
addTemplate = template.Must(template.New("").Parse(`<html><body>
Added {{.Title}}
</body></html>`))
contentTemplate = template.Must(template.New("").Parse(`<html><body>
<b>{{.Title}}</b><br><br>
{{.Content}}
</body></html>`))
searchTemplate = template.Must(template.New("").Parse(`<html><body>
Results for <b>{{.Query}}</b>:<br><br>
{{range .Results}}
<a href="/content?name={{.Title}}">{{.Title}}</a><br>
<i>{{.Snippet}}</i><br><br>
{{end}}
</body></html>`))
)
const (
indexColumnFamily = "i"
contentColumnFamily = "c"
mainPage = `
<html>
<head>
<title>Document Search</title>
</head>
<body>
Initialize and clear table:
<form action="/reset" method="post">
<div><input type="submit" value="Init"></div>
</form>
Search for documents:
<form action="/search" method="post">
<div><input type="text" name="q" size=80></div>
<div><input type="submit" value="Search"></div>
</form>
Add a document:
<form action="/add" method="post">
Document name:
<div><textarea name="name" rows="1" cols="80"></textarea></div>
Document text:
<div><textarea name="content" rows="20" cols="80"></textarea></div>
<div><input type="submit" value="Submit"></div>
</form>
Copy data from another table:
<form action="/copy" method="post">
Source table name:
<div><input type="text" name="name" size=80></div>
<div><input type="submit" value="Copy"></div>
</form>
</body>
</html>
`
)
func main() {
var (
project = flag.String("project", "", "The name of the project.")
instance = flag.String("instance", "", "The name of the Cloud Bigtable instance.")
tableName = flag.String("table", "docindex", "The name of the table containing the documents and index.")
port = flag.Int("port", 8080, "TCP port for server.")
)
flag.Parse()
// Make an admin client.
adminClient, err := bigtable.NewAdminClient(context.Background(), *project, *instance)
if err != nil {
log.Fatal("Bigtable NewAdminClient:", err)
}
// Make a regular client.
client, err := bigtable.NewClient(context.Background(), *project, *instance)
if err != nil {
log.Fatal("Bigtable NewClient:", err)
}
// Open the table.
table := client.Open(*tableName)
// Set up HTML handlers, and start the web server.
http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) { handleSearch(w, r, table) })
http.HandleFunc("/content", func(w http.ResponseWriter, r *http.Request) { handleContent(w, r, table) })
http.HandleFunc("/add", func(w http.ResponseWriter, r *http.Request) { handleAddDoc(w, r, table) })
http.HandleFunc("/reset", func(w http.ResponseWriter, r *http.Request) { handleReset(w, r, *tableName, adminClient) })
http.HandleFunc("/copy", func(w http.ResponseWriter, r *http.Request) { handleCopy(w, r, *tableName, client, adminClient) })
http.HandleFunc("/", handleMain)
log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
}
// handleMain outputs the home page.
func handleMain(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, mainPage)
}
// tokenize splits a string into tokens.
// This is very simple, it's not a good tokenization function.
func tokenize(s string) []string {
wordMap := make(map[string]bool)
f := strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) })
for _, word := range f {
word = strings.ToLower(word)
wordMap[word] = true
}
words := make([]string, 0, len(wordMap))
for word := range wordMap {
words = append(words, word)
}
return words
}
// handleContent fetches the content of a document from the Bigtable and returns it.
func handleContent(w http.ResponseWriter, r *http.Request, table *bigtable.Table) {
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
name := r.FormValue("name")
if len(name) == 0 {
http.Error(w, "No document name supplied.", http.StatusBadRequest)
return
}
row, err := table.ReadRow(ctx, name)
if err != nil {
http.Error(w, "Error reading content: "+err.Error(), http.StatusInternalServerError)
return
}
content := row[contentColumnFamily]
if len(content) == 0 {
http.Error(w, "Document not found.", http.StatusNotFound)
return
}
var buf bytes.Buffer
if err := contentTemplate.ExecuteTemplate(&buf, "", struct{ Title, Content string }{name, string(content[0].Value)}); err != nil {
http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError)
return
}
io.Copy(w, &buf)
}
// handleSearch responds to search queries, returning links and snippets for matching documents.
func handleSearch(w http.ResponseWriter, r *http.Request, table *bigtable.Table) {
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
query := r.FormValue("q")
// Split the query into words.
words := tokenize(query)
if len(words) == 0 {
http.Error(w, "Empty query.", http.StatusBadRequest)
return
}
// readRows reads from many rows concurrently.
readRows := func(rows []string) ([]bigtable.Row, error) {
results := make([]bigtable.Row, len(rows))
errors := make([]error, len(rows))
var wg sync.WaitGroup
for i, row := range rows {
wg.Add(1)
go func(i int, row string) {
defer wg.Done()
results[i], errors[i] = table.ReadRow(ctx, row, bigtable.RowFilter(bigtable.LatestNFilter(1)))
}(i, row)
}
wg.Wait()
for _, err := range errors {
if err != nil {
return nil, err
}
}
return results, nil
}
// For each query word, get the list of documents containing it.
results, err := readRows(words)
if err != nil {
http.Error(w, "Error reading index: "+err.Error(), http.StatusInternalServerError)
return
}
// Count how many of the query words each result contained.
hits := make(map[string]int)
for _, r := range results {
for _, r := range r[indexColumnFamily] {
hits[r.Column]++
}
}
// Build a slice of all the documents that matched every query word.
var matches []string
for doc, count := range hits {
if count == len(words) {
matches = append(matches, doc[len(indexColumnFamily+":"):])
}
}
// Fetch the content of those documents from the Bigtable.
content, err := readRows(matches)
if err != nil {
http.Error(w, "Error reading results: "+err.Error(), http.StatusInternalServerError)
return
}
type result struct{ Title, Snippet string }
data := struct {
Query string
Results []result
}{query, nil}
// Output links and snippets.
for i, doc := range matches {
var text string
c := content[i][contentColumnFamily]
if len(c) > 0 {
text = string(c[0].Value)
}
if len(text) > 100 {
text = text[:100] + "..."
}
data.Results = append(data.Results, result{doc, text})
}
var buf bytes.Buffer
if err := searchTemplate.ExecuteTemplate(&buf, "", data); err != nil {
http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError)
return
}
io.Copy(w, &buf)
}
// handleAddDoc adds a document to the index.
func handleAddDoc(w http.ResponseWriter, r *http.Request, table *bigtable.Table) {
if r.Method != "POST" {
http.Error(w, "POST requests only", http.StatusMethodNotAllowed)
return
}
ctx, _ := context.WithTimeout(context.Background(), time.Minute)
name := r.FormValue("name")
if len(name) == 0 {
http.Error(w, "Empty document name!", http.StatusBadRequest)
return
}
content := r.FormValue("content")
if len(content) == 0 {
http.Error(w, "Empty document content!", http.StatusBadRequest)
return
}
var (
writeErr error // Set if any write fails.
mu sync.Mutex // Protects writeErr
wg sync.WaitGroup // Used to wait for all writes to finish.
)
// writeOneColumn writes one column in one row, updates err if there is an error,
// and signals wg that one operation has finished.
writeOneColumn := func(row, family, column, value string, ts bigtable.Timestamp) {
mut := bigtable.NewMutation()
mut.Set(family, column, ts, []byte(value))
err := table.Apply(ctx, row, mut)
if err != nil {
mu.Lock()
writeErr = err
mu.Unlock()
}
}
// Start a write to store the document content.
wg.Add(1)
go func() {
writeOneColumn(name, contentColumnFamily, "", content, bigtable.Now())
wg.Done()
}()
// Start writes to store the document name in the index for each word in the document.
words := tokenize(content)
for _, word := range words {
var (
row = word
family = indexColumnFamily
column = name
value = ""
ts = bigtable.Now()
)
wg.Add(1)
go func() {
// TODO: should use a semaphore to limit the number of concurrent writes.
writeOneColumn(row, family, column, value, ts)
wg.Done()
}()
}
wg.Wait()
if writeErr != nil {
http.Error(w, "Error writing to Bigtable: "+writeErr.Error(), http.StatusInternalServerError)
return
}
var buf bytes.Buffer
if err := addTemplate.ExecuteTemplate(&buf, "", struct{ Title string }{name}); err != nil {
http.Error(w, "Error executing HTML template: "+err.Error(), http.StatusInternalServerError)
return
}
io.Copy(w, &buf)
}
// handleReset deletes the table if it exists, creates it again, and creates its column families.
func handleReset(w http.ResponseWriter, r *http.Request, table string, adminClient *bigtable.AdminClient) {
if r.Method != "POST" {
http.Error(w, "POST requests only", http.StatusMethodNotAllowed)
return
}
ctx, _ := context.WithTimeout(context.Background(), 5*time.Minute)
adminClient.DeleteTable(ctx, table)
if err := adminClient.CreateTable(ctx, table); err != nil {
http.Error(w, "Error creating Bigtable: "+err.Error(), http.StatusInternalServerError)
return
}
time.Sleep(20 * time.Second)
// Create two column families, and set the GC policy for each one to keep one version.
for _, family := range []string{indexColumnFamily, contentColumnFamily} {
if err := adminClient.CreateColumnFamily(ctx, table, family); err != nil {
http.Error(w, "Error creating column family: "+err.Error(), http.StatusInternalServerError)
return
}
if err := adminClient.SetGCPolicy(ctx, table, family, bigtable.MaxVersionsPolicy(1)); err != nil {
http.Error(w, "Error setting GC policy: "+err.Error(), http.StatusInternalServerError)
return
}
}
w.Write([]byte("<html><body>Done.</body></html>"))
return
}
// copyTable copies data from one table to another.
func copyTable(src, dst string, client *bigtable.Client, adminClient *bigtable.AdminClient) error {
if src == "" || src == dst {
return nil
}
ctx, _ := context.WithTimeout(context.Background(), time.Minute)
// Open the source and destination tables.
srcTable := client.Open(src)
dstTable := client.Open(dst)
var (
writeErr error // Set if any write fails.
mu sync.Mutex // Protects writeErr
wg sync.WaitGroup // Used to wait for all writes to finish.
)
copyRowToTable := func(row bigtable.Row) bool {
mu.Lock()
failed := writeErr != nil
mu.Unlock()
if failed {
return false
}
mut := bigtable.NewMutation()
for family, items := range row {
for _, item := range items {
// Get the column name, excluding the column family name and ':' character.
columnWithoutFamily := item.Column[len(family)+1:]
mut.Set(family, columnWithoutFamily, bigtable.Now(), item.Value)
}
}
wg.Add(1)
go func() {
// TODO: should use a semaphore to limit the number of concurrent writes.
if err := dstTable.Apply(ctx, row.Key(), mut); err != nil {
mu.Lock()
writeErr = err
mu.Unlock()
}
wg.Done()
}()
return true
}
// Create a filter that only accepts the column families we're interested in.
filter := bigtable.FamilyFilter(indexColumnFamily + "|" + contentColumnFamily)
// Read every row from srcTable, and call copyRowToTable to copy it to our table.
err := srcTable.ReadRows(ctx, bigtable.InfiniteRange(""), copyRowToTable, bigtable.RowFilter(filter))
wg.Wait()
if err != nil {
return err
}
return writeErr
}
// handleCopy copies data from one table to another.
func handleCopy(w http.ResponseWriter, r *http.Request, dst string, client *bigtable.Client, adminClient *bigtable.AdminClient) {
if r.Method != "POST" {
http.Error(w, "POST requests only", http.StatusMethodNotAllowed)
return
}
src := r.FormValue("name")
if src == "" {
http.Error(w, "No source table specified.", http.StatusBadRequest)
return
}
if err := copyTable(src, dst, client, adminClient); err != nil {
http.Error(w, "Failed to rebuild index: "+err.Error(), http.StatusInternalServerError)
return
}
fmt.Fprint(w, "Copied table.\n")
}
+29
View File
@@ -0,0 +1,29 @@
# User Counter
# (Cloud Bigtable on Managed VMs using Go)
This app counts how often each user visits. The app uses Cloud Bigtable to store the visit counts for each user.
## Prerequisites
1. Set up Cloud Console.
1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project.
You will need the project ID later.
1. Go to **Settings > Project Billing Settings** and enable billing.
1. Select **APIs & Auth > APIs**.
1. Enable the **Cloud Bigtable API** and the **Cloud Bigtable Admin API**.
(You may need to search for the API).
1. Set up gcloud.
1. `gcloud components update`
1. `gcloud auth login`
1. `gcloud config set project PROJECT_ID`
1. Download App Engine SDK for Go.
1. `go get -u google.golang.org/appengine/...`
1. In main.go, change the `project` and `instance` constants.
## Running locally
1. From the sample project folder, `dev_appserver.py app.yaml`.
## Deploying on Google App Engine flexible environment
Follow the [deployment instructions](https://cloud.google.com/appengine/docs/flexible/go/testing-and-deploying-your-app).
+11
View File
@@ -0,0 +1,11 @@
runtime: go
api_version: go1
vm: true
manual_scaling:
instances: 1
handlers:
# Serve only the web root.
- url: /
script: _go_app
+180
View File
@@ -0,0 +1,180 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
User counter is a program that tracks how often a user has visited the index page.
This program demonstrates usage of the Cloud Bigtable API for App Engine flexible environment and Go.
Instructions for running this program are in the README.md.
*/
package main
import (
"bytes"
"encoding/binary"
"html/template"
"log"
"net/http"
"cloud.google.com/go/bigtable"
"golang.org/x/net/context"
"google.golang.org/appengine"
aelog "google.golang.org/appengine/log"
"google.golang.org/appengine/user"
)
// User-provided constants.
const (
project = "PROJECT_ID"
instance = "INSTANCE"
)
var (
tableName = "user-visit-counter"
familyName = "emails"
// Client is initialized by main.
client *bigtable.Client
)
func main() {
ctx := context.Background()
// Set up admin client, tables, and column families.
// NewAdminClient uses Application Default Credentials to authenticate.
adminClient, err := bigtable.NewAdminClient(ctx, project, instance)
if err != nil {
log.Fatalf("Unable to create a table admin client. %v", err)
}
tables, err := adminClient.Tables(ctx)
if err != nil {
log.Fatalf("Unable to fetch table list. %v", err)
}
if !sliceContains(tables, tableName) {
if err := adminClient.CreateTable(ctx, tableName); err != nil {
log.Fatalf("Unable to create table: %v. %v", tableName, err)
}
}
tblInfo, err := adminClient.TableInfo(ctx, tableName)
if err != nil {
log.Fatalf("Unable to read info for table: %v. %v", tableName, err)
}
if !sliceContains(tblInfo.Families, familyName) {
if err := adminClient.CreateColumnFamily(ctx, tableName, familyName); err != nil {
log.Fatalf("Unable to create column family: %v. %v", familyName, err)
}
}
adminClient.Close()
// Set up Bigtable data operations client.
// NewClient uses Application Default Credentials to authenticate.
client, err = bigtable.NewClient(ctx, project, instance)
if err != nil {
log.Fatalf("Unable to create data operations client. %v", err)
}
http.Handle("/", appHandler(mainHandler))
appengine.Main() // Never returns.
}
// mainHandler tracks how many times each user has visited this page.
func mainHandler(w http.ResponseWriter, r *http.Request) *appError {
if r.URL.Path != "/" {
http.NotFound(w, r)
return nil
}
ctx := appengine.NewContext(r)
u := user.Current(ctx)
if u == nil {
login, err := user.LoginURL(ctx, r.URL.String())
if err != nil {
return &appError{err, "Error finding login URL", http.StatusInternalServerError}
}
http.Redirect(w, r, login, http.StatusFound)
return nil
}
logoutURL, err := user.LogoutURL(ctx, "/")
if err != nil {
return &appError{err, "Error finding logout URL", http.StatusInternalServerError}
}
// Increment visit count for user.
tbl := client.Open(tableName)
rmw := bigtable.NewReadModifyWrite()
rmw.Increment(familyName, u.Email, 1)
row, err := tbl.ApplyReadModifyWrite(ctx, u.Email, rmw)
if err != nil {
return &appError{err, "Error applying ReadModifyWrite to row: " + u.Email, http.StatusInternalServerError}
}
data := struct {
Username, Logout string
Visits uint64
}{
Username: u.Email,
// Retrieve the most recently edited column.
Visits: binary.BigEndian.Uint64(row[familyName][0].Value),
Logout: logoutURL,
}
// Display hello page.
var buf bytes.Buffer
if err := tmpl.Execute(&buf, data); err != nil {
return &appError{err, "Error writing template", http.StatusInternalServerError}
}
buf.WriteTo(w)
return nil
}
var tmpl = template.Must(template.New("").Parse(`
<html><body>
<p>
{{with .Username}} Hello {{.}}{{end}}
{{with .Logout}}<a href="{{.}}">Sign out</a>{{end}}
</p>
<p>
You have visited {{.Visits}}
</p>
</body></html>`))
// sliceContains reports whether the provided string is present in the given slice of strings.
func sliceContains(list []string, target string) bool {
for _, s := range list {
if s == target {
return true
}
}
return false
}
// More info about this method of error handling can be found at: http://blog.golang.org/error-handling-and-go
type appHandler func(http.ResponseWriter, *http.Request) *appError
type appError struct {
Error error
Message string
Code int
}
func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if e := fn(w, r); e != nil {
ctx := appengine.NewContext(r)
aelog.Errorf(ctx, "%v", e.Error)
http.Error(w, e.Message, e.Code)
}
}
+428
View File
@@ -0,0 +1,428 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//[START sample]
// Package gcsdemo is an example App Engine app using the Google Cloud Storage API.
package gcsdemo
//[START imports]
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
"cloud.google.com/go/storage"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
"google.golang.org/appengine"
"google.golang.org/appengine/file"
"google.golang.org/appengine/log"
)
//[END imports]
func init() {
http.HandleFunc("/", handler)
}
// demo struct holds information needed to run the various demo functions.
type demo struct {
client *storage.Client
bucketName string
bucket *storage.BucketHandle
w io.Writer
ctx context.Context
// cleanUp is a list of filenames that need cleaning up at the end of the demo.
cleanUp []string
// failed indicates that one or more of the demo steps failed.
failed bool
}
func (d *demo) errorf(format string, args ...interface{}) {
d.failed = true
fmt.Fprintln(d.w, fmt.Sprintf(format, args...))
log.Errorf(d.ctx, format, args...)
}
// handler is the main demo entry point that calls the GCS operations.
func handler(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
//[START get_default_bucket]
// Use `dev_appserver.py --default_gcs_bucket_name GCS_BUCKET_NAME`
// when running locally.
bucket, err := file.DefaultBucketName(ctx)
if err != nil {
log.Errorf(ctx, "failed to get default GCS bucket name: %v", err)
}
//[END get_default_bucket]
client, err := storage.NewClient(ctx)
if err != nil {
log.Errorf(ctx, "failed to create client: %v", err)
return
}
defer client.Close()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(ctx))
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)
buf := &bytes.Buffer{}
d := &demo{
w: buf,
ctx: ctx,
client: client,
bucket: client.Bucket(bucket),
bucketName: bucket,
}
n := "demo-testfile-go"
d.createFile(n)
d.readFile(n)
d.copyFile(n)
d.statFile(n)
d.createListFiles()
d.listBucket()
d.listBucketDirMode()
d.defaultACL()
d.putDefaultACLRule()
d.deleteDefaultACLRule()
d.bucketACL()
d.putBucketACLRule()
d.deleteBucketACLRule()
d.acl(n)
d.putACLRule(n)
d.deleteACLRule(n)
d.deleteFiles()
if d.failed {
w.WriteHeader(http.StatusInternalServerError)
buf.WriteTo(w)
fmt.Fprintf(w, "\nDemo failed.\n")
} else {
w.WriteHeader(http.StatusOK)
buf.WriteTo(w)
fmt.Fprintf(w, "\nDemo succeeded.\n")
}
}
//[START write]
// createFile creates a file in Google Cloud Storage.
func (d *demo) createFile(fileName string) {
fmt.Fprintf(d.w, "Creating file /%v/%v\n", d.bucketName, fileName)
wc := d.bucket.Object(fileName).NewWriter(d.ctx)
wc.ContentType = "text/plain"
wc.Metadata = map[string]string{
"x-goog-meta-foo": "foo",
"x-goog-meta-bar": "bar",
}
d.cleanUp = append(d.cleanUp, fileName)
if _, err := wc.Write([]byte("abcde\n")); err != nil {
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil {
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
if err := wc.Close(); err != nil {
d.errorf("createFile: unable to close bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
}
//[END write]
//[START read]
// readFile reads the named file in Google Cloud Storage.
func (d *demo) readFile(fileName string) {
io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n")
rc, err := d.bucket.Object(fileName).NewReader(d.ctx)
if err != nil {
d.errorf("readFile: unable to open file from bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
defer rc.Close()
slurp, err := ioutil.ReadAll(rc)
if err != nil {
d.errorf("readFile: unable to read data from bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0])
if len(slurp) > 1024 {
fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:])
} else {
fmt.Fprintf(d.w, "%s\n", slurp)
}
}
//[END read]
//[START copy]
// copyFile copies a file in Google Cloud Storage.
func (d *demo) copyFile(fileName string) {
copyName := fileName + "-copy"
fmt.Fprintf(d.w, "Copying file /%v/%v to /%v/%v:\n", d.bucketName, fileName, d.bucketName, copyName)
obj, err := d.bucket.Object(copyName).CopierFrom(d.bucket.Object(fileName)).Run(d.ctx)
if err != nil {
d.errorf("copyFile: unable to copy /%v/%v to bucket %q, file %q: %v", d.bucketName, fileName, d.bucketName, copyName, err)
return
}
d.cleanUp = append(d.cleanUp, copyName)
d.dumpStats(obj)
}
//[END copy]
func (d *demo) dumpStats(obj *storage.ObjectAttrs) {
fmt.Fprintf(d.w, "(filename: /%v/%v, ", obj.Bucket, obj.Name)
fmt.Fprintf(d.w, "ContentType: %q, ", obj.ContentType)
fmt.Fprintf(d.w, "ACL: %#v, ", obj.ACL)
fmt.Fprintf(d.w, "Owner: %v, ", obj.Owner)
fmt.Fprintf(d.w, "ContentEncoding: %q, ", obj.ContentEncoding)
fmt.Fprintf(d.w, "Size: %v, ", obj.Size)
fmt.Fprintf(d.w, "MD5: %q, ", obj.MD5)
fmt.Fprintf(d.w, "CRC32C: %q, ", obj.CRC32C)
fmt.Fprintf(d.w, "Metadata: %#v, ", obj.Metadata)
fmt.Fprintf(d.w, "MediaLink: %q, ", obj.MediaLink)
fmt.Fprintf(d.w, "StorageClass: %q, ", obj.StorageClass)
if !obj.Deleted.IsZero() {
fmt.Fprintf(d.w, "Deleted: %v, ", obj.Deleted)
}
fmt.Fprintf(d.w, "Updated: %v)\n", obj.Updated)
}
//[START file_metadata]
// statFile reads the stats of the named file in Google Cloud Storage.
func (d *demo) statFile(fileName string) {
io.WriteString(d.w, "\nFile stat:\n")
obj, err := d.bucket.Object(fileName).Attrs(d.ctx)
if err != nil {
d.errorf("statFile: unable to stat file from bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
d.dumpStats(obj)
}
//[END file_metadata]
// createListFiles creates files that will be used by listBucket.
func (d *demo) createListFiles() {
io.WriteString(d.w, "\nCreating more files for listbucket...\n")
for _, n := range []string{"foo1", "foo2", "bar", "bar/1", "bar/2", "boo/"} {
d.createFile(n)
}
}
//[START list_bucket]
// listBucket lists the contents of a bucket in Google Cloud Storage.
func (d *demo) listBucket() {
io.WriteString(d.w, "\nListbucket result:\n")
query := &storage.Query{Prefix: "foo"}
it := d.bucket.Objects(d.ctx, query)
for {
obj, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
d.errorf("listBucket: unable to list bucket %q: %v", d.bucketName, err)
return
}
d.dumpStats(obj)
}
}
//[END list_bucket]
func (d *demo) listDir(name, indent string) {
query := &storage.Query{Prefix: name, Delimiter: "/"}
it := d.bucket.Objects(d.ctx, query)
for {
obj, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
d.errorf("listBucketDirMode: unable to list bucket %q: %v", d.bucketName, err)
return
}
if obj.Prefix == "" {
fmt.Fprint(d.w, indent)
d.dumpStats(obj)
continue
}
fmt.Fprintf(d.w, "%v(directory: /%v/%v)\n", indent, d.bucketName, obj.Prefix)
d.listDir(obj.Prefix, indent+" ")
}
}
// listBucketDirMode lists the contents of a bucket in dir mode in Google Cloud Storage.
func (d *demo) listBucketDirMode() {
io.WriteString(d.w, "\nListbucket directory mode result:\n")
d.listDir("b", "")
}
// dumpDefaultACL prints out the default object ACL for this bucket.
func (d *demo) dumpDefaultACL() {
acl, err := d.bucket.ACL().List(d.ctx)
if err != nil {
d.errorf("defaultACL: unable to list default object ACL for bucket %q: %v", d.bucketName, err)
return
}
for _, v := range acl {
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
}
}
// defaultACL displays the default object ACL for this bucket.
func (d *demo) defaultACL() {
io.WriteString(d.w, "\nDefault object ACL:\n")
d.dumpDefaultACL()
}
// putDefaultACLRule adds the "allUsers" default object ACL rule for this bucket.
func (d *demo) putDefaultACLRule() {
io.WriteString(d.w, "\nPut Default object ACL Rule:\n")
err := d.bucket.DefaultObjectACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
if err != nil {
d.errorf("putDefaultACLRule: unable to save default object ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpDefaultACL()
}
// deleteDefaultACLRule deleted the "allUsers" default object ACL rule for this bucket.
func (d *demo) deleteDefaultACLRule() {
io.WriteString(d.w, "\nDelete Default object ACL Rule:\n")
err := d.bucket.DefaultObjectACL().Delete(d.ctx, storage.AllUsers)
if err != nil {
d.errorf("deleteDefaultACLRule: unable to delete default object ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpDefaultACL()
}
// dumpBucketACL prints out the bucket ACL.
func (d *demo) dumpBucketACL() {
acl, err := d.bucket.ACL().List(d.ctx)
if err != nil {
d.errorf("dumpBucketACL: unable to list bucket ACL for bucket %q: %v", d.bucketName, err)
return
}
for _, v := range acl {
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
}
}
// bucketACL displays the bucket ACL for this bucket.
func (d *demo) bucketACL() {
io.WriteString(d.w, "\nBucket ACL:\n")
d.dumpBucketACL()
}
// putBucketACLRule adds the "allUsers" bucket ACL rule for this bucket.
func (d *demo) putBucketACLRule() {
io.WriteString(d.w, "\nPut Bucket ACL Rule:\n")
err := d.bucket.ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
if err != nil {
d.errorf("putBucketACLRule: unable to save bucket ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpBucketACL()
}
// deleteBucketACLRule deleted the "allUsers" bucket ACL rule for this bucket.
func (d *demo) deleteBucketACLRule() {
io.WriteString(d.w, "\nDelete Bucket ACL Rule:\n")
err := d.bucket.ACL().Delete(d.ctx, storage.AllUsers)
if err != nil {
d.errorf("deleteBucketACLRule: unable to delete bucket ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpBucketACL()
}
// dumpACL prints out the ACL of the named file.
func (d *demo) dumpACL(fileName string) {
acl, err := d.bucket.Object(fileName).ACL().List(d.ctx)
if err != nil {
d.errorf("dumpACL: unable to list file ACL for bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
for _, v := range acl {
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
}
}
// acl displays the ACL for the named file.
func (d *demo) acl(fileName string) {
fmt.Fprintf(d.w, "\nACL for file %v:\n", fileName)
d.dumpACL(fileName)
}
// putACLRule adds the "allUsers" ACL rule for the named file.
func (d *demo) putACLRule(fileName string) {
fmt.Fprintf(d.w, "\nPut ACL rule for file %v:\n", fileName)
err := d.bucket.Object(fileName).ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
if err != nil {
d.errorf("putACLRule: unable to save ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
d.dumpACL(fileName)
}
// deleteACLRule deleted the "allUsers" ACL rule for the named file.
func (d *demo) deleteACLRule(fileName string) {
fmt.Fprintf(d.w, "\nDelete ACL rule for file %v:\n", fileName)
err := d.bucket.Object(fileName).ACL().Delete(d.ctx, storage.AllUsers)
if err != nil {
d.errorf("deleteACLRule: unable to delete ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
d.dumpACL(fileName)
}
// deleteFiles deletes all the temporary files from a bucket created by this demo.
func (d *demo) deleteFiles() {
io.WriteString(d.w, "\nDeleting files...\n")
for _, v := range d.cleanUp {
fmt.Fprintf(d.w, "Deleting file %v\n", v)
if err := d.bucket.Object(v).Delete(d.ctx); err != nil {
d.errorf("deleteFiles: unable to delete bucket %q, file %q: %v", d.bucketName, v, err)
return
}
}
}
//[END sample]
+8
View File
@@ -0,0 +1,8 @@
application: your-app-id
version: v1
runtime: go
api_version: go1
handlers:
- url: /.*
script: _go_app
+432
View File
@@ -0,0 +1,432 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package main is an example Mananged VM app using the Google Cloud Storage API.
package main
//[START imports]
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
"cloud.google.com/go/storage"
"golang.org/x/net/context"
"google.golang.org/api/iterator"
"google.golang.org/appengine"
"google.golang.org/appengine/file"
"google.golang.org/appengine/log"
)
//[END imports]
func main() {
http.HandleFunc("/", handler)
appengine.Main()
}
//[START bucket_struct]
// demo struct holds information needed to run the various demo functions.
type demo struct {
client *storage.Client
bucketName string
bucket *storage.BucketHandle
w io.Writer
ctx context.Context
// cleanUp is a list of filenames that need cleaning up at the end of the demo.
cleanUp []string
// failed indicates that one or more of the demo steps failed.
failed bool
}
//[END bucket_struct]
func (d *demo) errorf(format string, args ...interface{}) {
d.failed = true
fmt.Fprintln(d.w, fmt.Sprintf(format, args...))
log.Errorf(d.ctx, format, args...)
}
// handler is the main demo entry point that calls the GCS operations.
func handler(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
//[START get_default_bucket]
// Use `dev_appserver.py --default_gcs_bucket_name GCS_BUCKET_NAME`
// when running locally.
bucket, err := file.DefaultBucketName(ctx)
if err != nil {
log.Errorf(ctx, "failed to get default GCS bucket name: %v", err)
}
//[END get_default_bucket]
client, err := storage.NewClient(ctx)
if err != nil {
log.Errorf(ctx, "failed to create client: %v", err)
return
}
defer client.Close()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(ctx))
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)
buf := &bytes.Buffer{}
d := &demo{
w: buf,
ctx: ctx,
client: client,
bucket: client.Bucket(bucket),
bucketName: bucket,
}
n := "demo-testfile-go"
d.createFile(n)
d.readFile(n)
d.copyFile(n)
d.statFile(n)
d.createListFiles()
d.listBucket()
d.listBucketDirMode()
d.defaultACL()
d.putDefaultACLRule()
d.deleteDefaultACLRule()
d.bucketACL()
d.putBucketACLRule()
d.deleteBucketACLRule()
d.acl(n)
d.putACLRule(n)
d.deleteACLRule(n)
d.deleteFiles()
if d.failed {
w.WriteHeader(http.StatusInternalServerError)
buf.WriteTo(w)
fmt.Fprintf(w, "\nDemo failed.\n")
} else {
w.WriteHeader(http.StatusOK)
buf.WriteTo(w)
fmt.Fprintf(w, "\nDemo succeeded.\n")
}
}
//[START write]
// createFile creates a file in Google Cloud Storage.
func (d *demo) createFile(fileName string) {
fmt.Fprintf(d.w, "Creating file /%v/%v\n", d.bucket, fileName)
wc := d.bucket.Object(fileName).NewWriter(d.ctx)
wc.ContentType = "text/plain"
wc.Metadata = map[string]string{
"x-goog-meta-foo": "foo",
"x-goog-meta-bar": "bar",
}
d.cleanUp = append(d.cleanUp, fileName)
if _, err := wc.Write([]byte("abcde\n")); err != nil {
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil {
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
if err := wc.Close(); err != nil {
d.errorf("createFile: unable to close bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
}
//[END write]
//[START read]
// readFile reads the named file in Google Cloud Storage.
func (d *demo) readFile(fileName string) {
io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n")
rc, err := d.bucket.Object(fileName).NewReader(d.ctx)
if err != nil {
d.errorf("readFile: unable to open file from bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
defer rc.Close()
slurp, err := ioutil.ReadAll(rc)
if err != nil {
d.errorf("readFile: unable to read data from bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0])
if len(slurp) > 1024 {
fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:])
} else {
fmt.Fprintf(d.w, "%s\n", slurp)
}
}
//[END read]
//[START copy]
// copyFile copies a file in Google Cloud Storage.
func (d *demo) copyFile(fileName string) {
copyName := fileName + "-copy"
fmt.Fprintf(d.w, "Copying file /%v/%v to /%v/%v:\n", d.bucket, fileName, d.bucketName, copyName)
obj, err := d.bucket.Object(copyName).CopierFrom(d.bucket.Object(fileName)).Run(d.ctx)
if err != nil {
d.errorf("copyFile: unable to copy /%v/%v to bucket %q, file %q: %v", d.bucketName, fileName, d.bucketName, copyName, err)
return
}
d.cleanUp = append(d.cleanUp, copyName)
d.dumpStats(obj)
}
//[END copy]
func (d *demo) dumpStats(obj *storage.ObjectAttrs) {
fmt.Fprintf(d.w, "(filename: /%v/%v, ", obj.Bucket, obj.Name)
fmt.Fprintf(d.w, "ContentType: %q, ", obj.ContentType)
fmt.Fprintf(d.w, "ACL: %#v, ", obj.ACL)
fmt.Fprintf(d.w, "Owner: %v, ", obj.Owner)
fmt.Fprintf(d.w, "ContentEncoding: %q, ", obj.ContentEncoding)
fmt.Fprintf(d.w, "Size: %v, ", obj.Size)
fmt.Fprintf(d.w, "MD5: %q, ", obj.MD5)
fmt.Fprintf(d.w, "CRC32C: %q, ", obj.CRC32C)
fmt.Fprintf(d.w, "Metadata: %#v, ", obj.Metadata)
fmt.Fprintf(d.w, "MediaLink: %q, ", obj.MediaLink)
fmt.Fprintf(d.w, "StorageClass: %q, ", obj.StorageClass)
if !obj.Deleted.IsZero() {
fmt.Fprintf(d.w, "Deleted: %v, ", obj.Deleted)
}
fmt.Fprintf(d.w, "Updated: %v)\n", obj.Updated)
}
//[START file_metadata]
// statFile reads the stats of the named file in Google Cloud Storage.
func (d *demo) statFile(fileName string) {
io.WriteString(d.w, "\nFile stat:\n")
obj, err := d.bucket.Object(fileName).Attrs(d.ctx)
if err != nil {
d.errorf("statFile: unable to stat file from bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
d.dumpStats(obj)
}
//[END file_metadata]
// createListFiles creates files that will be used by listBucket.
func (d *demo) createListFiles() {
io.WriteString(d.w, "\nCreating more files for listbucket...\n")
for _, n := range []string{"foo1", "foo2", "bar", "bar/1", "bar/2", "boo/"} {
d.createFile(n)
}
}
//[START list_bucket]
// listBucket lists the contents of a bucket in Google Cloud Storage.
func (d *demo) listBucket() {
io.WriteString(d.w, "\nListbucket result:\n")
query := &storage.Query{Prefix: "foo"}
it := d.bucket.Objects(d.ctx, query)
for {
obj, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
d.errorf("listBucket: unable to list bucket %q: %v", d.bucketName, err)
return
}
d.dumpStats(obj)
}
}
//[END list_bucket]
func (d *demo) listDir(name, indent string) {
query := &storage.Query{Prefix: name, Delimiter: "/"}
it := d.bucket.Objects(d.ctx, query)
for {
obj, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
d.errorf("listBucketDirMode: unable to list bucket %q: %v", d.bucketName, err)
return
}
if obj.Prefix == "" {
fmt.Fprint(d.w, indent)
d.dumpStats(obj)
continue
}
fmt.Fprintf(d.w, "%v(directory: /%v/%v)\n", indent, d.bucketName, obj.Prefix)
d.listDir(obj.Prefix, indent+" ")
}
}
// listBucketDirMode lists the contents of a bucket in dir mode in Google Cloud Storage.
func (d *demo) listBucketDirMode() {
io.WriteString(d.w, "\nListbucket directory mode result:\n")
d.listDir("b", "")
}
// dumpDefaultACL prints out the default object ACL for this bucket.
func (d *demo) dumpDefaultACL() {
acl, err := d.bucket.ACL().List(d.ctx)
if err != nil {
d.errorf("defaultACL: unable to list default object ACL for bucket %q: %v", d.bucketName, err)
return
}
for _, v := range acl {
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
}
}
// defaultACL displays the default object ACL for this bucket.
func (d *demo) defaultACL() {
io.WriteString(d.w, "\nDefault object ACL:\n")
d.dumpDefaultACL()
}
// putDefaultACLRule adds the "allUsers" default object ACL rule for this bucket.
func (d *demo) putDefaultACLRule() {
io.WriteString(d.w, "\nPut Default object ACL Rule:\n")
err := d.bucket.DefaultObjectACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
if err != nil {
d.errorf("putDefaultACLRule: unable to save default object ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpDefaultACL()
}
// deleteDefaultACLRule deleted the "allUsers" default object ACL rule for this bucket.
func (d *demo) deleteDefaultACLRule() {
io.WriteString(d.w, "\nDelete Default object ACL Rule:\n")
err := d.bucket.DefaultObjectACL().Delete(d.ctx, storage.AllUsers)
if err != nil {
d.errorf("deleteDefaultACLRule: unable to delete default object ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpDefaultACL()
}
// dumpBucketACL prints out the bucket ACL.
func (d *demo) dumpBucketACL() {
acl, err := d.bucket.ACL().List(d.ctx)
if err != nil {
d.errorf("dumpBucketACL: unable to list bucket ACL for bucket %q: %v", d.bucketName, err)
return
}
for _, v := range acl {
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
}
}
// bucketACL displays the bucket ACL for this bucket.
func (d *demo) bucketACL() {
io.WriteString(d.w, "\nBucket ACL:\n")
d.dumpBucketACL()
}
// putBucketACLRule adds the "allUsers" bucket ACL rule for this bucket.
func (d *demo) putBucketACLRule() {
io.WriteString(d.w, "\nPut Bucket ACL Rule:\n")
err := d.bucket.ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
if err != nil {
d.errorf("putBucketACLRule: unable to save bucket ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpBucketACL()
}
// deleteBucketACLRule deleted the "allUsers" bucket ACL rule for this bucket.
func (d *demo) deleteBucketACLRule() {
io.WriteString(d.w, "\nDelete Bucket ACL Rule:\n")
err := d.bucket.ACL().Delete(d.ctx, storage.AllUsers)
if err != nil {
d.errorf("deleteBucketACLRule: unable to delete bucket ACL rule for bucket %q: %v", d.bucketName, err)
return
}
d.dumpBucketACL()
}
// dumpACL prints out the ACL of the named file.
func (d *demo) dumpACL(fileName string) {
acl, err := d.bucket.Object(fileName).ACL().List(d.ctx)
if err != nil {
d.errorf("dumpACL: unable to list file ACL for bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
for _, v := range acl {
fmt.Fprintf(d.w, "Scope: %q, Permission: %q\n", v.Entity, v.Role)
}
}
// acl displays the ACL for the named file.
func (d *demo) acl(fileName string) {
fmt.Fprintf(d.w, "\nACL for file %v:\n", fileName)
d.dumpACL(fileName)
}
// putACLRule adds the "allUsers" ACL rule for the named file.
func (d *demo) putACLRule(fileName string) {
fmt.Fprintf(d.w, "\nPut ACL rule for file %v:\n", fileName)
err := d.bucket.Object(fileName).ACL().Set(d.ctx, storage.AllUsers, storage.RoleReader)
if err != nil {
d.errorf("putACLRule: unable to save ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
d.dumpACL(fileName)
}
// deleteACLRule deleted the "allUsers" ACL rule for the named file.
func (d *demo) deleteACLRule(fileName string) {
fmt.Fprintf(d.w, "\nDelete ACL rule for file %v:\n", fileName)
err := d.bucket.Object(fileName).ACL().Delete(d.ctx, storage.AllUsers)
if err != nil {
d.errorf("deleteACLRule: unable to delete ACL rule for bucket %q, file %q: %v", d.bucketName, fileName, err)
return
}
d.dumpACL(fileName)
}
//[START delete]
// deleteFiles deletes all the temporary files from a bucket created by this demo.
func (d *demo) deleteFiles() {
io.WriteString(d.w, "\nDeleting files...\n")
for _, v := range d.cleanUp {
fmt.Fprintf(d.w, "Deleting file %v\n", v)
if err := d.bucket.Object(v).Delete(d.ctx); err != nil {
d.errorf("deleteFiles: unable to delete bucket %q, file %q: %v", d.bucketName, v, err)
return
}
}
}
//[END delete]
+10
View File
@@ -0,0 +1,10 @@
runtime: go
api_version: 1
vm: true
manual_scaling:
instances: 1
handlers:
- url: /.*
script: _go_app