├── key.json.enc
├── examples
    ├── storage
    │   ├── appengine
    │   │   └── app.yaml
    │   └── appenginevm
    │   │   └── app.yaml
    ├── bigtable
    │   └── bigtable-hello
    │   │   ├── app.yaml
    │   │   ├── README.md
    │   │   └── helloworld.go
    └── bigquery
    │   ├── query
    │       └── main.go
    │   ├── load
    │       └── main.go
    │   ├── concat_table
    │       └── main.go
    │   └── read
    │       └── main.go
├── .travis.yml
├── AUTHORS
├── internal
    ├── opts
    │   └── option.go
    ├── transport
    │   ├── cancelreq.go
    │   ├── cancelreq_legacy.go
    │   ├── proto.go
    │   └── dial.go
    ├── testutil
    │   └── context.go
    └── cloud.go
├── datastore
    ├── testdata
    │   └── index.yaml
    ├── time.go
    ├── errors.go
    ├── time_test.go
    └── save.go
├── bigquery
    ├── doc.go
    ├── error.go
    ├── dataset.go
    ├── utils_test.go
    ├── query.go
    ├── copy_op.go
    ├── create_table_test.go
    ├── extract_op.go
    ├── read_op.go
    ├── extract_test.go
    ├── copy_test.go
    ├── query_op.go
    ├── query_test.go
    ├── schema.go
    ├── dataset_test.go
    ├── job.go
    ├── load_op.go
    ├── gcs.go
    ├── value.go
    ├── schema_test.go
    ├── bigquery.go
    └── load_test.go
├── CONTRIBUTORS
├── bigtable
    ├── internal
    │   ├── empty
    │   │   ├── empty.proto
    │   │   └── empty.pb.go
    │   ├── service_proto
    │   │   └── bigtable_service.proto
    │   ├── regen.sh
    │   ├── table_service_proto
    │   │   ├── bigtable_table_service.proto
    │   │   └── bigtable_table_service_messages.proto
    │   ├── duration_proto
    │   │   ├── duration.proto
    │   │   └── duration.pb.go
    │   ├── cluster_data_proto
    │   │   └── bigtable_cluster_data.proto
    │   ├── cbtrc
    │   │   └── cbtrc.go
    │   ├── cluster_service_proto
    │   │   └── bigtable_cluster_service_messages.proto
    │   └── table_data_proto
    │   │   └── bigtable_table_data.proto
    ├── admin_test.go
    ├── cmd
    │   ├── loadtest
    │   │   ├── stats.go
    │   │   └── loadtest.go
    │   └── cbt
    │   │   └── cbtdoc.go
    ├── gc.go
    ├── doc.go
    └── filter.go
├── cloud_test.go
├── pubsub
    ├── pubsub_test.go
    ├── example_test.go
    └── integration_test.go
├── storage
    ├── writer_test.go
    ├── testdata
    │   ├── dummy_rsa
    │   └── dummy_pem
    ├── writer.go
    └── example_test.go
├── option_test.go
├── cloud.go
├── option.go
└── CONTRIBUTING.md


/key.json.enc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/fd/gcloud-golang/master/key.json.enc


--------------------------------------------------------------------------------
/examples/storage/appengine/app.yaml:
--------------------------------------------------------------------------------
1 | application: <your-app-id-here>
2 | version: v1
3 | runtime: go
4 | api_version: go1
5 | 
6 | handlers:
7 | - url: /.*
8 |   script: _go_app
9 | 


--------------------------------------------------------------------------------
/examples/storage/appenginevm/app.yaml:
--------------------------------------------------------------------------------
 1 | runtime: go
 2 | api_version: go1
 3 | vm: true
 4 | 
 5 | manual_scaling:
 6 |   instances: 1
 7 | 
 8 | handlers:
 9 | - url: /.*
10 |   script: _go_app
11 | 


--------------------------------------------------------------------------------
/examples/bigtable/bigtable-hello/app.yaml:
--------------------------------------------------------------------------------
 1 | runtime: go
 2 | api_version: go1
 3 | vm: true
 4 | 
 5 | manual_scaling:
 6 |   instances: 1
 7 | 
 8 | handlers:
 9 | # Serve only the web root.
10 | - url: /
11 |   script: _go_app
12 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | sudo: false
 2 | language: go
 3 | go:
 4 | - 1.4
 5 | - 1.5
 6 | install:
 7 | - go get -v google.golang.org/cloud/...
 8 | script:
 9 | - openssl aes-256-cbc -K $encrypted_912ff8fa81ad_key -iv $encrypted_912ff8fa81ad_iv -in key.json.enc -out key.json -d
10 | - GCLOUD_TESTS_GOLANG_PROJECT_ID="dulcet-port-762" GCLOUD_TESTS_GOLANG_KEY="$(pwd)/key.json"
11 |   go test -v -tags=integration google.golang.org/cloud/...
12 | 


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
 1 | # This is the official list of cloud authors for copyright purposes.
 2 | # This file is distinct from the CONTRIBUTORS files.
 3 | # See the latter for an explanation.
 4 | 
 5 | # Names should be added to this file as:
 6 | # Name or Organization <email address>
 7 | # The email address is not required for organizations.
 8 | 
 9 | Google Inc.
10 | Palm Stone Games, Inc.
11 | Péter Szilágyi <peterke@gmail.com>
12 | Tyler Treat <ttreat31@gmail.com>
13 | 


--------------------------------------------------------------------------------
/internal/opts/option.go:
--------------------------------------------------------------------------------
 1 | // Package opts holds the DialOpts struct, configurable by
 2 | // cloud.ClientOptions to set up transports for cloud packages.
 3 | //
 4 | // This is a separate page to prevent cycles between the core
 5 | // cloud packages.
 6 | package opts
 7 | 
 8 | import (
 9 | 	"net/http"
10 | 
11 | 	"golang.org/x/oauth2"
12 | 	"google.golang.org/grpc"
13 | )
14 | 
15 | type DialOpt struct {
16 | 	Endpoint  string
17 | 	Scopes    []string
18 | 	UserAgent string
19 | 
20 | 	TokenSource oauth2.TokenSource
21 | 
22 | 	HTTPClient   *http.Client
23 | 	GRPCClient   *grpc.ClientConn
24 | 	GRPCDialOpts []grpc.DialOption
25 | }
26 | 


--------------------------------------------------------------------------------
/datastore/testdata/index.yaml:
--------------------------------------------------------------------------------
 1 | indexes:
 2 | 
 3 | - kind: SQChild
 4 |   ancestor: yes
 5 |   properties:
 6 |   - name: T
 7 |   - name: I
 8 | 
 9 | - kind: SQChild
10 |   ancestor: yes
11 |   properties:
12 |   - name: T
13 |   - name: I
14 |     direction: desc
15 | 
16 | - kind: SQChild
17 |   ancestor: yes
18 |   properties:
19 |   - name: I
20 |   - name: T
21 |   - name: U
22 | 
23 | - kind: SQChild
24 |   ancestor: yes
25 |   properties:
26 |   - name: I
27 |   - name: T
28 |   - name: U
29 | 
30 | - kind: SQChild
31 |   ancestor: yes
32 |   properties:
33 |   - name: T
34 |   - name: J
35 | 
36 | - kind: SQChild
37 |   ancestor: yes
38 |   properties:
39 |   - name: T
40 |   - name: J
41 |   - name: U


--------------------------------------------------------------------------------
/bigquery/doc.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | // Package bigquery provides a client for the BigQuery service.
16 | //
17 | // Note: This package is a work-in-progress.  Backwards-incompatible changes should be expected.
18 | package bigquery // import "github.com/fd/gcloud-golang/bigquery"
19 | 


--------------------------------------------------------------------------------
/CONTRIBUTORS:
--------------------------------------------------------------------------------
 1 | # People who have agreed to one of the CLAs and can contribute patches.
 2 | # The AUTHORS file lists the copyright holders; this file
 3 | # lists people.  For example, Google employees are listed here
 4 | # but not in AUTHORS, because Google holds the copyright.
 5 | #
 6 | # https://developers.google.com/open-source/cla/individual
 7 | # https://developers.google.com/open-source/cla/corporate
 8 | #
 9 | # Names should be added to this file as:
10 | #     Name <email address>
11 | 
12 | # Keep the list alphabetically sorted.
13 | 
14 | Andrew Gerrand <adg@golang.org>
15 | Brad Fitzpatrick <bradfitz@golang.org>
16 | Burcu Dogan <jbd@google.com>
17 | Dave Day <djd@golang.org>
18 | David Symonds <dsymonds@golang.org>
19 | Glenn Lewis <gmlewis@google.com>
20 | Johan Euphrosine <proppy@google.com>
21 | Luna Duclos <luna.duclos@palmstonegames.com>
22 | Michael McGreevy <mcgreevy@golang.org>
23 | Péter Szilágyi <peterke@gmail.com>
24 | Tyler Treat <ttreat31@gmail.com>
25 | 


--------------------------------------------------------------------------------
/internal/transport/cancelreq.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | // +build go1.5
16 | 
17 | package transport
18 | 
19 | import "net/http"
20 | 
21 | // makeReqCancel returns a closure that cancels the given http.Request
22 | // when called.
23 | func makeReqCancel(req *http.Request) func(http.RoundTripper) {
24 | 	c := make(chan struct{})
25 | 	req.Cancel = c
26 | 	return func(http.RoundTripper) {
27 | 		close(c)
28 | 	}
29 | }
30 | 


--------------------------------------------------------------------------------
/internal/transport/cancelreq_legacy.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | // +build !go1.5
16 | 
17 | package transport
18 | 
19 | import "net/http"
20 | 
21 | // makeReqCancel returns a closure that cancels the given http.Request
22 | // when called.
23 | func makeReqCancel(req *http.Request) func(http.RoundTripper) {
24 | 	// Go 1.4 and prior do not have a reliable way of cancelling a request.
25 | 	// Transport.CancelRequest will only work if the request is already in-flight.
26 | 	return func(r http.RoundTripper) {
27 | 		if t, ok := r.(*http.Transport); ok {
28 | 			t.CancelRequest(req)
29 | 		}
30 | 	}
31 | }
32 | 


--------------------------------------------------------------------------------
/bigtable/internal/empty/empty.proto:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2015, Google Inc.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | syntax = "proto3";
16 | 
17 | package google.protobuf;
18 | 
19 | option java_multiple_files = true;
20 | option java_outer_classname = "EmptyProto";
21 | option java_package = "com.google.protobuf";
22 | 
23 | 
24 | // A generic empty message that you can re-use to avoid defining duplicated
25 | // empty messages in your APIs. A typical example is to use it as the request
26 | // or the response type of an API method. For instance:
27 | //
28 | //     service Foo {
29 | //       rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);
30 | //     }
31 | //
32 | message Empty {
33 | 
34 | }
35 | 


--------------------------------------------------------------------------------
/examples/bigtable/bigtable-hello/README.md:
--------------------------------------------------------------------------------
 1 | # Cloud Bigtable on Managed VMs using Go
 2 | # (Hello World for Cloud Bigtable)
 3 | 
 4 | This app counts how often each user visits.
 5 | 
 6 | ## Prerequisites
 7 | 
 8 | 1. Set up Cloud Console.
 9 |   1. Go to the [Cloud Console](https://cloud.google.com/console) and create or select your project.
10 |      You will need the project ID later.
11 |   1. Go to **Settings > Project Billing Settings** and enable billing.
12 |   1. Select **APIs & Auth > APIs**.
13 |   1. Enable the **Cloud Bigtable API** and the **Cloud Bigtable Admin API**.
14 |   (You may need to search for the API).
15 | 1. Set up gcloud.
16 |   1. `gcloud components update`
17 |   1. `gcloud auth login`
18 |   1. `gcloud config set project PROJECT_ID`
19 | 1. Download App Engine SDK for Go.
20 |   1. `go get -u google.golang.org/appengine/...`
21 | 1. In helloworld.go, change the constants `project`, `zone` and `cluster`
22 | 
23 | ## Running locally
24 | 
25 | 1. From the sample project folder, `gcloud preview app run app.yaml`
26 | 
27 | ## Deploying on Google App Engine Managed VM
28 | 
29 | 1. Install and start [Docker](https://cloud.google.com/appengine/docs/managed-vms/getting-started#install_docker).
30 | 1. From the sample project folder, `aedeploy gcloud preview app deploy app.yaml`
31 | 


--------------------------------------------------------------------------------
/cloud_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package cloud
16 | 
17 | import (
18 | 	"net/http"
19 | 	"testing"
20 | 
21 | 	"github.com/fd/gcloud-golang/internal"
22 | )
23 | 
24 | func TestClientTransportMutate(t *testing.T) {
25 | 	c := &http.Client{Transport: http.DefaultTransport}
26 | 	NewContext("project-id", c)
27 | 	NewContext("project-id", c)
28 | 
29 | 	tr, ok := c.Transport.(*internal.Transport)
30 | 	if !ok {
31 | 		t.Errorf("Transport is expected to be an internal.Transport, found to be a %T", c.Transport)
32 | 	}
33 | 	if _, ok := tr.Base.(*internal.Transport); ok {
34 | 		t.Errorf("Transport's Base shouldn't have been an internal.Transport, found to be a %T", tr.Base)
35 | 	}
36 | }
37 | 


--------------------------------------------------------------------------------
/datastore/time.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package datastore
16 | 
17 | import (
18 | 	"math"
19 | 	"time"
20 | )
21 | 
22 | var (
23 | 	minTime = time.Unix(int64(math.MinInt64)/1e6, (int64(math.MinInt64)%1e6)*1e3)
24 | 	maxTime = time.Unix(int64(math.MaxInt64)/1e6, (int64(math.MaxInt64)%1e6)*1e3)
25 | )
26 | 
27 | func toUnixMicro(t time.Time) int64 {
28 | 	// We cannot use t.UnixNano() / 1e3 because we want to handle times more than
29 | 	// 2^63 nanoseconds (which is about 292 years) away from 1970, and those cannot
30 | 	// be represented in the numerator of a single int64 divide.
31 | 	return t.Unix()*1e6 + int64(t.Nanosecond()/1e3)
32 | }
33 | 
34 | func fromUnixMicro(t int64) time.Time {
35 | 	return time.Unix(t/1e6, (t%1e6)*1e3)
36 | }
37 | 


--------------------------------------------------------------------------------
/bigquery/error.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import (
18 | 	"fmt"
19 | 
20 | 	bq "google.golang.org/api/bigquery/v2"
21 | )
22 | 
23 | // An Error contains detailed information about an error encountered while processing a job.
24 | type Error struct {
25 | 	// Mirrors bq.ErrorProto, but drops DebugInfo
26 | 	Location, Message, Reason string
27 | }
28 | 
29 | func (e Error) Error() string {
30 | 	return fmt.Sprintf("{Location: %q; Message: %q; Reason: %q}", e.Location, e.Message, e.Reason)
31 | }
32 | 
33 | func errorFromErrorProto(ep *bq.ErrorProto) *Error {
34 | 	if ep == nil {
35 | 		return nil
36 | 	}
37 | 	return &Error{
38 | 		Location: ep.Location,
39 | 		Message:  ep.Message,
40 | 		Reason:   ep.Reason,
41 | 	}
42 | }
43 | 


--------------------------------------------------------------------------------
/bigquery/dataset.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import "golang.org/x/net/context"
18 | 
19 | // Dataset is a reference to a BigQuery dataset.
20 | type Dataset struct {
21 | 	id     string
22 | 	client *Client
23 | }
24 | 
25 | // ListTables returns a list of all the tables contained in the Dataset.
26 | func (d *Dataset) ListTables(ctx context.Context) ([]*Table, error) {
27 | 	var tables []*Table
28 | 
29 | 	err := getPages("", func(pageToken string) (string, error) {
30 | 		ts, tok, err := d.client.service.listTables(ctx, d.client.projectID, d.id, pageToken)
31 | 		if err == nil {
32 | 			tables = append(tables, ts...)
33 | 		}
34 | 		return tok, err
35 | 	})
36 | 
37 | 	if err != nil {
38 | 		return nil, err
39 | 	}
40 | 	return tables, nil
41 | }
42 | 


--------------------------------------------------------------------------------
/datastore/errors.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | // This file provides error functions for common API failure modes.
16 | 
17 | package datastore
18 | 
19 | import (
20 | 	"fmt"
21 | )
22 | 
23 | // MultiError is returned by batch operations when there are errors with
24 | // particular elements. Errors will be in a one-to-one correspondence with
25 | // the input elements; successful elements will have a nil entry.
26 | type MultiError []error
27 | 
28 | func (m MultiError) Error() string {
29 | 	s, n := "", 0
30 | 	for _, e := range m {
31 | 		if e != nil {
32 | 			if n == 0 {
33 | 				s = e.Error()
34 | 			}
35 | 			n++
36 | 		}
37 | 	}
38 | 	switch n {
39 | 	case 0:
40 | 		return "(0 errors)"
41 | 	case 1:
42 | 		return s
43 | 	case 2:
44 | 		return s + " (and 1 other error)"
45 | 	}
46 | 	return fmt.Sprintf("%s (and %d other errors)", s, n-1)
47 | }
48 | 


--------------------------------------------------------------------------------
/bigquery/utils_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import (
18 | 	"golang.org/x/net/context"
19 | 	bq "google.golang.org/api/bigquery/v2"
20 | )
21 | 
22 | var defaultTable = &Table{
23 | 	ProjectID: "project-id",
24 | 	DatasetID: "dataset-id",
25 | 	TableID:   "table-id",
26 | }
27 | 
28 | var defaultGCS = &GCSReference{
29 | 	uris: []string{"uri"},
30 | }
31 | 
32 | var defaultQuery = &Query{
33 | 	Q:                "query string",
34 | 	DefaultProjectID: "def-project-id",
35 | 	DefaultDatasetID: "def-dataset-id",
36 | }
37 | 
38 | type testService struct {
39 | 	*bq.Job
40 | 
41 | 	service
42 | }
43 | 
44 | func (s *testService) insertJob(ctx context.Context, job *bq.Job, projectID string) (*Job, error) {
45 | 	s.Job = job
46 | 	return &Job{}, nil
47 | }
48 | 
49 | func (s *testService) jobStatus(ctx context.Context, projectID, jobID string) (*JobStatus, error) {
50 | 	return &JobStatus{State: Done}, nil
51 | }
52 | 


--------------------------------------------------------------------------------
/bigquery/query.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import bq "google.golang.org/api/bigquery/v2"
18 | 
19 | // Query represents a query to be executed.
20 | type Query struct {
21 | 	// The query to execute. See https://cloud.google.com/bigquery/query-reference for details.
22 | 	Q string
23 | 
24 | 	// DefaultProjectID and DefaultDatasetID specify the dataset to use for unqualified table names in the query.
25 | 	// If DefaultProjectID is set, DefaultDatasetID must also be set.
26 | 	DefaultProjectID string
27 | 	DefaultDatasetID string
28 | }
29 | 
30 | func (q *Query) implementsSource() {}
31 | 
32 | func (q *Query) implementsReadSource() {}
33 | 
34 | func (q *Query) customizeQuerySrc(conf *bq.JobConfigurationQuery, projectID string) {
35 | 	conf.Query = q.Q
36 | 	if q.DefaultProjectID != "" || q.DefaultDatasetID != "" {
37 | 		conf.DefaultDataset = &bq.DatasetReference{
38 | 			DatasetId: q.DefaultDatasetID,
39 | 			ProjectId: q.DefaultProjectID,
40 | 		}
41 | 	}
42 | }
43 | 


--------------------------------------------------------------------------------
/pubsub/pubsub_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package pubsub
16 | 
17 | import (
18 | 	"net/http"
19 | 	"strings"
20 | 	"testing"
21 | 	"time"
22 | 
23 | 	"github.com/fd/gcloud-golang"
24 | )
25 | 
26 | func TestIsSec(t *testing.T) {
27 | 	tests := map[time.Duration]bool{
28 | 		time.Second:                    true,
29 | 		5 * time.Second:                true,
30 | 		time.Hour:                      true,
31 | 		time.Millisecond:               false,
32 | 		time.Second + time.Microsecond: false,
33 | 	}
34 | 	for dur, expected := range tests {
35 | 		if isSec(dur) != expected {
36 | 			t.Errorf("%v is more precise than a second", dur)
37 | 		}
38 | 	}
39 | }
40 | 
41 | func TestEmptyAckID(t *testing.T) {
42 | 	ctx := cloud.NewContext("project-id", &http.Client{})
43 | 	id := []string{"test", ""}
44 | 	err := Ack(ctx, "sub", id...)
45 | 
46 | 	if err == nil || !strings.Contains(err.Error(), "index 1") {
47 | 		t.Errorf("Ack should report an error indicating the id is empty. Got: %v", err)
48 | 	}
49 | }
50 | 


--------------------------------------------------------------------------------
/bigquery/copy_op.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import (
18 | 	"fmt"
19 | 
20 | 	"golang.org/x/net/context"
21 | 	bq "google.golang.org/api/bigquery/v2"
22 | )
23 | 
24 | type copyOption interface {
25 | 	customizeCopy(conf *bq.JobConfigurationTableCopy, projectID string)
26 | }
27 | 
28 | func (c *Client) cp(ctx context.Context, dst *Table, src Tables, options []Option) (*Job, error) {
29 | 	job, options := initJobProto(c.projectID, options)
30 | 	payload := &bq.JobConfigurationTableCopy{}
31 | 
32 | 	dst.customizeCopyDst(payload, c.projectID)
33 | 	src.customizeCopySrc(payload, c.projectID)
34 | 
35 | 	for _, opt := range options {
36 | 		o, ok := opt.(copyOption)
37 | 		if !ok {
38 | 			return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src)
39 | 		}
40 | 		o.customizeCopy(payload, c.projectID)
41 | 	}
42 | 
43 | 	job.Configuration = &bq.JobConfiguration{
44 | 		Copy: payload,
45 | 	}
46 | 	return c.service.insertJob(ctx, job, c.projectID)
47 | }
48 | 


--------------------------------------------------------------------------------
/storage/writer_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package storage
16 | 
17 | import (
18 | 	"fmt"
19 | 	"net/http"
20 | 	"testing"
21 | 
22 | 	"golang.org/x/net/context"
23 | 
24 | 	"github.com/fd/gcloud-golang"
25 | )
26 | 
27 | type fakeTransport struct{}
28 | 
29 | func (t *fakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
30 | 	return nil, fmt.Errorf("error handling request")
31 | }
32 | 
33 | func TestErrorOnObjectsInsertCall(t *testing.T) {
34 | 	ctx := context.Background()
35 | 	hc := &http.Client{Transport: &fakeTransport{}}
36 | 	client, err := NewClient(ctx, cloud.WithBaseHTTP(hc))
37 | 	if err != nil {
38 | 		t.Errorf("error when creating client: %v", err)
39 | 	}
40 | 	wc := client.Bucket("bucketname").Object("filename1").NewWriter(ctx)
41 | 	wc.ContentType = "text/plain"
42 | 	if _, err := wc.Write([]byte("hello world")); err == nil {
43 | 		t.Errorf("expected error on write, got nil")
44 | 	}
45 | 	if err := wc.Close(); err == nil {
46 | 		t.Errorf("expected error on close, got nil")
47 | 	}
48 | }
49 | 


--------------------------------------------------------------------------------
/bigquery/create_table_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import (
18 | 	"reflect"
19 | 	"testing"
20 | 	"time"
21 | 
22 | 	"golang.org/x/net/context"
23 | )
24 | 
25 | type createTableRecorder struct {
26 | 	conf *createTableConf
27 | 	service
28 | }
29 | 
30 | func (rec *createTableRecorder) createTable(ctx context.Context, conf *createTableConf) error {
31 | 	rec.conf = conf
32 | 	return nil
33 | }
34 | 
35 | func TestCreateTableOptions(t *testing.T) {
36 | 	s := &createTableRecorder{}
37 | 	c := &Client{
38 | 		service: s,
39 | 	}
40 | 	exp := time.Now()
41 | 	q := "query"
42 | 	if _, err := c.CreateTable(context.Background(), "p", "d", "t", TableExpiration(exp), ViewQuery(q)); err != nil {
43 | 		t.Fatalf("err calling CreateTable: %v", err)
44 | 	}
45 | 	want := createTableConf{
46 | 		projectID:  "p",
47 | 		datasetID:  "d",
48 | 		tableID:    "t",
49 | 		expiration: exp,
50 | 		viewQuery:  q,
51 | 	}
52 | 	if !reflect.DeepEqual(*s.conf, want) {
53 | 		t.Errorf("createTableConf: got:\n%v\nwant:\n%v", *s.conf, want)
54 | 	}
55 | }
56 | 


--------------------------------------------------------------------------------
/option_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package cloud
16 | 
17 | import (
18 | 	"testing"
19 | 
20 | 	"github.com/fd/gcloud-golang/internal/opts"
21 | )
22 | 
23 | // Check that the slice passed into WithScopes is copied.
24 | func TestCopyScopes(t *testing.T) {
25 | 	o := &opts.DialOpt{}
26 | 
27 | 	scopes := []string{"a", "b"}
28 | 	WithScopes(scopes...).Resolve(o)
29 | 
30 | 	// Modify after using.
31 | 	scopes[1] = "c"
32 | 
33 | 	if o.Scopes[0] != "a" || o.Scopes[1] != "b" {
34 | 		t.Errorf("want ['a', 'b'], got %+v", o.Scopes)
35 | 	}
36 | }
37 | 
38 | // Check that resolution of WithScopes uses the most recent.
39 | // That is, it doesn't append scopes or ignore subsequent calls.
40 | func TestScopesOverride(t *testing.T) {
41 | 	o := &opts.DialOpt{}
42 | 
43 | 	WithScopes("a").Resolve(o)
44 | 	WithScopes("b").Resolve(o)
45 | 
46 | 	if want, got := 1, len(o.Scopes); want != got {
47 | 		t.Errorf("want %d scope, got %d", want, got)
48 | 	}
49 | 
50 | 	if want, got := "b", o.Scopes[0]; want != got {
51 | 		t.Errorf("want %s scope, got %s", want, got)
52 | 	}
53 | }
54 | 


--------------------------------------------------------------------------------
/storage/testdata/dummy_rsa:
--------------------------------------------------------------------------------
 1 | -----BEGIN RSA PRIVATE KEY-----
 2 | MIIEpAIBAAKCAQEAx4fm7dngEmOULNmAs1IGZ9Apfzh+BkaQ1dzkmbUgpcoghucE
 3 | DZRnAGd2aPyB6skGMXUytWQvNYav0WTR00wFtX1ohWTfv68HGXJ8QXCpyoSKSSFY
 4 | fuP9X36wBSkSX9J5DVgiuzD5VBdzUISSmapjKm+DcbRALjz6OUIPEWi1Tjl6p5RK
 5 | 1w41qdbmt7E5/kGhKLDuT7+M83g4VWhgIvaAXtnhklDAggilPPa8ZJ1IFe31lNlr
 6 | k4DRk38nc6sEutdf3RL7QoH7FBusI7uXV03DC6dwN1kP4GE7bjJhcRb/7jYt7CQ9
 7 | /E9Exz3c0yAp0yrTg0Fwh+qxfH9dKwN52S7SBwIDAQABAoIBAQCaCs26K07WY5Jt
 8 | 3a2Cw3y2gPrIgTCqX6hJs7O5ByEhXZ8nBwsWANBUe4vrGaajQHdLj5OKfsIDrOvn
 9 | 2NI1MqflqeAbu/kR32q3tq8/Rl+PPiwUsW3E6Pcf1orGMSNCXxeducF2iySySzh3
10 | nSIhCG5uwJDWI7a4+9KiieFgK1pt/Iv30q1SQS8IEntTfXYwANQrfKUVMmVF9aIK
11 | 6/WZE2yd5+q3wVVIJ6jsmTzoDCX6QQkkJICIYwCkglmVy5AeTckOVwcXL0jqw5Kf
12 | 5/soZJQwLEyBoQq7Kbpa26QHq+CJONetPP8Ssy8MJJXBT+u/bSseMb3Zsr5cr43e
13 | DJOhwsThAoGBAPY6rPKl2NT/K7XfRCGm1sbWjUQyDShscwuWJ5+kD0yudnT/ZEJ1
14 | M3+KS/iOOAoHDdEDi9crRvMl0UfNa8MAcDKHflzxg2jg/QI+fTBjPP5GOX0lkZ9g
15 | z6VePoVoQw2gpPFVNPPTxKfk27tEzbaffvOLGBEih0Kb7HTINkW8rIlzAoGBAM9y
16 | 1yr+jvfS1cGFtNU+Gotoihw2eMKtIqR03Yn3n0PK1nVCDKqwdUqCypz4+ml6cxRK
17 | J8+Pfdh7D+ZJd4LEG6Y4QRDLuv5OA700tUoSHxMSNn3q9As4+T3MUyYxWKvTeu3U
18 | f2NWP9ePU0lV8ttk7YlpVRaPQmc1qwooBA/z/8AdAoGAW9x0HWqmRICWTBnpjyxx
19 | QGlW9rQ9mHEtUotIaRSJ6K/F3cxSGUEkX1a3FRnp6kPLcckC6NlqdNgNBd6rb2rA
20 | cPl/uSkZP42Als+9YMoFPU/xrrDPbUhu72EDrj3Bllnyb168jKLa4VBOccUvggxr
21 | Dm08I1hgYgdN5huzs7y6GeUCgYEAj+AZJSOJ6o1aXS6rfV3mMRve9bQ9yt8jcKXw
22 | 5HhOCEmMtaSKfnOF1Ziih34Sxsb7O2428DiX0mV/YHtBnPsAJidL0SdLWIapBzeg
23 | KHArByIRkwE6IvJvwpGMdaex1PIGhx5i/3VZL9qiq/ElT05PhIb+UXgoWMabCp84
24 | OgxDK20CgYAeaFo8BdQ7FmVX2+EEejF+8xSge6WVLtkaon8bqcn6P0O8lLypoOhd
25 | mJAYH8WU+UAy9pecUnDZj14LAGNVmYcse8HFX71MoshnvCTFEPVo4rZxIAGwMpeJ
26 | 5jgQ3slYLpqrGlcbLgUXBUgzEO684Wk/UV9DFPlHALVqCfXQ9dpJPg==
27 | -----END RSA PRIVATE KEY-----
28 | 


--------------------------------------------------------------------------------
/bigtable/admin_test.go:
--------------------------------------------------------------------------------
 1 | package bigtable
 2 | 
 3 | import (
 4 | 	"reflect"
 5 | 	"sort"
 6 | 	"testing"
 7 | 	"time"
 8 | 
 9 | 	"golang.org/x/net/context"
10 | 	"github.com/fd/gcloud-golang"
11 | 	"github.com/fd/gcloud-golang/bigtable/bttest"
12 | 	"google.golang.org/grpc"
13 | )
14 | 
15 | func TestAdminIntegration(t *testing.T) {
16 | 	srv, err := bttest.NewServer()
17 | 	if err != nil {
18 | 		t.Fatal(err)
19 | 	}
20 | 	defer srv.Close()
21 | 	t.Logf("bttest.Server running on %s", srv.Addr)
22 | 
23 | 	ctx, _ := context.WithTimeout(context.Background(), 2*time.Second)
24 | 
25 | 	conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
26 | 	if err != nil {
27 | 		t.Fatalf("grpc.Dial: %v", err)
28 | 	}
29 | 
30 | 	adminClient, err := NewAdminClient(ctx, "proj", "zone", "cluster", cloud.WithBaseGRPC(conn))
31 | 	if err != nil {
32 | 		t.Fatalf("NewAdminClient: %v", err)
33 | 	}
34 | 	defer adminClient.Close()
35 | 
36 | 	list := func() []string {
37 | 		tbls, err := adminClient.Tables(ctx)
38 | 		if err != nil {
39 | 			t.Fatalf("Fetching list of tables: %v", err)
40 | 		}
41 | 		sort.Strings(tbls)
42 | 		return tbls
43 | 	}
44 | 	if err := adminClient.CreateTable(ctx, "mytable"); err != nil {
45 | 		t.Fatalf("Creating table: %v", err)
46 | 	}
47 | 	if err := adminClient.CreateTable(ctx, "myothertable"); err != nil {
48 | 		t.Fatalf("Creating table: %v", err)
49 | 	}
50 | 	if got, want := list(), []string{"myothertable", "mytable"}; !reflect.DeepEqual(got, want) {
51 | 		t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
52 | 	}
53 | 	if err := adminClient.DeleteTable(ctx, "myothertable"); err != nil {
54 | 		t.Fatalf("Deleting table: %v", err)
55 | 	}
56 | 	if got, want := list(), []string{"mytable"}; !reflect.DeepEqual(got, want) {
57 | 		t.Errorf("adminClient.Tables returned %#v, want %#v", got, want)
58 | 	}
59 | }
60 | 


--------------------------------------------------------------------------------
/bigquery/extract_op.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import (
18 | 	"fmt"
19 | 
20 | 	"golang.org/x/net/context"
21 | 	bq "google.golang.org/api/bigquery/v2"
22 | )
23 | 
24 | type extractOption interface {
25 | 	customizeExtract(conf *bq.JobConfigurationExtract, projectID string)
26 | }
27 | 
28 | // DisableHeader returns an Option that disables the printing of a header row in exported data.
29 | func DisableHeader() Option { return disableHeader{} }
30 | 
31 | type disableHeader struct{}
32 | 
33 | func (opt disableHeader) implementsOption() {}
34 | 
35 | func (opt disableHeader) customizeExtract(conf *bq.JobConfigurationExtract, projectID string) {
36 | 	f := false
37 | 	conf.PrintHeader = &f
38 | }
39 | 
40 | func (c *Client) extract(ctx context.Context, dst *GCSReference, src *Table, options []Option) (*Job, error) {
41 | 	job, options := initJobProto(c.projectID, options)
42 | 	payload := &bq.JobConfigurationExtract{}
43 | 
44 | 	dst.customizeExtractDst(payload, c.projectID)
45 | 	src.customizeExtractSrc(payload, c.projectID)
46 | 
47 | 	for _, opt := range options {
48 | 		o, ok := opt.(extractOption)
49 | 		if !ok {
50 | 			return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src)
51 | 		}
52 | 		o.customizeExtract(payload, c.projectID)
53 | 	}
54 | 
55 | 	job.Configuration = &bq.JobConfiguration{
56 | 		Extract: payload,
57 | 	}
58 | 	return c.service.insertJob(ctx, job, c.projectID)
59 | }
60 | 


--------------------------------------------------------------------------------
/cloud.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | // Package cloud contains Google Cloud Platform APIs related types
16 | // and common functions.
17 | package cloud // import "github.com/fd/gcloud-golang"
18 | 
19 | import (
20 | 	"net/http"
21 | 
22 | 	"github.com/fd/gcloud-golang/internal"
23 | 	"golang.org/x/net/context"
24 | )
25 | 
26 | // NewContext returns a new context that uses the provided http.Client.
27 | // Provided http.Client is responsible to authorize and authenticate
28 | // the requests made to the Google Cloud APIs.
29 | // It mutates the client's original Transport to append the cloud
30 | // package's user-agent to the outgoing requests.
31 | // You can obtain the project ID from the Google Developers Console,
32 | // https://console.developers.google.com.
33 | func NewContext(projID string, c *http.Client) context.Context {
34 | 	if c == nil {
35 | 		panic("invalid nil *http.Client passed to NewContext")
36 | 	}
37 | 	return WithContext(context.Background(), projID, c)
38 | }
39 | 
40 | // WithContext returns a new context in a similar way NewContext does,
41 | // but initiates the new context with the specified parent.
42 | func WithContext(parent context.Context, projID string, c *http.Client) context.Context {
43 | 	// TODO(bradfitz): delete internal.Transport. It's too wrappy for what it does.
44 | 	// Do User-Agent some other way.
45 | 	if _, ok := c.Transport.(*internal.Transport); !ok {
46 | 		c.Transport = &internal.Transport{Base: c.Transport}
47 | 	}
48 | 	return internal.WithContext(parent, projID, c)
49 | }
50 | 


--------------------------------------------------------------------------------
/internal/transport/proto.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package transport
16 | 
17 | import (
18 | 	"bytes"
19 | 	"io/ioutil"
20 | 	"net/http"
21 | 
22 | 	"github.com/golang/protobuf/proto"
23 | 	"golang.org/x/net/context"
24 | )
25 | 
26 | type ProtoClient struct {
27 | 	client    *http.Client
28 | 	endpoint  string
29 | 	userAgent string
30 | }
31 | 
32 | func (c *ProtoClient) Call(ctx context.Context, method string, req, resp proto.Message) error {
33 | 	payload, err := proto.Marshal(req)
34 | 	if err != nil {
35 | 		return err
36 | 	}
37 | 
38 | 	httpReq, err := http.NewRequest("POST", c.endpoint+method, bytes.NewReader(payload))
39 | 	if err != nil {
40 | 		return err
41 | 	}
42 | 	httpReq.Header.Set("Content-Type", "application/x-protobuf")
43 | 	if ua := c.userAgent; ua != "" {
44 | 		httpReq.Header.Set("User-Agent", ua)
45 | 	}
46 | 
47 | 	errc := make(chan error, 1)
48 | 	cancel := makeReqCancel(httpReq)
49 | 
50 | 	go func() {
51 | 		r, err := c.client.Do(httpReq)
52 | 		if err != nil {
53 | 			errc <- err
54 | 			return
55 | 		}
56 | 		defer r.Body.Close()
57 | 
58 | 		body, err := ioutil.ReadAll(r.Body)
59 | 		if r.StatusCode != http.StatusOK {
60 | 			err = &ErrHTTP{
61 | 				StatusCode: r.StatusCode,
62 | 				Body:       body,
63 | 				err:        err,
64 | 			}
65 | 		}
66 | 		if err != nil {
67 | 			errc <- err
68 | 			return
69 | 		}
70 | 		errc <- proto.Unmarshal(body, resp)
71 | 	}()
72 | 
73 | 	select {
74 | 	case <-ctx.Done():
75 | 		cancel(c.client.Transport) // Cancel the HTTP request.
76 | 		return ctx.Err()
77 | 	case err := <-errc:
78 | 		return err
79 | 	}
80 | }
81 | 


--------------------------------------------------------------------------------
/bigtable/internal/empty/empty.pb.go:
--------------------------------------------------------------------------------
 1 | // Code generated by protoc-gen-go.
 2 | // source: github.com/fd/gcloud-golang/bigtable/internal/empty/empty.proto
 3 | // DO NOT EDIT!
 4 | 
 5 | /*
 6 | Package google_protobuf is a generated protocol buffer package.
 7 | 
 8 | It is generated from these files:
 9 | 	github.com/fd/gcloud-golang/bigtable/internal/empty/empty.proto
10 | 
11 | It has these top-level messages:
12 | 	Empty
13 | */
14 | package google_protobuf
15 | 
16 | import proto "github.com/golang/protobuf/proto"
17 | import fmt "fmt"
18 | import math "math"
19 | 
20 | // Reference imports to suppress errors if they are not otherwise used.
21 | var _ = proto.Marshal
22 | var _ = fmt.Errorf
23 | var _ = math.Inf
24 | 
25 | // A generic empty message that you can re-use to avoid defining duplicated
26 | // empty messages in your APIs. A typical example is to use it as the request
27 | // or the response type of an API method. For instance:
28 | //
29 | //     service Foo {
30 | //       rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);
31 | //     }
32 | //
33 | type Empty struct {
34 | }
35 | 
36 | func (m *Empty) Reset()                    { *m = Empty{} }
37 | func (m *Empty) String() string            { return proto.CompactTextString(m) }
38 | func (*Empty) ProtoMessage()               {}
39 | func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
40 | 
41 | func init() {
42 | 	proto.RegisterType((*Empty)(nil), "google.protobuf.Empty")
43 | }
44 | 
45 | var fileDescriptor0 = []byte{
46 | 	// 120 bytes of a gzipped FileDescriptorProto
47 | 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xb2, 0x4e, 0xcf, 0xcf, 0x4f,
48 | 	0xcf, 0x49, 0xd5, 0x4b, 0xcf, 0xcf, 0x49, 0xcc, 0x4b, 0xd7, 0xcb, 0x2f, 0x4a, 0xd7, 0x4f, 0xce,
49 | 	0xc9, 0x2f, 0x4d, 0xd1, 0x4f, 0xca, 0x4c, 0x2f, 0x49, 0x4c, 0xca, 0x49, 0xd5, 0xcf, 0xcc, 0x2b,
50 | 	0x49, 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x4f, 0xcd, 0x2d, 0x28, 0xa9, 0x84, 0x90, 0x7a, 0x05, 0x45,
51 | 	0xf9, 0x25, 0xf9, 0x42, 0xfc, 0x50, 0xcd, 0x60, 0x5e, 0x52, 0x69, 0x9a, 0x12, 0x3b, 0x17, 0xab,
52 | 	0x2b, 0x48, 0xde, 0x49, 0x99, 0x4b, 0x38, 0x39, 0x3f, 0x57, 0x0f, 0x4d, 0xde, 0x89, 0x0b, 0x2c,
53 | 	0x1b, 0x00, 0xe2, 0x06, 0x30, 0x26, 0xb1, 0x81, 0xc5, 0x8d, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff,
54 | 	0xa0, 0x50, 0xb8, 0x83, 0x84, 0x00, 0x00, 0x00,
55 | }
56 | 


--------------------------------------------------------------------------------
/internal/testutil/context.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | // Package testutil contains helper functions for writing tests.
16 | package testutil
17 | 
18 | import (
19 | 	"io/ioutil"
20 | 	"log"
21 | 	"net/http"
22 | 	"os"
23 | 
24 | 	"golang.org/x/net/context"
25 | 	"golang.org/x/oauth2"
26 | 	"golang.org/x/oauth2/google"
27 | 	"github.com/fd/gcloud-golang"
28 | )
29 | 
30 | const (
31 | 	envProjID     = "GCLOUD_TESTS_GOLANG_PROJECT_ID"
32 | 	envPrivateKey = "GCLOUD_TESTS_GOLANG_KEY"
33 | )
34 | 
35 | func ProjID() string {
36 | 	projID := os.Getenv(envProjID)
37 | 	if projID == "" {
38 | 		log.Fatal(envProjID + " must be set. See CONTRIBUTING.md for details.")
39 | 	}
40 | 	return projID
41 | }
42 | 
43 | func TokenSource(ctx context.Context, scopes ...string) oauth2.TokenSource {
44 | 	key := os.Getenv(envPrivateKey)
45 | 	if key == "" {
46 | 		log.Fatal(envPrivateKey + " must be set. See CONTRIBUTING.md for details.")
47 | 	}
48 | 	jsonKey, err := ioutil.ReadFile(key)
49 | 	if err != nil {
50 | 		log.Fatalf("Cannot read the JSON key file, err: %v", err)
51 | 	}
52 | 	conf, err := google.JWTConfigFromJSON(jsonKey, scopes...)
53 | 	if err != nil {
54 | 		log.Fatalf("google.JWTConfigFromJSON: %v", err)
55 | 	}
56 | 	return conf.TokenSource(ctx)
57 | }
58 | 
59 | // TODO(djd): Delete this function when it's no longer used.
60 | func Context(scopes ...string) context.Context {
61 | 	ctx := oauth2.NoContext
62 | 	ts := TokenSource(ctx, scopes...)
63 | 	return cloud.NewContext(ProjID(), oauth2.NewClient(ctx, ts))
64 | }
65 | 
66 | // TODO(djd): Delete this function when it's no longer used.
67 | func NoAuthContext() context.Context {
68 | 	return cloud.NewContext(ProjID(), &http.Client{Transport: http.DefaultTransport})
69 | }
70 | 


--------------------------------------------------------------------------------
/storage/testdata/dummy_pem:
--------------------------------------------------------------------------------
 1 | Bag Attributes
 2 |     friendlyName: privatekey
 3 |     localKeyID: 54 69 6D 65 20 31 34 31 36 38 35 32 30 30 34 37 37 32 
 4 | Key Attributes: <No Attributes>
 5 | -----BEGIN RSA PRIVATE KEY-----
 6 | MIICXQIBAAKBgQCtCWMoJ2Bok2QoGFyU7A6IlGprO9QfUTT0jNrLkIbM5OWNIuDx
 7 | 64+PEaTS5g5m+2Hz/lmd5jJKanAH4dY9LZzsaYAPq1K17Gcmg1hEisYeKsgOcjYY
 8 | kwRkV+natCTsC+tfWmS0voRh0jA1rI1J4MikceoHtgWdEuoHrrptRVpWKwIDAQAB
 9 | AoGAKp3uQvx3vSnX+BwP6Um+RpsvHpwMoW3xue1bEdnVqW8SrlERz+NxZw40ZxDs
10 | KSbuuBZD4iTI7BUM5JQVnNm4FQY1YrPlWZLyI73Bj8RKTXrPdJheM/0r7xjiIXbQ
11 | 7w4cUSM9rVugnI/rxF2kPIQTGYI+EG/6+P+k6VvgPmC0T/ECQQDUPskiS18WaY+i
12 | Koalbrb3GakaBoHrC1b4ln4CAv7fq7H4WvFvqi/2rxLhHYq31iwxYy8s7J7Sba1+
13 | 5vwJ2TxZAkEA0LVfs3Q2VWZ+cM3bv0aYTalMXg6wT+LoNvk9HnOb0zQYajF3qm4G
14 | ZFdfEqvOkje0zQ4fcihARKyda/VY84UGIwJBAIZa0FvjNmgrnn7bSKzEbxHwrnkJ
15 | EYjGfuGR8mY3mzvfpiM+/oLfSslvfhX+62cALq18yco4ZzlxsFgaxAU//NECQDcS
16 | NN94YcHlGqYPW9W7/gI4EwOaoqFhwV6II71+SfbP/0U+KlJZV+xwNZEKrqZcdqPI
17 | /zkzL8ovNha/laokRrsCQQCyoPHGcBWj+VFbNoyQnX4tghc6rOY7n4pmpgQvU825
18 | TAM9vnYtSkKK/V56kEDNBO5LwiRsir95IUNclqqMKR1C
19 | -----END RSA PRIVATE KEY-----
20 | Bag Attributes
21 |     friendlyName: privatekey
22 |     localKeyID: 54 69 6D 65 20 31 34 31 36 38 35 32 30 30 34 37 37 32 
23 | subject=/CN=1079432350659-nvog0vmn9s6pqr3kr4v2avbc7nkhoa11.apps.googleusercontent.com
24 | issuer=/CN=1079432350659-nvog0vmn9s6pqr3kr4v2avbc7nkhoa11.apps.googleusercontent.com
25 | -----BEGIN CERTIFICATE-----
26 | MIICXTCCAcagAwIBAgIIHxTMQUVJRZ0wDQYJKoZIhvcNAQEFBQAwVDFSMFAGA1UE
27 | AxNJMTA3OTQzMjM1MDY1OS1udm9nMHZtbjlzNnBxcjNrcjR2MmF2YmM3bmtob2Ex
28 | MS5hcHBzLmdvb2dsZXVzZXJjb250ZW50LmNvbTAeFw0xNDExMjQxODAwMDRaFw0y
29 | NDExMjExODAwMDRaMFQxUjBQBgNVBAMTSTEwNzk0MzIzNTA2NTktbnZvZzB2bW45
30 | czZwcXIza3I0djJhdmJjN25raG9hMTEuYXBwcy5nb29nbGV1c2VyY29udGVudC5j
31 | b20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK0JYygnYGiTZCgYXJTsDoiU
32 | ams71B9RNPSM2suQhszk5Y0i4PHrj48RpNLmDmb7YfP+WZ3mMkpqcAfh1j0tnOxp
33 | gA+rUrXsZyaDWESKxh4qyA5yNhiTBGRX6dq0JOwL619aZLS+hGHSMDWsjUngyKRx
34 | 6ge2BZ0S6geuum1FWlYrAgMBAAGjODA2MAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/
35 | BAQDAgeAMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMA0GCSqGSIb3DQEBBQUAA4GB
36 | ACVvKkZkomHq3uffOQwdZ4VJYuxrvDGnZu/ExW9WngO2teEsjxABL41TNnRYHN5T
37 | lMC19poFA2tR/DySDLJ2XNs/hSvyQUL6HHCncVdR4Srpie88j48peY1MZSMP51Jv
38 | qagbbP5K5DSEu02/zZaV0kaCvLEN0KAtj/noDuOOnQU2
39 | -----END CERTIFICATE-----
40 | 


--------------------------------------------------------------------------------
/bigquery/read_op.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import "golang.org/x/net/context"
18 | 
19 | // RecordsPerRequest returns a ReadOption that sets the number of records to fetch per request when streaming data from BigQuery.
20 | func RecordsPerRequest(n int64) ReadOption { return recordsPerRequest(n) }
21 | 
22 | type recordsPerRequest int64
23 | 
24 | func (opt recordsPerRequest) customizeRead(conf *pagingConf) {
25 | 	conf.recordsPerRequest = int64(opt)
26 | 	conf.setRecordsPerRequest = true
27 | }
28 | 
29 | // StartIndex returns a ReadOption that sets the zero-based index of the row to start reading from.
30 | func StartIndex(i uint64) ReadOption { return startIndex(i) }
31 | 
32 | type startIndex uint64
33 | 
34 | func (opt startIndex) customizeRead(conf *pagingConf) {
35 | 	conf.startIndex = uint64(opt)
36 | }
37 | 
38 | func (conf *readTableConf) fetch(ctx context.Context, c *Client, token string) (*readDataResult, error) {
39 | 	return c.service.readTabledata(ctx, conf, token)
40 | }
41 | 
42 | func (c *Client) readTable(t *Table, options []ReadOption) (*Iterator, error) {
43 | 	conf := &readTableConf{}
44 | 	t.customizeReadSrc(conf)
45 | 
46 | 	for _, o := range options {
47 | 		o.customizeRead(&conf.paging)
48 | 	}
49 | 
50 | 	return newIterator(c, conf), nil
51 | }
52 | 
53 | func (conf *readQueryConf) fetch(ctx context.Context, c *Client, token string) (*readDataResult, error) {
54 | 	return c.service.readQuery(ctx, conf, token)
55 | }
56 | 
57 | func (c *Client) readQueryResults(job *Job, options []ReadOption) (*Iterator, error) {
58 | 	conf := &readQueryConf{}
59 | 	if err := job.customizeReadQuery(conf); err != nil {
60 | 		return nil, err
61 | 	}
62 | 
63 | 	for _, o := range options {
64 | 		o.customizeRead(&conf.paging)
65 | 	}
66 | 
67 | 	return newIterator(c, conf), nil
68 | }
69 | 


--------------------------------------------------------------------------------
/pubsub/example_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package pubsub_test
16 | 
17 | import (
18 | 	"io/ioutil"
19 | 	"log"
20 | 	"testing"
21 | 
22 | 	"golang.org/x/net/context"
23 | 	"golang.org/x/oauth2"
24 | 	"golang.org/x/oauth2/google"
25 | 	"github.com/fd/gcloud-golang"
26 | 	"github.com/fd/gcloud-golang/pubsub"
27 | )
28 | 
29 | // TODO(jbd): Remove after Go 1.4.
30 | // Related to https://codereview.appspot.com/107320046
31 | func TestA(t *testing.T) {}
32 | 
33 | func Example_auth() context.Context {
34 | 	// Initialize an authorized context with Google Developers Console
35 | 	// JSON key. Read the google package examples to learn more about
36 | 	// different authorization flows you can use.
37 | 	// http://godoc.org/golang.org/x/oauth2/google
38 | 	jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
39 | 	if err != nil {
40 | 		log.Fatal(err)
41 | 	}
42 | 	conf, err := google.JWTConfigFromJSON(
43 | 		jsonKey,
44 | 		pubsub.ScopeCloudPlatform,
45 | 		pubsub.ScopePubSub,
46 | 	)
47 | 	if err != nil {
48 | 		log.Fatal(err)
49 | 	}
50 | 	ctx := cloud.NewContext("project-id", conf.Client(oauth2.NoContext))
51 | 	// See the other samples to learn how to use the context.
52 | 	return ctx
53 | }
54 | 
55 | func ExamplePublish() {
56 | 	ctx := Example_auth()
57 | 
58 | 	msgIDs, err := pubsub.Publish(ctx, "topic1", &pubsub.Message{
59 | 		Data: []byte("hello world"),
60 | 	})
61 | 	if err != nil {
62 | 		log.Fatal(err)
63 | 	}
64 | 	log.Printf("Published a message with a message id: %s\n", msgIDs[0])
65 | }
66 | 
67 | func ExamplePull() {
68 | 	ctx := Example_auth()
69 | 
70 | 	// E.g. c.CreateSub("sub1", "topic1", time.Duration(0), "")
71 | 	msgs, err := pubsub.Pull(ctx, "sub1", 1)
72 | 	if err != nil {
73 | 		log.Fatal(err)
74 | 	}
75 | 	log.Printf("New message arrived: %v\n", msgs[0])
76 | 	if err := pubsub.Ack(ctx, "sub1", msgs[0].AckID); err != nil {
77 | 		log.Fatal(err)
78 | 	}
79 | 	log.Println("Acknowledged message")
80 | }
81 | 


--------------------------------------------------------------------------------
/bigtable/cmd/loadtest/stats.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"fmt"
 6 | 	"math"
 7 | 	"sort"
 8 | 	"text/tabwriter"
 9 | 	"time"
10 | )
11 | 
12 | type byDuration []time.Duration
13 | 
14 | func (data byDuration) Len() int           { return len(data) }
15 | func (data byDuration) Swap(i, j int)      { data[i], data[j] = data[j], data[i] }
16 | func (data byDuration) Less(i, j int) bool { return data[i] < data[j] }
17 | 
18 | // quantile returns a value representing the kth of q quantiles.
19 | // May alter the order of data.
20 | func quantile(data []time.Duration, k, q int) (quantile time.Duration, ok bool) {
21 | 	if len(data) < 1 {
22 | 		return 0, false
23 | 	}
24 | 	if k > q {
25 | 		return 0, false
26 | 	}
27 | 	if k < 0 || q < 1 {
28 | 		return 0, false
29 | 	}
30 | 
31 | 	sort.Sort(byDuration(data))
32 | 
33 | 	if k == 0 {
34 | 		return data[0], true
35 | 	}
36 | 	if k == q {
37 | 		return data[len(data)-1], true
38 | 	}
39 | 
40 | 	bucketSize := float64(len(data)-1) / float64(q)
41 | 	i := float64(k) * bucketSize
42 | 
43 | 	lower := int(math.Trunc(i))
44 | 	var upper int
45 | 	if i > float64(lower) && lower+1 < len(data) {
46 | 		// If the quantile lies between two elements
47 | 		upper = lower + 1
48 | 	} else {
49 | 		upper = lower
50 | 	}
51 | 	weightUpper := i - float64(lower)
52 | 	weightLower := 1 - weightUpper
53 | 	return time.Duration(weightLower*float64(data[lower]) + weightUpper*float64(data[upper])), true
54 | }
55 | 
56 | type aggregate struct {
57 | 	min, median, max time.Duration
58 | 	p95, p99         time.Duration // percentiles
59 | }
60 | 
61 | // newAggregate constructs an aggregate from latencies. Returns nil if latencies does not contain aggregateable data.
62 | func newAggregate(latencies []time.Duration) *aggregate {
63 | 	var agg aggregate
64 | 
65 | 	if len(latencies) == 0 {
66 | 		return nil
67 | 	}
68 | 	var ok bool
69 | 	if agg.min, ok = quantile(latencies, 0, 2); !ok {
70 | 		return nil
71 | 	}
72 | 	if agg.median, ok = quantile(latencies, 1, 2); !ok {
73 | 		return nil
74 | 	}
75 | 	if agg.max, ok = quantile(latencies, 2, 2); !ok {
76 | 		return nil
77 | 	}
78 | 	if agg.p95, ok = quantile(latencies, 95, 100); !ok {
79 | 		return nil
80 | 	}
81 | 	if agg.p99, ok = quantile(latencies, 99, 100); !ok {
82 | 		return nil
83 | 	}
84 | 	return &agg
85 | }
86 | 
87 | func (agg *aggregate) String() string {
88 | 	if agg == nil {
89 | 		return "no data"
90 | 	}
91 | 	var buf bytes.Buffer
92 | 	tw := tabwriter.NewWriter(&buf, 0, 0, 1, ' ', 0) // one-space padding
93 | 	fmt.Fprintf(tw, "min:\t%v\nmedian:\t%v\nmax:\t%v\n95th percentile:\t%v\n99th percentile:\t%v\n",
94 | 		agg.min, agg.median, agg.max, agg.p95, agg.p99)
95 | 	tw.Flush()
96 | 	return buf.String()
97 | }
98 | 


--------------------------------------------------------------------------------
/bigquery/extract_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import (
18 | 	"reflect"
19 | 	"testing"
20 | 
21 | 	"golang.org/x/net/context"
22 | 
23 | 	bq "google.golang.org/api/bigquery/v2"
24 | )
25 | 
26 | func defaultExtractJob() *bq.Job {
27 | 	return &bq.Job{
28 | 		Configuration: &bq.JobConfiguration{
29 | 			Extract: &bq.JobConfigurationExtract{
30 | 				SourceTable: &bq.TableReference{
31 | 					ProjectId: "project-id",
32 | 					DatasetId: "dataset-id",
33 | 					TableId:   "table-id",
34 | 				},
35 | 				DestinationUris: []string{"uri"},
36 | 			},
37 | 		},
38 | 	}
39 | }
40 | 
41 | func TestExtract(t *testing.T) {
42 | 	testCases := []struct {
43 | 		dst     *GCSReference
44 | 		src     *Table
45 | 		options []Option
46 | 		want    *bq.Job
47 | 	}{
48 | 		{
49 | 			dst:  defaultGCS,
50 | 			src:  defaultTable,
51 | 			want: defaultExtractJob(),
52 | 		},
53 | 		{
54 | 			dst: defaultGCS,
55 | 			src: defaultTable,
56 | 			options: []Option{
57 | 				DisableHeader(),
58 | 			},
59 | 			want: func() *bq.Job {
60 | 				j := defaultExtractJob()
61 | 				f := false
62 | 				j.Configuration.Extract.PrintHeader = &f
63 | 				return j
64 | 			}(),
65 | 		},
66 | 		{
67 | 			dst: &GCSReference{
68 | 				uris:              []string{"uri"},
69 | 				Compression:       Gzip,
70 | 				DestinationFormat: JSON,
71 | 				FieldDelimiter:    "\t",
72 | 			},
73 | 			src: defaultTable,
74 | 			want: func() *bq.Job {
75 | 				j := defaultExtractJob()
76 | 				j.Configuration.Extract.Compression = "GZIP"
77 | 				j.Configuration.Extract.DestinationFormat = "NEWLINE_DELIMITED_JSON"
78 | 				j.Configuration.Extract.FieldDelimiter = "\t"
79 | 				return j
80 | 			}(),
81 | 		},
82 | 	}
83 | 
84 | 	for _, tc := range testCases {
85 | 		s := &testService{}
86 | 		c := &Client{
87 | 			service: s,
88 | 		}
89 | 		if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil {
90 | 			t.Errorf("err calling extract: %v", err)
91 | 			continue
92 | 		}
93 | 		if !reflect.DeepEqual(s.Job, tc.want) {
94 | 			t.Errorf("extracting: got:\n%v\nwant:\n%v", s.Job, tc.want)
95 | 		}
96 | 	}
97 | }
98 | 


--------------------------------------------------------------------------------
/datastore/time_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2014 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package datastore
16 | 
17 | import (
18 | 	"testing"
19 | 	"time"
20 | )
21 | 
22 | func TestUnixMicro(t *testing.T) {
23 | 	// Test that all these time.Time values survive a round trip to unix micros.
24 | 	testCases := []time.Time{
25 | 		{},
26 | 		time.Date(2, 1, 1, 0, 0, 0, 0, time.UTC),
27 | 		time.Date(23, 1, 1, 0, 0, 0, 0, time.UTC),
28 | 		time.Date(234, 1, 1, 0, 0, 0, 0, time.UTC),
29 | 		time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC),
30 | 		time.Date(1600, 1, 1, 0, 0, 0, 0, time.UTC),
31 | 		time.Date(1700, 1, 1, 0, 0, 0, 0, time.UTC),
32 | 		time.Date(1800, 1, 1, 0, 0, 0, 0, time.UTC),
33 | 		time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC),
34 | 		time.Unix(-1e6, -1000),
35 | 		time.Unix(-1e6, 0),
36 | 		time.Unix(-1e6, +1000),
37 | 		time.Unix(-60, -1000),
38 | 		time.Unix(-60, 0),
39 | 		time.Unix(-60, +1000),
40 | 		time.Unix(-1, -1000),
41 | 		time.Unix(-1, 0),
42 | 		time.Unix(-1, +1000),
43 | 		time.Unix(0, -3000),
44 | 		time.Unix(0, -2000),
45 | 		time.Unix(0, -1000),
46 | 		time.Unix(0, 0),
47 | 		time.Unix(0, +1000),
48 | 		time.Unix(0, +2000),
49 | 		time.Unix(+60, -1000),
50 | 		time.Unix(+60, 0),
51 | 		time.Unix(+60, +1000),
52 | 		time.Unix(+1e6, -1000),
53 | 		time.Unix(+1e6, 0),
54 | 		time.Unix(+1e6, +1000),
55 | 		time.Date(1999, 12, 31, 23, 59, 59, 999000, time.UTC),
56 | 		time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC),
57 | 		time.Date(2006, 1, 2, 15, 4, 5, 678000, time.UTC),
58 | 		time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC),
59 | 		time.Date(3456, 1, 1, 0, 0, 0, 0, time.UTC),
60 | 	}
61 | 	for _, tc := range testCases {
62 | 		got := fromUnixMicro(toUnixMicro(tc))
63 | 		if !got.Equal(tc) {
64 | 			t.Errorf("got %q, want %q", got, tc)
65 | 		}
66 | 	}
67 | 
68 | 	// Test that a time.Time that isn't an integral number of microseconds
69 | 	// is not perfectly reconstructed after a round trip.
70 | 	t0 := time.Unix(0, 123)
71 | 	t1 := fromUnixMicro(toUnixMicro(t0))
72 | 	if t1.Nanosecond()%1000 != 0 || t0.Nanosecond()%1000 == 0 {
73 | 		t.Errorf("quantization to µs: got %q with %d ns, started with %d ns", t1, t1.Nanosecond(), t0.Nanosecond())
74 | 	}
75 | }
76 | 


--------------------------------------------------------------------------------
/bigtable/internal/service_proto/bigtable_service.proto:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2015, Google Inc.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | syntax = "proto3";
16 | 
17 | package google.bigtable.v1;
18 | 
19 | import "google.golang.org/cloud/bigtable/internal/data_proto/bigtable_data.proto";
20 | import "google.golang.org/cloud/bigtable/internal/service_proto/bigtable_service_messages.proto";
21 | import "google.golang.org/cloud/bigtable/internal/empty/empty.proto";
22 | 
23 | option java_generic_services = true;
24 | option java_multiple_files = true;
25 | option java_outer_classname = "BigtableServicesProto";
26 | option java_package = "com.google.bigtable.v1";
27 | 
28 | 
29 | // Service for reading from and writing to existing Bigtables.
30 | service BigtableService {
31 |   // Streams back the contents of all requested rows, optionally applying
32 |   // the same Reader filter to each. Depending on their size, rows may be
33 |   // broken up across multiple responses, but atomicity of each row will still
34 |   // be preserved.
35 |   rpc ReadRows(ReadRowsRequest) returns (stream ReadRowsResponse) {
36 |   }
37 | 
38 |   // Returns a sample of row keys in the table. The returned row keys will
39 |   // delimit contiguous sections of the table of approximately equal size,
40 |   // which can be used to break up the data for distributed tasks like
41 |   // mapreduces.
42 |   rpc SampleRowKeys(SampleRowKeysRequest) returns (stream SampleRowKeysResponse) {
43 |   }
44 | 
45 |   // Mutates a row atomically. Cells already present in the row are left
46 |   // unchanged unless explicitly changed by 'mutation'.
47 |   rpc MutateRow(MutateRowRequest) returns (google.protobuf.Empty) {
48 |   }
49 | 
50 |   // Mutates a row atomically based on the output of a predicate Reader filter.
51 |   rpc CheckAndMutateRow(CheckAndMutateRowRequest) returns (CheckAndMutateRowResponse) {
52 |   }
53 | 
54 |   // Modifies a row atomically, reading the latest existing timestamp/value from
55 |   // the specified columns and writing a new value at
56 |   // max(existing timestamp, current server time) based on pre-defined
57 |   // read/modify/write rules. Returns the new contents of all modified cells.
58 |   rpc ReadModifyWriteRow(ReadModifyWriteRowRequest) returns (Row) {
59 |   }
60 | }
61 | 


--------------------------------------------------------------------------------
/bigtable/internal/regen.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash -e
 2 | #
 3 | # This script rebuilds the generated code for the protocol buffers.
 4 | # To run this you will need protoc and goprotobuf installed;
 5 | # see https://github.com/golang/protobuf for instructions.
 6 | # You also need Go and Git installed.
 7 | 
 8 | PKG=google.golang.org/cloud/bigtable
 9 | UPSTREAM=https://github.com/GoogleCloudPlatform/cloud-bigtable-client
10 | UPSTREAM_SUBDIR=bigtable-protos/src/main/proto
11 | 
12 | function die() {
13 |   echo 1>&2 $*
14 |   exit 1
15 | }
16 | 
17 | # Sanity check that the right tools are accessible.
18 | for tool in go git protoc protoc-gen-go; do
19 |   q=$(which $tool) || die "didn't find $tool"
20 |   echo 1>&2 "$tool: $q"
21 | done
22 | 
23 | tmpdir=$(mktemp -d -t regen-cbt.XXXXXX)
24 | trap 'rm -rf $tmpdir' EXIT
25 | 
26 | echo -n 1>&2 "finding package dir... "
27 | pkgdir=$(go list -f '{{.Dir}}' $PKG)
28 | echo 1>&2 $pkgdir
29 | base=$(echo $pkgdir | sed "s,/$PKG\$,,")
30 | echo 1>&2 "base: $base"
31 | cd $base
32 | 
33 | echo 1>&2 "fetching latest protos... "
34 | git clone -q $UPSTREAM $tmpdir
35 | # Pass 1: build mapping from upstream filename to our filename.
36 | declare -A filename_map
37 | for f in $(cd $PKG && find internal -name '*.proto'); do
38 |   echo -n 1>&2 "looking for latest version of $f... "
39 |   up=$(cd $tmpdir/$UPSTREAM_SUBDIR && find * -name $(basename $f))
40 |   echo 1>&2 $up
41 |   if [ $(echo $up | wc -w) != "1" ]; then
42 |     die "not exactly one match"
43 |   fi
44 |   filename_map[$up]=$f
45 | done
46 | # Pass 2: build sed script for fixing imports.
47 | import_fixes=$tmpdir/fix_imports.sed
48 | for up in "${!filename_map[@]}"; do
49 |   f=${filename_map[$up]}
50 |   echo >>$import_fixes "s,\"$up\",\"$PKG/$f\","
51 | done
52 | cat $import_fixes | sed 's,^,### ,' 1>&2
53 | # Pass 3: copy files, making necessary adjustments.
54 | for up in "${!filename_map[@]}"; do
55 |   f=${filename_map[$up]}
56 |   cat $tmpdir/$UPSTREAM_SUBDIR/$up |
57 |     # Adjust proto imports.
58 |     sed -f $import_fixes |
59 |     # Drop the UndeleteCluster RPC method. It returns a google.longrunning.Operation.
60 |     sed '/^  rpc UndeleteCluster(/,/^  }$/d' |
61 |     # Drop annotations, long-running operations and timestamps. They aren't supported (yet).
62 |     sed '/"google\/longrunning\/operations.proto"/d' |
63 |     sed '/google.longrunning.Operation/d' |
64 |     sed '/"google\/protobuf\/timestamp.proto"/d' |
65 |     sed '/google\.protobuf\.Timestamp/d' |
66 |     sed '/"google\/api\/annotations.proto"/d' |
67 |     sed '/option.*google\.api\.http.*{.*};$/d' |
68 |     cat > $PKG/$f
69 | done
70 | 
71 | # Run protoc once per package.
72 | for dir in $(find $PKG/internal -name '*.proto' | xargs dirname | sort | uniq); do
73 |   echo 1>&2 "* $dir"
74 |   protoc --go_out=plugins=grpc:. $dir/*.proto
75 | done
76 | echo 1>&2 "All OK"
77 | 


--------------------------------------------------------------------------------
/bigtable/internal/table_service_proto/bigtable_table_service.proto:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2015, Google Inc.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | syntax = "proto3";
16 | 
17 | package google.bigtable.admin.table.v1;
18 | 
19 | import "google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto";
20 | import "google.golang.org/cloud/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto";
21 | import "google.golang.org/cloud/bigtable/internal/empty/empty.proto";
22 | 
23 | option java_multiple_files = true;
24 | option java_outer_classname = "BigtableTableServicesProto";
25 | option java_package = "com.google.bigtable.admin.table.v1";
26 | 
27 | 
28 | // Service for creating, configuring, and deleting Cloud Bigtable tables.
29 | // Provides access to the table schemas only, not the data stored within the tables.
30 | service BigtableTableService {
31 |   // Creates a new table, to be served from a specified cluster.
32 |   // The table can be created with a full set of initial column families,
33 |   // specified in the request.
34 |   rpc CreateTable(CreateTableRequest) returns (Table) {
35 |   }
36 | 
37 |   // Lists the names of all tables served from a specified cluster.
38 |   rpc ListTables(ListTablesRequest) returns (ListTablesResponse) {
39 |   }
40 | 
41 |   // Gets the schema of the specified table, including its column families.
42 |   rpc GetTable(GetTableRequest) returns (Table) {
43 |   }
44 | 
45 |   // Permanently deletes a specified table and all of its data.
46 |   rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) {
47 |   }
48 | 
49 |   // Changes the name of a specified table.
50 |   // Cannot be used to move tables between clusters, zones, or projects.
51 |   rpc RenameTable(RenameTableRequest) returns (google.protobuf.Empty) {
52 |   }
53 | 
54 |   // Creates a new column family within a specified table.
55 |   rpc CreateColumnFamily(CreateColumnFamilyRequest) returns (ColumnFamily) {
56 |   }
57 | 
58 |   // Changes the configuration of a specified column family.
59 |   rpc UpdateColumnFamily(ColumnFamily) returns (ColumnFamily) {
60 |   }
61 | 
62 |   // Permanently deletes a specified column family and all of its data.
63 |   rpc DeleteColumnFamily(DeleteColumnFamilyRequest) returns (google.protobuf.Empty) {
64 |   }
65 | }
66 | 


--------------------------------------------------------------------------------
/bigtable/cmd/cbt/cbtdoc.go:
--------------------------------------------------------------------------------
  1 | // DO NOT EDIT. THIS IS AUTOMATICALLY GENERATED.
  2 | // Run "go generate" to regenerate.
  3 | //go:generate go run cbt.go -o cbtdoc.go doc
  4 | 
  5 | /*
  6 | Cbt is a tool for doing basic interactions with Cloud Bigtable.
  7 | 
  8 | Usage:
  9 | 
 10 | 	cbt [options] command [arguments]
 11 | 
 12 | The commands are:
 13 | 
 14 | 	count                     Count rows in a table
 15 | 	createfamily              Create a column family
 16 | 	createtable               Create a table
 17 | 	deletefamily              Delete a column family
 18 | 	deleterow                 Delete a row
 19 | 	deletetable               Delete a table
 20 | 	doc                       Print documentation for cbt
 21 | 	help                      Print help text
 22 | 	listclusters              List clusters in a project
 23 | 	lookup                    Read from a single row
 24 | 	ls                        List tables and column families
 25 | 	read                      Read rows
 26 | 	set                       Set value of a cell
 27 | 
 28 | Use "cbt help <command>" for more information about a command.
 29 | 
 30 | 
 31 | Count rows in a table
 32 | 
 33 | Usage:
 34 | 	cbt count <table>
 35 | 
 36 | 
 37 | 
 38 | 
 39 | Create a column family
 40 | 
 41 | Usage:
 42 | 	cbt createfamily <table> <family>
 43 | 
 44 | 
 45 | 
 46 | 
 47 | Create a table
 48 | 
 49 | Usage:
 50 | 	cbt createtable <table>
 51 | 
 52 | 
 53 | 
 54 | 
 55 | Delete a column family
 56 | 
 57 | Usage:
 58 | 	cbt deletefamily <table> <family>
 59 | 
 60 | 
 61 | 
 62 | 
 63 | Delete a row
 64 | 
 65 | Usage:
 66 | 	cbt deleterow <table> <row>
 67 | 
 68 | 
 69 | 
 70 | 
 71 | Delete a table
 72 | 
 73 | Usage:
 74 | 	cbt deletetable <table>
 75 | 
 76 | 
 77 | 
 78 | 
 79 | Print documentation for cbt
 80 | 
 81 | Usage:
 82 | 	cbt doc
 83 | 
 84 | 
 85 | 
 86 | 
 87 | Print help text
 88 | 
 89 | Usage:
 90 | 	cbt help [command]
 91 | 
 92 | 
 93 | 
 94 | 
 95 | List clusters in a project
 96 | 
 97 | Usage:
 98 | 	cbt listclusters
 99 | 
100 | 
101 | 
102 | 
103 | Read from a single row
104 | 
105 | Usage:
106 | 	cbt lookup <table> <row>
107 | 
108 | 
109 | 
110 | 
111 | List tables and column families
112 | 
113 | Usage:
114 | 	cbt ls			List tables
115 | 	cbt ls <table>		List column families in <table>
116 | 
117 | 
118 | 
119 | 
120 | Read rows
121 | 
122 | Usage:
123 | 	cbt read <table> [start=<row>] [limit=<row>] [prefix=<prefix>]
124 | 	  start=<row>		Start reading at this row
125 | 	  limit=<row>		Stop reading before this row
126 | 	  prefix=<prefix>	Read rows with this prefix
127 | 
128 | 
129 | 
130 | 
131 | 
132 | Set value of a cell
133 | 
134 | Usage:
135 | 	cbt set <table> <row> family:column=val[@ts] ...
136 | 	  family:column=val[@ts] may be repeated to set multiple cells.
137 | 
138 | 	  ts is an optional integer timestamp.
139 | 	  If it cannot be parsed, the `@ts` part will be
140 | 	  interpreted as part of the value.
141 | 
142 | 
143 | 
144 | 
145 | */
146 | package main
147 | 


--------------------------------------------------------------------------------
/bigquery/copy_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"reflect"
 19 | 	"testing"
 20 | 
 21 | 	"golang.org/x/net/context"
 22 | 	bq "google.golang.org/api/bigquery/v2"
 23 | )
 24 | 
 25 | func defaultCopyJob() *bq.Job {
 26 | 	return &bq.Job{
 27 | 		Configuration: &bq.JobConfiguration{
 28 | 			Copy: &bq.JobConfigurationTableCopy{
 29 | 				DestinationTable: &bq.TableReference{
 30 | 					ProjectId: "d-project-id",
 31 | 					DatasetId: "d-dataset-id",
 32 | 					TableId:   "d-table-id",
 33 | 				},
 34 | 				SourceTables: []*bq.TableReference{
 35 | 					{
 36 | 						ProjectId: "s-project-id",
 37 | 						DatasetId: "s-dataset-id",
 38 | 						TableId:   "s-table-id",
 39 | 					},
 40 | 				},
 41 | 			},
 42 | 		},
 43 | 	}
 44 | }
 45 | 
 46 | func TestCopy(t *testing.T) {
 47 | 	testCases := []struct {
 48 | 		dst     *Table
 49 | 		src     Tables
 50 | 		options []Option
 51 | 		want    *bq.Job
 52 | 	}{
 53 | 		{
 54 | 			dst: &Table{
 55 | 				ProjectID: "d-project-id",
 56 | 				DatasetID: "d-dataset-id",
 57 | 				TableID:   "d-table-id",
 58 | 			},
 59 | 			src: Tables{
 60 | 				{
 61 | 					ProjectID: "s-project-id",
 62 | 					DatasetID: "s-dataset-id",
 63 | 					TableID:   "s-table-id",
 64 | 				},
 65 | 			},
 66 | 			want: defaultCopyJob(),
 67 | 		},
 68 | 		{
 69 | 			dst: &Table{
 70 | 				ProjectID: "d-project-id",
 71 | 				DatasetID: "d-dataset-id",
 72 | 				TableID:   "d-table-id",
 73 | 			},
 74 | 			src: Tables{
 75 | 				{
 76 | 					ProjectID: "s-project-id",
 77 | 					DatasetID: "s-dataset-id",
 78 | 					TableID:   "s-table-id",
 79 | 				},
 80 | 			},
 81 | 			options: []Option{CreateNever, WriteTruncate},
 82 | 			want: func() *bq.Job {
 83 | 				j := defaultCopyJob()
 84 | 				j.Configuration.Copy.CreateDisposition = "CREATE_NEVER"
 85 | 				j.Configuration.Copy.WriteDisposition = "WRITE_TRUNCATE"
 86 | 				return j
 87 | 			}(),
 88 | 		},
 89 | 	}
 90 | 
 91 | 	for _, tc := range testCases {
 92 | 		s := &testService{}
 93 | 		c := &Client{
 94 | 			service: s,
 95 | 		}
 96 | 		if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil {
 97 | 			t.Errorf("err calling cp: %v", err)
 98 | 			continue
 99 | 		}
100 | 		if !reflect.DeepEqual(s.Job, tc.want) {
101 | 			t.Errorf("copying: got:\n%v\nwant:\n%v", s.Job, tc.want)
102 | 		}
103 | 	}
104 | }
105 | 


--------------------------------------------------------------------------------
/bigtable/internal/duration_proto/duration.proto:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2015, Google Inc.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | syntax = "proto3";
16 | 
17 | package google.protobuf;
18 | 
19 | option java_generate_equals_and_hash = true;
20 | option java_multiple_files = true;
21 | option java_outer_classname = "DurationProto";
22 | option java_package = "com.google.protobuf";
23 | 
24 | 
25 | // A Duration represents a signed, fixed-length span of time represented
26 | // as a count of seconds and fractions of seconds at nanosecond
27 | // resolution. It is independent of any calendar and concepts like "day"
28 | // or "month". It is related to Timestamp in that the difference between
29 | // two Timestamp values is a Duration and it can be added or subtracted
30 | // from a Timestamp. Range is approximately +-10,000 years.
31 | //
32 | // Example 1: Compute Duration from two Timestamps in pseudo code.
33 | //
34 | //     Timestamp start = ...;
35 | //     Timestamp end = ...;
36 | //     Duration duration = ...;
37 | //
38 | //     duration.seconds = end.seconds - start.seconds;
39 | //     duration.nanos = end.nanos - start.nanos;
40 | //
41 | //     if (duration.seconds < 0 && duration.nanos > 0) {
42 | //       duration.seconds += 1;
43 | //       duration.nanos -= 1000000000;
44 | //     } else if (durations.seconds > 0 && duration.nanos < 0) {
45 | //       duration.seconds -= 1;
46 | //       duration.nanos += 1000000000;
47 | //     }
48 | //
49 | // Example 2: Compute Timestamp from Timestamp + Duration in pseudo code.
50 | //
51 | //     Timestamp start = ...;
52 | //     Duration duration = ...;
53 | //     Timestamp end = ...;
54 | //
55 | //     end.seconds = start.seconds + duration.seconds;
56 | //     end.nanos = start.nanos + duration.nanos;
57 | //
58 | //     if (end.nanos < 0) {
59 | //       end.seconds -= 1;
60 | //       end.nanos += 1000000000;
61 | //     } else if (end.nanos >= 1000000000) {
62 | //       end.seconds += 1;
63 | //       end.nanos -= 1000000000;
64 | //     }
65 | //
66 | message Duration {
67 |   // Signed seconds of the span of time. Must be from -315,576,000,000
68 |   // to +315,576,000,000 inclusive.
69 |   int64 seconds = 1;
70 | 
71 |   // Signed fractions of a second at nanosecond resolution of the span
72 |   // of time. Durations less than one second are represented with a 0
73 |   // `seconds` field and a positive or negative `nanos` field. For durations
74 |   // of one second or more, a non-zero value for the `nanos` field must be
75 |   // of the same sign as the `seconds` field. Must be from -999,999,999
76 |   // to +999,999,999 inclusive.
77 |   int32 nanos = 2;
78 | }
79 | 


--------------------------------------------------------------------------------
/bigquery/query_op.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2015 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //      http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | package bigquery
16 | 
17 | import (
18 | 	"fmt"
19 | 
20 | 	"golang.org/x/net/context"
21 | 	bq "google.golang.org/api/bigquery/v2"
22 | )
23 | 
24 | type queryOption interface {
25 | 	customizeQuery(conf *bq.JobConfigurationQuery, projectID string)
26 | }
27 | 
28 | // DisableQueryCache returns an Option that prevents results being fetched from the query cache.
29 | // If this Option is not used, results are fetched from the cache if they are available.
30 | // The query cache is a best-effort cache that is flushed whenever tables in the query are modified.
31 | // Cached results are only available when TableID is unspecified in the query's destination Table.
32 | // For more information, see https://cloud.google.com/bigquery/querying-data#querycaching
33 | func DisableQueryCache() Option { return disableQueryCache{} }
34 | 
35 | type disableQueryCache struct{}
36 | 
37 | func (opt disableQueryCache) implementsOption() {}
38 | 
39 | func (opt disableQueryCache) customizeQuery(conf *bq.JobConfigurationQuery, projectID string) {
40 | 	f := false
41 | 	conf.UseQueryCache = &f
42 | }
43 | 
44 | // JobPriority returns an Option that causes a query to be scheduled with the specified priority.
45 | // The default priority is InteractivePriority.
46 | // For more information, see https://cloud.google.com/bigquery/querying-data#batchqueries
47 | func JobPriority(priority string) Option { return jobPriority(priority) }
48 | 
49 | type jobPriority string
50 | 
51 | func (opt jobPriority) implementsOption() {}
52 | 
53 | func (opt jobPriority) customizeQuery(conf *bq.JobConfigurationQuery, projectID string) {
54 | 	conf.Priority = string(opt)
55 | }
56 | 
57 | const (
58 | 	BatchPriority       = "BATCH"
59 | 	InteractivePriority = "INTERACTIVE"
60 | )
61 | 
62 | // TODO(mcgreevy): support large results.
63 | // TODO(mcgreevy): support non-flattened results.
64 | 
65 | func (c *Client) query(ctx context.Context, dst *Table, src *Query, options []Option) (*Job, error) {
66 | 	job, options := initJobProto(c.projectID, options)
67 | 	payload := &bq.JobConfigurationQuery{}
68 | 
69 | 	dst.customizeQueryDst(payload, c.projectID)
70 | 	src.customizeQuerySrc(payload, c.projectID)
71 | 
72 | 	for _, opt := range options {
73 | 		o, ok := opt.(queryOption)
74 | 		if !ok {
75 | 			return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src)
76 | 		}
77 | 		o.customizeQuery(payload, c.projectID)
78 | 	}
79 | 
80 | 	job.Configuration = &bq.JobConfiguration{
81 | 		Query: payload,
82 | 	}
83 | 	j, err := c.service.insertJob(ctx, job, c.projectID)
84 | 	if err != nil {
85 | 		return nil, err
86 | 	}
87 | 	j.isQuery = true
88 | 	return j, nil
89 | }
90 | 


--------------------------------------------------------------------------------
/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto:
--------------------------------------------------------------------------------
 1 | // Copyright (c) 2015, Google Inc.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | 
15 | syntax = "proto3";
16 | 
17 | package google.bigtable.admin.cluster.v1;
18 | 
19 | 
20 | option java_multiple_files = true;
21 | option java_outer_classname = "BigtableClusterDataProto";
22 | option java_package = "com.google.bigtable.admin.cluster.v1";
23 | 
24 | 
25 | // A physical location in which a particular project can allocate Cloud BigTable
26 | // resources.
27 | message Zone {
28 |   // Possible states of a zone.
29 |   enum Status {
30 |     // The state of the zone is unknown or unspecified.
31 |     UNKNOWN = 0;
32 | 
33 |     // The zone is in a good state.
34 |     OK = 1;
35 | 
36 |     // The zone is down for planned maintenance.
37 |     PLANNED_MAINTENANCE = 2;
38 | 
39 |     // The zone is down for emergency or unplanned maintenance.
40 |     EMERGENCY_MAINENANCE = 3;
41 |   }
42 | 
43 |   // A permanent unique identifier for the zone.
44 |   // Values are of the form projects/<project>/zones/[a-z][-a-z0-9]*
45 |   string name = 1;
46 | 
47 |   // The name of this zone as it appears in UIs.
48 |   string display_name = 2;
49 | 
50 |   // The current state of this zone.
51 |   Status status = 3;
52 | }
53 | 
54 | // An isolated set of Cloud BigTable resources on which tables can be hosted.
55 | message Cluster {
56 |   // A permanent unique identifier for the cluster. For technical reasons, the
57 |   // zone in which the cluster resides is included here.
58 |   // Values are of the form
59 |   // projects/<project>/zones/<zone>/clusters/[a-z][-a-z0-9]*
60 |   string name = 1;
61 | 
62 |   // If this cluster has been deleted, the time at which its backup will
63 |   // be irrevocably destroyed. Omitted otherwise.
64 |   // This cannot be set directly, only through DeleteCluster.
65 | 
66 |   // The operation currently running on the cluster, if any.
67 |   // This cannot be set directly, only through CreateCluster, UpdateCluster,
68 |   // or UndeleteCluster. Calls to these methods will be rejected if
69 |   // "current_operation" is already set.
70 | 
71 |   // The descriptive name for this cluster as it appears in UIs.
72 |   // Must be unique per zone.
73 |   string display_name = 4;
74 | 
75 |   // The number of serve nodes allocated to this cluster.
76 |   int32 serve_nodes = 5;
77 | 
78 |   // What storage type to use for tables in this cluster. Only configurable at
79 |   // cluster creation time. If unspecified, STORAGE_SSD will be used.
80 |   StorageType default_storage_type = 8;
81 | }
82 | 
83 | enum StorageType {
84 |   // The storage type used is unspecified.
85 |   STORAGE_UNSPECIFIED = 0;
86 | 
87 |   // Data will be stored in SSD, providing low and consistent latencies.
88 |   STORAGE_SSD = 1;
89 | }
90 | 


--------------------------------------------------------------------------------
/bigquery/query_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"reflect"
 19 | 	"testing"
 20 | 
 21 | 	"golang.org/x/net/context"
 22 | 
 23 | 	bq "google.golang.org/api/bigquery/v2"
 24 | )
 25 | 
 26 | func defaultQueryJob() *bq.Job {
 27 | 	return &bq.Job{
 28 | 		Configuration: &bq.JobConfiguration{
 29 | 			Query: &bq.JobConfigurationQuery{
 30 | 				DestinationTable: &bq.TableReference{
 31 | 					ProjectId: "project-id",
 32 | 					DatasetId: "dataset-id",
 33 | 					TableId:   "table-id",
 34 | 				},
 35 | 				Query: "query string",
 36 | 				DefaultDataset: &bq.DatasetReference{
 37 | 					ProjectId: "def-project-id",
 38 | 					DatasetId: "def-dataset-id",
 39 | 				},
 40 | 			},
 41 | 		},
 42 | 	}
 43 | }
 44 | 
 45 | func TestQuery(t *testing.T) {
 46 | 	testCases := []struct {
 47 | 		dst     *Table
 48 | 		src     *Query
 49 | 		options []Option
 50 | 		want    *bq.Job
 51 | 	}{
 52 | 		{
 53 | 			dst:  defaultTable,
 54 | 			src:  defaultQuery,
 55 | 			want: defaultQueryJob(),
 56 | 		},
 57 | 		{
 58 | 			dst: defaultTable,
 59 | 			src: &Query{
 60 | 				Q: "query string",
 61 | 			},
 62 | 			want: func() *bq.Job {
 63 | 				j := defaultQueryJob()
 64 | 				j.Configuration.Query.DefaultDataset = nil
 65 | 				return j
 66 | 			}(),
 67 | 		},
 68 | 		{
 69 | 			dst: &Table{},
 70 | 			src: defaultQuery,
 71 | 			want: func() *bq.Job {
 72 | 				j := defaultQueryJob()
 73 | 				j.Configuration.Query.DestinationTable = nil
 74 | 				return j
 75 | 			}(),
 76 | 		},
 77 | 		{
 78 | 			dst: &Table{
 79 | 				ProjectID: "project-id",
 80 | 				DatasetID: "dataset-id",
 81 | 				TableID:   "table-id",
 82 | 			},
 83 | 			src:     defaultQuery,
 84 | 			options: []Option{CreateNever, WriteTruncate},
 85 | 			want: func() *bq.Job {
 86 | 				j := defaultQueryJob()
 87 | 				j.Configuration.Query.WriteDisposition = "WRITE_TRUNCATE"
 88 | 				j.Configuration.Query.CreateDisposition = "CREATE_NEVER"
 89 | 				return j
 90 | 			}(),
 91 | 		},
 92 | 		{
 93 | 			dst:     defaultTable,
 94 | 			src:     defaultQuery,
 95 | 			options: []Option{DisableQueryCache()},
 96 | 			want: func() *bq.Job {
 97 | 				j := defaultQueryJob()
 98 | 				f := false
 99 | 				j.Configuration.Query.UseQueryCache = &f
100 | 				return j
101 | 			}(),
102 | 		},
103 | 	}
104 | 
105 | 	for _, tc := range testCases {
106 | 		s := &testService{}
107 | 		c := &Client{
108 | 			service: s,
109 | 		}
110 | 		if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil {
111 | 			t.Errorf("err calling query: %v", err)
112 | 			continue
113 | 		}
114 | 		if !reflect.DeepEqual(s.Job, tc.want) {
115 | 			t.Errorf("querying: got:\n%v\nwant:\n%v", s.Job, tc.want)
116 | 		}
117 | 	}
118 | }
119 | 


--------------------------------------------------------------------------------
/examples/bigquery/query/main.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | // query is an example client of the bigquery client library.
 16 | // It submits a query and writes the result to a table.
 17 | package main
 18 | 
 19 | import (
 20 | 	"flag"
 21 | 	"fmt"
 22 | 	"log"
 23 | 	"os"
 24 | 	"time"
 25 | 
 26 | 	"golang.org/x/net/context"
 27 | 	"golang.org/x/oauth2/google"
 28 | 	"github.com/fd/gcloud-golang/bigquery"
 29 | )
 30 | 
 31 | var (
 32 | 	project = flag.String("project", "", "The ID of a Google Cloud Platform project")
 33 | 	dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
 34 | 	q       = flag.String("q", "", "The query string")
 35 | 	dest    = flag.String("dest", "", "The ID of the BigQuery table to write the result to.  If unset, an ephemeral table ID will be generated.")
 36 | 	pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
 37 | 	wait    = flag.Bool("wait", false, "Whether to wait for the query job to complete.")
 38 | )
 39 | 
 40 | func main() {
 41 | 	flag.Parse()
 42 | 
 43 | 	flagsOk := true
 44 | 	for _, f := range []string{"project", "dataset", "q"} {
 45 | 		if flag.Lookup(f).Value.String() == "" {
 46 | 			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
 47 | 			flagsOk = false
 48 | 		}
 49 | 	}
 50 | 	if !flagsOk {
 51 | 		os.Exit(1)
 52 | 	}
 53 | 
 54 | 	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
 55 | 	if err != nil {
 56 | 		log.Fatalf("Creating http client: %v", err)
 57 | 	}
 58 | 
 59 | 	client, err := bigquery.NewClient(httpClient, *project)
 60 | 	if err != nil {
 61 | 		log.Fatalf("Creating bigquery client: %v", err)
 62 | 	}
 63 | 
 64 | 	d := &bigquery.Table{}
 65 | 
 66 | 	if *dest != "" {
 67 | 		d.ProjectID = *project
 68 | 		d.DatasetID = *dataset
 69 | 		d.TableID = *dest
 70 | 	}
 71 | 
 72 | 	query := &bigquery.Query{
 73 | 		Q:                *q,
 74 | 		DefaultProjectID: *project,
 75 | 		DefaultDatasetID: *dataset,
 76 | 	}
 77 | 
 78 | 	// Query data.
 79 | 	job, err := client.Copy(context.Background(), d, query, bigquery.WriteTruncate)
 80 | 
 81 | 	if err != nil {
 82 | 		log.Fatalf("Querying: %v", err)
 83 | 	}
 84 | 
 85 | 	fmt.Printf("Submitted query. Job ID: %s\n", job.ID())
 86 | 	if !*wait {
 87 | 		return
 88 | 	}
 89 | 
 90 | 	fmt.Printf("Waiting for job to complete.\n")
 91 | 
 92 | 	for range time.Tick(*pollint) {
 93 | 		status, err := job.Status(context.Background())
 94 | 		if err != nil {
 95 | 			fmt.Printf("Failure determining status: %v", err)
 96 | 			break
 97 | 		}
 98 | 		if !status.Done() {
 99 | 			continue
100 | 		}
101 | 		if err := status.Err(); err == nil {
102 | 			fmt.Printf("Success\n")
103 | 		} else {
104 | 			fmt.Printf("Failure: %+v\n", err)
105 | 		}
106 | 		break
107 | 	}
108 | }
109 | 


--------------------------------------------------------------------------------
/bigtable/internal/cbtrc/cbtrc.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | // Package cbtrc encapsulates common code for reading .cbtrc files.
 18 | package cbtrc
 19 | 
 20 | import (
 21 | 	"bufio"
 22 | 	"bytes"
 23 | 	"flag"
 24 | 	"fmt"
 25 | 	"io/ioutil"
 26 | 	"os"
 27 | 	"path/filepath"
 28 | 	"strings"
 29 | )
 30 | 
 31 | // Config represents a configuration.
 32 | type Config struct {
 33 | 	Project, Zone, Cluster string // required
 34 | 	Creds                  string // optional
 35 | }
 36 | 
 37 | // RegisterFlags registers a set of standard flags for this config.
 38 | // It should be called before flag.Parse.
 39 | func (c *Config) RegisterFlags() {
 40 | 	flag.StringVar(&c.Project, "project", c.Project, "project ID")
 41 | 	flag.StringVar(&c.Zone, "zone", c.Zone, "CBT zone")
 42 | 	flag.StringVar(&c.Cluster, "cluster", c.Cluster, "CBT cluster")
 43 | 	flag.StringVar(&c.Creds, "creds", c.Creds, "if set, use application credentials in this file")
 44 | }
 45 | 
 46 | // CheckFlags checks that the required config values are set.
 47 | func (c *Config) CheckFlags() error {
 48 | 	var missing []string
 49 | 	if c.Project == "" {
 50 | 		missing = append(missing, "-project")
 51 | 	}
 52 | 	if c.Zone == "" {
 53 | 		missing = append(missing, "-zone")
 54 | 	}
 55 | 	if c.Cluster == "" {
 56 | 		missing = append(missing, "-cluster")
 57 | 	}
 58 | 	if len(missing) > 0 {
 59 | 		return fmt.Errorf("Missing %s", strings.Join(missing, " and "))
 60 | 	}
 61 | 	return nil
 62 | }
 63 | 
 64 | // Filename returns the filename consulted for standard configuration.
 65 | func Filename() string {
 66 | 	// TODO(dsymonds): Might need tweaking for Windows.
 67 | 	return filepath.Join(os.Getenv("HOME"), ".cbtrc")
 68 | }
 69 | 
 70 | // Load loads a .cbtrc file.
 71 | // If the file is not present, an empty config is returned.
 72 | func Load() (*Config, error) {
 73 | 	filename := Filename()
 74 | 	data, err := ioutil.ReadFile(filename)
 75 | 	if err != nil {
 76 | 		// silent fail if the file isn't there
 77 | 		if os.IsNotExist(err) {
 78 | 			return &Config{}, nil
 79 | 		}
 80 | 		return nil, fmt.Errorf("Reading %s: %v", filename, err)
 81 | 	}
 82 | 	c := new(Config)
 83 | 	s := bufio.NewScanner(bytes.NewReader(data))
 84 | 	for s.Scan() {
 85 | 		line := s.Text()
 86 | 		i := strings.Index(line, "=")
 87 | 		if i < 0 {
 88 | 			return nil, fmt.Errorf("Bad line in %s: %q", filename, line)
 89 | 		}
 90 | 		key, val := strings.TrimSpace(line[:i]), strings.TrimSpace(line[i+1:])
 91 | 		switch key {
 92 | 		default:
 93 | 			return nil, fmt.Errorf("Unknown key in %s: %q", filename, key)
 94 | 		case "project":
 95 | 			c.Project = val
 96 | 		case "zone":
 97 | 			c.Zone = val
 98 | 		case "cluster":
 99 | 			c.Cluster = val
100 | 		case "creds":
101 | 			c.Creds = val
102 | 		}
103 | 	}
104 | 	return c, s.Err()
105 | }
106 | 


--------------------------------------------------------------------------------
/storage/writer.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2014 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package storage
 16 | 
 17 | import (
 18 | 	"fmt"
 19 | 	"io"
 20 | 	"unicode/utf8"
 21 | 
 22 | 	"golang.org/x/net/context"
 23 | )
 24 | 
 25 | // A Writer writes a Cloud Storage object.
 26 | type Writer struct {
 27 | 	// ObjectAttrs are optional attributes to set on the object. Any attributes
 28 | 	// must be initialized before the first Write call. Nil or zero-valued
 29 | 	// attributes are ignored.
 30 | 	ObjectAttrs
 31 | 
 32 | 	ctx    context.Context
 33 | 	client *Client
 34 | 	bucket string
 35 | 	name   string
 36 | 
 37 | 	opened bool
 38 | 	pw     *io.PipeWriter
 39 | 
 40 | 	donec chan struct{} // closed after err and obj are set.
 41 | 	err   error
 42 | 	obj   *ObjectAttrs
 43 | }
 44 | 
 45 | func (w *Writer) open() error {
 46 | 	attrs := w.ObjectAttrs
 47 | 	// Always set the name, otherwise the backend
 48 | 	// rejects the request and responds with an HTTP 400.
 49 | 	if attrs.Name == "" {
 50 | 		attrs.Name = w.name
 51 | 	}
 52 | 	if !utf8.ValidString(attrs.Name) {
 53 | 		return fmt.Errorf("storage: object name %q is not valid UTF-8", attrs.Name)
 54 | 	}
 55 | 	pr, pw := io.Pipe()
 56 | 	r := &contentTyper{pr, attrs.ContentType}
 57 | 	w.pw = pw
 58 | 	w.opened = true
 59 | 
 60 | 	go func() {
 61 | 		resp, err := w.client.raw.Objects.Insert(
 62 | 			w.bucket, attrs.toRawObject(w.bucket)).Media(r).Projection("full").Context(w.ctx).Do()
 63 | 		w.err = err
 64 | 		if err == nil {
 65 | 			w.obj = newObject(resp)
 66 | 		} else {
 67 | 			pr.CloseWithError(w.err)
 68 | 		}
 69 | 		close(w.donec)
 70 | 	}()
 71 | 	return nil
 72 | }
 73 | 
 74 | // Write appends to w.
 75 | func (w *Writer) Write(p []byte) (n int, err error) {
 76 | 	if w.err != nil {
 77 | 		return 0, w.err
 78 | 	}
 79 | 	if !w.opened {
 80 | 		if err := w.open(); err != nil {
 81 | 			return 0, err
 82 | 		}
 83 | 	}
 84 | 	return w.pw.Write(p)
 85 | }
 86 | 
 87 | // Close completes the write operation and flushes any buffered data.
 88 | // If Close doesn't return an error, metadata about the written object
 89 | // can be retrieved by calling Object.
 90 | func (w *Writer) Close() error {
 91 | 	if !w.opened {
 92 | 		if err := w.open(); err != nil {
 93 | 			return err
 94 | 		}
 95 | 	}
 96 | 	if err := w.pw.Close(); err != nil {
 97 | 		return err
 98 | 	}
 99 | 	<-w.donec
100 | 	return w.err
101 | }
102 | 
103 | // CloseWithError aborts the write operation with the provided error.
104 | // CloseWithError always returns nil.
105 | func (w *Writer) CloseWithError(err error) error {
106 | 	if !w.opened {
107 | 		return nil
108 | 	}
109 | 	return w.pw.CloseWithError(err)
110 | }
111 | 
112 | // ObjectAttrs returns metadata about a successfully-written object.
113 | // It's only valid to call it after Close returns nil.
114 | func (w *Writer) Attrs() *ObjectAttrs {
115 | 	return w.obj
116 | }
117 | 


--------------------------------------------------------------------------------
/bigquery/schema.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import bq "google.golang.org/api/bigquery/v2"
 18 | 
 19 | // Schema describes the fields in a table or query result.
 20 | type Schema []*FieldSchema
 21 | 
 22 | // TODO(mcgreevy): add a function to generate a schema from a struct.
 23 | 
 24 | type FieldSchema struct {
 25 | 	// The field name.
 26 | 	// Must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_),
 27 | 	// and must start with a letter or underscore.
 28 | 	// The maximum length is 128 characters.
 29 | 	Name string
 30 | 
 31 | 	// A description of the field. The maximum length is 16,384 characters.
 32 | 	Description string
 33 | 
 34 | 	// Whether the field may contain multiple values.
 35 | 	Repeated bool
 36 | 	// Whether the field is required.  Ignored if Repeated is true.
 37 | 	Required bool
 38 | 
 39 | 	// The field data type.  If Type is Record, then this field contains a nested schema,
 40 | 	// which is described by Schema.
 41 | 	Type FieldType
 42 | 	// Describes the nested schema if Type is set to Record.
 43 | 	Schema Schema
 44 | }
 45 | 
 46 | func (fs *FieldSchema) asTableFieldSchema() *bq.TableFieldSchema {
 47 | 	tfs := &bq.TableFieldSchema{
 48 | 		Description: fs.Description,
 49 | 		Name:        fs.Name,
 50 | 		Type:        string(fs.Type),
 51 | 	}
 52 | 
 53 | 	if fs.Repeated {
 54 | 		tfs.Mode = "REPEATED"
 55 | 	} else if fs.Required {
 56 | 		tfs.Mode = "REQUIRED"
 57 | 	} // else leave as default, which is interpreted as NULLABLE.
 58 | 
 59 | 	for _, f := range fs.Schema {
 60 | 		tfs.Fields = append(tfs.Fields, f.asTableFieldSchema())
 61 | 	}
 62 | 
 63 | 	return tfs
 64 | }
 65 | 
 66 | func (s Schema) asTableSchema() *bq.TableSchema {
 67 | 	var fields []*bq.TableFieldSchema
 68 | 	for _, f := range s {
 69 | 		fields = append(fields, f.asTableFieldSchema())
 70 | 	}
 71 | 	return &bq.TableSchema{Fields: fields}
 72 | }
 73 | 
 74 | func convertTableFieldSchema(tfs *bq.TableFieldSchema) *FieldSchema {
 75 | 	fs := &FieldSchema{
 76 | 		Description: tfs.Description,
 77 | 		Name:        tfs.Name,
 78 | 		Repeated:    tfs.Mode == "REPEATED",
 79 | 		Required:    tfs.Mode == "REQUIRED",
 80 | 		Type:        FieldType(tfs.Type),
 81 | 	}
 82 | 
 83 | 	for _, f := range tfs.Fields {
 84 | 		fs.Schema = append(fs.Schema, convertTableFieldSchema(f))
 85 | 	}
 86 | 	return fs
 87 | }
 88 | 
 89 | func convertTableSchema(ts *bq.TableSchema) Schema {
 90 | 	var s Schema
 91 | 	for _, f := range ts.Fields {
 92 | 		s = append(s, convertTableFieldSchema(f))
 93 | 	}
 94 | 	return s
 95 | }
 96 | 
 97 | type FieldType string
 98 | 
 99 | const (
100 | 	StringFieldType    FieldType = "STRING"
101 | 	IntegerFieldType   FieldType = "INTEGER"
102 | 	FloatFieldType     FieldType = "FLOAT"
103 | 	BooleanFieldType   FieldType = "BOOLEAN"
104 | 	TimestampFieldType FieldType = "TIMESTAMP"
105 | 	RecordFieldType    FieldType = "RECORD"
106 | )
107 | 


--------------------------------------------------------------------------------
/examples/bigquery/load/main.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | // load is an example client of the bigquery client library.
 16 | // It loads a file from Google Cloud Storage into a BigQuery table.
 17 | package main
 18 | 
 19 | import (
 20 | 	"flag"
 21 | 	"fmt"
 22 | 	"log"
 23 | 	"os"
 24 | 	"time"
 25 | 
 26 | 	"golang.org/x/net/context"
 27 | 	"golang.org/x/oauth2/google"
 28 | 	"github.com/fd/gcloud-golang/bigquery"
 29 | )
 30 | 
 31 | var (
 32 | 	project  = flag.String("project", "", "The ID of a Google Cloud Platform project")
 33 | 	dataset  = flag.String("dataset", "", "The ID of a BigQuery dataset")
 34 | 	table    = flag.String("table", "", "The ID of a BigQuery table to load data into")
 35 | 	bucket   = flag.String("bucket", "", "The name of a Google Cloud Storage bucket to load data from")
 36 | 	object   = flag.String("object", "", "The name of a Google Cloud Storage object to load data from. Must exist within the bucket specified by --bucket")
 37 | 	skiprows = flag.Int64("skiprows", 0, "The number of rows of the source data to skip when loading")
 38 | 	pollint  = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
 39 | )
 40 | 
 41 | func main() {
 42 | 	flag.Parse()
 43 | 
 44 | 	flagsOk := true
 45 | 	for _, f := range []string{"project", "dataset", "table", "bucket", "object"} {
 46 | 		if flag.Lookup(f).Value.String() == "" {
 47 | 			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
 48 | 			flagsOk = false
 49 | 		}
 50 | 	}
 51 | 	if !flagsOk {
 52 | 		os.Exit(1)
 53 | 	}
 54 | 
 55 | 	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
 56 | 	if err != nil {
 57 | 		log.Fatalf("Creating http client: %v", err)
 58 | 	}
 59 | 
 60 | 	client, err := bigquery.NewClient(httpClient, *project)
 61 | 	if err != nil {
 62 | 		log.Fatalf("Creating bigquery client: %v", err)
 63 | 	}
 64 | 
 65 | 	table := &bigquery.Table{
 66 | 		ProjectID: *project,
 67 | 		DatasetID: *dataset,
 68 | 		TableID:   *table,
 69 | 	}
 70 | 
 71 | 	gcs := client.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object))
 72 | 	gcs.SkipLeadingRows = *skiprows
 73 | 
 74 | 	// Load data from Google Cloud Storage into a BigQuery table.
 75 | 	job, err := client.Copy(
 76 | 		context.Background(), table, gcs,
 77 | 		bigquery.MaxBadRecords(1),
 78 | 		bigquery.AllowQuotedNewlines(),
 79 | 		bigquery.WriteTruncate)
 80 | 
 81 | 	if err != nil {
 82 | 		log.Fatalf("Loading data: %v", err)
 83 | 	}
 84 | 
 85 | 	fmt.Printf("Job for data load operation: %+v\n", job)
 86 | 	fmt.Printf("Waiting for job to complete.\n")
 87 | 
 88 | 	for range time.Tick(*pollint) {
 89 | 		status, err := job.Status(context.Background())
 90 | 		if err != nil {
 91 | 			fmt.Printf("Failure determining status: %v", err)
 92 | 			break
 93 | 		}
 94 | 		if !status.Done() {
 95 | 			continue
 96 | 		}
 97 | 		if err := status.Err(); err == nil {
 98 | 			fmt.Printf("Success\n")
 99 | 		} else {
100 | 			fmt.Printf("Failure: %+v\n", err)
101 | 		}
102 | 		break
103 | 	}
104 | }
105 | 


--------------------------------------------------------------------------------
/examples/bigquery/concat_table/main.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | // concat_table is an example client of the bigquery client library.
 16 | // It concatenates two BigQuery tables and writes the result to another table.
 17 | package main
 18 | 
 19 | import (
 20 | 	"flag"
 21 | 	"fmt"
 22 | 	"log"
 23 | 	"os"
 24 | 	"time"
 25 | 
 26 | 	"golang.org/x/net/context"
 27 | 	"golang.org/x/oauth2/google"
 28 | 	"github.com/fd/gcloud-golang/bigquery"
 29 | )
 30 | 
 31 | var (
 32 | 	project = flag.String("project", "", "The ID of a Google Cloud Platform project")
 33 | 	dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
 34 | 	src1    = flag.String("src1", "", "The ID of the first BigQuery table to concatenate")
 35 | 	src2    = flag.String("src2", "", "The ID of the second BigQuery table to concatenate")
 36 | 	dest    = flag.String("dest", "", "The ID of the BigQuery table to write the result to")
 37 | 	pollint = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
 38 | )
 39 | 
 40 | func main() {
 41 | 	flag.Parse()
 42 | 
 43 | 	flagsOk := true
 44 | 	for _, f := range []string{"project", "dataset", "src1", "src2", "dest"} {
 45 | 		if flag.Lookup(f).Value.String() == "" {
 46 | 			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
 47 | 			flagsOk = false
 48 | 		}
 49 | 	}
 50 | 	if !flagsOk {
 51 | 		os.Exit(1)
 52 | 	}
 53 | 	if *src1 == *src2 || *src1 == *dest || *src2 == *dest {
 54 | 		log.Fatalf("Different values must be supplied for each of --src1, --src2 and --dest")
 55 | 	}
 56 | 
 57 | 	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
 58 | 	if err != nil {
 59 | 		log.Fatalf("Creating http client: %v", err)
 60 | 	}
 61 | 
 62 | 	client, err := bigquery.NewClient(httpClient, *project)
 63 | 	if err != nil {
 64 | 		log.Fatalf("Creating bigquery client: %v", err)
 65 | 	}
 66 | 
 67 | 	s1 := &bigquery.Table{
 68 | 		ProjectID: *project,
 69 | 		DatasetID: *dataset,
 70 | 		TableID:   *src1,
 71 | 	}
 72 | 
 73 | 	s2 := &bigquery.Table{
 74 | 		ProjectID: *project,
 75 | 		DatasetID: *dataset,
 76 | 		TableID:   *src2,
 77 | 	}
 78 | 
 79 | 	d := &bigquery.Table{
 80 | 		ProjectID: *project,
 81 | 		DatasetID: *dataset,
 82 | 		TableID:   *dest,
 83 | 	}
 84 | 
 85 | 	// Concatenate data.
 86 | 	job, err := client.Copy(context.Background(), d, bigquery.Tables{s1, s2}, bigquery.WriteTruncate)
 87 | 
 88 | 	if err != nil {
 89 | 		log.Fatalf("Concatenating: %v", err)
 90 | 	}
 91 | 
 92 | 	fmt.Printf("Job for concatenation operation: %+v\n", job)
 93 | 	fmt.Printf("Waiting for job to complete.\n")
 94 | 
 95 | 	for range time.Tick(*pollint) {
 96 | 		status, err := job.Status(context.Background())
 97 | 		if err != nil {
 98 | 			fmt.Printf("Failure determining status: %v", err)
 99 | 			break
100 | 		}
101 | 		if !status.Done() {
102 | 			continue
103 | 		}
104 | 		if err := status.Err(); err == nil {
105 | 			fmt.Printf("Success\n")
106 | 		} else {
107 | 			fmt.Printf("Failure: %+v\n", err)
108 | 		}
109 | 		break
110 | 	}
111 | }
112 | 


--------------------------------------------------------------------------------
/bigquery/dataset_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"errors"
 19 | 	"reflect"
 20 | 	"testing"
 21 | 
 22 | 	"golang.org/x/net/context"
 23 | )
 24 | 
 25 | // readServiceStub services read requests by returning data from an in-memory list of values.
 26 | type listTablesServiceStub struct {
 27 | 	expectedProject, expectedDataset string
 28 | 	values                           [][]*Table        // contains pages of tables.
 29 | 	pageTokens                       map[string]string // maps incoming page token to returned page token.
 30 | 
 31 | 	service
 32 | }
 33 | 
 34 | func (s *listTablesServiceStub) listTables(ctx context.Context, projectID, datasetID, pageToken string) ([]*Table, string, error) {
 35 | 	if projectID != s.expectedProject {
 36 | 		return nil, "", errors.New("wrong project id")
 37 | 	}
 38 | 	if datasetID != s.expectedDataset {
 39 | 		return nil, "", errors.New("wrong dataset id")
 40 | 	}
 41 | 
 42 | 	tables := s.values[0]
 43 | 	s.values = s.values[1:]
 44 | 	return tables, s.pageTokens[pageToken], nil
 45 | }
 46 | 
 47 | func TestListTables(t *testing.T) {
 48 | 	t1 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t1"}
 49 | 	t2 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t2"}
 50 | 	t3 := &Table{ProjectID: "p1", DatasetID: "d1", TableID: "t3"}
 51 | 	testCases := []struct {
 52 | 		data       [][]*Table
 53 | 		pageTokens map[string]string
 54 | 		want       []*Table
 55 | 	}{
 56 | 		{
 57 | 			data:       [][]*Table{{t1, t2}, {t3}},
 58 | 			pageTokens: map[string]string{"": "a", "a": ""},
 59 | 			want:       []*Table{t1, t2, t3},
 60 | 		},
 61 | 		{
 62 | 			data:       [][]*Table{{t1, t2}, {t3}},
 63 | 			pageTokens: map[string]string{"": ""}, // no more pages after first one.
 64 | 			want:       []*Table{t1, t2},
 65 | 		},
 66 | 	}
 67 | 
 68 | 	for _, tc := range testCases {
 69 | 		c := &Client{
 70 | 			service: &listTablesServiceStub{
 71 | 				expectedProject: "x",
 72 | 				expectedDataset: "y",
 73 | 				values:          tc.data,
 74 | 				pageTokens:      tc.pageTokens,
 75 | 			},
 76 | 			projectID: "x",
 77 | 		}
 78 | 		got, err := c.Dataset("y").ListTables(context.Background())
 79 | 		if err != nil {
 80 | 			t.Errorf("err calling ListTables: %v", err)
 81 | 			continue
 82 | 		}
 83 | 
 84 | 		if !reflect.DeepEqual(got, tc.want) {
 85 | 			t.Errorf("reading: got:\n%v\nwant:\n%v", got, tc.want)
 86 | 		}
 87 | 	}
 88 | }
 89 | 
 90 | func TestListTablesError(t *testing.T) {
 91 | 	c := &Client{
 92 | 		service: &listTablesServiceStub{
 93 | 			expectedProject: "x",
 94 | 			expectedDataset: "y",
 95 | 		},
 96 | 		projectID: "x",
 97 | 	}
 98 | 	// Test that service read errors are propagated back to the caller.
 99 | 	// Passing "not y" as the dataset id will cause the service to return an error.
100 | 	_, err := c.Dataset("not y").ListTables(context.Background())
101 | 	if err == nil {
102 | 		// Read should not return an error; only Err should.
103 | 		t.Errorf("ListTables expected: non-nil err, got: nil")
104 | 	}
105 | }
106 | 


--------------------------------------------------------------------------------
/option.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package cloud
 16 | 
 17 | import (
 18 | 	"net/http"
 19 | 
 20 | 	"github.com/fd/gcloud-golang/internal/opts"
 21 | 	"golang.org/x/oauth2"
 22 | 	"google.golang.org/grpc"
 23 | )
 24 | 
 25 | // ClientOption is used when construct clients for each cloud service.
 26 | type ClientOption interface {
 27 | 	// Resolve configures the given DialOpts for this option.
 28 | 	Resolve(*opts.DialOpt)
 29 | }
 30 | 
 31 | // WithTokenSource returns a ClientOption that specifies an OAuth2 token
 32 | // source to be used as the basis for authentication.
 33 | func WithTokenSource(s oauth2.TokenSource) ClientOption {
 34 | 	return withTokenSource{s}
 35 | }
 36 | 
 37 | type withTokenSource struct{ ts oauth2.TokenSource }
 38 | 
 39 | func (w withTokenSource) Resolve(o *opts.DialOpt) {
 40 | 	o.TokenSource = w.ts
 41 | }
 42 | 
 43 | // WithEndpoint returns a ClientOption that overrides the default endpoint
 44 | // to be used for a service.
 45 | func WithEndpoint(url string) ClientOption {
 46 | 	return withEndpoint(url)
 47 | }
 48 | 
 49 | type withEndpoint string
 50 | 
 51 | func (w withEndpoint) Resolve(o *opts.DialOpt) {
 52 | 	o.Endpoint = string(w)
 53 | }
 54 | 
 55 | // WithScopes returns a ClientOption that overrides the default OAuth2 scopes
 56 | // to be used for a service.
 57 | func WithScopes(scope ...string) ClientOption {
 58 | 	return withScopes(scope)
 59 | }
 60 | 
 61 | type withScopes []string
 62 | 
 63 | func (w withScopes) Resolve(o *opts.DialOpt) {
 64 | 	s := make([]string, len(w))
 65 | 	copy(s, w)
 66 | 	o.Scopes = s
 67 | }
 68 | 
 69 | // WithUserAgent returns a ClientOption that sets the User-Agent.
 70 | func WithUserAgent(ua string) ClientOption {
 71 | 	return withUA(ua)
 72 | }
 73 | 
 74 | type withUA string
 75 | 
 76 | func (w withUA) Resolve(o *opts.DialOpt) { o.UserAgent = string(w) }
 77 | 
 78 | // WithBaseHTTP returns a ClientOption that specifies the HTTP client to
 79 | // use as the basis of communications. This option may only be used with
 80 | // services that support HTTP as their communication transport.
 81 | func WithBaseHTTP(client *http.Client) ClientOption {
 82 | 	return withBaseHTTP{client}
 83 | }
 84 | 
 85 | type withBaseHTTP struct{ client *http.Client }
 86 | 
 87 | func (w withBaseHTTP) Resolve(o *opts.DialOpt) {
 88 | 	o.HTTPClient = w.client
 89 | }
 90 | 
 91 | // WithBaseGRPC returns a ClientOption that specifies the gRPC client
 92 | // connection to use as the basis of communications. This option many only be
 93 | // used with services that support gRPC as their communication transport.
 94 | func WithBaseGRPC(client *grpc.ClientConn) ClientOption {
 95 | 	return withBaseGRPC{client}
 96 | }
 97 | 
 98 | type withBaseGRPC struct{ client *grpc.ClientConn }
 99 | 
100 | func (w withBaseGRPC) Resolve(o *opts.DialOpt) {
101 | 	o.GRPCClient = w.client
102 | }
103 | 
104 | // WithGRPCDialOption returns a ClientOption that appends a new grpc.DialOption
105 | // to an underlying gRPC dial. It does not work with WithBaseGRPC.
106 | func WithGRPCDialOption(opt grpc.DialOption) ClientOption {
107 | 	return withGRPCDialOption{opt}
108 | }
109 | 
110 | type withGRPCDialOption struct{ opt grpc.DialOption }
111 | 
112 | func (w withGRPCDialOption) Resolve(o *opts.DialOpt) {
113 | 	o.GRPCDialOpts = append(o.GRPCDialOpts, w.opt)
114 | }
115 | 


--------------------------------------------------------------------------------
/bigtable/internal/table_service_proto/bigtable_table_service_messages.proto:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2015, Google Inc.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | syntax = "proto3";
 16 | 
 17 | package google.bigtable.admin.table.v1;
 18 | 
 19 | import "google.golang.org/cloud/bigtable/internal/table_data_proto/bigtable_table_data.proto";
 20 | 
 21 | option java_multiple_files = true;
 22 | option java_outer_classname = "BigtableTableServiceMessagesProto";
 23 | option java_package = "com.google.bigtable.admin.table.v1";
 24 | 
 25 | 
 26 | message CreateTableRequest {
 27 |   // The unique name of the cluster in which to create the new table.
 28 |   string name = 1;
 29 | 
 30 |   // The name by which the new table should be referred to within the cluster,
 31 |   // e.g. "foobar" rather than "<cluster_name>/tables/foobar".
 32 |   string table_id = 2;
 33 | 
 34 |   // The Table to create. The `name` field of the Table and all of its
 35 |   // ColumnFamilies must be left blank, and will be populated in the response.
 36 |   Table table = 3;
 37 | 
 38 |   // The optional list of row keys that will be used to initially split the
 39 |   // table into several tablets (Tablets are similar to HBase regions).
 40 |   // Given two split keys, "s1" and "s2", three tablets will be created,
 41 |   // spanning the key ranges: [, s1), [s1, s2), [s2, ).
 42 |   //
 43 |   // Example:
 44 |   //  * Row keys := ["a", "apple", "custom", "customer_1", "customer_2",
 45 |   //                 "other", "zz"]
 46 |   //  * initial_split_keys := ["apple", "customer_1", "customer_2", "other"]
 47 |   //  * Key assignment:
 48 |   //    - Tablet 1 [, apple)                => {"a"}.
 49 |   //    - Tablet 2 [apple, customer_1)      => {"apple", "custom"}.
 50 |   //    - Tablet 3 [customer_1, customer_2) => {"customer_1"}.
 51 |   //    - Tablet 4 [customer_2, other)      => {"customer_2"}.
 52 |   //    - Tablet 5 [other, )                => {"other", "zz"}.
 53 |   repeated string initial_split_keys = 4;
 54 | }
 55 | 
 56 | message ListTablesRequest {
 57 |   // The unique name of the cluster for which tables should be listed.
 58 |   string name = 1;
 59 | }
 60 | 
 61 | message ListTablesResponse {
 62 |   // The tables present in the requested cluster.
 63 |   // At present, only the names of the tables are populated.
 64 |   repeated Table tables = 1;
 65 | }
 66 | 
 67 | message GetTableRequest {
 68 |   // The unique name of the requested table.
 69 |   string name = 1;
 70 | }
 71 | 
 72 | message DeleteTableRequest {
 73 |   // The unique name of the table to be deleted.
 74 |   string name = 1;
 75 | }
 76 | 
 77 | message RenameTableRequest {
 78 |   // The current unique name of the table.
 79 |   string name = 1;
 80 | 
 81 |   // The new name by which the table should be referred to within its containing
 82 |   // cluster, e.g. "foobar" rather than "<cluster_name>/tables/foobar".
 83 |   string new_id = 2;
 84 | }
 85 | 
 86 | message CreateColumnFamilyRequest {
 87 |   // The unique name of the table in which to create the new column family.
 88 |   string name = 1;
 89 | 
 90 |   // The name by which the new column family should be referred to within the
 91 |   // table, e.g. "foobar" rather than "<table_name>/columnFamilies/foobar".
 92 |   string column_family_id = 2;
 93 | 
 94 |   // The column family to create. The `name` field must be left blank.
 95 |   ColumnFamily column_family = 3;
 96 | }
 97 | 
 98 | message DeleteColumnFamilyRequest {
 99 |   // The unique name of the column family to be deleted.
100 |   string name = 1;
101 | }
102 | 


--------------------------------------------------------------------------------
/bigquery/job.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"errors"
 19 | 
 20 | 	"golang.org/x/net/context"
 21 | 	bq "google.golang.org/api/bigquery/v2"
 22 | )
 23 | 
 24 | // A Job represents an operation which has been submitted to BigQuery for processing.
 25 | type Job struct {
 26 | 	service   service
 27 | 	projectID string
 28 | 	jobID     string
 29 | 
 30 | 	isQuery bool
 31 | }
 32 | 
 33 | // JobFromID creates a Job which refers to an existing BigQuery job. The job
 34 | // need not have been created by this package. For example, the job may have
 35 | // been created in the BigQuery console.
 36 | func (c *Client) JobFromID(ctx context.Context, id string) (*Job, error) {
 37 | 	jobType, err := c.service.getJobType(ctx, c.projectID, id)
 38 | 	if err != nil {
 39 | 		return nil, err
 40 | 	}
 41 | 
 42 | 	return &Job{
 43 | 		service:   c.service,
 44 | 		projectID: c.projectID,
 45 | 		jobID:     id,
 46 | 		isQuery:   jobType == queryJobType,
 47 | 	}, nil
 48 | }
 49 | 
 50 | func (j *Job) ID() string {
 51 | 	return j.jobID
 52 | }
 53 | 
 54 | // State is one of a sequence of states that a Job progresses through as it is processed.
 55 | type State int
 56 | 
 57 | const (
 58 | 	Pending State = iota
 59 | 	Running
 60 | 	Done
 61 | )
 62 | 
 63 | // JobStatus contains the current State of a job, and errors encountered while processing that job.
 64 | type JobStatus struct {
 65 | 	State State
 66 | 
 67 | 	err error
 68 | 
 69 | 	// All errors encountered during the running of the job.
 70 | 	// Not all Errors are fatal, so errors here do not necessarily mean that the job has completed or was unsuccessful.
 71 | 	Errors []*Error
 72 | }
 73 | 
 74 | // jobOption is an Option which modifies a bq.Job proto.
 75 | // This is used for configuring values that apply to all operations, such as setting a jobReference.
 76 | type jobOption interface {
 77 | 	customizeJob(job *bq.Job, projectID string)
 78 | }
 79 | 
 80 | type jobID string
 81 | 
 82 | // JobID returns an Option that sets the job ID of a BigQuery job.
 83 | // If this Option is not used, a job ID is generated automatically.
 84 | func JobID(ID string) Option {
 85 | 	return jobID(ID)
 86 | }
 87 | 
 88 | func (opt jobID) implementsOption() {}
 89 | 
 90 | func (opt jobID) customizeJob(job *bq.Job, projectID string) {
 91 | 	job.JobReference = &bq.JobReference{
 92 | 		JobId:     string(opt),
 93 | 		ProjectId: projectID,
 94 | 	}
 95 | }
 96 | 
 97 | // Done reports whether the job has completed.
 98 | // After Done returns true, the Err method will return an error if the job completed unsuccesfully.
 99 | func (s *JobStatus) Done() bool {
100 | 	return s.State == Done
101 | }
102 | 
103 | // Err returns the error that caused the job to complete unsuccesfully (if any).
104 | func (s *JobStatus) Err() error {
105 | 	return s.err
106 | }
107 | 
108 | // Status returns the current status of the job.  It fails if the Status could not be determined.
109 | func (j *Job) Status(ctx context.Context) (*JobStatus, error) {
110 | 	return j.service.jobStatus(ctx, j.projectID, j.jobID)
111 | }
112 | 
113 | func (j *Job) implementsReadSource() {}
114 | 
115 | func (j *Job) customizeReadQuery(cursor *readQueryConf) error {
116 | 	// There are mulitple kinds of jobs, but only a query job is suitable for reading.
117 | 	if !j.isQuery {
118 | 		return errors.New("Cannot read from a non-query job")
119 | 	}
120 | 
121 | 	cursor.projectID = j.projectID
122 | 	cursor.jobID = j.jobID
123 | 	return nil
124 | }
125 | 


--------------------------------------------------------------------------------
/bigtable/gc.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package bigtable
 18 | 
 19 | import (
 20 | 	"fmt"
 21 | 	"strings"
 22 | 	"time"
 23 | 
 24 | 	durpb "github.com/fd/gcloud-golang/bigtable/internal/duration_proto"
 25 | 	bttdpb "github.com/fd/gcloud-golang/bigtable/internal/table_data_proto"
 26 | )
 27 | 
 28 | // A GCPolicy represents a rule that determines which cells are eligible for garbage collection.
 29 | type GCPolicy interface {
 30 | 	String() string
 31 | 	proto() *bttdpb.GcRule
 32 | }
 33 | 
 34 | // IntersectionPolicy returns a GC policy that only applies when all its sub-policies apply.
 35 | func IntersectionPolicy(sub ...GCPolicy) GCPolicy { return intersectionPolicy{sub} }
 36 | 
 37 | type intersectionPolicy struct {
 38 | 	sub []GCPolicy
 39 | }
 40 | 
 41 | func (ip intersectionPolicy) String() string {
 42 | 	var ss []string
 43 | 	for _, sp := range ip.sub {
 44 | 		ss = append(ss, sp.String())
 45 | 	}
 46 | 	return "(" + strings.Join(ss, " && ") + ")"
 47 | }
 48 | 
 49 | func (ip intersectionPolicy) proto() *bttdpb.GcRule {
 50 | 	inter := &bttdpb.GcRule_Intersection{}
 51 | 	for _, sp := range ip.sub {
 52 | 		inter.Rules = append(inter.Rules, sp.proto())
 53 | 	}
 54 | 	return &bttdpb.GcRule{
 55 | 		Rule: &bttdpb.GcRule_Intersection_{inter},
 56 | 	}
 57 | }
 58 | 
 59 | // UnionPolicy returns a GC policy that applies when any of its sub-policies apply.
 60 | func UnionPolicy(sub ...GCPolicy) GCPolicy { return unionPolicy{sub} }
 61 | 
 62 | type unionPolicy struct {
 63 | 	sub []GCPolicy
 64 | }
 65 | 
 66 | func (up unionPolicy) String() string {
 67 | 	var ss []string
 68 | 	for _, sp := range up.sub {
 69 | 		ss = append(ss, sp.String())
 70 | 	}
 71 | 	return "(" + strings.Join(ss, " || ") + ")"
 72 | }
 73 | 
 74 | func (up unionPolicy) proto() *bttdpb.GcRule {
 75 | 	union := &bttdpb.GcRule_Union{}
 76 | 	for _, sp := range up.sub {
 77 | 		union.Rules = append(union.Rules, sp.proto())
 78 | 	}
 79 | 	return &bttdpb.GcRule{
 80 | 		Rule: &bttdpb.GcRule_Union_{union},
 81 | 	}
 82 | }
 83 | 
 84 | // MaxVersionsPolicy returns a GC policy that applies to all versions of a cell
 85 | // except for the most recent n.
 86 | func MaxVersionsPolicy(n int) GCPolicy { return maxVersionsPolicy(n) }
 87 | 
 88 | type maxVersionsPolicy int
 89 | 
 90 | func (mvp maxVersionsPolicy) String() string { return fmt.Sprintf("versions() > %d", int(mvp)) }
 91 | 
 92 | func (mvp maxVersionsPolicy) proto() *bttdpb.GcRule {
 93 | 	return &bttdpb.GcRule{Rule: &bttdpb.GcRule_MaxNumVersions{int32(mvp)}}
 94 | }
 95 | 
 96 | // MaxAgePolicy returns a GC policy that applies to all cells
 97 | // older than the given age.
 98 | func MaxAgePolicy(d time.Duration) GCPolicy { return maxAgePolicy(d) }
 99 | 
100 | type maxAgePolicy time.Duration
101 | 
102 | var units = []struct {
103 | 	d      time.Duration
104 | 	suffix string
105 | }{
106 | 	{24 * time.Hour, "d"},
107 | 	{time.Hour, "h"},
108 | 	{time.Minute, "m"},
109 | }
110 | 
111 | func (ma maxAgePolicy) String() string {
112 | 	d := time.Duration(ma)
113 | 	for _, u := range units {
114 | 		if d%u.d == 0 {
115 | 			return fmt.Sprintf("age() > %d%s", d/u.d, u.suffix)
116 | 		}
117 | 	}
118 | 	return fmt.Sprintf("age() > %d", d/time.Microsecond)
119 | }
120 | 
121 | func (ma maxAgePolicy) proto() *bttdpb.GcRule {
122 | 	// This doesn't handle overflows, etc.
123 | 	// Fix this if people care about GC policies over 290 years.
124 | 	ns := time.Duration(ma).Nanoseconds()
125 | 	return &bttdpb.GcRule{
126 | 		Rule: &bttdpb.GcRule_MaxAge{&durpb.Duration{
127 | 			Seconds: ns / 1e9,
128 | 			Nanos:   int32(ns % 1e9),
129 | 		}},
130 | 	}
131 | }
132 | 


--------------------------------------------------------------------------------
/internal/cloud.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2014 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | // Package internal provides support for the cloud packages.
 16 | //
 17 | // Users should not import this package directly.
 18 | package internal
 19 | 
 20 | import (
 21 | 	"fmt"
 22 | 	"net/http"
 23 | 	"sync"
 24 | 
 25 | 	"golang.org/x/net/context"
 26 | )
 27 | 
 28 | type contextKey struct{}
 29 | 
 30 | func WithContext(parent context.Context, projID string, c *http.Client) context.Context {
 31 | 	if c == nil {
 32 | 		panic("nil *http.Client passed to WithContext")
 33 | 	}
 34 | 	if projID == "" {
 35 | 		panic("empty project ID passed to WithContext")
 36 | 	}
 37 | 	return context.WithValue(parent, contextKey{}, &cloudContext{
 38 | 		ProjectID:  projID,
 39 | 		HTTPClient: c,
 40 | 	})
 41 | }
 42 | 
 43 | const userAgent = "gcloud-golang/0.1"
 44 | 
 45 | type cloudContext struct {
 46 | 	ProjectID  string
 47 | 	HTTPClient *http.Client
 48 | 
 49 | 	mu  sync.Mutex             // guards svc
 50 | 	svc map[string]interface{} // e.g. "storage" => *rawStorage.Service
 51 | }
 52 | 
 53 | // Service returns the result of the fill function if it's never been
 54 | // called before for the given name (which is assumed to be an API
 55 | // service name, like "datastore"). If it has already been cached, the fill
 56 | // func is not run.
 57 | // It's safe for concurrent use by multiple goroutines.
 58 | func Service(ctx context.Context, name string, fill func(*http.Client) interface{}) interface{} {
 59 | 	return cc(ctx).service(name, fill)
 60 | }
 61 | 
 62 | func (c *cloudContext) service(name string, fill func(*http.Client) interface{}) interface{} {
 63 | 	c.mu.Lock()
 64 | 	defer c.mu.Unlock()
 65 | 
 66 | 	if c.svc == nil {
 67 | 		c.svc = make(map[string]interface{})
 68 | 	} else if v, ok := c.svc[name]; ok {
 69 | 		return v
 70 | 	}
 71 | 	v := fill(c.HTTPClient)
 72 | 	c.svc[name] = v
 73 | 	return v
 74 | }
 75 | 
 76 | // Transport is an http.RoundTripper that appends
 77 | // Google Cloud client's user-agent to the original
 78 | // request's user-agent header.
 79 | type Transport struct {
 80 | 	// Base represents the actual http.RoundTripper
 81 | 	// the requests will be delegated to.
 82 | 	Base http.RoundTripper
 83 | }
 84 | 
 85 | // RoundTrip appends a user-agent to the existing user-agent
 86 | // header and delegates the request to the base http.RoundTripper.
 87 | func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
 88 | 	req = cloneRequest(req)
 89 | 	ua := req.Header.Get("User-Agent")
 90 | 	if ua == "" {
 91 | 		ua = userAgent
 92 | 	} else {
 93 | 		ua = fmt.Sprintf("%s %s", ua, userAgent)
 94 | 	}
 95 | 	req.Header.Set("User-Agent", ua)
 96 | 	return t.Base.RoundTrip(req)
 97 | }
 98 | 
 99 | // cloneRequest returns a clone of the provided *http.Request.
100 | // The clone is a shallow copy of the struct and its Header map.
101 | func cloneRequest(r *http.Request) *http.Request {
102 | 	// shallow copy of the struct
103 | 	r2 := new(http.Request)
104 | 	*r2 = *r
105 | 	// deep copy of the Header
106 | 	r2.Header = make(http.Header)
107 | 	for k, s := range r.Header {
108 | 		r2.Header[k] = s
109 | 	}
110 | 	return r2
111 | }
112 | 
113 | func ProjID(ctx context.Context) string {
114 | 	return cc(ctx).ProjectID
115 | }
116 | 
117 | func HTTPClient(ctx context.Context) *http.Client {
118 | 	return cc(ctx).HTTPClient
119 | }
120 | 
121 | // cc returns the internal *cloudContext (cc) state for a context.Context.
122 | // It panics if the user did it wrong.
123 | func cc(ctx context.Context) *cloudContext {
124 | 	if c, ok := ctx.Value(contextKey{}).(*cloudContext); ok {
125 | 		return c
126 | 	}
127 | 	panic("invalid context.Context type; it should be created with cloud.NewContext")
128 | }
129 | 


--------------------------------------------------------------------------------
/pubsub/integration_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2014 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | // +build integration
 16 | 
 17 | package pubsub
 18 | 
 19 | import (
 20 | 	"fmt"
 21 | 	"testing"
 22 | 	"time"
 23 | 
 24 | 	"github.com/fd/gcloud-golang/internal/testutil"
 25 | )
 26 | 
 27 | func TestAll(t *testing.T) {
 28 | 	ctx := testutil.Context(ScopePubSub, ScopeCloudPlatform)
 29 | 	now := time.Now()
 30 | 	topic := fmt.Sprintf("topic-%d", now.Unix())
 31 | 	subscription := fmt.Sprintf("subscription-%d", now.Unix())
 32 | 
 33 | 	if err := CreateTopic(ctx, topic); err != nil {
 34 | 		t.Errorf("CreateTopic error: %v", err)
 35 | 	}
 36 | 
 37 | 	if err := CreateSub(ctx, subscription, topic, time.Duration(0), ""); err != nil {
 38 | 		t.Errorf("CreateSub error: %v", err)
 39 | 	}
 40 | 
 41 | 	exists, err := TopicExists(ctx, topic)
 42 | 	if err != nil {
 43 | 		t.Fatalf("TopicExists error: %v", err)
 44 | 	}
 45 | 	if !exists {
 46 | 		t.Errorf("topic %s should exist, but it doesn't", topic)
 47 | 	}
 48 | 
 49 | 	exists, err = SubExists(ctx, subscription)
 50 | 	if err != nil {
 51 | 		t.Fatalf("SubExists error: %v", err)
 52 | 	}
 53 | 	if !exists {
 54 | 		t.Errorf("subscription %s should exist, but it doesn't", subscription)
 55 | 	}
 56 | 
 57 | 	max := 10
 58 | 	msgs := make([]*Message, max)
 59 | 	expectedMsgs := make(map[string]bool, max)
 60 | 	for i := 0; i < max; i++ {
 61 | 		text := fmt.Sprintf("a message with an index %d", i)
 62 | 		attrs := make(map[string]string)
 63 | 		attrs["foo"] = "bar"
 64 | 		msgs[i] = &Message{
 65 | 			Data:       []byte(text),
 66 | 			Attributes: attrs,
 67 | 		}
 68 | 		expectedMsgs[text] = false
 69 | 	}
 70 | 
 71 | 	ids, err := Publish(ctx, topic, msgs...)
 72 | 	if err != nil {
 73 | 		t.Fatalf("Publish (1) error: %v", err)
 74 | 	}
 75 | 
 76 | 	if len(ids) != max {
 77 | 		t.Errorf("unexpected number of message IDs received; %d, want %d", len(ids), max)
 78 | 	}
 79 | 
 80 | 	expectedIDs := make(map[string]bool, max)
 81 | 	for _, id := range ids {
 82 | 		expectedIDs[id] = false
 83 | 	}
 84 | 
 85 | 	received, err := PullWait(ctx, subscription, max)
 86 | 	if err != nil {
 87 | 		t.Fatalf("PullWait error: %v", err)
 88 | 	}
 89 | 
 90 | 	if len(received) != max {
 91 | 		t.Errorf("unexpected number of messages received; %d, want %d", len(received), max)
 92 | 	}
 93 | 
 94 | 	for _, msg := range received {
 95 | 		expectedMsgs[string(msg.Data)] = true
 96 | 		expectedIDs[msg.ID] = true
 97 | 		if msg.Attributes["foo"] != "bar" {
 98 | 			t.Errorf("message attribute foo is expected to be 'bar', found '%s'", msg.Attributes["foo"])
 99 | 		}
100 | 	}
101 | 
102 | 	for msg, found := range expectedMsgs {
103 | 		if !found {
104 | 			t.Errorf("message '%s' should be received", msg)
105 | 		}
106 | 	}
107 | 
108 | 	for id, found := range expectedIDs {
109 | 		if !found {
110 | 			t.Errorf("message with the message id '%s' should be received", id)
111 | 		}
112 | 	}
113 | 
114 | 	// base64 test
115 | 	data := "=@~"
116 | 	msg := &Message{
117 | 		Data: []byte(data),
118 | 	}
119 | 	_, err = Publish(ctx, topic, msg)
120 | 	if err != nil {
121 | 		t.Fatalf("Publish (2) error: %v", err)
122 | 	}
123 | 
124 | 	received, err = PullWait(ctx, subscription, 1)
125 | 	if err != nil {
126 | 		t.Fatalf("PullWait error: %v", err)
127 | 	}
128 | 	if len(received) != 1 {
129 | 		t.Fatalf("unexpected number of messages received; %d, want %d", len(received), 1)
130 | 	}
131 | 	if string(received[0].Data) != data {
132 | 		t.Errorf("unexpexted message received; %s, want %s", string(received[0].Data), data)
133 | 	}
134 | 
135 | 	err = DeleteSub(ctx, subscription)
136 | 	if err != nil {
137 | 		t.Errorf("DeleteSub error: %v", err)
138 | 	}
139 | 
140 | 	err = DeleteTopic(ctx, topic)
141 | 	if err != nil {
142 | 		t.Errorf("DeleteTopic error: %v", err)
143 | 	}
144 | }
145 | 


--------------------------------------------------------------------------------
/examples/bigquery/read/main.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | // read is an example client of the bigquery client library.
 16 | // It reads from a table, returning the data via an Iterator.
 17 | package main
 18 | 
 19 | import (
 20 | 	"flag"
 21 | 	"fmt"
 22 | 	"log"
 23 | 	"os"
 24 | 	"regexp"
 25 | 	"strings"
 26 | 	"text/tabwriter"
 27 | 
 28 | 	"golang.org/x/net/context"
 29 | 	"golang.org/x/oauth2/google"
 30 | 	"github.com/fd/gcloud-golang/bigquery"
 31 | )
 32 | 
 33 | var (
 34 | 	project = flag.String("project", "", "The ID of a Google Cloud Platform project")
 35 | 	dataset = flag.String("dataset", "", "The ID of a BigQuery dataset")
 36 | 	table   = flag.String("table", ".*", "A regular expression to match the IDs of tables to read.")
 37 | 	jobID   = flag.String("jobid", "", "The ID of a query job that has already been submitted."+
 38 | 		" If set, --dataset, --table will be ignored, and results will be read from the specified job.")
 39 | )
 40 | 
 41 | func printValues(it *bigquery.Iterator) {
 42 | 	// one-space padding.
 43 | 	tw := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
 44 | 
 45 | 	for it.Next(context.Background()) {
 46 | 		var vals bigquery.ValueList
 47 | 		if err := it.Get(&vals); err != nil {
 48 | 			fmt.Printf("err calling get: %v\n", err)
 49 | 		} else {
 50 | 			sep := ""
 51 | 			for _, v := range vals {
 52 | 				fmt.Fprintf(tw, "%s%v", sep, v)
 53 | 				sep = "\t"
 54 | 			}
 55 | 			fmt.Fprintf(tw, "\n")
 56 | 		}
 57 | 	}
 58 | 	tw.Flush()
 59 | 
 60 | 	fmt.Printf("\n")
 61 | 	if err := it.Err(); err != nil {
 62 | 		fmt.Printf("err reading: %v\n", err)
 63 | 	}
 64 | }
 65 | 
 66 | func printTable(client *bigquery.Client, t *bigquery.Table) {
 67 | 	it, err := client.Read(context.Background(), t)
 68 | 	if err != nil {
 69 | 		log.Fatalf("Reading: %v", err)
 70 | 	}
 71 | 
 72 | 	id := t.FullyQualifiedName()
 73 | 	fmt.Printf("%s\n%s\n", id, strings.Repeat("-", len(id)))
 74 | 	printValues(it)
 75 | }
 76 | 
 77 | func printQueryResults(client *bigquery.Client, queryJobID string) {
 78 | 	job, err := client.JobFromID(context.Background(), queryJobID)
 79 | 	if err != nil {
 80 | 		log.Fatalf("Loading job: %v", err)
 81 | 	}
 82 | 
 83 | 	it, err := client.Read(context.Background(), job)
 84 | 	if err != nil {
 85 | 		log.Fatalf("Reading: %v", err)
 86 | 	}
 87 | 
 88 | 	// TODO: print schema.
 89 | 	printValues(it)
 90 | }
 91 | 
 92 | func main() {
 93 | 	flag.Parse()
 94 | 
 95 | 	flagsOk := true
 96 | 	if flag.Lookup("project").Value.String() == "" {
 97 | 		fmt.Fprintf(os.Stderr, "Flag --project is required\n")
 98 | 		flagsOk = false
 99 | 	}
100 | 
101 | 	var sourceFlagCount int
102 | 	if flag.Lookup("dataset").Value.String() != "" {
103 | 		sourceFlagCount++
104 | 	}
105 | 	if flag.Lookup("jobid").Value.String() != "" {
106 | 		sourceFlagCount++
107 | 	}
108 | 	if sourceFlagCount != 1 {
109 | 		fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n")
110 | 		flagsOk = false
111 | 	}
112 | 
113 | 	if !flagsOk {
114 | 		os.Exit(1)
115 | 	}
116 | 
117 | 	tableRE, err := regexp.Compile(*table)
118 | 	if err != nil {
119 | 		fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
120 | 		os.Exit(1)
121 | 	}
122 | 
123 | 	httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
124 | 	if err != nil {
125 | 		log.Fatalf("Creating http client: %v", err)
126 | 	}
127 | 
128 | 	client, err := bigquery.NewClient(httpClient, *project)
129 | 	if err != nil {
130 | 		log.Fatalf("Creating bigquery client: %v", err)
131 | 	}
132 | 
133 | 	if *jobID != "" {
134 | 		printQueryResults(client, *jobID)
135 | 		return
136 | 	}
137 | 	ds := client.Dataset(*dataset)
138 | 	var tables []*bigquery.Table
139 | 	tables, err = ds.ListTables(context.Background())
140 | 	if err != nil {
141 | 		log.Fatalf("Listing tables: %v", err)
142 | 	}
143 | 	for _, t := range tables {
144 | 		if tableRE.MatchString(t.TableID) {
145 | 			printTable(client, t)
146 | 		}
147 | 	}
148 | }
149 | 


--------------------------------------------------------------------------------
/bigquery/load_op.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"fmt"
 19 | 
 20 | 	"golang.org/x/net/context"
 21 | 	bq "google.golang.org/api/bigquery/v2"
 22 | )
 23 | 
 24 | type loadOption interface {
 25 | 	customizeLoad(conf *bq.JobConfigurationLoad, projectID string)
 26 | }
 27 | 
 28 | // DestinationSchema returns an Option that specifies the schema to use when loading data into a new table.
 29 | // A DestinationSchema Option must be supplied when loading data from Google Cloud Storage into a non-existent table.
 30 | // Caveat: DestinationSchema is not required if the data being loaded is a datastore backup.
 31 | // schema must not be nil.
 32 | func DestinationSchema(schema Schema) Option { return destSchema{Schema: schema} }
 33 | 
 34 | type destSchema struct {
 35 | 	Schema
 36 | }
 37 | 
 38 | func (opt destSchema) implementsOption() {}
 39 | 
 40 | func (opt destSchema) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
 41 | 	conf.Schema = opt.asTableSchema()
 42 | }
 43 | 
 44 | // MaxBadRecords returns an Option that sets the maximum number of bad records that will be ignored.
 45 | // If this maximum is exceeded, the operation will be unsuccessful.
 46 | func MaxBadRecords(n int64) Option { return maxBadRecords(n) }
 47 | 
 48 | type maxBadRecords int64
 49 | 
 50 | func (opt maxBadRecords) implementsOption() {}
 51 | 
 52 | func (opt maxBadRecords) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
 53 | 	conf.MaxBadRecords = int64(opt)
 54 | }
 55 | 
 56 | // AllowJaggedRows returns an Option that causes missing trailing optional columns to be tolerated in CSV data.  Missing values are treated as nulls.
 57 | func AllowJaggedRows() Option { return allowJaggedRows{} }
 58 | 
 59 | type allowJaggedRows struct{}
 60 | 
 61 | func (opt allowJaggedRows) implementsOption() {}
 62 | 
 63 | func (opt allowJaggedRows) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
 64 | 	conf.AllowJaggedRows = true
 65 | }
 66 | 
 67 | // AllowQuotedNewlines returns an Option that allows quoted data sections containing newlines in CSV data.
 68 | func AllowQuotedNewlines() Option { return allowQuotedNewlines{} }
 69 | 
 70 | type allowQuotedNewlines struct{}
 71 | 
 72 | func (opt allowQuotedNewlines) implementsOption() {}
 73 | 
 74 | func (opt allowQuotedNewlines) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
 75 | 	conf.AllowQuotedNewlines = true
 76 | }
 77 | 
 78 | // IgnoreUnknownValues returns an Option that causes values not matching the schema to be tolerated.
 79 | // Unknown values are ignored. For CSV this ignores extra values at the end of a line.
 80 | // For JSON this ignores named values that do not match any column name.
 81 | // If this Option is not used, records containing unknown values are treated as bad records.
 82 | // The MaxBadRecords Option can be used to customize how bad records are handled.
 83 | func IgnoreUnknownValues() Option { return ignoreUnknownValues{} }
 84 | 
 85 | type ignoreUnknownValues struct{}
 86 | 
 87 | func (opt ignoreUnknownValues) implementsOption() {}
 88 | 
 89 | func (opt ignoreUnknownValues) customizeLoad(conf *bq.JobConfigurationLoad, projectID string) {
 90 | 	conf.IgnoreUnknownValues = true
 91 | }
 92 | 
 93 | func (c *Client) load(ctx context.Context, dst *Table, src *GCSReference, options []Option) (*Job, error) {
 94 | 	job, options := initJobProto(c.projectID, options)
 95 | 	payload := &bq.JobConfigurationLoad{}
 96 | 
 97 | 	dst.customizeLoadDst(payload, c.projectID)
 98 | 	src.customizeLoadSrc(payload, c.projectID)
 99 | 
100 | 	for _, opt := range options {
101 | 		o, ok := opt.(loadOption)
102 | 		if !ok {
103 | 			return nil, fmt.Errorf("option (%#v) not applicable to dst/src pair: dst: %T ; src: %T", opt, dst, src)
104 | 		}
105 | 		o.customizeLoad(payload, c.projectID)
106 | 	}
107 | 
108 | 	job.Configuration = &bq.JobConfiguration{
109 | 		Load: payload,
110 | 	}
111 | 	return c.service.insertJob(ctx, job, c.projectID)
112 | }
113 | 


--------------------------------------------------------------------------------
/bigtable/internal/duration_proto/duration.pb.go:
--------------------------------------------------------------------------------
  1 | // Code generated by protoc-gen-go.
  2 | // source: github.com/fd/gcloud-golang/bigtable/internal/duration_proto/duration.proto
  3 | // DO NOT EDIT!
  4 | 
  5 | /*
  6 | Package google_protobuf is a generated protocol buffer package.
  7 | 
  8 | It is generated from these files:
  9 | 	github.com/fd/gcloud-golang/bigtable/internal/duration_proto/duration.proto
 10 | 
 11 | It has these top-level messages:
 12 | 	Duration
 13 | */
 14 | package google_protobuf
 15 | 
 16 | import proto "github.com/golang/protobuf/proto"
 17 | import fmt "fmt"
 18 | import math "math"
 19 | 
 20 | // Reference imports to suppress errors if they are not otherwise used.
 21 | var _ = proto.Marshal
 22 | var _ = fmt.Errorf
 23 | var _ = math.Inf
 24 | 
 25 | // A Duration represents a signed, fixed-length span of time represented
 26 | // as a count of seconds and fractions of seconds at nanosecond
 27 | // resolution. It is independent of any calendar and concepts like "day"
 28 | // or "month". It is related to Timestamp in that the difference between
 29 | // two Timestamp values is a Duration and it can be added or subtracted
 30 | // from a Timestamp. Range is approximately +-10,000 years.
 31 | //
 32 | // Example 1: Compute Duration from two Timestamps in pseudo code.
 33 | //
 34 | //     Timestamp start = ...;
 35 | //     Timestamp end = ...;
 36 | //     Duration duration = ...;
 37 | //
 38 | //     duration.seconds = end.seconds - start.seconds;
 39 | //     duration.nanos = end.nanos - start.nanos;
 40 | //
 41 | //     if (duration.seconds < 0 && duration.nanos > 0) {
 42 | //       duration.seconds += 1;
 43 | //       duration.nanos -= 1000000000;
 44 | //     } else if (durations.seconds > 0 && duration.nanos < 0) {
 45 | //       duration.seconds -= 1;
 46 | //       duration.nanos += 1000000000;
 47 | //     }
 48 | //
 49 | // Example 2: Compute Timestamp from Timestamp + Duration in pseudo code.
 50 | //
 51 | //     Timestamp start = ...;
 52 | //     Duration duration = ...;
 53 | //     Timestamp end = ...;
 54 | //
 55 | //     end.seconds = start.seconds + duration.seconds;
 56 | //     end.nanos = start.nanos + duration.nanos;
 57 | //
 58 | //     if (end.nanos < 0) {
 59 | //       end.seconds -= 1;
 60 | //       end.nanos += 1000000000;
 61 | //     } else if (end.nanos >= 1000000000) {
 62 | //       end.seconds += 1;
 63 | //       end.nanos -= 1000000000;
 64 | //     }
 65 | //
 66 | type Duration struct {
 67 | 	// Signed seconds of the span of time. Must be from -315,576,000,000
 68 | 	// to +315,576,000,000 inclusive.
 69 | 	Seconds int64 `protobuf:"varint,1,opt,name=seconds" json:"seconds,omitempty"`
 70 | 	// Signed fractions of a second at nanosecond resolution of the span
 71 | 	// of time. Durations less than one second are represented with a 0
 72 | 	// `seconds` field and a positive or negative `nanos` field. For durations
 73 | 	// of one second or more, a non-zero value for the `nanos` field must be
 74 | 	// of the same sign as the `seconds` field. Must be from -999,999,999
 75 | 	// to +999,999,999 inclusive.
 76 | 	Nanos int32 `protobuf:"varint,2,opt,name=nanos" json:"nanos,omitempty"`
 77 | }
 78 | 
 79 | func (m *Duration) Reset()                    { *m = Duration{} }
 80 | func (m *Duration) String() string            { return proto.CompactTextString(m) }
 81 | func (*Duration) ProtoMessage()               {}
 82 | func (*Duration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
 83 | 
 84 | func init() {
 85 | 	proto.RegisterType((*Duration)(nil), "google.protobuf.Duration")
 86 | }
 87 | 
 88 | var fileDescriptor0 = []byte{
 89 | 	// 160 bytes of a gzipped FileDescriptorProto
 90 | 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x72, 0x4f, 0xcf, 0xcf, 0x4f,
 91 | 	0xcf, 0x49, 0xd5, 0x4b, 0xcf, 0xcf, 0x49, 0xcc, 0x4b, 0xd7, 0xcb, 0x2f, 0x4a, 0xd7, 0x4f, 0xce,
 92 | 	0xc9, 0x2f, 0x4d, 0xd1, 0x4f, 0xca, 0x4c, 0x2f, 0x49, 0x4c, 0xca, 0x49, 0xd5, 0xcf, 0xcc, 0x2b,
 93 | 	0x49, 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x4f, 0x29, 0x2d, 0x4a, 0x2c, 0xc9, 0xcc, 0xcf, 0x8b, 0x2f,
 94 | 	0x28, 0xca, 0x2f, 0xc9, 0x87, 0x73, 0xf5, 0xc0, 0x5c, 0x21, 0x7e, 0xa8, 0x41, 0x60, 0x5e, 0x52,
 95 | 	0x69, 0x9a, 0x92, 0x16, 0x17, 0x87, 0x0b, 0x54, 0x89, 0x10, 0x3f, 0x17, 0x7b, 0x71, 0x6a, 0x72,
 96 | 	0x7e, 0x5e, 0x4a, 0xb1, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0xb3, 0x10, 0x2f, 0x17, 0x6b, 0x5e, 0x62,
 97 | 	0x5e, 0x7e, 0xb1, 0x04, 0x13, 0x90, 0xcb, 0xea, 0xa4, 0xc9, 0x25, 0x9c, 0x9c, 0x9f, 0xab, 0x87,
 98 | 	0x66, 0x84, 0x13, 0x2f, 0xcc, 0x80, 0x00, 0x90, 0x48, 0x00, 0xe3, 0x02, 0x46, 0xc6, 0x24, 0x36,
 99 | 	0xb0, 0xac, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xc3, 0x14, 0xb7, 0x46, 0xb9, 0x00, 0x00, 0x00,
100 | }
101 | 


--------------------------------------------------------------------------------
/internal/transport/dial.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package transport
 16 | 
 17 | import (
 18 | 	"errors"
 19 | 	"fmt"
 20 | 	"net/http"
 21 | 
 22 | 	"golang.org/x/net/context"
 23 | 	"golang.org/x/oauth2"
 24 | 	"golang.org/x/oauth2/google"
 25 | 	"github.com/fd/gcloud-golang"
 26 | 	"github.com/fd/gcloud-golang/internal/opts"
 27 | 	"google.golang.org/grpc"
 28 | 	"google.golang.org/grpc/credentials"
 29 | 	"google.golang.org/grpc/credentials/oauth"
 30 | )
 31 | 
 32 | // ErrHTTP is returned when on a non-200 HTTP response.
 33 | type ErrHTTP struct {
 34 | 	StatusCode int
 35 | 	Body       []byte
 36 | 	err        error
 37 | }
 38 | 
 39 | func (e *ErrHTTP) Error() string {
 40 | 	if e.err == nil {
 41 | 		return fmt.Sprintf("error during call, http status code: %v %s", e.StatusCode, e.Body)
 42 | 	}
 43 | 	return e.err.Error()
 44 | }
 45 | 
 46 | // NewHTTPClient returns an HTTP client for use communicating with a Google cloud
 47 | // service, configured with the given ClientOptions. It also returns the endpoint
 48 | // for the service as specified in the options.
 49 | func NewHTTPClient(ctx context.Context, opt ...cloud.ClientOption) (*http.Client, string, error) {
 50 | 	var o opts.DialOpt
 51 | 	for _, opt := range opt {
 52 | 		opt.Resolve(&o)
 53 | 	}
 54 | 	if o.GRPCClient != nil {
 55 | 		return nil, "", errors.New("unsupported GRPC base transport specified")
 56 | 	}
 57 | 	// TODO(djd): Wrap all http.Clients with appropriate internal version to add
 58 | 	// UserAgent header and prepend correct endpoint.
 59 | 	if o.HTTPClient != nil {
 60 | 		return o.HTTPClient, o.Endpoint, nil
 61 | 	}
 62 | 	if o.TokenSource == nil {
 63 | 		var err error
 64 | 		o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...)
 65 | 		if err != nil {
 66 | 			return nil, "", fmt.Errorf("google.DefaultTokenSource: %v", err)
 67 | 		}
 68 | 	}
 69 | 	return oauth2.NewClient(ctx, o.TokenSource), o.Endpoint, nil
 70 | }
 71 | 
 72 | // NewProtoClient returns a ProtoClient for communicating with a Google cloud service,
 73 | // configured with the given ClientOptions.
 74 | func NewProtoClient(ctx context.Context, opt ...cloud.ClientOption) (*ProtoClient, error) {
 75 | 	var o opts.DialOpt
 76 | 	for _, opt := range opt {
 77 | 		opt.Resolve(&o)
 78 | 	}
 79 | 	if o.GRPCClient != nil {
 80 | 		return nil, errors.New("unsupported GRPC base transport specified")
 81 | 	}
 82 | 	var client *http.Client
 83 | 	switch {
 84 | 	case o.HTTPClient != nil:
 85 | 		if o.TokenSource != nil {
 86 | 			return nil, errors.New("at most one of WithTokenSource or WithBaseHTTP may be provided")
 87 | 		}
 88 | 		client = o.HTTPClient
 89 | 	case o.TokenSource != nil:
 90 | 		client = oauth2.NewClient(ctx, o.TokenSource)
 91 | 	default:
 92 | 		var err error
 93 | 		client, err = google.DefaultClient(ctx, o.Scopes...)
 94 | 		if err != nil {
 95 | 			return nil, err
 96 | 		}
 97 | 	}
 98 | 
 99 | 	return &ProtoClient{
100 | 		client:    client,
101 | 		endpoint:  o.Endpoint,
102 | 		userAgent: o.UserAgent,
103 | 	}, nil
104 | }
105 | 
106 | // DialGRPC returns a GRPC connection for use communicating with a Google cloud
107 | // service, configured with the given ClientOptions.
108 | func DialGRPC(ctx context.Context, opt ...cloud.ClientOption) (*grpc.ClientConn, error) {
109 | 	var o opts.DialOpt
110 | 	for _, opt := range opt {
111 | 		opt.Resolve(&o)
112 | 	}
113 | 	if o.HTTPClient != nil {
114 | 		return nil, errors.New("unsupported HTTP base transport specified")
115 | 	}
116 | 	if o.GRPCClient != nil {
117 | 		return o.GRPCClient, nil
118 | 	}
119 | 	if o.TokenSource == nil {
120 | 		var err error
121 | 		o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...)
122 | 		if err != nil {
123 | 			return nil, fmt.Errorf("google.DefaultTokenSource: %v", err)
124 | 		}
125 | 	}
126 | 	grpcOpts := []grpc.DialOption{
127 | 		grpc.WithPerRPCCredentials(oauth.TokenSource{o.TokenSource}),
128 | 		grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")),
129 | 	}
130 | 	grpcOpts = append(grpcOpts, o.GRPCDialOpts...)
131 | 	if o.UserAgent != "" {
132 | 		grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent))
133 | 	}
134 | 	return grpc.Dial(o.Endpoint, grpcOpts...)
135 | }
136 | 


--------------------------------------------------------------------------------
/bigquery/gcs.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import bq "google.golang.org/api/bigquery/v2"
 18 | 
 19 | // GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute
 20 | // an input or output to a BigQuery operation.
 21 | type GCSReference struct {
 22 | 	uris []string
 23 | 
 24 | 	// FieldDelimiter is the separator for fields in a CSV file, used when loading or exporting data.
 25 | 	// The default is ",".
 26 | 	FieldDelimiter string
 27 | 
 28 | 	// The number of rows at the top of a CSV file that BigQuery will skip when loading the data.
 29 | 	SkipLeadingRows int64
 30 | 
 31 | 	// SourceFormat is the format of the GCS data to be loaded into BigQuery.
 32 | 	// Allowed values are: CSV, JSON, DatastoreBackup.  The default is CSV.
 33 | 	SourceFormat DataFormat
 34 | 	// Only used when loading data.
 35 | 	Encoding Encoding
 36 | 
 37 | 	// Quote is the value used to quote data sections in a CSV file.
 38 | 	// The default quotation character is the double quote ("), which is used if both Quote and ForceZeroQuote are unset.
 39 | 	// To specify that no character should be interpreted as a quotation character, set ForceZeroQuote to true.
 40 | 	// Only used when loading data.
 41 | 	Quote          string
 42 | 	ForceZeroQuote bool
 43 | 
 44 | 	// DestinationFormat is the format to use when writing exported files.
 45 | 	// Allowed values are: CSV, Avro, JSON.  The default is CSV.
 46 | 	// CSV is not supported for tables with nested or repeated fields.
 47 | 	DestinationFormat DataFormat
 48 | 	// Only used when writing data.  Default is None.
 49 | 	Compression Compression
 50 | }
 51 | 
 52 | func (gcs *GCSReference) implementsSource()      {}
 53 | func (gcs *GCSReference) implementsDestination() {}
 54 | 
 55 | // NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination.
 56 | // In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object.
 57 | // Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided.
 58 | // Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name.
 59 | // For more information about the treatment of wildcards and multiple URIs,
 60 | // see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple
 61 | func (c *Client) NewGCSReference(uri ...string) *GCSReference {
 62 | 	return &GCSReference{uris: uri}
 63 | }
 64 | 
 65 | type DataFormat string
 66 | 
 67 | const (
 68 | 	CSV             DataFormat = "CSV"
 69 | 	Avro            DataFormat = "AVRO"
 70 | 	JSON            DataFormat = "NEWLINE_DELIMITED_JSON"
 71 | 	DatastoreBackup DataFormat = "DATASTORE_BACKUP"
 72 | )
 73 | 
 74 | // Encoding specifies the character encoding of data to be loaded into BigQuery.
 75 | // See https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.encoding
 76 | // for more details about how this is used.
 77 | type Encoding string
 78 | 
 79 | const (
 80 | 	UTF_8      Encoding = "UTF-8"
 81 | 	ISO_8859_1 Encoding = "ISO-8859-1"
 82 | )
 83 | 
 84 | // Compression is the type of compression to apply when writing data to Google Cloud Storage.
 85 | type Compression string
 86 | 
 87 | const (
 88 | 	None Compression = "NONE"
 89 | 	Gzip Compression = "GZIP"
 90 | )
 91 | 
 92 | func (gcs *GCSReference) customizeLoadSrc(conf *bq.JobConfigurationLoad, projectID string) {
 93 | 	conf.SourceUris = gcs.uris
 94 | 	conf.SkipLeadingRows = gcs.SkipLeadingRows
 95 | 	conf.SourceFormat = string(gcs.SourceFormat)
 96 | 	conf.Encoding = string(gcs.Encoding)
 97 | 	conf.FieldDelimiter = gcs.FieldDelimiter
 98 | 
 99 | 	if gcs.ForceZeroQuote {
100 | 		quote := ""
101 | 		conf.Quote = &quote
102 | 	} else if gcs.Quote != "" {
103 | 		conf.Quote = &gcs.Quote
104 | 	}
105 | }
106 | 
107 | func (gcs *GCSReference) customizeExtractDst(conf *bq.JobConfigurationExtract, projectID string) {
108 | 	conf.DestinationUris = gcs.uris
109 | 	conf.Compression = string(gcs.Compression)
110 | 	conf.DestinationFormat = string(gcs.DestinationFormat)
111 | 	conf.FieldDelimiter = gcs.FieldDelimiter
112 | }
113 | 


--------------------------------------------------------------------------------
/bigquery/value.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"errors"
 19 | 	"fmt"
 20 | 	"strconv"
 21 | 	"time"
 22 | 
 23 | 	bq "google.golang.org/api/bigquery/v2"
 24 | )
 25 | 
 26 | // Value stores the contents of a single cell from a BigQuery result.
 27 | type Value interface{}
 28 | 
 29 | // ValueLoader stores a slice of Values representing a result row from a Read operation.
 30 | // See Iterator.Get for more information.
 31 | type ValueLoader interface {
 32 | 	Load(v []Value) error
 33 | }
 34 | 
 35 | // ValueList converts a []Value to implement ValueLoader.
 36 | type ValueList []Value
 37 | 
 38 | // Load stores a sequence of values in a ValueList.
 39 | func (vs *ValueList) Load(v []Value) error {
 40 | 	*vs = append(*vs, v...)
 41 | 	return nil
 42 | }
 43 | 
 44 | // convertRows converts a series of TableRows into a series of Value slices.
 45 | // schema is used to interpret the data from rows; its length must match the
 46 | // length of each row.
 47 | func convertRows(rows []*bq.TableRow, schema Schema) ([][]Value, error) {
 48 | 	var rs [][]Value
 49 | 	for _, r := range rows {
 50 | 		row, err := convertRow(r, schema)
 51 | 		if err != nil {
 52 | 			return nil, err
 53 | 		}
 54 | 		rs = append(rs, row)
 55 | 	}
 56 | 	return rs, nil
 57 | }
 58 | 
 59 | func convertRow(r *bq.TableRow, schema Schema) ([]Value, error) {
 60 | 	if len(schema) != len(r.F) {
 61 | 		return nil, errors.New("schema length does not match row length")
 62 | 	}
 63 | 	var values []Value
 64 | 	for i, cell := range r.F {
 65 | 		fs := schema[i]
 66 | 		v, err := convertValue(cell.V, fs.Type, fs.Schema)
 67 | 		if err != nil {
 68 | 			return nil, err
 69 | 		}
 70 | 		values = append(values, v)
 71 | 	}
 72 | 	return values, nil
 73 | }
 74 | 
 75 | func convertValue(val interface{}, typ FieldType, schema Schema) (Value, error) {
 76 | 	switch val := val.(type) {
 77 | 	case nil:
 78 | 		return nil, nil
 79 | 	case []interface{}:
 80 | 		return convertRepeatedRecord(val, typ, schema)
 81 | 	case map[string]interface{}:
 82 | 		return convertNestedRecord(val, schema)
 83 | 	case string:
 84 | 		return convertBasicType(val, typ)
 85 | 	default:
 86 | 		return nil, fmt.Errorf("got value %v; expected a value of type %s", val, typ)
 87 | 	}
 88 | }
 89 | 
 90 | func convertRepeatedRecord(vals []interface{}, typ FieldType, schema Schema) (Value, error) {
 91 | 	var values []Value
 92 | 	for _, cell := range vals {
 93 | 		// each cell contains a single entry, keyed by "v"
 94 | 		val := cell.(map[string]interface{})["v"]
 95 | 		v, err := convertValue(val, typ, schema)
 96 | 		if err != nil {
 97 | 			return nil, err
 98 | 		}
 99 | 		values = append(values, v)
100 | 	}
101 | 	return values, nil
102 | }
103 | 
104 | func convertNestedRecord(val map[string]interface{}, schema Schema) (Value, error) {
105 | 	// convertNestedRecord is similar to convertRow, as a record has the same structure as a row.
106 | 
107 | 	// Nested records are wrapped in a map with a single key, "f".
108 | 	record := val["f"].([]interface{})
109 | 	if len(record) != len(schema) {
110 | 		return nil, errors.New("schema length does not match record length")
111 | 	}
112 | 
113 | 	var values []Value
114 | 	for i, cell := range record {
115 | 		// each cell contains a single entry, keyed by "v"
116 | 		val := cell.(map[string]interface{})["v"]
117 | 
118 | 		fs := schema[i]
119 | 		v, err := convertValue(val, fs.Type, fs.Schema)
120 | 		if err != nil {
121 | 			return nil, err
122 | 		}
123 | 		values = append(values, v)
124 | 	}
125 | 	return values, nil
126 | }
127 | 
128 | // convertBasicType returns val as an interface with a concrete type specified by typ.
129 | func convertBasicType(val string, typ FieldType) (Value, error) {
130 | 	switch typ {
131 | 	case StringFieldType:
132 | 		return val, nil
133 | 	case IntegerFieldType:
134 | 		return strconv.Atoi(val)
135 | 	case FloatFieldType:
136 | 		return strconv.ParseFloat(val, 64)
137 | 	case BooleanFieldType:
138 | 		return strconv.ParseBool(val)
139 | 	case TimestampFieldType:
140 | 		f, err := strconv.ParseFloat(val, 64)
141 | 		return Value(time.Unix(0, int64(f*1e9))), err
142 | 	default:
143 | 		return nil, errors.New("unrecognized type")
144 | 	}
145 | }
146 | 


--------------------------------------------------------------------------------
/bigquery/schema_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"reflect"
 19 | 	"testing"
 20 | 
 21 | 	bq "google.golang.org/api/bigquery/v2"
 22 | )
 23 | 
 24 | func bqTableFieldSchema(desc, name, typ, mode string) *bq.TableFieldSchema {
 25 | 	return &bq.TableFieldSchema{
 26 | 		Description: desc,
 27 | 		Name:        name,
 28 | 		Mode:        mode,
 29 | 		Type:        typ,
 30 | 	}
 31 | }
 32 | 
 33 | func fieldSchema(desc, name, typ string, repeated, required bool) *FieldSchema {
 34 | 	return &FieldSchema{
 35 | 		Description: desc,
 36 | 		Name:        name,
 37 | 		Repeated:    repeated,
 38 | 		Required:    required,
 39 | 		Type:        FieldType(typ),
 40 | 	}
 41 | }
 42 | 
 43 | func TestSchemaConversion(t *testing.T) {
 44 | 	testCases := []struct {
 45 | 		schema   Schema
 46 | 		bqSchema *bq.TableSchema
 47 | 	}{
 48 | 		{
 49 | 			// required
 50 | 			bqSchema: &bq.TableSchema{
 51 | 				Fields: []*bq.TableFieldSchema{
 52 | 					bqTableFieldSchema("desc", "name", "STRING", "REQUIRED"),
 53 | 				},
 54 | 			},
 55 | 			schema: Schema{
 56 | 				fieldSchema("desc", "name", "STRING", false, true),
 57 | 			},
 58 | 		},
 59 | 		{
 60 | 			// repeated
 61 | 			bqSchema: &bq.TableSchema{
 62 | 				Fields: []*bq.TableFieldSchema{
 63 | 					bqTableFieldSchema("desc", "name", "STRING", "REPEATED"),
 64 | 				},
 65 | 			},
 66 | 			schema: Schema{
 67 | 				fieldSchema("desc", "name", "STRING", true, false),
 68 | 			},
 69 | 		},
 70 | 		{
 71 | 			// nullable, string
 72 | 			bqSchema: &bq.TableSchema{
 73 | 				Fields: []*bq.TableFieldSchema{
 74 | 					bqTableFieldSchema("desc", "name", "STRING", ""),
 75 | 				},
 76 | 			},
 77 | 			schema: Schema{
 78 | 				fieldSchema("desc", "name", "STRING", false, false),
 79 | 			},
 80 | 		},
 81 | 		{
 82 | 			// integer
 83 | 			bqSchema: &bq.TableSchema{
 84 | 				Fields: []*bq.TableFieldSchema{
 85 | 					bqTableFieldSchema("desc", "name", "INTEGER", ""),
 86 | 				},
 87 | 			},
 88 | 			schema: Schema{
 89 | 				fieldSchema("desc", "name", "INTEGER", false, false),
 90 | 			},
 91 | 		},
 92 | 		{
 93 | 			// float
 94 | 			bqSchema: &bq.TableSchema{
 95 | 				Fields: []*bq.TableFieldSchema{
 96 | 					bqTableFieldSchema("desc", "name", "FLOAT", ""),
 97 | 				},
 98 | 			},
 99 | 			schema: Schema{
100 | 				fieldSchema("desc", "name", "FLOAT", false, false),
101 | 			},
102 | 		},
103 | 		{
104 | 			// boolean
105 | 			bqSchema: &bq.TableSchema{
106 | 				Fields: []*bq.TableFieldSchema{
107 | 					bqTableFieldSchema("desc", "name", "BOOLEAN", ""),
108 | 				},
109 | 			},
110 | 			schema: Schema{
111 | 				fieldSchema("desc", "name", "BOOLEAN", false, false),
112 | 			},
113 | 		},
114 | 		{
115 | 			// timestamp
116 | 			bqSchema: &bq.TableSchema{
117 | 				Fields: []*bq.TableFieldSchema{
118 | 					bqTableFieldSchema("desc", "name", "TIMESTAMP", ""),
119 | 				},
120 | 			},
121 | 			schema: Schema{
122 | 				fieldSchema("desc", "name", "TIMESTAMP", false, false),
123 | 			},
124 | 		},
125 | 		{
126 | 			// nested
127 | 			bqSchema: &bq.TableSchema{
128 | 				Fields: []*bq.TableFieldSchema{
129 | 					&bq.TableFieldSchema{
130 | 						Description: "An outer schema wrapping a nested schema",
131 | 						Name:        "outer",
132 | 						Mode:        "REQUIRED",
133 | 						Type:        "RECORD",
134 | 						Fields: []*bq.TableFieldSchema{
135 | 							bqTableFieldSchema("inner field", "inner", "STRING", ""),
136 | 						},
137 | 					},
138 | 				},
139 | 			},
140 | 			schema: Schema{
141 | 				&FieldSchema{
142 | 					Description: "An outer schema wrapping a nested schema",
143 | 					Name:        "outer",
144 | 					Required:    true,
145 | 					Type:        "RECORD",
146 | 					Schema: []*FieldSchema{
147 | 						&FieldSchema{
148 | 							Description: "inner field",
149 | 							Name:        "inner",
150 | 							Type:        "STRING",
151 | 						},
152 | 					},
153 | 				},
154 | 			},
155 | 		},
156 | 	}
157 | 
158 | 	for _, tc := range testCases {
159 | 		bqSchema := tc.schema.asTableSchema()
160 | 		if !reflect.DeepEqual(bqSchema, tc.bqSchema) {
161 | 			t.Errorf("converting to TableSchema: got:\n%v\nwant:\n%v", bqSchema, tc.bqSchema)
162 | 		}
163 | 		schema := convertTableSchema(tc.bqSchema)
164 | 		if !reflect.DeepEqual(schema, tc.schema) {
165 | 			t.Errorf("converting to Schema: got:\n%v\nwant:\n%v", schema, tc.schema)
166 | 		}
167 | 	}
168 | }
169 | 


--------------------------------------------------------------------------------
/bigquery/bigquery.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | // TODO(mcgreevy): support dry-run mode when creating jobs.
 18 | 
 19 | import (
 20 | 	"fmt"
 21 | 	"net/http"
 22 | 
 23 | 	"golang.org/x/net/context"
 24 | 	bq "google.golang.org/api/bigquery/v2"
 25 | )
 26 | 
 27 | // A Source is a source of data for the Copy function.
 28 | type Source interface {
 29 | 	implementsSource()
 30 | }
 31 | 
 32 | // A Destination is a destination of data for the Copy function.
 33 | type Destination interface {
 34 | 	implementsDestination()
 35 | }
 36 | 
 37 | // An Option is an optional argument to Copy.
 38 | type Option interface {
 39 | 	implementsOption()
 40 | }
 41 | 
 42 | // A ReadSource is a source of data for the Read function.
 43 | type ReadSource interface {
 44 | 	implementsReadSource()
 45 | }
 46 | 
 47 | // A ReadOption is an optional argument to Read.
 48 | type ReadOption interface {
 49 | 	customizeRead(conf *pagingConf)
 50 | }
 51 | 
 52 | const Scope = "https://www.googleapis.com/auth/bigquery"
 53 | 
 54 | // Client may be used to perform BigQuery operations.
 55 | type Client struct {
 56 | 	service   service
 57 | 	projectID string
 58 | }
 59 | 
 60 | // Note: many of the methods on *Client appear in the various *_op.go source files.
 61 | 
 62 | // NewClient constructs a new Client which can perform BigQuery operations.
 63 | // Operations performed via the client are billed to the specified GCP project.
 64 | // The supplied http.Client is used for making requests to the BigQuery server and must be capable of
 65 | // authenticating requests with Scope.
 66 | func NewClient(client *http.Client, projectID string) (*Client, error) {
 67 | 	bqService, err := newBigqueryService(client)
 68 | 	if err != nil {
 69 | 		return nil, fmt.Errorf("constructing bigquery client: %v", err)
 70 | 	}
 71 | 
 72 | 	c := &Client{
 73 | 		service:   bqService,
 74 | 		projectID: projectID,
 75 | 	}
 76 | 	return c, nil
 77 | }
 78 | 
 79 | // initJobProto creates and returns a bigquery Job proto.
 80 | // The proto is customized using any jobOptions in options.
 81 | // The list of Options is returned with the jobOptions removed.
 82 | func initJobProto(projectID string, options []Option) (*bq.Job, []Option) {
 83 | 	job := &bq.Job{}
 84 | 
 85 | 	var other []Option
 86 | 	for _, opt := range options {
 87 | 		if o, ok := opt.(jobOption); ok {
 88 | 			o.customizeJob(job, projectID)
 89 | 		} else {
 90 | 			other = append(other, opt)
 91 | 		}
 92 | 	}
 93 | 	return job, other
 94 | }
 95 | 
 96 | // Copy starts a BigQuery operation to copy data from a Source to a Destination.
 97 | func (c *Client) Copy(ctx context.Context, dst Destination, src Source, options ...Option) (*Job, error) {
 98 | 	switch dst := dst.(type) {
 99 | 	case *Table:
100 | 		switch src := src.(type) {
101 | 		case *GCSReference:
102 | 			return c.load(ctx, dst, src, options)
103 | 		case *Table:
104 | 			return c.cp(ctx, dst, Tables{src}, options)
105 | 		case Tables:
106 | 			return c.cp(ctx, dst, src, options)
107 | 		case *Query:
108 | 			return c.query(ctx, dst, src, options)
109 | 		}
110 | 	case *GCSReference:
111 | 		if src, ok := src.(*Table); ok {
112 | 			return c.extract(ctx, dst, src, options)
113 | 		}
114 | 	}
115 | 	return nil, fmt.Errorf("no Copy operation matches dst/src pair: dst: %T ; src: %T", dst, src)
116 | }
117 | 
118 | // Read fetches data from a ReadSource and returns the data via an Iterator.
119 | func (c *Client) Read(ctx context.Context, src ReadSource, options ...ReadOption) (*Iterator, error) {
120 | 	switch src := src.(type) {
121 | 	case *Job:
122 | 		return c.readQueryResults(src, options)
123 | 	case *Query:
124 | 		return c.executeQuery(ctx, src, options...)
125 | 	case *Table:
126 | 		return c.readTable(src, options)
127 | 	}
128 | 	return nil, fmt.Errorf("src (%T) does not support the Read operation", src)
129 | }
130 | 
131 | // executeQuery submits a query for execution and returns the results via an Iterator.
132 | func (c *Client) executeQuery(ctx context.Context, q *Query, options ...ReadOption) (*Iterator, error) {
133 | 	dest := &Table{}
134 | 	job, err := c.Copy(ctx, dest, q, WriteTruncate)
135 | 	if err != nil {
136 | 		return nil, err
137 | 	}
138 | 
139 | 	return c.Read(ctx, job, options...)
140 | }
141 | 
142 | func (c *Client) Dataset(id string) *Dataset {
143 | 	return &Dataset{
144 | 		id:     id,
145 | 		client: c,
146 | 	}
147 | }
148 | 


--------------------------------------------------------------------------------
/bigtable/internal/cluster_service_proto/bigtable_cluster_service_messages.proto:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2015, Google Inc.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | syntax = "proto3";
 16 | 
 17 | package google.bigtable.admin.cluster.v1;
 18 | 
 19 | import "google.golang.org/cloud/bigtable/internal/cluster_data_proto/bigtable_cluster_data.proto";
 20 | 
 21 | option java_multiple_files = true;
 22 | option java_outer_classname = "BigtableClusterServiceMessagesProto";
 23 | option java_package = "com.google.bigtable.admin.cluster.v1";
 24 | 
 25 | 
 26 | // Request message for BigtableClusterService.ListZones.
 27 | message ListZonesRequest {
 28 |   // The unique name of the project for which a list of supported zones is
 29 |   // requested.
 30 |   // Values are of the form projects/<project>
 31 |   string name = 1;
 32 | }
 33 | 
 34 | // Response message for BigtableClusterService.ListZones.
 35 | message ListZonesResponse {
 36 |   // The list of requested zones.
 37 |   repeated Zone zones = 1;
 38 | }
 39 | 
 40 | // Request message for BigtableClusterService.GetCluster.
 41 | message GetClusterRequest {
 42 |   // The unique name of the requested cluster.
 43 |   // Values are of the form projects/<project>/zones/<zone>/clusters/<cluster>
 44 |   string name = 1;
 45 | }
 46 | 
 47 | // Request message for BigtableClusterService.ListClusters.
 48 | message ListClustersRequest {
 49 |   // The unique name of the project for which a list of clusters is requested.
 50 |   // Values are of the form projects/<project>
 51 |   string name = 1;
 52 | }
 53 | 
 54 | // Response message for BigtableClusterService.ListClusters.
 55 | message ListClustersResponse {
 56 |   // The list of requested Clusters.
 57 |   repeated Cluster clusters = 1;
 58 | 
 59 |   // The zones for which clusters could not be retrieved.
 60 |   repeated Zone failed_zones = 2;
 61 | }
 62 | 
 63 | // Request message for BigtableClusterService.CreateCluster.
 64 | message CreateClusterRequest {
 65 |   // The unique name of the zone in which to create the cluster.
 66 |   // Values are of the form projects/<project>/zones/<zone>
 67 |   string name = 1;
 68 | 
 69 |   // The id to be used when referring to the new cluster within its zone,
 70 |   // e.g. just the "test-cluster" section of the full name
 71 |   // "projects/<project>/zones/<zone>/clusters/test-cluster".
 72 |   string cluster_id = 2;
 73 | 
 74 |   // The cluster to create.
 75 |   // The "name", "delete_time", and "current_operation" fields must be left
 76 |   // blank.
 77 |   Cluster cluster = 3;
 78 | }
 79 | 
 80 | // Metadata type for the operation returned by
 81 | // BigtableClusterService.CreateCluster.
 82 | message CreateClusterMetadata {
 83 |   // The request which prompted the creation of this operation.
 84 |   CreateClusterRequest original_request = 1;
 85 | 
 86 |   // The time at which original_request was received.
 87 | 
 88 |   // The time at which this operation failed or was completed successfully.
 89 | }
 90 | 
 91 | // Metadata type for the operation returned by
 92 | // BigtableClusterService.UpdateCluster.
 93 | message UpdateClusterMetadata {
 94 |   // The request which prompted the creation of this operation.
 95 |   Cluster original_request = 1;
 96 | 
 97 |   // The time at which original_request was received.
 98 | 
 99 |   // The time at which this operation was cancelled. If set, this operation is
100 |   // in the process of undoing itself (which is guaranteed to succeed) and
101 |   // cannot be cancelled again.
102 | 
103 |   // The time at which this operation failed or was completed successfully.
104 | }
105 | 
106 | // Request message for BigtableClusterService.DeleteCluster.
107 | message DeleteClusterRequest {
108 |   // The unique name of the cluster to be deleted.
109 |   // Values are of the form projects/<project>/zones/<zone>/clusters/<cluster>
110 |   string name = 1;
111 | }
112 | 
113 | // Request message for BigtableClusterService.UndeleteCluster.
114 | message UndeleteClusterRequest {
115 |   // The unique name of the cluster to be un-deleted.
116 |   // Values are of the form projects/<project>/zones/<zone>/clusters/<cluster>
117 |   string name = 1;
118 | }
119 | 
120 | // Metadata type for the operation returned by
121 | // BigtableClusterService.UndeleteCluster.
122 | message UndeleteClusterMetadata {
123 |   // The time at which the original request was received.
124 | 
125 |   // The time at which this operation failed or was completed successfully.
126 | }
127 | 


--------------------------------------------------------------------------------
/bigtable/cmd/loadtest/loadtest.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | /*
 18 | Loadtest does some load testing through the Go client library for Cloud Bigtable.
 19 | */
 20 | package main
 21 | 
 22 | import (
 23 | 	"bytes"
 24 | 	"flag"
 25 | 	"fmt"
 26 | 	"log"
 27 | 	"math/rand"
 28 | 	"os"
 29 | 	"sync"
 30 | 	"sync/atomic"
 31 | 	"time"
 32 | 
 33 | 	"golang.org/x/net/context"
 34 | 	"github.com/fd/gcloud-golang/bigtable"
 35 | 	"github.com/fd/gcloud-golang/bigtable/internal/cbtrc"
 36 | )
 37 | 
 38 | var (
 39 | 	runFor       = flag.Duration("run_for", 5*time.Second, "how long to run the load test for")
 40 | 	scratchTable = flag.String("scratch_table", "loadtest-scratch", "name of table to use; should not already exist")
 41 | 
 42 | 	config      *cbtrc.Config
 43 | 	client      *bigtable.Client
 44 | 	adminClient *bigtable.AdminClient
 45 | )
 46 | 
 47 | func main() {
 48 | 	var err error
 49 | 	config, err = cbtrc.Load()
 50 | 	if err != nil {
 51 | 		log.Fatal(err)
 52 | 	}
 53 | 	config.RegisterFlags()
 54 | 
 55 | 	flag.Parse()
 56 | 	if err := config.CheckFlags(); err != nil {
 57 | 		log.Fatal(err)
 58 | 	}
 59 | 	if config.Creds != "" {
 60 | 		os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.Creds)
 61 | 	}
 62 | 	if flag.NArg() != 0 {
 63 | 		flag.Usage()
 64 | 		os.Exit(1)
 65 | 	}
 66 | 
 67 | 	log.Printf("Dialing connections...")
 68 | 	client, err = bigtable.NewClient(context.Background(), config.Project, config.Zone, config.Cluster)
 69 | 	if err != nil {
 70 | 		log.Fatalf("Making bigtable.Client: %v", err)
 71 | 	}
 72 | 	defer client.Close()
 73 | 	adminClient, err = bigtable.NewAdminClient(context.Background(), config.Project, config.Zone, config.Cluster)
 74 | 	if err != nil {
 75 | 		log.Fatalf("Making bigtable.AdminClient: %v", err)
 76 | 	}
 77 | 	defer adminClient.Close()
 78 | 
 79 | 	// Create a scratch table.
 80 | 	log.Printf("Setting up scratch table...")
 81 | 	if err := adminClient.CreateTable(context.Background(), *scratchTable); err != nil {
 82 | 		log.Fatalf("Making scratch table %q: %v", *scratchTable, err)
 83 | 	}
 84 | 	if err := adminClient.CreateColumnFamily(context.Background(), *scratchTable, "f"); err != nil {
 85 | 		log.Fatalf("Making scratch table column family: %v", err)
 86 | 	}
 87 | 	// Upon a successful run, delete the table. Don't bother checking for errors.
 88 | 	defer adminClient.DeleteTable(context.Background(), *scratchTable)
 89 | 
 90 | 	log.Printf("Starting load test... (run for %v)", *runFor)
 91 | 	tbl := client.Open(*scratchTable)
 92 | 	sem := make(chan int, 100) // limit the number of requests happening at once
 93 | 	var reads, writes stats
 94 | 	stopTime := time.Now().Add(*runFor)
 95 | 	var wg sync.WaitGroup
 96 | 	for time.Now().Before(stopTime) {
 97 | 		sem <- 1
 98 | 		wg.Add(1)
 99 | 		go func() {
100 | 			defer wg.Done()
101 | 			defer func() { <-sem }()
102 | 
103 | 			ok := true
104 | 			opStart := time.Now()
105 | 			var stats *stats
106 | 			defer func() {
107 | 				stats.Record(ok, time.Since(opStart))
108 | 			}()
109 | 
110 | 			row := fmt.Sprintf("row%d", rand.Intn(100)) // operate on 1 of 100 rows
111 | 
112 | 			switch rand.Intn(10) {
113 | 			default:
114 | 				// read
115 | 				stats = &reads
116 | 				_, err := tbl.ReadRow(context.Background(), row, bigtable.RowFilter(bigtable.LatestNFilter(1)))
117 | 				if err != nil {
118 | 					log.Printf("Error doing read: %v", err)
119 | 					ok = false
120 | 				}
121 | 			case 0, 1, 2, 3, 4:
122 | 				// write
123 | 				stats = &writes
124 | 				mut := bigtable.NewMutation()
125 | 				mut.Set("f", "col", bigtable.Now(), bytes.Repeat([]byte("0"), 1<<10)) // 1 KB write
126 | 				if err := tbl.Apply(context.Background(), row, mut); err != nil {
127 | 					log.Printf("Error doing mutation: %v", err)
128 | 					ok = false
129 | 				}
130 | 			}
131 | 		}()
132 | 	}
133 | 	wg.Wait()
134 | 
135 | 	log.Printf("Reads (%d ok / %d tries):\n%v", reads.ok, reads.tries, newAggregate(reads.ds))
136 | 	log.Printf("Writes (%d ok / %d tries):\n%v", writes.ok, writes.tries, newAggregate(writes.ds))
137 | }
138 | 
139 | var allStats int64 // atomic
140 | 
141 | type stats struct {
142 | 	mu        sync.Mutex
143 | 	tries, ok int
144 | 	ds        []time.Duration
145 | }
146 | 
147 | func (s *stats) Record(ok bool, d time.Duration) {
148 | 	s.mu.Lock()
149 | 	s.tries++
150 | 	if ok {
151 | 		s.ok++
152 | 	}
153 | 	s.ds = append(s.ds, d)
154 | 	s.mu.Unlock()
155 | 
156 | 	if n := atomic.AddInt64(&allStats, 1); n%1000 == 0 {
157 | 		log.Printf("Progress: done %d ops", n)
158 | 	}
159 | }
160 | 


--------------------------------------------------------------------------------
/bigtable/doc.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | /*
 18 | Package bigtable is an API to Google Cloud Bigtable.
 19 | 
 20 | See https://cloud.google.com/bigtable/docs/ for general product documentation.
 21 | 
 22 | Setup and Credentials
 23 | 
 24 | Use NewClient or NewAdminClient to create a client that can be used to access
 25 | the data or admin APIs respectively. Both require credentials that have permission
 26 | to access the Cloud Bigtable API.
 27 | 
 28 | If your program is run on Google App Engine or Google Compute Engine, using the Application Default Credentials
 29 | (https://developers.google.com/accounts/docs/application-default-credentials)
 30 | is the simplest option. Those credentials will be used by default when NewClient or NewAdminClient are called.
 31 | 
 32 | To use alternate credentials, pass them to NewClient or NewAdminClient using cloud.WithTokenSource.
 33 | For instance, you can use service account credentials by visiting
 34 | https://cloud.google.com/console/project/MYPROJECT/apiui/credential,
 35 | creating a new OAuth "Client ID", storing the JSON key somewhere accessible, and writing
 36 | 	jsonKey, err := ioutil.ReadFile(pathToKeyFile)
 37 | 	...
 38 | 	config, err := google.JWTConfigFromJSON(jsonKey, bigtable.Scope) // or bigtable.AdminScope, etc.
 39 | 	...
 40 | 	client, err := bigtable.NewClient(ctx, project, zone, cluster, cloud.WithTokenSource(config.TokenSource(ctx)))
 41 | 	...
 42 | Here, `google` means the golang.org/x/oauth2/google package
 43 | and `cloud` means the github.com/fd/gcloud-golang package.
 44 | 
 45 | Reading
 46 | 
 47 | The principal way to read from a Bigtable is to use the ReadRows method on *Table.
 48 | A RowRange specifies a contiguous portion of a table. A Filter may be provided through
 49 | RowFilter to limit or transform the data that is returned.
 50 | 	tbl := client.Open("mytable")
 51 | 	...
 52 | 	// Read all the rows starting with "com.google.",
 53 | 	// but only fetch the columns in the "links" family.
 54 | 	rr := bigtable.PrefixRange("com.google.")
 55 | 	err := tbl.ReadRows(ctx, rr, func(r Row) bool {
 56 | 		// do something with r
 57 | 		return true // keep going
 58 | 	}, bigtable.RowFilter(bigtable.FamilyFilter("links")))
 59 | 	...
 60 | 
 61 | To read a single row, use the ReadRow helper method.
 62 | 	r, err := tbl.ReadRow(ctx, "com.google.cloud") // "com.google.cloud" is the entire row key
 63 | 	...
 64 | 
 65 | Writing
 66 | 
 67 | This API exposes two distinct forms of writing to a Bigtable: a Mutation and a ReadModifyWrite.
 68 | The former expresses idempotent operations.
 69 | The latter expresses non-idempotent operations and returns the new values of updated cells.
 70 | These operations are performed by creating a Mutation or ReadModifyWrite (with NewMutation or NewReadModifyWrite),
 71 | building up one or more operations on that, and then using the Apply or ApplyReadModifyWrite
 72 | methods on a Table.
 73 | 
 74 | For instance, to set a couple of cells in a table,
 75 | 	tbl := client.Open("mytable")
 76 | 	mut := bigtable.NewMutation()
 77 | 	mut.Set("links", "maps.google.com", bigtable.Now(), []byte("1"))
 78 | 	mut.Set("links", "golang.org", bigtable.Now(), []byte("1"))
 79 | 	err := tbl.Apply(ctx, "com.google.cloud", mut)
 80 | 	...
 81 | 
 82 | To increment an encoded value in one cell,
 83 | 	tbl := client.Open("mytable")
 84 | 	rmw := bigtable.NewReadModifyWrite()
 85 | 	rmw.Increment("links", "golang.org", 12) // add 12 to the cell in column "links:golang.org"
 86 | 	r, err := tbl.ApplyReadModifyWrite(ctx, "com.google.cloud", rmw)
 87 | 	...
 88 | */
 89 | package bigtable // import "github.com/fd/gcloud-golang/bigtable"
 90 | 
 91 | // Scope constants for authentication credentials.
 92 | // These should be used when using credential creation functions such as oauth.NewServiceAccountFromFile.
 93 | const (
 94 | 	// Scope is the OAuth scope for Cloud Bigtable data operations.
 95 | 	Scope = "https://www.googleapis.com/auth/bigtable.data"
 96 | 	// ReadonlyScope is the OAuth scope for Cloud Bigtable read-only data operations.
 97 | 	ReadonlyScope = "https://www.googleapis.com/auth/bigtable.readonly"
 98 | 
 99 | 	// AdminScope is the OAuth scope for Cloud Bigtable table admin operations.
100 | 	AdminScope = "https://www.googleapis.com/auth/bigtable.admin.table"
101 | 
102 | 	// ClusterAdminScope is the OAuth scope for Cloud Bigtable cluster admin operations.
103 | 	ClusterAdminScope = "https://www.googleapis.com/auth/bigtable.admin.cluster"
104 | )
105 | 
106 | // clientUserAgent identifies the version of this package.
107 | // It should be bumped upon significant changes only.
108 | const clientUserAgent = "cbt-go/20150727"
109 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
  1 | # Contributing
  2 | 
  3 | 1. Sign one of the contributor license agreements below.
  4 | 1. `go get golang.org/x/review/git-codereview` to install the code reviewing tool.
  5 | 1. Get the cloud package by running `go get -d google.golang.org/cloud`.
  6 |     1. If you have already checked out the source, make sure that the remote git
  7 |        origin is https://code.googlesource.com/gocloud:
  8 | 
  9 |             git remote set-url origin https://code.googlesource.com/gocloud
 10 | 1. Make changes and create a change by running `git codereview change <name>`,
 11 | provide a command message, and use `git codereview mail` to create a Gerrit CL.
 12 | 1. Keep amending to the change and mail as your recieve feedback.
 13 | 
 14 | ## Integration Tests
 15 | 
 16 | Additional to the unit tests, you may run the integration test suite.
 17 | 
 18 | To run the integrations tests, creating and configuration of a project in the
 19 | Google Developers Console is required. Once you create a project, set the
 20 | following environment variables to be able to run the against the actual APIs.
 21 | 
 22 | - **GCLOUD_TESTS_GOLANG_PROJECT_ID**: Developers Console project's ID (e.g. bamboo-shift-455)
 23 | - **GCLOUD_TESTS_GOLANG_KEY**: The path to the JSON key file.
 24 | 
 25 | Create a storage bucket with the same name as the project id set in **GCLOUD_TESTS_GOLANG_PROJECT_ID**.
 26 | The storage integration test will create and delete some objects in this bucket.
 27 | 
 28 | Install the [gcloud command-line tool][gcloudcli] to your machine and use it
 29 | to create the indexes used in the datastore integration tests with indexes
 30 | found in `datastore/testdata/index.yaml`:
 31 | 
 32 | From the project's root directory:
 33 | 
 34 | ``` sh
 35 | # Install the app component
 36 | $ gcloud components update app
 37 | 
 38 | # Set the default project in your env
 39 | $ gcloud config set project $GCLOUD_TESTS_GOLANG_PROJECT_ID
 40 | 
 41 | # Authenticate the gcloud tool with your account
 42 | $ gcloud auth login
 43 | 
 44 | # Create the indexes
 45 | $ gcloud preview datastore create-indexes datastore/testdata/index.yaml
 46 | 
 47 | ```
 48 | 
 49 | You can run the integration tests by running:
 50 | 
 51 | ``` sh
 52 | $ go test -v -tags=integration google.golang.org/cloud/...
 53 | ```
 54 | 
 55 | ## Contributor License Agreements
 56 | 
 57 | Before we can accept your pull requests you'll need to sign a Contributor
 58 | License Agreement (CLA):
 59 | 
 60 | - **If you are an individual writing original source code** and **you own the
 61 | - intellectual property**, then you'll need to sign an [individual CLA][indvcla].
 62 | - **If you work for a company that wants to allow you to contribute your work**,
 63 | then you'll need to sign a [corporate CLA][corpcla].
 64 | 
 65 | You can sign these electronically (just scroll to the bottom). After that,
 66 | we'll be able to accept your pull requests.
 67 | 
 68 | ## Contributor Code of Conduct
 69 | 
 70 | As contributors and maintainers of this project,
 71 | and in the interest of fostering an open and welcoming community,
 72 | we pledge to respect all people who contribute through reporting issues,
 73 | posting feature requests, updating documentation,
 74 | submitting pull requests or patches, and other activities.
 75 | 
 76 | We are committed to making participation in this project
 77 | a harassment-free experience for everyone,
 78 | regardless of level of experience, gender, gender identity and expression,
 79 | sexual orientation, disability, personal appearance,
 80 | body size, race, ethnicity, age, religion, or nationality.
 81 | 
 82 | Examples of unacceptable behavior by participants include:
 83 | 
 84 | * The use of sexualized language or imagery
 85 | * Personal attacks
 86 | * Trolling or insulting/derogatory comments
 87 | * Public or private harassment
 88 | * Publishing other's private information,
 89 | such as physical or electronic
 90 | addresses, without explicit permission
 91 | * Other unethical or unprofessional conduct.
 92 | 
 93 | Project maintainers have the right and responsibility to remove, edit, or reject
 94 | comments, commits, code, wiki edits, issues, and other contributions
 95 | that are not aligned to this Code of Conduct.
 96 | By adopting this Code of Conduct,
 97 | project maintainers commit themselves to fairly and consistently
 98 | applying these principles to every aspect of managing this project.
 99 | Project maintainers who do not follow or enforce the Code of Conduct
100 | may be permanently removed from the project team.
101 | 
102 | This code of conduct applies both within project spaces and in public spaces
103 | when an individual is representing the project or its community.
104 | 
105 | Instances of abusive, harassing, or otherwise unacceptable behavior
106 | may be reported by opening an issue
107 | or contacting one or more of the project maintainers.
108 | 
109 | This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0,
110 | available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/)
111 | 
112 | [gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/
113 | [indvcla]: https://developers.google.com/open-source/cla/individual
114 | [corpcla]: https://developers.google.com/open-source/cla/corporate
115 | 


--------------------------------------------------------------------------------
/storage/example_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2014 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package storage_test
 16 | 
 17 | import (
 18 | 	"io/ioutil"
 19 | 	"log"
 20 | 
 21 | 	"golang.org/x/net/context"
 22 | 	"golang.org/x/oauth2/google"
 23 | 	"github.com/fd/gcloud-golang"
 24 | 	"github.com/fd/gcloud-golang/storage"
 25 | )
 26 | 
 27 | func Example_auth() {
 28 | 	// Initialize an authorized context with Google Developers Console
 29 | 	// JSON key. Read the google package examples to learn more about
 30 | 	// different authorization flows you can use.
 31 | 	// http://godoc.org/golang.org/x/oauth2/google
 32 | 	jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
 33 | 	if err != nil {
 34 | 		log.Fatal(err)
 35 | 	}
 36 | 	conf, err := google.JWTConfigFromJSON(
 37 | 		jsonKey,
 38 | 		storage.ScopeFullControl,
 39 | 	)
 40 | 	if err != nil {
 41 | 		log.Fatal(err)
 42 | 	}
 43 | 	ctx := context.Background()
 44 | 	client, err := storage.NewClient(ctx, cloud.WithTokenSource(conf.TokenSource(ctx)))
 45 | 	if err != nil {
 46 | 		log.Fatal(err)
 47 | 	}
 48 | 
 49 | 	// Use the client (see other examples)
 50 | 	doSomething(client)
 51 | 
 52 | 	// After using the client, free any resources (e.g. network connections).
 53 | 	client.Close()
 54 | }
 55 | 
 56 | func ExampleListObjects() {
 57 | 	ctx := context.Background()
 58 | 	var client *storage.Client // See Example (Auth)
 59 | 
 60 | 	var query *storage.Query
 61 | 	for {
 62 | 		// If you are using this package on App Engine Managed VMs runtime,
 63 | 		// you can init a bucket client with your app's default bucket name.
 64 | 		// See http://godoc.org/google.golang.org/appengine/file#DefaultBucketName.
 65 | 		objects, err := client.Bucket("bucketname").List(ctx, query)
 66 | 		if err != nil {
 67 | 			log.Fatal(err)
 68 | 		}
 69 | 		for _, obj := range objects.Results {
 70 | 			log.Printf("object name: %s, size: %v", obj.Name, obj.Size)
 71 | 		}
 72 | 		// If there are more results, objects.Next will be non-nil.
 73 | 		if objects.Next == nil {
 74 | 			break
 75 | 		}
 76 | 		query = objects.Next
 77 | 	}
 78 | 
 79 | 	log.Println("paginated through all object items in the bucket you specified.")
 80 | }
 81 | 
 82 | func ExampleNewReader() {
 83 | 	ctx := context.Background()
 84 | 	var client *storage.Client // See Example (Auth)
 85 | 
 86 | 	rc, err := client.Bucket("bucketname").Object("filename1").NewReader(ctx)
 87 | 	if err != nil {
 88 | 		log.Fatal(err)
 89 | 	}
 90 | 	slurp, err := ioutil.ReadAll(rc)
 91 | 	rc.Close()
 92 | 	if err != nil {
 93 | 		log.Fatal(err)
 94 | 	}
 95 | 
 96 | 	log.Println("file contents:", slurp)
 97 | }
 98 | 
 99 | func ExampleNewWriter() {
100 | 	ctx := context.Background()
101 | 	var client *storage.Client // See Example (Auth)
102 | 
103 | 	wc := client.Bucket("bucketname").Object("filename1").NewWriter(ctx)
104 | 	wc.ContentType = "text/plain"
105 | 	wc.ACL = []storage.ACLRule{{storage.AllUsers, storage.RoleReader}}
106 | 	if _, err := wc.Write([]byte("hello world")); err != nil {
107 | 		log.Fatal(err)
108 | 	}
109 | 	if err := wc.Close(); err != nil {
110 | 		log.Fatal(err)
111 | 	}
112 | 	log.Println("updated object:", wc.Attrs())
113 | }
114 | 
115 | func ExampleCopyObject() {
116 | 	ctx := context.Background()
117 | 	var client *storage.Client // See Example (Auth)
118 | 
119 | 	o, err := client.CopyObject(ctx, "bucketname", "file1", "another-bucketname", "file2", nil)
120 | 	if err != nil {
121 | 		log.Fatal(err)
122 | 	}
123 | 	log.Println("copied file:", o)
124 | }
125 | 
126 | func ExampleDeleteObject() {
127 | 	ctx := context.Background()
128 | 	var client *storage.Client // See Example (Auth)
129 | 
130 | 	// To delete multiple objects in a bucket, first List then Delete them.
131 | 
132 | 	// If you are using this package on App Engine Managed VMs runtime,
133 | 	// you can init a bucket client with your app's default bucket name.
134 | 	// See http://godoc.org/google.golang.org/appengine/file#DefaultBucketName.
135 | 	bucket := client.Bucket("bucketname")
136 | 
137 | 	var query *storage.Query // Set up query as desired.
138 | 	for {
139 | 		objects, err := bucket.List(ctx, query)
140 | 		if err != nil {
141 | 			log.Fatal(err)
142 | 		}
143 | 		for _, obj := range objects.Results {
144 | 			log.Printf("deleting object name: %q, size: %v", obj.Name, obj.Size)
145 | 			if err := bucket.Object(obj.Name).Delete(ctx); err != nil {
146 | 				log.Fatalf("unable to delete %q: %v", obj.Name, err)
147 | 			}
148 | 		}
149 | 		// If there are more results, objects.Next will be non-nil.
150 | 		if objects.Next == nil {
151 | 			break
152 | 		}
153 | 		query = objects.Next
154 | 	}
155 | 
156 | 	log.Println("deleted all object items in the bucket specified.")
157 | }
158 | 
159 | func doSomething(c *storage.Client) {}
160 | 


--------------------------------------------------------------------------------
/bigtable/internal/table_data_proto/bigtable_table_data.proto:
--------------------------------------------------------------------------------
  1 | // Copyright (c) 2015, Google Inc.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | syntax = "proto3";
 16 | 
 17 | package google.bigtable.admin.table.v1;
 18 | 
 19 | import "google.golang.org/cloud/bigtable/internal/duration_proto/duration.proto";
 20 | 
 21 | option java_multiple_files = true;
 22 | option java_outer_classname = "BigtableTableDataProto";
 23 | option java_package = "com.google.bigtable.admin.table.v1";
 24 | 
 25 | 
 26 | // A collection of user data indexed by row, column, and timestamp.
 27 | // Each table is served using the resources of its parent cluster.
 28 | message Table {
 29 |   enum TimestampGranularity {
 30 |     MILLIS = 0;
 31 |   }
 32 | 
 33 |   // A unique identifier of the form
 34 |   // <cluster_name>/tables/[_a-zA-Z0-9][-_.a-zA-Z0-9]*
 35 |   string name = 1;
 36 | 
 37 |   // If this Table is in the process of being created, the Operation used to
 38 |   // track its progress. As long as this operation is present, the Table will
 39 |   // not accept any Table Admin or Read/Write requests.
 40 | 
 41 |   // The column families configured for this table, mapped by column family id.
 42 |   map<string, ColumnFamily> column_families = 3;
 43 | 
 44 |   // The granularity (e.g. MILLIS, MICROS) at which timestamps are stored in
 45 |   // this table. Timestamps not matching the granularity will be rejected.
 46 |   // Cannot be changed once the table is created.
 47 |   TimestampGranularity granularity = 4;
 48 | }
 49 | 
 50 | // A set of columns within a table which share a common configuration.
 51 | message ColumnFamily {
 52 |   // A unique identifier of the form <table_name>/columnFamilies/[-_.a-zA-Z0-9]+
 53 |   // The last segment is the same as the "name" field in
 54 |   // google.bigtable.v1.Family.
 55 |   string name = 1;
 56 | 
 57 |   // Garbage collection expression specified by the following grammar:
 58 |   //   GC = EXPR
 59 |   //      | "" ;
 60 |   //   EXPR = EXPR, "||", EXPR              (* lowest precedence *)
 61 |   //        | EXPR, "&&", EXPR
 62 |   //        | "(", EXPR, ")"                (* highest precedence *)
 63 |   //        | PROP ;
 64 |   //   PROP = "version() >", NUM32
 65 |   //        | "age() >", NUM64, [ UNIT ] ;
 66 |   //   NUM32 = non-zero-digit { digit } ;    (* # NUM32 <= 2^32 - 1 *)
 67 |   //   NUM64 = non-zero-digit { digit } ;    (* # NUM64 <= 2^63 - 1 *)
 68 |   //   UNIT =  "d" | "h" | "m"  (* d=days, h=hours, m=minutes, else micros *)
 69 |   // GC expressions can be up to 500 characters in length
 70 |   //
 71 |   // The different types of PROP are defined as follows:
 72 |   //   version() - cell index, counting from most recent and starting at 1
 73 |   //   age() - age of the cell (current time minus cell timestamp)
 74 |   //
 75 |   // Example: "version() > 3 || (age() > 3d && version() > 1)"
 76 |   //   drop cells beyond the most recent three, and drop cells older than three
 77 |   //   days unless they're the most recent cell in the row/column
 78 |   //
 79 |   // Garbage collection executes opportunistically in the background, and so
 80 |   // it's possible for reads to return a cell even if it matches the active GC
 81 |   // expression for its family.
 82 |   string gc_expression = 2;
 83 | 
 84 |   // Garbage collection rule specified as a protobuf.
 85 |   // Supersedes `gc_expression`.
 86 |   // Must serialize to at most 500 bytes.
 87 |   //
 88 |   // NOTE: Garbage collection executes opportunistically in the background, and
 89 |   // so it's possible for reads to return a cell even if it matches the active
 90 |   // GC expression for its family.
 91 |   GcRule gc_rule = 3;
 92 | }
 93 | 
 94 | // Rule for determining which cells to delete during garbage collection.
 95 | message GcRule {
 96 |   // A GcRule which deletes cells matching all of the given rules.
 97 |   message Intersection {
 98 |     // Only delete cells which would be deleted by every element of `rules`.
 99 |     repeated GcRule rules = 1;
100 |   }
101 | 
102 |   // A GcRule which deletes cells matching any of the given rules.
103 |   message Union {
104 |     // Delete cells which would be deleted by any element of `rules`.
105 |     repeated GcRule rules = 1;
106 |   }
107 | 
108 |   oneof rule {
109 |     // Delete all cells in a column except the most recent N.
110 |     int32 max_num_versions = 1;
111 | 
112 |     // Delete cells in a column older than the given age.
113 |     // Values must be at least one millisecond, and will be truncated to
114 |     // microsecond granularity.
115 |     google.protobuf.Duration max_age = 2;
116 | 
117 |     // Delete cells that would be deleted by every nested rule.
118 |     Intersection intersection = 3;
119 | 
120 |     // Delete cells that would be deleted by any nested rule.
121 |     Union union = 4;
122 |   }
123 | }
124 | 


--------------------------------------------------------------------------------
/examples/bigtable/bigtable-hello/helloworld.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All rights reserved.
  2 | // Use of this source code is governed by the Apache 2.0
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | /*
  6 | helloworld tracks how often a user has visited the index page.
  7 | 
  8 | This program demonstrates usage of the Cloud Bigtable API for Managed VMs and Go.
  9 | Instructions for running this program are in the README.md.
 10 | */
 11 | package main
 12 | 
 13 | import (
 14 | 	"bytes"
 15 | 	"encoding/binary"
 16 | 	"html/template"
 17 | 	"log"
 18 | 	"net/http"
 19 | 
 20 | 	"golang.org/x/net/context"
 21 | 	"google.golang.org/appengine"
 22 | 	aelog "google.golang.org/appengine/log"
 23 | 	"google.golang.org/appengine/user"
 24 | 	"github.com/fd/gcloud-golang/bigtable"
 25 | )
 26 | 
 27 | // User-provided constants.
 28 | const (
 29 | 	project = "PROJECT_ID"
 30 | 	zone    = "CLUSTER_ZONE"
 31 | 	cluster = "CLUSTER_NAME"
 32 | )
 33 | 
 34 | var (
 35 | 	tableName  = "bigtable-hello"
 36 | 	familyName = "emails"
 37 | 
 38 | 	// Client is initialized by main.
 39 | 	client *bigtable.Client
 40 | )
 41 | 
 42 | func main() {
 43 | 	ctx := context.Background()
 44 | 
 45 | 	// Set up admin client, tables, and column families.
 46 | 	// NewAdminClient uses Application Default Credentials to authenticate.
 47 | 	adminClient, err := bigtable.NewAdminClient(ctx, project, zone, cluster)
 48 | 	if err != nil {
 49 | 		log.Fatalf("Unable to create a table admin client. %v", err)
 50 | 	}
 51 | 	tables, err := adminClient.Tables(ctx)
 52 | 	if err != nil {
 53 | 		log.Fatalf("Unable to fetch table list. %v", err)
 54 | 	}
 55 | 	if !sliceContains(tables, tableName) {
 56 | 		if err := adminClient.CreateTable(ctx, tableName); err != nil {
 57 | 			log.Fatalf("Unable to create table: %v. %v", tableName, err)
 58 | 		}
 59 | 	}
 60 | 	tblInfo, err := adminClient.TableInfo(ctx, tableName)
 61 | 	if err != nil {
 62 | 		log.Fatalf("Unable to read info for table: %v. %v", tableName, err)
 63 | 	}
 64 | 	if !sliceContains(tblInfo.Families, familyName) {
 65 | 		if err := adminClient.CreateColumnFamily(ctx, tableName, familyName); err != nil {
 66 | 			log.Fatalf("Unable to create column family: %v. %v", familyName, err)
 67 | 		}
 68 | 	}
 69 | 	adminClient.Close()
 70 | 
 71 | 	// Set up Bigtable data operations client.
 72 | 	// NewClient uses Application Default Credentials to authenticate.
 73 | 	client, err = bigtable.NewClient(ctx, project, zone, cluster)
 74 | 	if err != nil {
 75 | 		log.Fatalf("Unable to create data operations client. %v", err)
 76 | 	}
 77 | 
 78 | 	http.Handle("/", appHandler(mainHandler))
 79 | 	appengine.Main() // Never returns.
 80 | }
 81 | 
 82 | // mainHandler tracks how many times each user has visited this page.
 83 | func mainHandler(w http.ResponseWriter, r *http.Request) *appError {
 84 | 	if r.URL.Path != "/" {
 85 | 		http.NotFound(w, r)
 86 | 		return nil
 87 | 	}
 88 | 
 89 | 	ctx := appengine.NewContext(r)
 90 | 	u := user.Current(ctx)
 91 | 	if u == nil {
 92 | 		login, err := user.LoginURL(ctx, r.URL.String())
 93 | 		if err != nil {
 94 | 			return &appError{err, "Error finding login URL", http.StatusInternalServerError}
 95 | 		}
 96 | 		http.Redirect(w, r, login, http.StatusFound)
 97 | 		return nil
 98 | 	}
 99 | 	logoutURL, err := user.LogoutURL(ctx, "/")
100 | 	if err != nil {
101 | 		return &appError{err, "Error finding logout URL", http.StatusInternalServerError}
102 | 	}
103 | 
104 | 	// Display hello page.
105 | 	tbl := client.Open(tableName)
106 | 	rmw := bigtable.NewReadModifyWrite()
107 | 	rmw.Increment(familyName, u.Email, 1)
108 | 	row, err := tbl.ApplyReadModifyWrite(ctx, u.Email, rmw)
109 | 	if err != nil {
110 | 		return &appError{err, "Error applying ReadModifyWrite to row: " + u.Email, http.StatusInternalServerError}
111 | 	}
112 | 	data := struct {
113 | 		Username, Logout string
114 | 		Visits           uint64
115 | 	}{
116 | 		Username: u.Email,
117 | 		// Retrieve the most recently edited column.
118 | 		Visits: binary.BigEndian.Uint64(row[familyName][0].Value),
119 | 		Logout: logoutURL,
120 | 	}
121 | 	var buf bytes.Buffer
122 | 	if err := tmpl.Execute(&buf, data); err != nil {
123 | 		return &appError{err, "Error writing template", http.StatusInternalServerError}
124 | 	}
125 | 	buf.WriteTo(w)
126 | 	return nil
127 | }
128 | 
129 | var tmpl = template.Must(template.New("").Parse(`
130 | <html><body>
131 | 
132 | <p>
133 | {{with .Username}} Hello {{.}}{{end}}
134 | {{with .Logout}}<a href="{{.}}">Sign out</a>{{end}}
135 | 
136 | </p>
137 | 
138 | <p>
139 | You have visited {{.Visits}}
140 | </p>
141 | 
142 | </body></html>`))
143 | 
144 | // sliceContains reports whether the provided string is present in the given slice of strings.
145 | func sliceContains(list []string, target string) bool {
146 | 	for _, s := range list {
147 | 		if s == target {
148 | 			return true
149 | 		}
150 | 	}
151 | 	return false
152 | }
153 | 
154 | // More info about this method of error handling can be found at: http://blog.golang.org/error-handling-and-go
155 | type appHandler func(http.ResponseWriter, *http.Request) *appError
156 | 
157 | type appError struct {
158 | 	Error   error
159 | 	Message string
160 | 	Code    int
161 | }
162 | 
163 | func (fn appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
164 | 	if e := fn(w, r); e != nil {
165 | 		ctx := appengine.NewContext(r)
166 | 		aelog.Errorf(ctx, "%v", e.Error)
167 | 		http.Error(w, e.Message, e.Code)
168 | 	}
169 | }
170 | 


--------------------------------------------------------------------------------
/bigquery/load_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package bigquery
 16 | 
 17 | import (
 18 | 	"reflect"
 19 | 	"testing"
 20 | 
 21 | 	"golang.org/x/net/context"
 22 | 
 23 | 	bq "google.golang.org/api/bigquery/v2"
 24 | )
 25 | 
 26 | func defaultLoadJob() *bq.Job {
 27 | 	return &bq.Job{
 28 | 		Configuration: &bq.JobConfiguration{
 29 | 			Load: &bq.JobConfigurationLoad{
 30 | 				DestinationTable: &bq.TableReference{
 31 | 					ProjectId: "project-id",
 32 | 					DatasetId: "dataset-id",
 33 | 					TableId:   "table-id",
 34 | 				},
 35 | 				SourceUris: []string{"uri"},
 36 | 			},
 37 | 		},
 38 | 	}
 39 | }
 40 | 
 41 | func stringFieldSchema() *FieldSchema {
 42 | 	return &FieldSchema{Name: "fieldname", Type: StringFieldType}
 43 | }
 44 | 
 45 | func nestedFieldSchema() *FieldSchema {
 46 | 	return &FieldSchema{
 47 | 		Name:   "nested",
 48 | 		Type:   RecordFieldType,
 49 | 		Schema: Schema{stringFieldSchema()},
 50 | 	}
 51 | }
 52 | 
 53 | func bqStringFieldSchema() *bq.TableFieldSchema {
 54 | 	return &bq.TableFieldSchema{
 55 | 		Name: "fieldname",
 56 | 		Type: "STRING",
 57 | 	}
 58 | }
 59 | 
 60 | func bqNestedFieldSchema() *bq.TableFieldSchema {
 61 | 	return &bq.TableFieldSchema{
 62 | 		Name:   "nested",
 63 | 		Type:   "RECORD",
 64 | 		Fields: []*bq.TableFieldSchema{bqStringFieldSchema()},
 65 | 	}
 66 | }
 67 | 
 68 | func TestLoad(t *testing.T) {
 69 | 	testCases := []struct {
 70 | 		dst     *Table
 71 | 		src     *GCSReference
 72 | 		options []Option
 73 | 		want    *bq.Job
 74 | 	}{
 75 | 		{
 76 | 			dst:  defaultTable,
 77 | 			src:  defaultGCS,
 78 | 			want: defaultLoadJob(),
 79 | 		},
 80 | 		{
 81 | 			dst: defaultTable,
 82 | 			src: defaultGCS,
 83 | 			options: []Option{
 84 | 				MaxBadRecords(1),
 85 | 				AllowJaggedRows(),
 86 | 				AllowQuotedNewlines(),
 87 | 				IgnoreUnknownValues(),
 88 | 			},
 89 | 			want: func() *bq.Job {
 90 | 				j := defaultLoadJob()
 91 | 				j.Configuration.Load.MaxBadRecords = 1
 92 | 				j.Configuration.Load.AllowJaggedRows = true
 93 | 				j.Configuration.Load.AllowQuotedNewlines = true
 94 | 				j.Configuration.Load.IgnoreUnknownValues = true
 95 | 				return j
 96 | 			}(),
 97 | 		},
 98 | 		{
 99 | 			dst: &Table{
100 | 				ProjectID: "project-id",
101 | 				DatasetID: "dataset-id",
102 | 				TableID:   "table-id",
103 | 			},
104 | 			options: []Option{CreateNever, WriteTruncate},
105 | 			src:     defaultGCS,
106 | 			want: func() *bq.Job {
107 | 				j := defaultLoadJob()
108 | 				j.Configuration.Load.CreateDisposition = "CREATE_NEVER"
109 | 				j.Configuration.Load.WriteDisposition = "WRITE_TRUNCATE"
110 | 				return j
111 | 			}(),
112 | 		},
113 | 		{
114 | 			dst: &Table{
115 | 				ProjectID: "project-id",
116 | 				DatasetID: "dataset-id",
117 | 				TableID:   "table-id",
118 | 			},
119 | 			src: defaultGCS,
120 | 			options: []Option{
121 | 				DestinationSchema(Schema{
122 | 					stringFieldSchema(),
123 | 					nestedFieldSchema(),
124 | 				}),
125 | 			},
126 | 			want: func() *bq.Job {
127 | 				j := defaultLoadJob()
128 | 				j.Configuration.Load.Schema = &bq.TableSchema{
129 | 					Fields: []*bq.TableFieldSchema{
130 | 						bqStringFieldSchema(),
131 | 						bqNestedFieldSchema(),
132 | 					}}
133 | 				return j
134 | 			}(),
135 | 		},
136 | 		{
137 | 			dst: defaultTable,
138 | 			src: &GCSReference{
139 | 				uris:            []string{"uri"},
140 | 				SkipLeadingRows: 1,
141 | 				SourceFormat:    JSON,
142 | 				Encoding:        UTF_8,
143 | 				FieldDelimiter:  "\t",
144 | 				Quote:           "-",
145 | 			},
146 | 			want: func() *bq.Job {
147 | 				j := defaultLoadJob()
148 | 				j.Configuration.Load.SkipLeadingRows = 1
149 | 				j.Configuration.Load.SourceFormat = "NEWLINE_DELIMITED_JSON"
150 | 				j.Configuration.Load.Encoding = "UTF-8"
151 | 				j.Configuration.Load.FieldDelimiter = "\t"
152 | 				hyphen := "-"
153 | 				j.Configuration.Load.Quote = &hyphen
154 | 				return j
155 | 			}(),
156 | 		},
157 | 		{
158 | 			dst: defaultTable,
159 | 			src: &GCSReference{
160 | 				uris:  []string{"uri"},
161 | 				Quote: "",
162 | 			},
163 | 			want: func() *bq.Job {
164 | 				j := defaultLoadJob()
165 | 				j.Configuration.Load.Quote = nil
166 | 				return j
167 | 			}(),
168 | 		},
169 | 		{
170 | 			dst: defaultTable,
171 | 			src: &GCSReference{
172 | 				uris:           []string{"uri"},
173 | 				Quote:          "",
174 | 				ForceZeroQuote: true,
175 | 			},
176 | 			want: func() *bq.Job {
177 | 				j := defaultLoadJob()
178 | 				empty := ""
179 | 				j.Configuration.Load.Quote = &empty
180 | 				return j
181 | 			}(),
182 | 		},
183 | 	}
184 | 
185 | 	for _, tc := range testCases {
186 | 		s := &testService{}
187 | 		c := &Client{
188 | 			service: s,
189 | 		}
190 | 		if _, err := c.Copy(context.Background(), tc.dst, tc.src, tc.options...); err != nil {
191 | 			t.Errorf("err calling load: %v", err)
192 | 			continue
193 | 		}
194 | 		if !reflect.DeepEqual(s.Job, tc.want) {
195 | 			t.Errorf("loading: got:\n%v\nwant:\n%v", s.Job, tc.want)
196 | 		}
197 | 	}
198 | }
199 | 


--------------------------------------------------------------------------------
/bigtable/filter.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | Copyright 2015 Google Inc. All Rights Reserved.
  3 | 
  4 | Licensed under the Apache License, Version 2.0 (the "License");
  5 | you may not use this file except in compliance with the License.
  6 | You may obtain a copy of the License at
  7 | 
  8 |     http://www.apache.org/licenses/LICENSE-2.0
  9 | 
 10 | Unless required by applicable law or agreed to in writing, software
 11 | distributed under the License is distributed on an "AS IS" BASIS,
 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | See the License for the specific language governing permissions and
 14 | limitations under the License.
 15 | */
 16 | 
 17 | package bigtable
 18 | 
 19 | import (
 20 | 	"fmt"
 21 | 	"strings"
 22 | 
 23 | 	btdpb "github.com/fd/gcloud-golang/bigtable/internal/data_proto"
 24 | )
 25 | 
 26 | // A Filter represents a row filter.
 27 | type Filter interface {
 28 | 	String() string
 29 | 	proto() *btdpb.RowFilter
 30 | }
 31 | 
 32 | // ChainFilters returns a filter that applies a sequence of filters.
 33 | func ChainFilters(sub ...Filter) Filter { return chainFilter{sub} }
 34 | 
 35 | type chainFilter struct {
 36 | 	sub []Filter
 37 | }
 38 | 
 39 | func (cf chainFilter) String() string {
 40 | 	var ss []string
 41 | 	for _, sf := range cf.sub {
 42 | 		ss = append(ss, sf.String())
 43 | 	}
 44 | 	return "(" + strings.Join(ss, " | ") + ")"
 45 | }
 46 | 
 47 | func (cf chainFilter) proto() *btdpb.RowFilter {
 48 | 	chain := &btdpb.RowFilter_Chain{}
 49 | 	for _, sf := range cf.sub {
 50 | 		chain.Filters = append(chain.Filters, sf.proto())
 51 | 	}
 52 | 	return &btdpb.RowFilter{
 53 | 		Filter: &btdpb.RowFilter_Chain_{chain},
 54 | 	}
 55 | }
 56 | 
 57 | // InterleaveFilters returns a filter that applies a set of filters in parallel
 58 | // and interleaves the results.
 59 | func InterleaveFilters(sub ...Filter) Filter { return interleaveFilter{sub} }
 60 | 
 61 | type interleaveFilter struct {
 62 | 	sub []Filter
 63 | }
 64 | 
 65 | func (ilf interleaveFilter) String() string {
 66 | 	var ss []string
 67 | 	for _, sf := range ilf.sub {
 68 | 		ss = append(ss, sf.String())
 69 | 	}
 70 | 	return "(" + strings.Join(ss, " + ") + ")"
 71 | }
 72 | 
 73 | func (ilf interleaveFilter) proto() *btdpb.RowFilter {
 74 | 	inter := &btdpb.RowFilter_Interleave{}
 75 | 	for _, sf := range ilf.sub {
 76 | 		inter.Filters = append(inter.Filters, sf.proto())
 77 | 	}
 78 | 	return &btdpb.RowFilter{
 79 | 		Filter: &btdpb.RowFilter_Interleave_{inter},
 80 | 	}
 81 | }
 82 | 
 83 | // RowKeyFilter returns a filter that matches cells from rows whose
 84 | // key matches the provided RE2 pattern.
 85 | // See https://github.com/google/re2/wiki/Syntax for the accepted syntax.
 86 | func RowKeyFilter(pattern string) Filter { return rowKeyFilter(pattern) }
 87 | 
 88 | type rowKeyFilter string
 89 | 
 90 | func (rkf rowKeyFilter) String() string { return fmt.Sprintf("row(%s)", string(rkf)) }
 91 | 
 92 | func (rkf rowKeyFilter) proto() *btdpb.RowFilter {
 93 | 	return &btdpb.RowFilter{Filter: &btdpb.RowFilter_RowKeyRegexFilter{[]byte(rkf)}}
 94 | }
 95 | 
 96 | // FamilyFilter returns a filter that matches cells whose family name
 97 | // matches the provided RE2 pattern.
 98 | // See https://github.com/google/re2/wiki/Syntax for the accepted syntax.
 99 | func FamilyFilter(pattern string) Filter { return familyFilter(pattern) }
100 | 
101 | type familyFilter string
102 | 
103 | func (ff familyFilter) String() string { return fmt.Sprintf("col(%s:)", string(ff)) }
104 | 
105 | func (ff familyFilter) proto() *btdpb.RowFilter {
106 | 	return &btdpb.RowFilter{Filter: &btdpb.RowFilter_FamilyNameRegexFilter{string(ff)}}
107 | }
108 | 
109 | // ColumnFilter returns a filter that matches cells whose column name
110 | // matches the provided RE2 pattern.
111 | // See https://github.com/google/re2/wiki/Syntax for the accepted syntax.
112 | func ColumnFilter(pattern string) Filter { return columnFilter(pattern) }
113 | 
114 | type columnFilter string
115 | 
116 | func (cf columnFilter) String() string { return fmt.Sprintf("col(.*:%s)", string(cf)) }
117 | 
118 | func (cf columnFilter) proto() *btdpb.RowFilter {
119 | 	return &btdpb.RowFilter{Filter: &btdpb.RowFilter_ColumnQualifierRegexFilter{[]byte(cf)}}
120 | }
121 | 
122 | // ValueFilter returns a filter that matches cells whose value
123 | // matches the provided RE2 pattern.
124 | // See https://github.com/google/re2/wiki/Syntax for the accepted syntax.
125 | func ValueFilter(pattern string) Filter { return valueFilter(pattern) }
126 | 
127 | type valueFilter string
128 | 
129 | func (vf valueFilter) String() string { return fmt.Sprintf("value_match(%s)", string(vf)) }
130 | 
131 | func (vf valueFilter) proto() *btdpb.RowFilter {
132 | 	return &btdpb.RowFilter{Filter: &btdpb.RowFilter_ValueRegexFilter{[]byte(vf)}}
133 | }
134 | 
135 | // LatestNFilter returns a filter that matches the most recent N cells in each column.
136 | func LatestNFilter(n int) Filter { return latestNFilter(n) }
137 | 
138 | type latestNFilter int32
139 | 
140 | func (lnf latestNFilter) String() string { return fmt.Sprintf("col(*,%d)", lnf) }
141 | 
142 | func (lnf latestNFilter) proto() *btdpb.RowFilter {
143 | 	return &btdpb.RowFilter{Filter: &btdpb.RowFilter_CellsPerColumnLimitFilter{int32(lnf)}}
144 | }
145 | 
146 | // StripValueFilter returns a filter that replaces each value with the empty string.
147 | func StripValueFilter() Filter { return stripValueFilter{} }
148 | 
149 | type stripValueFilter struct{}
150 | 
151 | func (stripValueFilter) String() string { return "strip_value()" }
152 | func (stripValueFilter) proto() *btdpb.RowFilter {
153 | 	return &btdpb.RowFilter{Filter: &btdpb.RowFilter_StripValueTransformer{true}}
154 | }
155 | 
156 | // TODO(dsymonds): More filters: cond, col/ts/value range, sampling
157 | 


--------------------------------------------------------------------------------
/datastore/save.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2014 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //      http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | 
 15 | package datastore
 16 | 
 17 | import (
 18 | 	"errors"
 19 | 	"fmt"
 20 | 	"reflect"
 21 | 	"time"
 22 | 
 23 | 	"github.com/golang/protobuf/proto"
 24 | 	pb "github.com/fd/gcloud-golang/internal/datastore"
 25 | )
 26 | 
 27 | // saveEntity saves an EntityProto into a PropertyLoadSaver or struct pointer.
 28 | func saveEntity(key *Key, src interface{}) (*pb.Entity, error) {
 29 | 	var err error
 30 | 	var props []Property
 31 | 	if e, ok := src.(PropertyLoadSaver); ok {
 32 | 		props, err = e.Save()
 33 | 	} else {
 34 | 		props, err = SaveStruct(src)
 35 | 	}
 36 | 	if err != nil {
 37 | 		return nil, err
 38 | 	}
 39 | 	return propertiesToProto(key, props)
 40 | }
 41 | 
 42 | func saveStructProperty(props *[]Property, name string, noIndex, multiple bool, v reflect.Value) error {
 43 | 	p := Property{
 44 | 		Name:     name,
 45 | 		NoIndex:  noIndex,
 46 | 		Multiple: multiple,
 47 | 	}
 48 | 
 49 | 	switch x := v.Interface().(type) {
 50 | 	case *Key, time.Time:
 51 | 		p.Value = x
 52 | 	default:
 53 | 		switch v.Kind() {
 54 | 		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
 55 | 			p.Value = v.Int()
 56 | 		case reflect.Bool:
 57 | 			p.Value = v.Bool()
 58 | 		case reflect.String:
 59 | 			p.Value = v.String()
 60 | 		case reflect.Float32, reflect.Float64:
 61 | 			p.Value = v.Float()
 62 | 		case reflect.Slice:
 63 | 			if v.Type().Elem().Kind() == reflect.Uint8 {
 64 | 				p.Value = v.Bytes()
 65 | 			}
 66 | 		case reflect.Struct:
 67 | 			if !v.CanAddr() {
 68 | 				return fmt.Errorf("datastore: unsupported struct field: value is unaddressable")
 69 | 			}
 70 | 			sub, err := newStructPLS(v.Addr().Interface())
 71 | 			if err != nil {
 72 | 				return fmt.Errorf("datastore: unsupported struct field: %v", err)
 73 | 			}
 74 | 			return sub.(structPLS).save(props, name, noIndex, multiple)
 75 | 		}
 76 | 	}
 77 | 	if p.Value == nil {
 78 | 		return fmt.Errorf("datastore: unsupported struct field type: %v", v.Type())
 79 | 	}
 80 | 	*props = append(*props, p)
 81 | 	return nil
 82 | }
 83 | 
 84 | func (s structPLS) Save() ([]Property, error) {
 85 | 	var props []Property
 86 | 	if err := s.save(&props, "", false, false); err != nil {
 87 | 		return nil, err
 88 | 	}
 89 | 	return props, nil
 90 | }
 91 | 
 92 | func (s structPLS) save(props *[]Property, prefix string, noIndex, multiple bool) error {
 93 | 	for i, t := range s.codec.byIndex {
 94 | 		if t.name == "-" {
 95 | 			continue
 96 | 		}
 97 | 		name := t.name
 98 | 		if prefix != "" {
 99 | 			name = prefix + name
100 | 		}
101 | 		v := s.v.Field(i)
102 | 		if !v.IsValid() || !v.CanSet() {
103 | 			continue
104 | 		}
105 | 		noIndex1 := noIndex || t.noIndex
106 | 		// For slice fields that aren't []byte, save each element.
107 | 		if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
108 | 			for j := 0; j < v.Len(); j++ {
109 | 				if err := saveStructProperty(props, name, noIndex1, true, v.Index(j)); err != nil {
110 | 					return err
111 | 				}
112 | 			}
113 | 			continue
114 | 		}
115 | 		// Otherwise, save the field itself.
116 | 		if err := saveStructProperty(props, name, noIndex1, multiple, v); err != nil {
117 | 			return err
118 | 		}
119 | 	}
120 | 	return nil
121 | }
122 | 
123 | func propertiesToProto(key *Key, props []Property) (*pb.Entity, error) {
124 | 	e := &pb.Entity{
125 | 		Key: keyToProto(key),
126 | 	}
127 | 	indexedProps := 0
128 | 	prevMultiple := make(map[string]*pb.Property)
129 | 	for _, p := range props {
130 | 		val, err := interfaceToProto(p.Value)
131 | 		if err != "" {
132 | 			return nil, fmt.Errorf("datastore: %s for a Property with Name %q", err, p.Name)
133 | 		}
134 | 		if !p.NoIndex {
135 | 			rVal := reflect.ValueOf(p.Value)
136 | 			if rVal.Kind() == reflect.Slice && rVal.Type().Elem().Kind() != reflect.Uint8 {
137 | 				indexedProps += rVal.Len()
138 | 			} else {
139 | 				indexedProps++
140 | 			}
141 | 		}
142 | 		if indexedProps > maxIndexedProperties {
143 | 			return nil, errors.New("datastore: too many indexed properties")
144 | 		}
145 | 		switch v := p.Value.(type) {
146 | 		case string:
147 | 		case []byte:
148 | 			if len(v) > 1500 && !p.NoIndex {
149 | 				return nil, fmt.Errorf("datastore: cannot index a Property with Name %q", p.Name)
150 | 			}
151 | 		}
152 | 		val.Indexed = proto.Bool(!p.NoIndex)
153 | 		if p.Multiple {
154 | 			x, ok := prevMultiple[p.Name]
155 | 			if !ok {
156 | 				x = &pb.Property{
157 | 					Name:  proto.String(p.Name),
158 | 					Value: &pb.Value{},
159 | 				}
160 | 				prevMultiple[p.Name] = x
161 | 				e.Property = append(e.Property, x)
162 | 			}
163 | 			x.Value.ListValue = append(x.Value.ListValue, val)
164 | 		} else {
165 | 			e.Property = append(e.Property, &pb.Property{
166 | 				Name:  proto.String(p.Name),
167 | 				Value: val,
168 | 			})
169 | 		}
170 | 	}
171 | 	return e, nil
172 | }
173 | 
174 | func interfaceToProto(iv interface{}) (p *pb.Value, errStr string) {
175 | 	val := new(pb.Value)
176 | 	switch v := iv.(type) {
177 | 	case int:
178 | 		val.IntegerValue = proto.Int64(int64(v))
179 | 	case int32:
180 | 		val.IntegerValue = proto.Int64(int64(v))
181 | 	case int64:
182 | 		val.IntegerValue = proto.Int64(v)
183 | 	case bool:
184 | 		val.BooleanValue = proto.Bool(v)
185 | 	case string:
186 | 		val.StringValue = proto.String(v)
187 | 	case float32:
188 | 		val.DoubleValue = proto.Float64(float64(v))
189 | 	case float64:
190 | 		val.DoubleValue = proto.Float64(v)
191 | 	case *Key:
192 | 		if v != nil {
193 | 			val.KeyValue = keyToProto(v)
194 | 		}
195 | 	case time.Time:
196 | 		if v.Before(minTime) || v.After(maxTime) {
197 | 			return nil, fmt.Sprintf("time value out of range")
198 | 		}
199 | 		val.TimestampMicrosecondsValue = proto.Int64(toUnixMicro(v))
200 | 	case []byte:
201 | 		val.BlobValue = v
202 | 	default:
203 | 		if iv != nil {
204 | 			return nil, fmt.Sprintf("invalid Value type %t", iv)
205 | 		}
206 | 	}
207 | 	// TODO(jbd): Support ListValue and EntityValue.
208 | 	// TODO(jbd): Support types whose underlying type is one of the types above.
209 | 	return val, ""
210 | }
211 | 


--------------------------------------------------------------------------------