├── .circleci └── config.yml ├── .github └── dependabot.yml ├── .gitignore ├── LICENSE ├── README.md ├── bigtable ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── bttest │ ├── .gitignore │ ├── inmem.go │ ├── inmem_test.go │ ├── instance_server.go │ ├── leveldb_test.go │ ├── merge_test.go │ ├── remote_test.go │ ├── setup_test.go │ ├── storage.go │ ├── store_btree.go │ ├── store_leveldb.go │ ├── store_leveldb_disk.go │ ├── store_leveldb_mem.go │ ├── timestamp_test.go │ ├── validation.go │ └── validation_test.go ├── cmd │ └── cbtemulator │ │ └── cbtemulator.go ├── go.mod ├── go.sum └── releasing │ ├── README.md │ ├── RELEASE_NOTES.md │ └── do-release.sh ├── examples ├── Makefile ├── bigtable │ └── example_test.go ├── go.mod ├── go.sum └── storage │ └── example_test.go └── storage ├── Dockerfile ├── Makefile ├── cmd └── gcsemulator │ └── gcsemulator.go ├── gcsemu ├── batch.go ├── client.go ├── errors.go ├── filestore.go ├── filestore_test.go ├── gcsemu.go ├── gcsemu_test.go ├── http_wrappers.go ├── memstore.go ├── memstore_test.go ├── meta.go ├── multipart.go ├── parse.go ├── range.go ├── range_test.go ├── raw_http_test.go ├── remote_test.go ├── server.go ├── store.go ├── util.go └── walk.go ├── gcsutil ├── counted_lock.go ├── doc.go ├── gcspagetoken.go ├── gcspagetoken.pb.go ├── gcspagetoken.proto ├── gcspagetoken_test.go ├── transient_lock_map.go └── transient_lock_map_test.go ├── go.mod ├── go.sum └── releasing ├── README.md ├── RELEASE_NOTES.md └── do-release.sh /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | shared_configs: 2 | simple_job_steps: &simple_job_steps 3 | - checkout 4 | - run: 5 | name: Run tests 6 | command: | 7 | make -C bigtable test 8 | make -C storage test 9 | make -C examples test 10 | 11 | # Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference 12 | version: 2.1 13 | jobs: 14 | build-1-22: 15 | working_directory: ~/repo 16 | docker: 17 | - image: cimg/go:1.22 18 | steps: *simple_job_steps 19 | 20 | build-1-23: 21 | working_directory: ~/repo 22 | docker: 23 | - image: cimg/go:1.23 24 | steps: *simple_job_steps 25 | 26 | build-1-24: 27 | working_directory: ~/repo 28 | docker: 29 | - image: cimg/go:1.24 30 | steps: 31 | - checkout 32 | - run: 33 | name: Run tests and linters 34 | command: | 35 | make -C bigtable ci 36 | make -C storage ci 37 | make -C examples ci 38 | 39 | workflows: 40 | pr-build-test: 41 | jobs: 42 | - build-1-22 43 | - build-1-23 44 | - build-1-24 45 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directories: 5 | - "/bigtable" 6 | - "/examples" 7 | - "/storage" 8 | # Check for updates once a week 9 | schedule: 10 | interval: "weekly" 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | dist/ 18 | .idea/ 19 | VERSION 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Engineering at Fullstory 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # emulators 2 | High quality Google Cloud service emulators for local development stacks 3 | 4 | ## Why? 5 | 6 | At Fullstory, our entire product and backend software stack runs in each engineer's local workstation. This high-quality local development experience keeps our engineers happy and productive, because they are able to: 7 | 8 | - build and test features locally 9 | - reproduce and fix bugs quickly and easily 10 | - run high-quality services in unit and integration tests 11 | 12 | Our local development and testing story is simpler when our live code can rely on expected services to exist, and we don't have to write as many mocks. 13 | 14 | Some of the ways we achieve this: 15 | 16 | - Our own backend services operate in a reasonable manner in a local environment. 17 | - Open source, third party services (such as Redis, Zookeeper, or Solr) run locally. 18 | - We emulate Google Cloud infrastructure. 19 | 20 | ## What Google Cloud services do we emulate? 21 | 22 | | Service | Persistence? | Status | Notes | 23 | |----------------------------|--------------|-------------------------|-------------------------------------------------------------------------------------------------------------------------------| 24 | | Google Bigtable | Yes | Shipped, see below | Fork of [bigtable/bttest](https://github.com/googleapis/google-cloud-go/tree/master/bigtable/bttest) | 25 | | Google Cloud Storage (GCS) | Yes | Shipped, see below | Written from scratch | 26 | | Google Pubsub | No | Considering persistence | Vanilla [pubsub/pstest](https://github.com/googleapis/google-cloud-go/tree/master/pubsub/pstest) | 27 | | Google Cloud Functions | n/a | In consideration | Thin wrapper that manages `node` processes. | 28 | | Google Datastore | Yes | - | Google's [Datastore emulator](https://cloud.google.com/datastore/docs/tools/datastore-emulator) (written in Java) works great | 29 | 30 | ## Google Bigtable Emulator 31 | 32 | Our bigtable emulator is a fork of [bigtable/bttest](https://github.com/googleapis/google-cloud-go/tree/master/bigtable/bttest). A summary of the changes we made: 33 | - The core operates directly on Bigtable protobuf types, such as Table and Row, instead of bespoke types. 34 | - The storage layer is pluggable and operates on protos. 35 | - Leveldb is the default storage implementation, and runs either in-memory (transient for unit tests) or on disk (long running, persistence). 36 | 37 | ### Installing 38 | 39 | ```sh 40 | go install github.com/fullstorydev/emulators/bigtable/...@latest 41 | ``` 42 | 43 | ### Running, out of process 44 | 45 | Example, running on a specific port, with persistence: 46 | ```sh 47 | > cbtemulator -port 8888 -dir var/bigtable 48 | Writing to: var/bigtable 49 | Cloud Bigtable emulator running on 127.0.0.1:8888 50 | ``` 51 | 52 | Usage: 53 | ``` 54 | -dir string 55 | if set, use persistence in the given directory 56 | -host string 57 | the address to bind to on the local machine (default "localhost") 58 | -port int 59 | the port number to bind to on the local machine (default 9000) 60 | ``` 61 | 62 | ### Running, in process 63 | 64 | You can link bigtable emulator into existing Go binaries as a drop-in replacement for `bigtable/bttest`. 65 | 66 | For unit tests: 67 | ```go 68 | // start an in-memory leveldb BigTable test server (for unit tests) 69 | srv, err := bttest.NewServer("127.0.0.1:0", grpc.MaxRecvMsgSize(math.MaxInt32)) 70 | if err != nil { 71 | // ... 72 | } 73 | defer srv.Close() 74 | // bigtable.NewClient (via DefaultClientOptions) will look at this env var to figure out what host to talk to 75 | os.Setenv("BIGTABLE_EMULATOR_HOST", svr.Addr) 76 | ``` 77 | 78 | For on-disk persistence: 79 | ```go 80 | // start an leveldb-backed BigTable test server 81 | srv, err := bttest.NewServerWithOptions(fmt.Sprintf("127.0.0.1:%d", *btport), bttest.Options{ 82 | Storage: bttest.LeveldbDiskStorage{ 83 | Root: bigtableStorageDir, 84 | ErrLog: func(err error, msg string) { 85 | // wire into logging 86 | }, 87 | }, 88 | GrpcOpts: []grpc.ServerOption{grpc.MaxRecvMsgSize(maxGrpcMessageSize)}, 89 | }) 90 | ``` 91 | 92 | ### Connecting to the Bigtable emulator from Go 93 | 94 | ```go 95 | // assuming BIGTABLE_EMULATOR_HOST is already set... 96 | conn, err := grpc.Dial(os.Getenv("BIGTABLE_EMULATOR_HOST"), grpc.WithInsecure()) 97 | if err != nil { 98 | // ... 99 | } 100 | defer conn.Close() // only if the life cycle is scoped to this call 101 | 102 | client, err := bigtable.NewClient(ctx, project, instance, option.WithGRPCConn(conn)) 103 | if err != nil { 104 | // ... 105 | } 106 | tbl := client.Open("example") 107 | ``` 108 | 109 | ## Google Cloud Storage Emulator 110 | 111 | Our storage emulator was written in house. 112 | - Supports basic file operations, iteration, attributes, copying, and some conditionals. 113 | - The storage layer is pluggable. 114 | - In memory btree (transient for unit tests) or disk-based storage (long running, persistence). 115 | 116 | ### Installing 117 | 118 | ```sh 119 | go install github.com/fullstorydev/emulators/storage/...@latest 120 | ``` 121 | 122 | ### Running, out of process 123 | 124 | Example, running on a specific port, with persistence: 125 | ```sh 126 | > gcsemulator -port 8888 -dir var/storage 127 | Writing to: var/storage 128 | Cloud Storage emulator running on http://127.0.0.1:8888 129 | ``` 130 | 131 | Usage: 132 | ``` 133 | -dir string 134 | if set, use persistence in the given directory 135 | -host string 136 | the address to bind to on the local machine (default "localhost") 137 | -port int 138 | the port number to bind to on the local machine (default 9000) 139 | ``` 140 | 141 | For unit tests: 142 | ```go 143 | // start an in-memory Storage test server (for unit tests) 144 | svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) 145 | if err != nil { 146 | // ... 147 | } 148 | defer svr.Close() 149 | // gcsemu.NewClient will look at this env var to figure out what host/port to talk to 150 | os.Setenv("GCS_EMULATOR_HOST", svr.Addr) 151 | ``` 152 | 153 | For on-disk persistence: 154 | ```go 155 | // start an on-disk Storage test server 156 | svr, err := gcsemu.NewServer(fmt.Sprintf("127.0.0.1:%d", *port), gcsemu.Options{ 157 | Store: gcsemu.NewFileStore(*gcsDir), 158 | }) 159 | ``` 160 | 161 | ### Connecting to the GCS emulator from Go 162 | 163 | ```go 164 | // assuming GCS_EMULATOR_HOST is already set... 165 | client, err := gcsemu.NewClient(ctx) 166 | if err != nil { 167 | // ... 168 | } 169 | defer client.Close() // only if the life cycle is scoped to this call 170 | ``` 171 | 172 | #### NOTE #### 173 | 174 | Do NOT use `STORAGE_EMULATOR_HOST`, as defined in `cloud.google.com/go/storage`. There are unresolved issues in the Go 175 | client implementation. `STORAGE_EMULATOR_HOST` is supported inconsistently, and even has some bugs that can cause 176 | data races when using the same `*storage.Client` for different types of access. 177 | 178 | See: 179 | - [storage: when using an emulator, it is not possible to use the same Client object for both uploading and other operations #2476](https://github.com/googleapis/google-cloud-go/issues/2476) 180 | - [Storage: empty readHost when STORAGE_EMULATOR_HOST is set to host:port #4444](https://github.com/googleapis/google-cloud-go/issues/4444) 181 | 182 | Instead, use our `gcsemu.NewClient(ctx)` method which swaps out the entire HTTP transport. 183 | -------------------------------------------------------------------------------- /bigtable/.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - binary: cbtemulator 3 | main: ./cmd/cbtemulator 4 | goos: 5 | - linux 6 | - darwin 7 | - windows 8 | goarch: 9 | - amd64 10 | - 386 11 | - arm 12 | - arm64 13 | - s390x 14 | - ppc64le 15 | goarm: 16 | - 5 17 | - 6 18 | - 7 19 | ignore: 20 | - goos: darwin 21 | goarch: 386 22 | - goos: windows 23 | goarch: arm64 24 | - goos: darwin 25 | goarch: arm 26 | - goos: windows 27 | goarch: arm 28 | - goos: darwin 29 | goarch: s390x 30 | - goos: windows 31 | goarch: s390x 32 | - goos: darwin 33 | goarch: ppc64le 34 | - goos: windows 35 | goarch: ppc64le 36 | ldflags: 37 | - -s -w -X main.version=v{{.Version}} 38 | 39 | archives: 40 | - format: tar.gz 41 | name_template: >- 42 | {{ .Binary }}_{{ .Version }}_ 43 | {{- if eq .Os "darwin" }}osx{{ else }}{{ .Os }}{{ end }}_ 44 | {{- if eq .Arch "amd64" }}x86_64 45 | {{- else if eq .Arch "386" }}x86_32 46 | {{- else }}{{ .Arch }}{{ end }} 47 | {{- with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }} 48 | format_overrides: 49 | - goos: windows 50 | format: zip 51 | files: 52 | - LICENSE 53 | -------------------------------------------------------------------------------- /bigtable/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23-alpine As builder 2 | MAINTAINER Fullstory Engineering 3 | 4 | # create non-privileged group and user 5 | RUN addgroup -S emulators && adduser -S emulators -G emulators 6 | RUN mkdir -p /data 7 | # copy just the files/sources we need to build the emulator 8 | WORKDIR /tmp/fullstorydev/bigtable 9 | COPY VERSION *.go go.* /tmp/fullstorydev/bigtable/ 10 | COPY bttest /tmp/fullstorydev/bigtable/bttest 11 | COPY cmd /tmp/fullstorydev/bigtable/cmd 12 | # and build a completely static binary (so we can use 13 | # scratch as basis for the final image) 14 | ENV CGO_ENABLED=0 15 | ENV GO111MODULE=on 16 | RUN go build -o /cbtemulator \ 17 | -ldflags "-w -extldflags \"-static\" -X \"main.version=$(cat VERSION)\"" \ 18 | ./cmd/cbtemulator 19 | 20 | # New FROM so we have a nice'n'tiny image 21 | FROM scratch AS cbtemulator 22 | WORKDIR / 23 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 24 | COPY --from=builder /etc/passwd /etc/passwd 25 | COPY --from=builder /cbtemulator /bin/cbtemulator 26 | COPY --from=builder --chown=emulators /data /data 27 | EXPOSE 9000 28 | USER emulators 29 | ENTRYPOINT ["/bin/cbtemulator", "-port", "9000", "-host", "0.0.0.0", "-dir", "/data"] 30 | -------------------------------------------------------------------------------- /bigtable/LICENSE: -------------------------------------------------------------------------------- 1 | ../LICENSE -------------------------------------------------------------------------------- /bigtable/Makefile: -------------------------------------------------------------------------------- 1 | # dev_build_prefixed_version has the format "bigtable/" 2 | dev_build_prefixed_version=$(shell git describe --tags --always --match 'bigtable/*' --dirty) 3 | # dev_build_version is like dev_build_prefixed_version, but without the "bigtable/" prefix 4 | dev_build_version=$(shell echo $(dev_build_prefixed_version) | sed 's/^bigtable\///') 5 | 6 | .PHONY: ci 7 | ci: deps checkgofmt vet staticcheck ineffassign predeclared golint errcheck test 8 | 9 | .PHONY: deps 10 | deps: 11 | go mod download 12 | 13 | .PHONY: updatedeps 14 | updatedeps: 15 | go get -d -v -t -u -f ./... 16 | go mod tidy 17 | 18 | .PHONY: checkgofmt 19 | checkgofmt: 20 | gofmt -s -l . 21 | @if [ -n "$$(gofmt -s -l .)" ]; then \ 22 | exit 1; \ 23 | fi 24 | 25 | .PHONY: vet 26 | vet: 27 | go vet ./... 28 | 29 | .PHONY: staticcheck 30 | staticcheck: 31 | @go install honnef.co/go/tools/cmd/staticcheck@v0.6.0 32 | staticcheck ./... 33 | 34 | .PHONY: ineffassign 35 | ineffassign: 36 | @go install github.com/gordonklaus/ineffassign@7953dde2c7bf 37 | ineffassign . 38 | 39 | .PHONY: predeclared 40 | predeclared: 41 | @go install github.com/nishanths/predeclared@51e8c974458a0f93dc03fe356f91ae1a6d791e6f 42 | predeclared ./... 43 | 44 | .PHONY: golint 45 | golint: 46 | @go install golang.org/x/lint/golint@v0.0.0-20210508222113-6edffad5e616 47 | golint -min_confidence 0.9 -set_exit_status ./... 48 | 49 | .PHONY: errcheck 50 | errcheck: 51 | @go install github.com/kisielk/errcheck@v1.2.0 52 | errcheck ./... 53 | 54 | .PHONY: test 55 | test: deps 56 | go test -race ./... 57 | 58 | .PHONY: install 59 | install: 60 | go install -ldflags '-X "main.version=dev build $(dev_build_version)"' ./... 61 | 62 | .PHONY: release 63 | release: 64 | @go install github.com/goreleaser/goreleaser@v1.21.0 65 | goreleaser release --clean 66 | 67 | .PHONY: docker 68 | docker: 69 | @echo $(dev_build_version) > VERSION 70 | docker build -t fullstorydev/cbtemulator:$(dev_build_version) . 71 | @rm VERSION 72 | -------------------------------------------------------------------------------- /bigtable/bttest/.gitignore: -------------------------------------------------------------------------------- 1 | test-out 2 | -------------------------------------------------------------------------------- /bigtable/bttest/instance_server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bttest 16 | 17 | import ( 18 | "context" 19 | 20 | btapb "cloud.google.com/go/bigtable/admin/apiv2/adminpb" 21 | iampb "cloud.google.com/go/iam/apiv1/iampb" 22 | "google.golang.org/grpc/codes" 23 | "google.golang.org/grpc/status" 24 | ) 25 | 26 | var _ btapb.BigtableTableAdminServer = (*server)(nil) 27 | var _ btapb.BigtableInstanceAdminServer = (*server)(nil) 28 | 29 | // Must tie-break methods implemented by both BigtableTableAdminServer and BigtableInstanceAdminServer 30 | 31 | func (s *server) GetIamPolicy(_ context.Context, _ *iampb.GetIamPolicyRequest) (*iampb.Policy, error) { 32 | return nil, status.Errorf(codes.Unimplemented, "method GetIamPolicy not implemented") 33 | } 34 | 35 | func (s *server) SetIamPolicy(_ context.Context, _ *iampb.SetIamPolicyRequest) (*iampb.Policy, error) { 36 | return nil, status.Errorf(codes.Unimplemented, "method SetIamPolicy not implemented") 37 | } 38 | 39 | func (s *server) TestIamPermissions(_ context.Context, _ *iampb.TestIamPermissionsRequest) (*iampb.TestIamPermissionsResponse, error) { 40 | return nil, status.Errorf(codes.Unimplemented, "method TestIamPermissions not implemented") 41 | } 42 | -------------------------------------------------------------------------------- /bigtable/bttest/leveldb_test.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "cloud.google.com/go/bigtable" 9 | ) 10 | 11 | var ( 12 | testMeta = []struct { 13 | name string 14 | f func(*testing.T) 15 | }{ 16 | {"TestConcurrentMutationsReadModify", TestConcurrentMutationsReadModify}, 17 | {"TestCreateTableResponse", TestCreateTableResponse}, 18 | {"TestCreateTableWithFamily", TestCreateTableWithFamily}, 19 | {"TestSampleRowKeys", TestSampleRowKeys}, 20 | {"TestTableRowsConcurrent", TestTableRowsConcurrent}, 21 | {"TestModifyColumnFamilies", TestModifyColumnFamilies}, 22 | {"TestDropRowRange", TestDropRowRange}, 23 | {"TestCheckTimestampMaxValue", TestCheckTimestampMaxValue}, 24 | {"TestReadRows", TestReadRows}, 25 | {"TestReadRowsError", TestReadRowsError}, 26 | {"TestReadRowsAfterDeletion", TestReadRowsAfterDeletion}, 27 | {"TestReadRowsOrder", TestReadRowsOrder}, 28 | {"TestReadRowsWithlabelTransformer", TestReadRowsWithlabelTransformer}, 29 | {"TestCheckAndMutateRowWithoutPredicate", TestCheckAndMutateRowWithoutPredicate}, 30 | {"TestCheckAndMutateRowWithPredicate", TestCheckAndMutateRowWithPredicate}, 31 | {"TestServer_ReadModifyWriteRow", TestServer_ReadModifyWriteRow}, 32 | {"TestFilters", TestFilters}, 33 | {"Test_Mutation_DeleteFromColumn", Test_Mutation_DeleteFromColumn}, 34 | {"TestFilterRowWithSingleColumnQualifier", TestFilterRowWithSingleColumnQualifier}, 35 | {"TestValueFilterRowWithAlternationInRegex", TestValueFilterRowWithAlternationInRegex}, 36 | } 37 | ) 38 | 39 | func TestLevelDbMem(t *testing.T) { 40 | clientIntfFuncs[t.Name()] = func(t *testing.T, name string) (context.Context, *clientIntf, bool) { 41 | ctx := context.Background() 42 | 43 | svr := &server{ 44 | tables: make(map[string]*table), 45 | storage: LeveldbMemStorage{}, 46 | clock: func() bigtable.Timestamp { 47 | return 0 48 | }, 49 | } 50 | 51 | cl := &clientIntf{ 52 | parent: fmt.Sprintf("projects/%s/instances/%s", "project", "cluster"), 53 | name: name, 54 | tblName: fmt.Sprintf("projects/%s/instances/%s/tables/%s", "project", "cluster", name), 55 | BigtableClient: btServer2Client{s: svr}, 56 | BigtableTableAdminClient: btServer2AdminClient{s: svr}, 57 | } 58 | 59 | return ctx, cl, false 60 | } 61 | for _, tc := range testMeta { 62 | t.Run(tc.name, tc.f) 63 | } 64 | } 65 | 66 | func TestLevelDbDisk(t *testing.T) { 67 | clientIntfFuncs[t.Name()] = func(t *testing.T, name string) (context.Context, *clientIntf, bool) { 68 | ctx := context.Background() 69 | 70 | svr := &server{ 71 | tables: make(map[string]*table), 72 | storage: LeveldbDiskStorage{Root: "./test-out"}, 73 | clock: func() bigtable.Timestamp { 74 | return 0 75 | }, 76 | } 77 | 78 | cl := &clientIntf{ 79 | parent: fmt.Sprintf("projects/%s/instances/%s", "project", "cluster"), 80 | name: name, 81 | tblName: fmt.Sprintf("projects/%s/instances/%s/tables/%s", "project", "cluster", name), 82 | BigtableClient: btServer2Client{s: svr}, 83 | BigtableTableAdminClient: btServer2AdminClient{s: svr}, 84 | } 85 | 86 | return ctx, cl, false 87 | } 88 | for _, tc := range testMeta { 89 | t.Run(tc.name, tc.f) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /bigtable/bttest/merge_test.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestMergeRanges(t *testing.T) { 10 | // disjoint, start overlap, end overlap, equal, fully contained 11 | type rangeString struct { 12 | start string 13 | end string 14 | } 15 | 16 | tcs := []struct { 17 | desc string 18 | a, b rangeString 19 | want *rangeString 20 | }{ 21 | {"disjoint", 22 | rangeString{"a", "b"}, rangeString{"c", "d"}, nil}, 23 | 24 | {"disjoint infinite", 25 | rangeString{"", "b"}, rangeString{"c", ""}, nil}, 26 | 27 | {"same start", 28 | rangeString{"a", "b"}, rangeString{"a", "d"}, &rangeString{"a", "d"}}, 29 | 30 | {"same start infinite", 31 | rangeString{"", "b"}, rangeString{"", "d"}, &rangeString{"", "d"}}, 32 | 33 | {"same end", 34 | rangeString{"a", "d"}, rangeString{"c", "d"}, &rangeString{"a", "d"}}, 35 | 36 | {"same end infinite", 37 | rangeString{"a", ""}, rangeString{"c", ""}, &rangeString{"a", ""}}, 38 | 39 | {"eq", 40 | rangeString{"a", "d"}, rangeString{"a", "d"}, &rangeString{"a", "d"}}, 41 | 42 | {"eq start infinite", 43 | rangeString{"", "d"}, rangeString{"", "d"}, &rangeString{"", "d"}}, 44 | 45 | {"eq end infinite", 46 | rangeString{"a", ""}, rangeString{"a", ""}, &rangeString{"a", ""}}, 47 | 48 | {"eq both infinite", 49 | rangeString{"", ""}, rangeString{"", ""}, &rangeString{"", ""}}, 50 | 51 | {"a contains b", 52 | rangeString{"a", "d"}, rangeString{"b", "c"}, &rangeString{"a", "d"}}, 53 | 54 | {"a contains b start infinite", 55 | rangeString{"", "d"}, rangeString{"b", "c"}, &rangeString{"", "d"}}, 56 | 57 | {"a contains b end infinite", 58 | rangeString{"a", ""}, rangeString{"b", "c"}, &rangeString{"a", ""}}, 59 | 60 | {"a contains b both infinite", 61 | rangeString{"", ""}, rangeString{"b", "c"}, &rangeString{"", ""}}, 62 | } 63 | 64 | for _, tc := range tcs { 65 | t.Run(tc.desc, func(t *testing.T) { 66 | result := mergeSimpleRanges([]simpleRange{{keyType(tc.a.start), keyType(tc.a.end)}, {keyType(tc.b.start), keyType(tc.b.end)}}) 67 | if tc.want == nil { 68 | if len(result) != 2 { 69 | t.Errorf("expected to not merge, was %d %+v", len(result), result) 70 | } 71 | } else { 72 | if len(result) != 1 { 73 | t.Errorf("expected merge, was %d %+v", len(result), result) 74 | } else { 75 | got := result[0] 76 | if tc.want.start != string(got.start) { 77 | t.Errorf("start want=%q, got=%q", tc.want.start, string(got.start)) 78 | } 79 | if tc.want.end != string(got.end) { 80 | t.Errorf("end want=%q, got=%q", tc.want.end, string(got.end)) 81 | } 82 | } 83 | 84 | } 85 | }) 86 | } 87 | } 88 | 89 | func TestMergeRangesMultiple(t *testing.T) { 90 | got := mergeSimpleRanges(nil) 91 | if len(got) != 0 { 92 | t.Errorf("want=0, got=%d", len(got)) 93 | } 94 | 95 | in := []simpleRange{ 96 | {keyType(""), keyType("a")}, 97 | {keyType("a"), keyType("b")}, // merges 98 | {keyType("c"), keyType("e")}, 99 | {keyType("d"), keyType("e")}, // merges 100 | {keyType("f"), keyType("i")}, 101 | {keyType("g"), keyType("h")}, // merges 102 | {keyType("j"), keyType("k")}, 103 | {keyType("k"), keyType("")}, // merges 104 | } 105 | 106 | want := []simpleRange{ 107 | {keyType(""), keyType("b")}, 108 | {keyType("c"), keyType("e")}, 109 | {keyType("f"), keyType("i")}, 110 | {keyType("j"), keyType("")}, 111 | } 112 | 113 | rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 114 | rnd.Shuffle(len(in), func(i, j int) { 115 | in[i], in[j] = in[j], in[i] 116 | }) 117 | got = mergeSimpleRanges(in) 118 | if len(got) != len(want) { 119 | t.Fatalf("want=%d, got=%d", len(got), len(want)) 120 | } 121 | 122 | for i := range want { 123 | want := want[i] 124 | got := got[i] 125 | if string(want.start) != string(got.start) { 126 | t.Errorf("start want=%q, got=%q", string(want.start), string(got.start)) 127 | } 128 | if string(want.end) != string(got.end) { 129 | t.Errorf("end want=%q, got=%q", string(want.end), string(got.end)) 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /bigtable/bttest/remote_test.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "testing" 9 | 10 | "cloud.google.com/go/bigtable" 11 | btapb "cloud.google.com/go/bigtable/admin/apiv2/adminpb" 12 | btpb "cloud.google.com/go/bigtable/apiv2/bigtablepb" 13 | "google.golang.org/api/bigtableadmin/v2" 14 | "google.golang.org/api/option" 15 | "google.golang.org/api/option/internaloption" 16 | gtransport "google.golang.org/api/transport/grpc" 17 | "google.golang.org/grpc" 18 | "google.golang.org/grpc/codes" 19 | "google.golang.org/grpc/metadata" 20 | "google.golang.org/grpc/status" 21 | ) 22 | 23 | // e.g. PROJECT_ID=fs-playpen INSTANCE_ID=playpen-test1 go test -v fs/gcloud/bt/bttest -run TestRemote 24 | 25 | var ( 26 | remoteTestMeta = []struct { 27 | name string 28 | f func(*testing.T) 29 | }{ 30 | // {"TestConcurrentMutationsReadModify", TestConcurrentMutationsReadModify}, 31 | {"TestCreateTableResponse", TestCreateTableResponse}, 32 | {"TestCreateTableWithFamily", TestCreateTableWithFamily}, 33 | // {"TestSampleRowKeys", TestSampleRowKeys}, cannot make strong guarantees on real bigtable 34 | // {"TestTableRowsConcurrent", TestTableRowsConcurrent}, 35 | {"TestModifyColumnFamilies", TestModifyColumnFamilies}, 36 | {"TestDropRowRange", TestDropRowRange}, 37 | {"TestCheckTimestampMaxValue", TestCheckTimestampMaxValue}, 38 | {"TestReadRows", TestReadRows}, 39 | {"TestReadRowsError", TestReadRowsError}, 40 | {"TestReadRowsAfterDeletion", TestReadRowsAfterDeletion}, 41 | {"TestReadRowsOrder", TestReadRowsOrder}, 42 | {"TestReadRowsWithlabelTransformer", TestReadRowsWithlabelTransformer}, 43 | {"TestCheckAndMutateRowWithoutPredicate", TestCheckAndMutateRowWithoutPredicate}, 44 | {"TestCheckAndMutateRowWithPredicate", TestCheckAndMutateRowWithPredicate}, 45 | {"TestServer_ReadModifyWriteRow", TestServer_ReadModifyWriteRow}, 46 | {"TestFilters", TestFilters}, 47 | {"Test_Mutation_DeleteFromColumn", Test_Mutation_DeleteFromColumn}, 48 | {"TestFilterRowWithSingleColumnQualifier", TestFilterRowWithSingleColumnQualifier}, 49 | {"TestValueFilterRowWithAlternationInRegex", TestValueFilterRowWithAlternationInRegex}, 50 | } 51 | ) 52 | 53 | func TestRemote(t *testing.T) { 54 | project := os.Getenv("PROJECT_ID") 55 | instance := os.Getenv("INSTANCE_ID") 56 | if project == "" || instance == "" { 57 | t.Skip("PROJECT_ID and INSTANCE_ID must be set to run this") 58 | } 59 | 60 | ctx := context.Background() 61 | btcPool, err := newClientPool(ctx) 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | btcaPool, err := newAdminPool(ctx) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | 70 | clientIntfFuncs[t.Name()] = func(t *testing.T, name string) (context.Context, *clientIntf, bool) { 71 | return newRemoteServer(t, name, btcPool, btcaPool, project, instance) 72 | } 73 | for _, tc := range remoteTestMeta { 74 | t.Run(tc.name, tc.f) 75 | } 76 | } 77 | 78 | func newClientPool(ctx context.Context) (grpc.ClientConnInterface, error) { 79 | o := []option.ClientOption{ 80 | internaloption.WithDefaultEndpointTemplate("bigtable.UNIVERSE_DOMAIN:443"), 81 | internaloption.WithDefaultUniverseDomain("googleapis.com"), 82 | internaloption.WithDefaultMTLSEndpoint("bigtable.mtls.googleapis.com:443"), 83 | option.WithScopes(bigtable.Scope), 84 | option.WithUserAgent("cbt-go/v1.6.0"), 85 | option.WithGRPCConnectionPool(4), 86 | option.WithGRPCDialOption(grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(1<<28), grpc.MaxCallRecvMsgSize(1<<28))), 87 | } 88 | // Attempts direct access to spanner service over gRPC to improve throughput, 89 | // whether the attempt is allowed is totally controlled by service owner. 90 | // o = append(o, internaloption.EnableDirectPath(true)) 91 | return gtransport.DialPool(ctx, o...) 92 | } 93 | 94 | func newAdminPool(ctx context.Context) (grpc.ClientConnInterface, error) { 95 | o := []option.ClientOption{ 96 | internaloption.WithDefaultEndpointTemplate("bigtableadmin.UNIVERSE_DOMAIN:443"), 97 | internaloption.WithDefaultUniverseDomain("googleapis.com"), 98 | internaloption.WithDefaultMTLSEndpoint("bigtableadmin.mtls.googleapis.com:443"), 99 | option.WithScopes(bigtable.AdminScope), 100 | option.WithUserAgent("cbt-go/v1.6.0"), 101 | option.WithGRPCConnectionPool(4), 102 | } 103 | o = append(o, option.WithScopes(bigtableadmin.CloudPlatformScope)) 104 | return gtransport.DialPool(ctx, o...) 105 | } 106 | 107 | func newRemoteServer(t *testing.T, name string, btcPool, btcaPool grpc.ClientConnInterface, project, instance string) (context.Context, *clientIntf, bool) { 108 | nameParts := strings.Split(t.Name(), "/") 109 | parent := fmt.Sprintf("projects/%s/instances/%s", project, instance) 110 | tbl := fmt.Sprintf("projects/%s/instances/%s/tables/%s", project, instance, nameParts[len(nameParts)-1]) 111 | ret := &clientIntf{ 112 | parent: parent, 113 | name: name, 114 | tblName: tbl, 115 | BigtableClient: btpb.NewBigtableClient(btcPool), 116 | BigtableTableAdminClient: btapb.NewBigtableTableAdminClient(btcaPool), 117 | } 118 | 119 | md := metadata.New(map[string]string{"google-cloud-resource-prefix": ret.parent}) 120 | ctx := metadata.NewOutgoingContext(context.Background(), md) 121 | 122 | _, err := ret.DropRowRange(ctx, &btapb.DropRowRangeRequest{ 123 | Name: ret.tblName, 124 | Target: &btapb.DropRowRangeRequest_DeleteAllDataFromTable{DeleteAllDataFromTable: true}, 125 | }) 126 | if err != nil { 127 | if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { 128 | return ctx, ret, false 129 | } 130 | t.Fatal(err) 131 | } 132 | 133 | return ctx, ret, true 134 | } 135 | -------------------------------------------------------------------------------- /bigtable/bttest/setup_test.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "reflect" 8 | "strings" 9 | "testing" 10 | 11 | "cloud.google.com/go/bigtable" 12 | btapb "cloud.google.com/go/bigtable/admin/apiv2/adminpb" 13 | btpb "cloud.google.com/go/bigtable/apiv2/bigtablepb" 14 | emptypb "github.com/golang/protobuf/ptypes/empty" 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/metadata" 17 | "google.golang.org/protobuf/proto" 18 | ) 19 | 20 | type clientIntfFunc func(t *testing.T, name string) (context.Context, *clientIntf, bool) 21 | 22 | var ( 23 | clientIntfFuncs = map[string]clientIntfFunc{} 24 | ) 25 | 26 | func newClient(t *testing.T) (context.Context, *clientIntf, bool) { 27 | t.Parallel() 28 | ctx := context.Background() 29 | 30 | parts := strings.SplitN(t.Name(), "/", 2) 31 | if len(parts) == 2 { 32 | return clientIntfFuncs[parts[0]](t, parts[1]) 33 | } 34 | 35 | svr := &server{ 36 | tables: make(map[string]*table), 37 | storage: BtreeStorage{}, 38 | clock: func() bigtable.Timestamp { 39 | return 0 40 | }, 41 | } 42 | 43 | cl := &clientIntf{ 44 | parent: fmt.Sprintf("projects/%s/instances/%s", "project", "cluster"), 45 | name: t.Name(), 46 | tblName: fmt.Sprintf("projects/%s/instances/%s/tables/%s", "project", "cluster", t.Name()), 47 | BigtableClient: btServer2Client{s: svr}, 48 | BigtableTableAdminClient: btServer2AdminClient{s: svr}, 49 | } 50 | 51 | return ctx, cl, false 52 | } 53 | 54 | type streamAdapter struct { 55 | ctx context.Context 56 | msgs []proto.Message 57 | } 58 | 59 | func (a *streamAdapter) SetHeader(md metadata.MD) error { 60 | return nil 61 | } 62 | 63 | func (a *streamAdapter) SendHeader(md metadata.MD) error { 64 | return nil 65 | } 66 | 67 | func (a *streamAdapter) SetTrailer(md metadata.MD) { 68 | } 69 | 70 | func (a *streamAdapter) Header() (metadata.MD, error) { 71 | return nil, nil 72 | } 73 | 74 | func (a *streamAdapter) Trailer() metadata.MD { 75 | return nil 76 | } 77 | 78 | func (a *streamAdapter) CloseSend() error { 79 | return nil 80 | } 81 | 82 | func (a *streamAdapter) Context() context.Context { 83 | return a.ctx 84 | } 85 | 86 | func (a *streamAdapter) SendMsg(m interface{}) error { 87 | if err := a.ctx.Err(); err != nil { 88 | return err 89 | } 90 | a.msgs = append(a.msgs, m.(proto.Message)) 91 | return nil 92 | } 93 | 94 | func (a *streamAdapter) RecvMsg(m interface{}) error { 95 | if len(a.msgs) == 0 { 96 | return io.EOF 97 | } 98 | if err := a.ctx.Err(); err != nil { 99 | return err 100 | } 101 | ret := a.msgs[0] 102 | reflect.ValueOf(m).Elem().Set(reflect.ValueOf(ret).Elem()) 103 | a.msgs = a.msgs[1:] 104 | return nil 105 | } 106 | 107 | var _ grpc.ClientStream = (*streamAdapter)(nil) 108 | var _ grpc.ServerStream = (*streamAdapter)(nil) 109 | 110 | type btServer2Client struct { 111 | s btpb.BigtableServer 112 | btpb.BigtableClient 113 | } 114 | 115 | type rrAdapter struct { 116 | streamAdapter 117 | } 118 | 119 | func (r *rrAdapter) Send(response *btpb.ReadRowsResponse) error { 120 | return r.streamAdapter.SendMsg(response) 121 | 122 | } 123 | func (r *rrAdapter) Recv() (*btpb.ReadRowsResponse, error) { 124 | ret := &btpb.ReadRowsResponse{} 125 | return ret, r.streamAdapter.RecvMsg(ret) 126 | } 127 | 128 | func (b btServer2Client) ReadRows(ctx context.Context, in *btpb.ReadRowsRequest, _ ...grpc.CallOption) (btpb.Bigtable_ReadRowsClient, error) { 129 | cl := &rrAdapter{streamAdapter{ctx: ctx}} 130 | err := b.s.ReadRows(in, cl) 131 | return cl, err 132 | } 133 | 134 | type srkAdapter struct { 135 | streamAdapter 136 | } 137 | 138 | func (r *srkAdapter) Send(response *btpb.SampleRowKeysResponse) error { 139 | return r.streamAdapter.SendMsg(response) 140 | 141 | } 142 | func (r *srkAdapter) Recv() (*btpb.SampleRowKeysResponse, error) { 143 | ret := &btpb.SampleRowKeysResponse{} 144 | return ret, r.streamAdapter.RecvMsg(ret) 145 | } 146 | 147 | func (b btServer2Client) SampleRowKeys(ctx context.Context, in *btpb.SampleRowKeysRequest, _ ...grpc.CallOption) (btpb.Bigtable_SampleRowKeysClient, error) { 148 | cl := &srkAdapter{streamAdapter{ctx: ctx}} 149 | err := b.s.SampleRowKeys(in, cl) 150 | return cl, err 151 | } 152 | 153 | func (b btServer2Client) MutateRow(ctx context.Context, in *btpb.MutateRowRequest, _ ...grpc.CallOption) (*btpb.MutateRowResponse, error) { 154 | return b.s.MutateRow(ctx, in) 155 | } 156 | 157 | type mrAdapter struct { 158 | streamAdapter 159 | } 160 | 161 | func (r *mrAdapter) Send(response *btpb.MutateRowsResponse) error { 162 | return r.streamAdapter.SendMsg(response) 163 | 164 | } 165 | func (r *mrAdapter) Recv() (*btpb.MutateRowsResponse, error) { 166 | ret := &btpb.MutateRowsResponse{} 167 | return ret, r.streamAdapter.RecvMsg(ret) 168 | } 169 | 170 | func (b btServer2Client) MutateRows(ctx context.Context, in *btpb.MutateRowsRequest, _ ...grpc.CallOption) (btpb.Bigtable_MutateRowsClient, error) { 171 | cl := &mrAdapter{streamAdapter{ctx: ctx}} 172 | err := b.s.MutateRows(in, cl) 173 | return cl, err 174 | } 175 | 176 | func (b btServer2Client) CheckAndMutateRow(ctx context.Context, in *btpb.CheckAndMutateRowRequest, _ ...grpc.CallOption) (*btpb.CheckAndMutateRowResponse, error) { 177 | return b.s.CheckAndMutateRow(ctx, in) 178 | } 179 | 180 | func (b btServer2Client) ReadModifyWriteRow(ctx context.Context, in *btpb.ReadModifyWriteRowRequest, _ ...grpc.CallOption) (*btpb.ReadModifyWriteRowResponse, error) { 181 | return b.s.ReadModifyWriteRow(ctx, in) 182 | } 183 | 184 | type btServer2AdminClient struct { 185 | s btapb.BigtableTableAdminServer 186 | btapb.BigtableTableAdminClient 187 | } 188 | 189 | func (b btServer2AdminClient) CreateTable(ctx context.Context, in *btapb.CreateTableRequest, _ ...grpc.CallOption) (*btapb.Table, error) { 190 | return b.s.CreateTable(ctx, in) 191 | } 192 | 193 | func (b btServer2AdminClient) ListTables(ctx context.Context, in *btapb.ListTablesRequest, _ ...grpc.CallOption) (*btapb.ListTablesResponse, error) { 194 | return b.s.ListTables(ctx, in) 195 | } 196 | 197 | func (b btServer2AdminClient) GetTable(ctx context.Context, in *btapb.GetTableRequest, _ ...grpc.CallOption) (*btapb.Table, error) { 198 | return b.s.GetTable(ctx, in) 199 | } 200 | 201 | func (b btServer2AdminClient) DeleteTable(ctx context.Context, in *btapb.DeleteTableRequest, _ ...grpc.CallOption) (*emptypb.Empty, error) { 202 | return b.s.DeleteTable(ctx, in) 203 | } 204 | 205 | func (b btServer2AdminClient) ModifyColumnFamilies(ctx context.Context, in *btapb.ModifyColumnFamiliesRequest, _ ...grpc.CallOption) (*btapb.Table, error) { 206 | return b.s.ModifyColumnFamilies(ctx, in) 207 | } 208 | 209 | func (b btServer2AdminClient) DropRowRange(ctx context.Context, in *btapb.DropRowRangeRequest, _ ...grpc.CallOption) (*emptypb.Empty, error) { 210 | return b.s.DropRowRange(ctx, in) 211 | } 212 | 213 | func (b btServer2AdminClient) GenerateConsistencyToken(ctx context.Context, in *btapb.GenerateConsistencyTokenRequest, _ ...grpc.CallOption) (*btapb.GenerateConsistencyTokenResponse, error) { 214 | return b.s.GenerateConsistencyToken(ctx, in) 215 | } 216 | 217 | func (b btServer2AdminClient) CheckConsistency(ctx context.Context, in *btapb.CheckConsistencyRequest, _ ...grpc.CallOption) (*btapb.CheckConsistencyResponse, error) { 218 | return b.s.CheckConsistency(ctx, in) 219 | } 220 | -------------------------------------------------------------------------------- /bigtable/bttest/storage.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | btapb "cloud.google.com/go/bigtable/admin/apiv2/adminpb" 5 | btpb "cloud.google.com/go/bigtable/apiv2/bigtablepb" 6 | ) 7 | 8 | // Storage implements a storage layer for all bigtable emulator data. 9 | type Storage interface { 10 | // Create a new table, destroying any existing table. 11 | Create(tbl *btapb.Table) Rows 12 | // GetTables returns metadata about all stored tables. 13 | GetTables() []*btapb.Table 14 | // Open the given table, which must have been previously returned by GetTables(). 15 | Open(tbl *btapb.Table) Rows 16 | // SetTableMeta persists metadata about a table. 17 | SetTableMeta(tbl *btapb.Table) 18 | } 19 | 20 | type keyType = []byte 21 | 22 | // Rows implements storage algorithms per table. 23 | type Rows interface { 24 | // Ascend calls the iterator for every row in the table within the range 25 | // [first, last], until iterator returns false. 26 | Ascend(iterator RowIterator) 27 | 28 | // AscendRange calls the iterator for every row in the table within the range 29 | // [greaterOrEqual, lessThan), until iterator returns false. 30 | AscendRange(greaterOrEqual, lessThan keyType, iterator RowIterator) 31 | 32 | // AscendLessThan calls the iterator for every row in the table within the range 33 | // [first, pivot), until iterator returns false. 34 | AscendLessThan(lessThan keyType, iterator RowIterator) 35 | 36 | // AscendGreaterOrEqual calls the iterator for every row in the table within 37 | // the range [pivot, last], until iterator returns false. 38 | AscendGreaterOrEqual(greaterOrEqual keyType, iterator RowIterator) 39 | 40 | // Clear removes all rows from the table. 41 | Clear() 42 | 43 | // Delete removes a row whose key is equal to given key. 44 | Delete(key keyType) 45 | 46 | // Get looks for a row whose key is equal to the given key, returning it. 47 | // Returns nil if unable to find that row. 48 | Get(key keyType) *btpb.Row 49 | 50 | // ReplaceOrInsert adds the given row to the table. If a row in the table 51 | // already equals the given one, it is removed from the table. 52 | // 53 | // nil cannot be added to the table (will panic). 54 | ReplaceOrInsert(r *btpb.Row) 55 | 56 | Close() 57 | } 58 | 59 | // RowIterator is a callback function that receives a Row. 60 | type RowIterator = func(r *btpb.Row) bool 61 | -------------------------------------------------------------------------------- /bigtable/bttest/store_btree.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | "bytes" 5 | 6 | btapb "cloud.google.com/go/bigtable/admin/apiv2/adminpb" 7 | btpb "cloud.google.com/go/bigtable/apiv2/bigtablepb" 8 | "github.com/google/btree" 9 | "google.golang.org/protobuf/proto" 10 | ) 11 | 12 | const btreeDegree = 16 13 | 14 | // BtreeStorage stores data in an in-memory btree. This implementation is here for historical reference 15 | // and should not generally be used; prefer LeveldbMemStorage. BtreeStorage's row scans do not work well 16 | // in the face of concurrent insertions and deletions. Although no data races occur, changes to the Btree's 17 | // internal structure break iteration in surprising ways, resulting in unpredictable rowscan results. 18 | type BtreeStorage struct { 19 | } 20 | 21 | var _ Storage = BtreeStorage{} 22 | 23 | // Create a new table, destroying any existing table. 24 | func (BtreeStorage) Create(_ *btapb.Table) Rows { 25 | return btreeRows{btree.New(btreeDegree)} 26 | } 27 | 28 | // GetTables returns metadata about all stored tables. 29 | func (BtreeStorage) GetTables() []*btapb.Table { 30 | return nil 31 | } 32 | 33 | // Open the given table, which must have been previously returned by GetTables(). 34 | func (BtreeStorage) Open(_ *btapb.Table) Rows { 35 | panic("should not get here") 36 | } 37 | 38 | // SetTableMeta persists metadata about a table. 39 | func (f BtreeStorage) SetTableMeta(_ *btapb.Table) { 40 | } 41 | 42 | type btreeRows struct { 43 | tree *btree.BTree 44 | } 45 | 46 | var _ Rows = btreeRows{} 47 | 48 | func (b btreeRows) Ascend(iterator RowIterator) { 49 | b.tree.Ascend(b.adaptIterator(iterator)) 50 | } 51 | 52 | func (b btreeRows) AscendRange(greaterOrEqual, lessThan keyType, iterator RowIterator) { 53 | b.tree.AscendRange(b.key(greaterOrEqual), b.key(lessThan), b.adaptIterator(iterator)) 54 | } 55 | 56 | func (b btreeRows) AscendLessThan(lessThan keyType, iterator RowIterator) { 57 | b.tree.AscendLessThan(b.key(lessThan), b.adaptIterator(iterator)) 58 | } 59 | 60 | func (b btreeRows) AscendGreaterOrEqual(greaterOrEqual keyType, iterator RowIterator) { 61 | b.tree.AscendGreaterOrEqual(b.key(greaterOrEqual), b.adaptIterator(iterator)) 62 | } 63 | 64 | func (b btreeRows) Delete(key keyType) { 65 | b.tree.Delete(b.key(key)) 66 | } 67 | 68 | func (b btreeRows) Get(key keyType) *btpb.Row { 69 | item := b.tree.Get(b.key(key)) 70 | if item == nil { 71 | return nil 72 | } 73 | return fromProto(item.(protoItem).buf) 74 | } 75 | 76 | func (b btreeRows) ReplaceOrInsert(r *btpb.Row) { 77 | b.tree.ReplaceOrInsert(protoItem{ 78 | key: r.Key, 79 | buf: toProto(r), 80 | }) 81 | } 82 | 83 | func (b btreeRows) Clear() { 84 | b.tree.Clear(false) 85 | } 86 | 87 | func (b btreeRows) Close() { 88 | } 89 | 90 | func (b btreeRows) key(key keyType) protoItem { 91 | return protoItem{key: key} 92 | } 93 | 94 | func (b btreeRows) adaptIterator(iterator RowIterator) btree.ItemIterator { 95 | return func(i btree.Item) bool { 96 | r := fromProto(i.(protoItem).buf) 97 | return iterator(r) 98 | } 99 | } 100 | 101 | func fromProto(buf []byte) *btpb.Row { 102 | var p btpb.Row 103 | if err := proto.Unmarshal(buf, &p); err != nil { 104 | panic(err) 105 | } 106 | return &p 107 | } 108 | 109 | func toProto(r *btpb.Row) []byte { 110 | if buf, err := proto.Marshal(r); err != nil { 111 | panic(err) 112 | } else { 113 | return buf 114 | } 115 | } 116 | 117 | type protoItem struct { 118 | key keyType 119 | buf []byte 120 | } 121 | 122 | var _ btree.Item = protoItem{} 123 | 124 | // Less implements btree.Item. 125 | func (bi protoItem) Less(i btree.Item) bool { 126 | return bytes.Compare(bi.key, i.(protoItem).key) < 0 127 | } 128 | -------------------------------------------------------------------------------- /bigtable/bttest/store_leveldb.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | btpb "cloud.google.com/go/bigtable/apiv2/bigtablepb" 5 | "github.com/syndtr/goleveldb/leveldb" 6 | "github.com/syndtr/goleveldb/leveldb/util" 7 | ) 8 | 9 | type leveldbRows struct { 10 | db *leveldb.DB 11 | newFunc func(nuke bool) *leveldb.DB 12 | } 13 | 14 | var _ Rows = &leveldbRows{} 15 | 16 | func (rows *leveldbRows) Ascend(iterator RowIterator) { 17 | rows.ascendRange(nil, iterator) 18 | } 19 | 20 | func (rows *leveldbRows) AscendRange(greaterOrEqual, lessThan keyType, iterator RowIterator) { 21 | rows.ascendRange(&util.Range{ 22 | Start: greaterOrEqual, 23 | Limit: lessThan, 24 | }, iterator) 25 | } 26 | 27 | func (rows *leveldbRows) AscendLessThan(lessThan keyType, iterator RowIterator) { 28 | rows.ascendRange(&util.Range{ 29 | Limit: lessThan, 30 | }, iterator) 31 | } 32 | 33 | func (rows *leveldbRows) AscendGreaterOrEqual(greaterOrEqual keyType, iterator RowIterator) { 34 | rows.ascendRange(&util.Range{ 35 | Start: greaterOrEqual, 36 | }, iterator) 37 | } 38 | 39 | func (rows *leveldbRows) Delete(key keyType) { 40 | err := rows.db.Delete(key, nil) 41 | if err != nil { 42 | panic(err) 43 | } 44 | } 45 | 46 | func (rows *leveldbRows) Get(key keyType) *btpb.Row { 47 | item, err := rows.db.Get(key, nil) 48 | if err == leveldb.ErrNotFound { 49 | return nil 50 | } else if err != nil { 51 | panic(err) 52 | } 53 | return fromProto(item) 54 | } 55 | 56 | func (rows *leveldbRows) ReplaceOrInsert(r *btpb.Row) { 57 | err := rows.db.Put(r.Key, toProto(r), nil) 58 | if err != nil { 59 | panic(err) 60 | } 61 | } 62 | 63 | func (rows *leveldbRows) Clear() { 64 | if err := rows.db.Close(); err != nil { 65 | panic(err) 66 | } 67 | rows.db = rows.newFunc(true) 68 | } 69 | 70 | func (rows *leveldbRows) Close() { 71 | if err := rows.db.Close(); err != nil { 72 | panic(err) 73 | } 74 | } 75 | 76 | func (rows *leveldbRows) ascendRange(rng *util.Range, iterator RowIterator) { 77 | it := rows.db.NewIterator(rng, nil) 78 | defer it.Release() 79 | for ok := it.First(); ok; ok = it.Next() { 80 | iterator(fromProto(it.Value())) 81 | } 82 | if err := it.Error(); err != nil { 83 | panic(err) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /bigtable/bttest/store_leveldb_disk.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | 9 | btapb "cloud.google.com/go/bigtable/admin/apiv2/adminpb" 10 | "github.com/syndtr/goleveldb/leveldb" 11 | "github.com/syndtr/goleveldb/leveldb/comparer" 12 | "github.com/syndtr/goleveldb/leveldb/opt" 13 | "google.golang.org/protobuf/proto" 14 | ) 15 | 16 | // LeveldbDiskStorage stores data persistently on leveldb. 17 | type LeveldbDiskStorage struct { 18 | // A root directory under which all data is stored. 19 | Root string 20 | 21 | // Optional error logger. 22 | ErrLog func(err error, msg string) 23 | 24 | // TODO: options like compression? 25 | } 26 | 27 | // Create a new table, destroying any existing table. 28 | func (f LeveldbDiskStorage) Create(tbl *btapb.Table) Rows { 29 | f.SetTableMeta(tbl) 30 | path := filepath.Join(f.Root, tbl.Name) 31 | newFunc := func(nuke bool) *leveldb.DB { 32 | return newDiskDb(path, nuke) 33 | } 34 | 35 | return &leveldbRows{ 36 | db: newFunc(true), 37 | newFunc: newFunc, 38 | } 39 | } 40 | 41 | // GetTables returns metadata about all stored tables. 42 | func (f LeveldbDiskStorage) GetTables() []*btapb.Table { 43 | // Ignore any errors, just return 44 | var ret []*btapb.Table 45 | err := filepath.Walk(f.Root, func(path string, info os.FileInfo, err error) error { 46 | if err != nil { 47 | return err 48 | } 49 | if !strings.HasSuffix(path, ".table.proto") { 50 | return nil 51 | } 52 | var tbl btapb.Table 53 | buf, err := os.ReadFile(path) 54 | if err != nil { 55 | f.errLog(err, "openq %q", path) 56 | return nil 57 | } 58 | if err := proto.Unmarshal(buf, &tbl); err != nil { 59 | f.errLog(err, "unmarshal %q", path) 60 | return nil 61 | } 62 | ret = append(ret, &tbl) 63 | return nil 64 | }) 65 | if err != nil { 66 | f.errLog(err, "walk %q", f.Root) 67 | } 68 | return ret 69 | } 70 | 71 | // Open the given table, which must have been previously returned by GetTables(). 72 | func (f LeveldbDiskStorage) Open(tbl *btapb.Table) Rows { 73 | path := filepath.Join(f.Root, tbl.Name) 74 | newFunc := func(nuke bool) *leveldb.DB { 75 | return newDiskDb(path, nuke) 76 | } 77 | 78 | return &leveldbRows{ 79 | db: newFunc(false), 80 | newFunc: newFunc, 81 | } 82 | } 83 | 84 | // SetTableMeta persists metadata about a table. 85 | func (f LeveldbDiskStorage) SetTableMeta(tbl *btapb.Table) { 86 | path := filepath.Join(f.Root, tbl.Name) 87 | if err := os.MkdirAll(path, 0777); err != nil { 88 | f.errLog(err, "os.MkdirAll %q", path) 89 | } 90 | buf, err := proto.Marshal(tbl) 91 | if err != nil { 92 | panic(err) // should not fail 93 | } 94 | 95 | outPath := filepath.Join(path + ".table.proto") 96 | tmpPath := filepath.Join(path + ".table.proto.tmp") 97 | if err := os.WriteFile(tmpPath, buf, 0666); err != nil { 98 | f.errLog(err, "ioutil.WriteFile %q", tmpPath) 99 | return 100 | } 101 | 102 | if err := os.Rename(tmpPath, outPath); err != nil { 103 | f.errLog(err, "os.Rename %q -> %q", tmpPath, outPath) 104 | return 105 | } 106 | } 107 | 108 | func (f LeveldbDiskStorage) errLog(err error, format string, args ...interface{}) { 109 | if f.ErrLog != nil { 110 | f.ErrLog(err, fmt.Sprintf(format, args...)) 111 | } 112 | } 113 | 114 | var _ Storage = LeveldbDiskStorage{} 115 | 116 | func newDiskDb(path string, nuke bool) *leveldb.DB { 117 | if nuke { 118 | _ = os.RemoveAll(path) 119 | } 120 | 121 | db, err := leveldb.OpenFile(path, &opt.Options{ 122 | Comparer: comparer.DefaultComparer, 123 | Compression: opt.NoCompression, 124 | DisableBufferPool: true, 125 | DisableLargeBatchTransaction: true, 126 | }) 127 | if err != nil { 128 | panic(err) 129 | } 130 | return db 131 | } 132 | -------------------------------------------------------------------------------- /bigtable/bttest/store_leveldb_mem.go: -------------------------------------------------------------------------------- 1 | package bttest 2 | 3 | import ( 4 | btapb "cloud.google.com/go/bigtable/admin/apiv2/adminpb" 5 | "github.com/syndtr/goleveldb/leveldb" 6 | "github.com/syndtr/goleveldb/leveldb/comparer" 7 | "github.com/syndtr/goleveldb/leveldb/opt" 8 | "github.com/syndtr/goleveldb/leveldb/storage" 9 | ) 10 | 11 | // LeveldbMemStorage stores data in an in-memory level db. This is the default. 12 | // Unlike BtreeStorage, LeveldbMemStorage is resilient against concurrent insertions and deletions during 13 | // row scans. Concurrently added and deleted rows may or may be scanned (as with real bigtable), but the 14 | // general row scan semantics should hold. 15 | type LeveldbMemStorage struct { 16 | } 17 | 18 | // Create a new table, destroying any existing table. 19 | func (f LeveldbMemStorage) Create(_ *btapb.Table) Rows { 20 | newFunc := func(nuke bool) *leveldb.DB { 21 | return newMemDb(nuke) 22 | } 23 | return &leveldbRows{ 24 | db: newFunc(false), 25 | newFunc: newFunc, 26 | } 27 | } 28 | 29 | // GetTables returns metadata about all stored tables. 30 | func (f LeveldbMemStorage) GetTables() []*btapb.Table { 31 | return nil 32 | } 33 | 34 | // Open the given table, which must have been previously returned by GetTables(). 35 | func (f LeveldbMemStorage) Open(_ *btapb.Table) Rows { 36 | panic("should not get here") 37 | } 38 | 39 | // SetTableMeta persists metadata about a table. 40 | func (f LeveldbMemStorage) SetTableMeta(_ *btapb.Table) { 41 | } 42 | 43 | var _ Storage = LeveldbMemStorage{} 44 | 45 | func newMemDb(_ bool) *leveldb.DB { 46 | db, err := leveldb.Open(storage.NewMemStorage(), &opt.Options{ 47 | Comparer: comparer.DefaultComparer, 48 | Compression: opt.NoCompression, 49 | DisableBufferPool: true, 50 | DisableLargeBatchTransaction: true, 51 | }) 52 | if err != nil { 53 | panic(err) 54 | } 55 | return db 56 | } 57 | -------------------------------------------------------------------------------- /bigtable/bttest/timestamp_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package bttest 15 | 16 | import ( 17 | "math" 18 | "testing" 19 | "time" 20 | 21 | "cloud.google.com/go/bigtable" 22 | ) 23 | 24 | func TestTimestampConversion(t *testing.T) { 25 | // 1. Test a Timestamp converting to Time. 26 | var ts1 bigtable.Timestamp = 1583863200000000 27 | t1 := ts1.Time().UTC() 28 | want1 := time.Date(2020, time.March, 10, 18, 0, 0, 0, time.UTC) 29 | 30 | if !want1.Equal(t1) { 31 | t.Errorf("Mismatched time got %v wanted %v", t1, want1) 32 | } 33 | // 2. Test a reversed Timestamp converting to Time. 34 | reverse := math.MaxInt64 - ts1 35 | 36 | got2 := reverse.Time().UTC() 37 | want2 := time.Date(294196, time.October, 31, 10, 0, 54, 775807000, time.UTC) 38 | 39 | if !want2.Equal(got2) { 40 | t.Errorf("Mismatched time got %v wanted %v", got2, want2) 41 | } 42 | 43 | // 3. Test a Time converted to Timestamp then converted back to Time. 44 | t2 := time.Date(2016, time.October, 3, 14, 7, 7, 0, time.UTC) 45 | ts2 := bigtable.Timestamp(t2.UnixNano() / 1000) 46 | 47 | got3 := ts2.Time().UTC() 48 | want3 := time.Date(2016, time.October, 3, 14, 7, 7, 0, time.UTC) 49 | if !want3.Equal(got3) { 50 | t.Errorf("Mismatched time got %v wanted %v", got3, want3) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /bigtable/bttest/validation.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package bttest 18 | 19 | import ( 20 | "bytes" 21 | 22 | btpb "cloud.google.com/go/bigtable/apiv2/bigtablepb" 23 | "google.golang.org/grpc/codes" 24 | "google.golang.org/grpc/status" 25 | ) 26 | 27 | // validateRowRanges returns a status.Error for req if: 28 | // - both start_qualifier_closed and start_qualifier_open are set 29 | // - both end_qualifier_closed and end_qualifier_open are set 30 | // - start_qualifier_closed > end_qualifier_closed 31 | // - start_qualifier_closed > end_qualifier_open 32 | // - start_qualifier_open > end_qualifier_closed 33 | // - start_qualifier_open > end_qualifier_open 34 | func validateRowRanges(req *btpb.ReadRowsRequest) error { 35 | rowRanges := req.GetRows().GetRowRanges() 36 | if len(rowRanges) == 0 { 37 | return nil 38 | } 39 | 40 | for i, rowRange := range rowRanges { 41 | skC := rowRange.GetStartKeyClosed() 42 | ekC := rowRange.GetEndKeyClosed() 43 | skO := rowRange.GetStartKeyOpen() 44 | ekO := rowRange.GetEndKeyOpen() 45 | 46 | if msg := messageOnInvalidKeyRanges(skC, skO, ekC, ekO); msg != "" { 47 | return status.Errorf(codes.InvalidArgument, "Error in element #%d: %s", i, msg) 48 | } 49 | } 50 | return nil 51 | } 52 | 53 | func messageOnInvalidKeyRanges(startKeyClosed, startKeyOpen, endKeyClosed, endKeyOpen []byte) string { 54 | switch { 55 | case len(startKeyClosed) != 0 && len(startKeyOpen) != 0: 56 | return "both start_key_closed and start_key_open cannot be set" 57 | 58 | case len(endKeyClosed) != 0 && len(endKeyOpen) != 0: 59 | return "both end_key_closed and end_key_open cannot be set" 60 | 61 | case keysOutOfRange(startKeyClosed, endKeyClosed): 62 | return "start_key_closed must be less than end_key_closed" 63 | 64 | case keysOutOfRange(startKeyOpen, endKeyOpen): 65 | return "start_key_open must be less than end_key_open" 66 | 67 | case keysOutOfRange(startKeyClosed, endKeyOpen): 68 | return "start_key_closed must be less than end_key_open" 69 | 70 | case keysOutOfRange(startKeyOpen, endKeyClosed): 71 | return "start_key_open must be less than end_key_closed" 72 | } 73 | return "" 74 | } 75 | 76 | func keysOutOfRange(start, end []byte) bool { 77 | if len(start) == 0 && len(end) == 0 { 78 | // Neither keys have been set, this is an implicit indefinite range. 79 | return false 80 | } 81 | if len(start) == 0 || len(end) == 0 { 82 | // Either of the keys have been set so this is an explicit indefinite range. 83 | return false 84 | } 85 | // Both keys have been set now check if start > end. 86 | return bytes.Compare(start, end) > 0 87 | } 88 | -------------------------------------------------------------------------------- /bigtable/bttest/validation_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package bttest 18 | 19 | import ( 20 | "testing" 21 | 22 | btpb "cloud.google.com/go/bigtable/apiv2/bigtablepb" 23 | "google.golang.org/grpc/codes" 24 | "google.golang.org/grpc/status" 25 | ) 26 | 27 | func TestMessageOnInvalidKeyRange(t *testing.T) { 28 | tests := []struct { 29 | startOpen, startClosed, endOpen, endClosed string 30 | wantErr string 31 | }{ 32 | { 33 | startOpen: "A", startClosed: "A", 34 | wantErr: "both start_key_closed and start_key_open cannot be set", 35 | }, 36 | { 37 | endOpen: "Z", endClosed: "Z", 38 | wantErr: "both end_key_closed and end_key_open cannot be set", 39 | }, 40 | { 41 | startClosed: "Z", endClosed: "A", 42 | wantErr: "start_key_closed must be less than end_key_closed", 43 | }, 44 | { 45 | startClosed: "Z", endOpen: "A", 46 | wantErr: "start_key_closed must be less than end_key_open", 47 | }, 48 | 49 | { 50 | startOpen: "Z", endClosed: "A", 51 | wantErr: "start_key_open must be less than end_key_closed", 52 | }, 53 | { 54 | startOpen: "Z", endOpen: "A", 55 | wantErr: "start_key_open must be less than end_key_open", 56 | }, 57 | 58 | // All values not set. 59 | { 60 | startClosed: "", startOpen: "", endClosed: "", endOpen: "", wantErr: "", 61 | }, 62 | 63 | // Indefinite ranges on each side. 64 | { 65 | startClosed: "A", startOpen: "", endClosed: "", endOpen: "", wantErr: "", 66 | }, 67 | { 68 | startClosed: "", startOpen: "A", endClosed: "", endOpen: "", wantErr: "", 69 | }, 70 | { 71 | startClosed: "", startOpen: "", endClosed: "A", endOpen: "", wantErr: "", 72 | }, 73 | { 74 | startClosed: "", startOpen: "", endClosed: "", endOpen: "A", wantErr: "", 75 | }, 76 | 77 | // startClosed, endClosed ranges properly set. 78 | { 79 | startClosed: "A", startOpen: "", endClosed: "Z", endOpen: "", wantErr: "", 80 | }, 81 | // startClosed, endOpen range. 82 | { 83 | startClosed: "A", startOpen: "", endClosed: "", endOpen: "Z", wantErr: "", 84 | }, 85 | // startOpen, endClosed ranges properly set. 86 | { 87 | startClosed: "", startOpen: "A", endClosed: "Z", endOpen: "", wantErr: "", 88 | }, 89 | // startOpen, endOpen range. 90 | { 91 | startClosed: "", startOpen: "A", endClosed: "", endOpen: "Z", wantErr: "", 92 | }, 93 | } 94 | 95 | for i, tt := range tests { 96 | gotErr := messageOnInvalidKeyRanges( 97 | []byte(tt.startClosed), 98 | []byte(tt.startOpen), 99 | []byte(tt.endClosed), 100 | []byte(tt.endOpen)) 101 | 102 | if gotErr != tt.wantErr { 103 | t.Errorf("#%d. Error mismatch\nGot: %q\nGiven:\n%+v", i, gotErr, tt) 104 | } 105 | } 106 | } 107 | 108 | // This test is just to ensure that the emulator always sends back 109 | // an RPC error with a status code just like Google Bigtable does. 110 | func TestValidateReadRowsRequestSendsRPCError(t *testing.T) { 111 | tableName := "foo.org/bar" 112 | // Minimal server to reproduce failures. 113 | srv := &server{ 114 | tables: map[string]*table{tableName: new(table)}, 115 | } 116 | 117 | badValues := []struct { 118 | startKeyClosed, startKeyOpen, endKeyClosed, endKeyOpen string 119 | }{ 120 | {startKeyClosed: "Z", endKeyClosed: "A"}, 121 | {startKeyClosed: "Z", endKeyOpen: "A"}, 122 | {startKeyOpen: "Z", endKeyClosed: "A"}, 123 | {startKeyOpen: "Z", endKeyOpen: "A"}, 124 | } 125 | 126 | for i, tt := range badValues { 127 | badReq := &btpb.ReadRowsRequest{ 128 | TableName: tableName, 129 | Rows: &btpb.RowSet{ 130 | RowRanges: []*btpb.RowRange{ 131 | { 132 | StartKey: &btpb.RowRange_StartKeyClosed{ 133 | StartKeyClosed: []byte(tt.startKeyClosed), 134 | }, 135 | EndKey: &btpb.RowRange_EndKeyClosed{ 136 | EndKeyClosed: []byte(tt.endKeyClosed), 137 | }, 138 | }, 139 | { 140 | StartKey: &btpb.RowRange_StartKeyClosed{ 141 | StartKeyClosed: []byte(tt.startKeyClosed), 142 | }, 143 | EndKey: &btpb.RowRange_EndKeyOpen{ 144 | EndKeyOpen: []byte(tt.endKeyOpen), 145 | }, 146 | }, 147 | { 148 | StartKey: &btpb.RowRange_StartKeyOpen{ 149 | StartKeyOpen: []byte(tt.startKeyOpen), 150 | }, 151 | EndKey: &btpb.RowRange_EndKeyOpen{ 152 | EndKeyOpen: []byte(tt.endKeyOpen), 153 | }, 154 | }, 155 | { 156 | StartKey: &btpb.RowRange_StartKeyOpen{ 157 | StartKeyOpen: []byte(tt.startKeyOpen), 158 | }, 159 | EndKey: &btpb.RowRange_EndKeyClosed{ 160 | EndKeyClosed: []byte(tt.endKeyClosed), 161 | }, 162 | }, 163 | }, 164 | }, 165 | } 166 | 167 | err := srv.ReadRows(badReq, nil) 168 | if err == nil { 169 | t.Errorf("#%d: unexpectedly returned nil error", i) 170 | continue 171 | } 172 | 173 | status, ok := status.FromError(err) 174 | if !ok { 175 | t.Errorf("#%d: wrong error type %T, expected status.Error", i, err) 176 | continue 177 | } 178 | if g, w := status.Code(), codes.InvalidArgument; g != w { 179 | t.Errorf("#%d: wrong error code\nGot %d %s\nWant %d %s", i, g, g, w, w) 180 | } 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /bigtable/cmd/cbtemulator/cbtemulator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | cbtemulator launches the Cloud Bigtable emulator on the given address. 17 | */ 18 | package main 19 | 20 | import ( 21 | "flag" 22 | "fmt" 23 | "log" 24 | "os" 25 | 26 | "github.com/fullstorydev/emulators/bigtable/bttest" 27 | "google.golang.org/grpc" 28 | ) 29 | 30 | var ( 31 | host = flag.String("host", "localhost", "the address to bind to on the local machine") 32 | port = flag.Int("port", 9000, "the port number to bind to on the local machine") 33 | dir = flag.String("dir", "", "if set, use persistence in the given directory") 34 | ) 35 | 36 | const ( 37 | maxMsgSize = 256 * 1024 * 1024 // 256 MiB 38 | ) 39 | 40 | func main() { 41 | grpc.EnableTracing = false 42 | flag.Parse() 43 | 44 | opts := bttest.Options{ 45 | Storage: nil, 46 | GrpcOpts: []grpc.ServerOption{ 47 | grpc.MaxRecvMsgSize(maxMsgSize), 48 | grpc.MaxSendMsgSize(maxMsgSize), 49 | }, 50 | } 51 | 52 | if *dir != "" { 53 | _ = os.Mkdir(*dir, 0777) 54 | fmt.Printf("Writing to: %s\n", *dir) 55 | opts.Storage = bttest.LeveldbDiskStorage{ 56 | Root: *dir, 57 | ErrLog: func(err error, msg string) { 58 | fmt.Printf("%s: %v\n", msg, err) 59 | }, 60 | } 61 | } 62 | 63 | srv, err := bttest.NewServerWithOptions(fmt.Sprintf("%s:%d", *host, *port), opts) 64 | if err != nil { 65 | log.Fatalf("failed to start emulator: %v", err) 66 | } 67 | defer srv.Close() 68 | 69 | fmt.Printf("Cloud Bigtable emulator running on %s\n", srv.Addr) 70 | select {} 71 | } 72 | -------------------------------------------------------------------------------- /bigtable/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/fullstorydev/emulators/bigtable 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | cloud.google.com/go/bigtable v1.37.0 9 | cloud.google.com/go/iam v1.5.2 10 | github.com/golang/protobuf v1.5.4 11 | github.com/google/btree v1.1.3 12 | github.com/google/go-cmp v0.7.0 13 | github.com/syndtr/goleveldb v1.0.0 14 | golang.org/x/sync v0.14.0 15 | google.golang.org/api v0.235.0 16 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 17 | google.golang.org/grpc v1.72.2 18 | google.golang.org/protobuf v1.36.6 19 | rsc.io/binaryregexp v0.2.0 20 | ) 21 | 22 | require ( 23 | cel.dev/expr v0.20.0 // indirect 24 | cloud.google.com/go v0.120.0 // indirect 25 | cloud.google.com/go/auth v0.16.1 // indirect 26 | cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect 27 | cloud.google.com/go/compute/metadata v0.7.0 // indirect 28 | cloud.google.com/go/longrunning v0.6.7 // indirect 29 | cloud.google.com/go/monitoring v1.24.2 // indirect 30 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 31 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect 32 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect 33 | github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect 34 | github.com/felixge/httpsnoop v1.0.4 // indirect 35 | github.com/go-jose/go-jose/v4 v4.0.5 // indirect 36 | github.com/go-logr/logr v1.4.2 // indirect 37 | github.com/go-logr/stdr v1.2.2 // indirect 38 | github.com/golang/snappy v0.0.4 // indirect 39 | github.com/google/s2a-go v0.1.9 // indirect 40 | github.com/google/uuid v1.6.0 // indirect 41 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect 42 | github.com/googleapis/gax-go/v2 v2.14.2 // indirect 43 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 44 | github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect 45 | github.com/zeebo/errs v1.4.0 // indirect 46 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 47 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect 48 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect 49 | go.opentelemetry.io/otel v1.35.0 // indirect 50 | go.opentelemetry.io/otel/metric v1.35.0 // indirect 51 | go.opentelemetry.io/otel/sdk v1.35.0 // indirect 52 | go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect 53 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 54 | golang.org/x/crypto v0.38.0 // indirect 55 | golang.org/x/net v0.40.0 // indirect 56 | golang.org/x/oauth2 v0.30.0 // indirect 57 | golang.org/x/sys v0.33.0 // indirect 58 | golang.org/x/text v0.25.0 // indirect 59 | golang.org/x/time v0.11.0 // indirect 60 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect 61 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /bigtable/go.sum: -------------------------------------------------------------------------------- 1 | cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= 2 | cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= 3 | cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= 4 | cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= 5 | cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= 6 | cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= 7 | cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= 8 | cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= 9 | cloud.google.com/go/bigtable v1.37.0 h1:Q+x7y04lQ0B+WXp03wc1/FLhFt4CwcQdkwWT0M4Jp3w= 10 | cloud.google.com/go/bigtable v1.37.0/go.mod h1:HXqddP6hduwzrtiTCqZPpj9ij4hGZb4Zy1WF/dT+yaU= 11 | cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= 12 | cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= 13 | cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= 14 | cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= 15 | cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= 16 | cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= 17 | cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= 18 | cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= 19 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 20 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 21 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= 22 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= 23 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 24 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 25 | github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= 26 | github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= 27 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= 28 | github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= 29 | github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= 30 | github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= 31 | github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= 32 | github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= 33 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 34 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 35 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 36 | github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= 37 | github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= 38 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 39 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 40 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 41 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 42 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 43 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 44 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 45 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 46 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 47 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 48 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 49 | github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= 50 | github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= 51 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 52 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 53 | github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= 54 | github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= 55 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 56 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 57 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= 58 | github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= 59 | github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= 60 | github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= 61 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 62 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 63 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 64 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 65 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 66 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 67 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 68 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= 69 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= 70 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 71 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 72 | github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= 73 | github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= 74 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 75 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 76 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 77 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 78 | github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= 79 | github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= 80 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 81 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 82 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= 83 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= 84 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= 85 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= 86 | go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= 87 | go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= 88 | go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= 89 | go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= 90 | go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= 91 | go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= 92 | go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= 93 | go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= 94 | go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= 95 | go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= 96 | golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= 97 | golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= 98 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 99 | golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= 100 | golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= 101 | golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= 102 | golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= 103 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 104 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 105 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 106 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 107 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 108 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 109 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 110 | golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= 111 | golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= 112 | golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= 113 | golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= 114 | google.golang.org/api v0.235.0 h1:C3MkpQSRxS1Jy6AkzTGKKrpSCOd2WOGrezZ+icKSkKo= 115 | google.golang.org/api v0.235.0/go.mod h1:QpeJkemzkFKe5VCE/PMv7GsUfn9ZF+u+q1Q7w6ckxTg= 116 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= 117 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= 118 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 h1:vPV0tzlsK6EzEDHNNH5sa7Hs9bd7iXR7B1tSiPepkV0= 119 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw= 120 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 h1:IkAfh6J/yllPtpYFU0zZN1hUPYdT0ogkBT/9hMxHjvg= 121 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= 122 | google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= 123 | google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= 124 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 125 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 126 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 127 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 128 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 129 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 130 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 131 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 132 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 133 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 134 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 135 | rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= 136 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 137 | -------------------------------------------------------------------------------- /bigtable/releasing/README.md: -------------------------------------------------------------------------------- 1 | # Releases of emulators/bigtable 2 | 3 | This document provides instructions for building a release of `emulators/bigtable`. 4 | 5 | The release process consists of a handful of tasks: 6 | 1. Drop a release tag in git. 7 | 2. Build binaries for various platforms. This is done using the local go tool and uses GOOS and GOARCH environment variables to cross-compile for supported platforms. 8 | 3. Creates a release in GitHub, uploads the binaries, and creates provisional release notes (in the form of a change log). 9 | 4. Build a docker image for the new release. 10 | 5. Push the docker image to Docker Hub, with both a version tag and the "latest" tag. 11 | 12 | Most of this is automated via a script in this same directory. The main thing you will need is a GitHub personal access token, which will be used for creating the release in GitHub (so you need write access to the fullstorydev/emulators repo). 13 | 14 | ## Creating a new release 15 | 16 | So, to actually create a new release, just run the script in this directory. 17 | 18 | First, you need a version number for the new release, following sem-ver format: `v..`. Second, you need a personal access token for GitHub. 19 | 20 | ```sh 21 | # from the root of the package 22 | GITHUB_TOKEN= ./releasing/do-release.sh v.. 23 | ``` 24 | 25 | Wasn't that easy! There is one last step: update the release notes in GitHub. By default, the script just records a change log of commit descriptions. Use that log (and, if necessary, drill into individual PRs included in the release) to flesh out notes in the format of the `RELEASE_NOTES.md` file _in this directory_. Then login to GitHub, go to the new release, edit the notes, and paste in the markdown you just wrote. 26 | 27 | That should be all there is to it! 28 | -------------------------------------------------------------------------------- /bigtable/releasing/RELEASE_NOTES.md: -------------------------------------------------------------------------------- 1 | ## Changes 2 | 3 | ### Go package "github.com/fullstorydev/emulators/bigtable" 4 | 5 | * _In this list, describe the changes in this repo: "github.com/fullstorydev/emulators/bigtable"._ 6 | * _Use one bullet per change. Include both bug-fixes and improvements._ 7 | * -------------------------------------------------------------------------------- /bigtable/releasing/do-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # strict mode 4 | set -euo pipefail 5 | IFS=$'\n\t' 6 | 7 | if [[ -z ${DRY_RUN:-} ]]; then 8 | PREFIX="" 9 | else 10 | PREFIX="echo" 11 | fi 12 | 13 | # input validation 14 | if [[ -z ${GITHUB_TOKEN:-} ]]; then 15 | echo "GITHUB_TOKEN environment variable must be set before running." >&2 16 | exit 1 17 | fi 18 | if [[ $# -ne 1 || $1 == "" ]]; then 19 | echo "This program requires one argument: the version number, in 'vM.N.P' format." >&2 20 | exit 1 21 | fi 22 | VERSION=$1 23 | 24 | # Change to root of the repo 25 | cd "$(dirname "$0")/.." 26 | 27 | # Docker release 28 | 29 | # make sure credentials are valid for later push steps; this might 30 | # be interactive since this will prompt for username and password 31 | # if there are no valid current credentials. 32 | $PREFIX docker login 33 | echo "$VERSION" > VERSION 34 | # Docker Buildx support is included in Docker 19.03 35 | # Below step installs emulators for different architectures on the host 36 | # This enables running and building containers for below architectures mentioned using --platforms 37 | $PREFIX docker run --privileged --rm tonistiigi/binfmt:qemu-v6.1.0 --install all 38 | # Create a new builder instance 39 | export DOCKER_CLI_EXPERIMENTAL=enabled 40 | $PREFIX docker buildx create --use --name multiarch-builder --node multiarch-builder0 41 | # push to docker hub, both the given version as a tag and for "latest" tag 42 | $PREFIX docker buildx build --target=cbtemulator --platform linux/amd64,linux/s390x,linux/arm64,linux/ppc64le --tag fullstorydev/cbtemulator:${VERSION} --tag fullstorydev/cbtemulator:latest --push --progress plain --no-cache . 43 | rm VERSION 44 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: ci 2 | ci: deps checkgofmt vet staticcheck ineffassign predeclared golint errcheck test 3 | 4 | .PHONY: deps 5 | deps: 6 | go get -d -v -t ./... 7 | go mod tidy 8 | 9 | .PHONY: updatedeps 10 | updatedeps: 11 | go get -d -v -t -u -f ./... 12 | go mod tidy 13 | 14 | .PHONY: checkgofmt 15 | checkgofmt: 16 | gofmt -s -l . 17 | @if [ -n "$$(gofmt -s -l .)" ]; then \ 18 | exit 1; \ 19 | fi 20 | 21 | .PHONY: vet 22 | vet: 23 | go vet ./... 24 | 25 | .PHONY: staticcheck 26 | staticcheck: 27 | @go install honnef.co/go/tools/cmd/staticcheck@v0.6.0 28 | staticcheck ./... 29 | 30 | .PHONY: ineffassign 31 | ineffassign: 32 | @go install github.com/gordonklaus/ineffassign@7953dde2c7bf 33 | ineffassign . 34 | 35 | .PHONY: predeclared 36 | predeclared: 37 | @go install github.com/nishanths/predeclared@51e8c974458a0f93dc03fe356f91ae1a6d791e6f 38 | predeclared ./... 39 | 40 | .PHONY: golint 41 | golint: 42 | @go install golang.org/x/lint/golint@v0.0.0-20210508222113-6edffad5e616 43 | golint -min_confidence 0.9 -set_exit_status ./... 44 | 45 | .PHONY: errcheck 46 | errcheck: 47 | @go install github.com/kisielk/errcheck@v1.2.0 48 | errcheck ./... 49 | 50 | .PHONY: test 51 | test: deps 52 | go test -race ./... 53 | -------------------------------------------------------------------------------- /examples/bigtable/example_test.go: -------------------------------------------------------------------------------- 1 | package bigtable 2 | 3 | import ( 4 | "cloud.google.com/go/bigtable" 5 | "context" 6 | "fmt" 7 | "io" 8 | "log" 9 | "testing" 10 | 11 | "github.com/fullstorydev/emulators/bigtable/bttest" 12 | "google.golang.org/api/option" 13 | "google.golang.org/grpc" 14 | "google.golang.org/grpc/credentials/insecure" 15 | ) 16 | 17 | func TestLocalServer(t *testing.T) { 18 | srv, err := bttest.NewServer("localhost:0") 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | err = validateServer(srv.Addr) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | } 28 | 29 | func validateServer(srvAddr string) error { 30 | ctx, cancel := context.WithCancel(context.Background()) 31 | defer cancel() 32 | 33 | conn, err := grpc.NewClient(srvAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) 34 | if err != nil { 35 | return err 36 | } 37 | defer silentClose(conn) 38 | 39 | proj, instance := "proj", "instance" 40 | 41 | adminClient, err := bigtable.NewAdminClient(ctx, proj, instance, option.WithGRPCConn(conn)) 42 | if err != nil { 43 | return err 44 | } 45 | defer silentClose(adminClient) 46 | 47 | if err = adminClient.CreateTable(ctx, "example"); err != nil { 48 | return err 49 | } 50 | 51 | if err = adminClient.CreateColumnFamily(ctx, "example", "links"); err != nil { 52 | return err 53 | } 54 | 55 | clientConfig := bigtable.ClientConfig{MetricsProvider: bigtable.NoopMetricsProvider{}} 56 | client, err := bigtable.NewClientWithConfig(ctx, proj, instance, clientConfig, option.WithGRPCConn(conn)) 57 | if err != nil { 58 | log.Fatalln(err) 59 | } 60 | defer silentClose(client) 61 | 62 | tbl := client.Open("example") 63 | 64 | mut := bigtable.NewMutation() 65 | mut.Set("links", "golang.org", bigtable.Now(), []byte("Gophers!")) 66 | if err = tbl.Apply(ctx, "com.google.cloud", mut); err != nil { 67 | return err 68 | } 69 | 70 | row, err := tbl.ReadRow(ctx, "com.google.cloud") 71 | if err != nil { 72 | return err 73 | } 74 | for _, column := range row["links"] { 75 | if column.Column != "links:golang.org" { 76 | return fmt.Errorf("response [%s] != [links:golang.org]", column.Column) 77 | } 78 | if string(column.Value) != "Gophers!" { 79 | return fmt.Errorf("response [%s] != [Gophers!]", string(column.Value)) 80 | } 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func silentClose(c io.Closer) { 87 | _ = c.Close() 88 | } 89 | -------------------------------------------------------------------------------- /examples/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/fullstorydev/emulators/examples 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | cloud.google.com/go/bigtable v1.37.0 9 | github.com/fullstorydev/emulators/bigtable v0.0.0 10 | github.com/fullstorydev/emulators/storage v0.0.0 11 | google.golang.org/api v0.235.0 12 | google.golang.org/grpc v1.72.2 13 | ) 14 | 15 | require ( 16 | cel.dev/expr v0.20.0 // indirect 17 | cloud.google.com/go v0.121.0 // indirect 18 | cloud.google.com/go/auth v0.16.1 // indirect 19 | cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect 20 | cloud.google.com/go/compute/metadata v0.7.0 // indirect 21 | cloud.google.com/go/iam v1.5.2 // indirect 22 | cloud.google.com/go/longrunning v0.6.7 // indirect 23 | cloud.google.com/go/monitoring v1.24.2 // indirect 24 | cloud.google.com/go/storage v1.54.0 // indirect 25 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect 26 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect 27 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect 28 | github.com/bluele/gcache v0.0.2 // indirect 29 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 30 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect 31 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect 32 | github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect 33 | github.com/felixge/httpsnoop v1.0.4 // indirect 34 | github.com/go-jose/go-jose/v4 v4.0.5 // indirect 35 | github.com/go-logr/logr v1.4.2 // indirect 36 | github.com/go-logr/stdr v1.2.2 // indirect 37 | github.com/golang/protobuf v1.5.4 // indirect 38 | github.com/golang/snappy v0.0.4 // indirect 39 | github.com/google/btree v1.1.3 // indirect 40 | github.com/google/s2a-go v0.1.9 // indirect 41 | github.com/google/uuid v1.6.0 // indirect 42 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect 43 | github.com/googleapis/gax-go/v2 v2.14.2 // indirect 44 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 45 | github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect 46 | github.com/syndtr/goleveldb v1.0.0 // indirect 47 | github.com/zeebo/errs v1.4.0 // indirect 48 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 49 | go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect 50 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect 51 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect 52 | go.opentelemetry.io/otel v1.35.0 // indirect 53 | go.opentelemetry.io/otel/metric v1.35.0 // indirect 54 | go.opentelemetry.io/otel/sdk v1.35.0 // indirect 55 | go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect 56 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 57 | golang.org/x/crypto v0.38.0 // indirect 58 | golang.org/x/net v0.40.0 // indirect 59 | golang.org/x/oauth2 v0.30.0 // indirect 60 | golang.org/x/sync v0.14.0 // indirect 61 | golang.org/x/sys v0.33.0 // indirect 62 | golang.org/x/text v0.25.0 // indirect 63 | golang.org/x/time v0.11.0 // indirect 64 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect 65 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect 66 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 // indirect 67 | google.golang.org/protobuf v1.36.6 // indirect 68 | gopkg.in/yaml.v2 v2.4.0 // indirect 69 | rsc.io/binaryregexp v0.2.0 // indirect 70 | ) 71 | 72 | replace ( 73 | github.com/fullstorydev/emulators/bigtable => ../bigtable 74 | github.com/fullstorydev/emulators/storage => ../storage 75 | ) 76 | -------------------------------------------------------------------------------- /examples/go.sum: -------------------------------------------------------------------------------- 1 | cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= 2 | cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= 3 | cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg= 4 | cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q= 5 | cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= 6 | cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= 7 | cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= 8 | cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= 9 | cloud.google.com/go/bigtable v1.37.0 h1:Q+x7y04lQ0B+WXp03wc1/FLhFt4CwcQdkwWT0M4Jp3w= 10 | cloud.google.com/go/bigtable v1.37.0/go.mod h1:HXqddP6hduwzrtiTCqZPpj9ij4hGZb4Zy1WF/dT+yaU= 11 | cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= 12 | cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= 13 | cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= 14 | cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= 15 | cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= 16 | cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= 17 | cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= 18 | cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= 19 | cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= 20 | cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= 21 | cloud.google.com/go/storage v1.54.0 h1:Du3XEyliAiftfyW0bwfdppm2MMLdpVAfiIg4T2nAI+0= 22 | cloud.google.com/go/storage v1.54.0/go.mod h1:hIi9Boe8cHxTyaeqh7KMMwKg088VblFK46C2x/BWaZE= 23 | cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= 24 | cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= 25 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= 26 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= 27 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= 28 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= 29 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= 30 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= 31 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= 32 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= 33 | github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= 34 | github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= 35 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 36 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 37 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= 38 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= 39 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 40 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 41 | github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= 42 | github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= 43 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= 44 | github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= 45 | github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= 46 | github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= 47 | github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= 48 | github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= 49 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 50 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 51 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 52 | github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= 53 | github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= 54 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 55 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 56 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 57 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 58 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 59 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 60 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 61 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 62 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 63 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 64 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 65 | github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= 66 | github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= 67 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 68 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 69 | github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= 70 | github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= 71 | github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= 72 | github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= 73 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 74 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 75 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= 76 | github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= 77 | github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= 78 | github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= 79 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 80 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 81 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 82 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 83 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 84 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 85 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 86 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= 87 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= 88 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 89 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 90 | github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= 91 | github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= 92 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 93 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 94 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 95 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 96 | github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= 97 | github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= 98 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 99 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 100 | go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= 101 | go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= 102 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= 103 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= 104 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= 105 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= 106 | go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= 107 | go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= 108 | go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= 109 | go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= 110 | go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= 111 | go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= 112 | go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= 113 | go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= 114 | go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= 115 | go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= 116 | go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= 117 | go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= 118 | golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= 119 | golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= 120 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 121 | golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= 122 | golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= 123 | golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= 124 | golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= 125 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 126 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 127 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 128 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 129 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 130 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 131 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 132 | golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= 133 | golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= 134 | golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= 135 | golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= 136 | google.golang.org/api v0.235.0 h1:C3MkpQSRxS1Jy6AkzTGKKrpSCOd2WOGrezZ+icKSkKo= 137 | google.golang.org/api v0.235.0/go.mod h1:QpeJkemzkFKe5VCE/PMv7GsUfn9ZF+u+q1Q7w6ckxTg= 138 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= 139 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= 140 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 h1:vPV0tzlsK6EzEDHNNH5sa7Hs9bd7iXR7B1tSiPepkV0= 141 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw= 142 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 h1:IkAfh6J/yllPtpYFU0zZN1hUPYdT0ogkBT/9hMxHjvg= 143 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= 144 | google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= 145 | google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= 146 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 147 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 148 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 149 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 150 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 151 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 152 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 153 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 154 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 155 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 156 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 157 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 158 | gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= 159 | gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= 160 | rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= 161 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 162 | -------------------------------------------------------------------------------- /examples/storage/example_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/fullstorydev/emulators/storage/gcsemu" 12 | ) 13 | 14 | func TestLocalServer(t *testing.T) { 15 | srv, err := gcsemu.NewServer("localhost:0", gcsemu.Options{}) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | defer srv.Close() 20 | 21 | err = validateServer(srv.Addr) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | } 26 | 27 | func validateServer(srvAddr string) error { 28 | // gcsemu.NewClient will look at this env var to figure out what host/port to talk to 29 | _ = os.Setenv("GCS_EMULATOR_HOST", srvAddr) 30 | 31 | ctx, cancel := context.WithCancel(context.Background()) 32 | defer cancel() 33 | fileContent := "Fullstory\n" + 34 | "Google Cloud Storage Emulator\n" + 35 | "Gophers!\n" 36 | 37 | client, err := gcsemu.NewClient(ctx) 38 | if err != nil { 39 | return err 40 | } 41 | defer silentClose(client) 42 | 43 | o := client.Bucket("test").Object("data/test.txt") 44 | writer := o.NewWriter(ctx) 45 | 46 | _, err = io.Copy(writer, strings.NewReader(fileContent)) 47 | if err != nil { 48 | return err 49 | } 50 | err = writer.Close() 51 | if err != nil { 52 | return err 53 | } 54 | 55 | reader, err := o.NewReader(ctx) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | res, err := io.ReadAll(reader) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | if string(res) != fileContent { 66 | return fmt.Errorf("response [%s] != file content [%s]", string(res), fileContent) 67 | } 68 | 69 | return nil 70 | } 71 | 72 | func silentClose(c io.Closer) { 73 | _ = c.Close() 74 | } 75 | -------------------------------------------------------------------------------- /storage/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23-alpine As builder 2 | MAINTAINER Fullstory Engineering 3 | 4 | # create non-privileged group and user 5 | RUN addgroup -S emulators && adduser -S emulators -G emulators 6 | RUN mkdir -p /data 7 | # copy just the files/sources we need to build the emulator 8 | WORKDIR /tmp/fullstorydev/storage 9 | COPY VERSION *.go go.* /tmp/fullstorydev/storage/ 10 | COPY gcsemu /tmp/fullstorydev/storage/gcsemu 11 | COPY gcsutil /tmp/fullstorydev/storage/gcsutil 12 | COPY cmd /tmp/fullstorydev/storage/cmd 13 | # and build a completely static binary (so we can use 14 | # scratch as basis for the final image) 15 | ENV CGO_ENABLED=0 16 | ENV GO111MODULE=on 17 | RUN go build -o /gcsemulator \ 18 | -ldflags "-w -extldflags \"-static\" -X \"main.version=$(cat VERSION)\"" \ 19 | ./cmd/gcsemulator 20 | 21 | # New FROM so we have a nice'n'tiny image 22 | FROM scratch AS gcsemulator 23 | WORKDIR / 24 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 25 | COPY --from=builder /etc/passwd /etc/passwd 26 | COPY --from=builder /gcsemulator /bin/gcsemulator 27 | COPY --from=builder --chown=emulators /data /data 28 | EXPOSE 9000 29 | USER emulators 30 | ENTRYPOINT ["/bin/gcsemulator", "-port", "9000", "-host", "0.0.0.0", "-dir", "/data"] 31 | -------------------------------------------------------------------------------- /storage/Makefile: -------------------------------------------------------------------------------- 1 | # dev_build_prefixed_version has the format "storage/" 2 | dev_build_prefixed_version=$(shell git describe --tags --always --match 'storage/*' --dirty) 3 | # dev_build_version is like dev_build_prefixed_version, but without the "storage/" prefix 4 | dev_build_version=$(shell echo $(dev_build_prefixed_version) | sed 's/^storage\///') 5 | 6 | .PHONY: ci 7 | ci: deps checkgofmt vet staticcheck ineffassign predeclared golint errcheck test 8 | 9 | .PHONY: deps 10 | deps: 11 | go mod download 12 | 13 | .PHONY: updatedeps 14 | updatedeps: 15 | go get -d -v -t -u -f ./... 16 | go mod tidy 17 | 18 | .PHONY: checkgofmt 19 | checkgofmt: 20 | gofmt -s -l . 21 | @if [ -n "$$(gofmt -s -l .)" ]; then \ 22 | exit 1; \ 23 | fi 24 | 25 | .PHONY: vet 26 | vet: 27 | go vet ./... 28 | 29 | .PHONY: staticcheck 30 | staticcheck: 31 | @go install honnef.co/go/tools/cmd/staticcheck@v0.6.0 32 | staticcheck ./... 33 | 34 | .PHONY: ineffassign 35 | ineffassign: 36 | @go install github.com/gordonklaus/ineffassign@7953dde2c7bf 37 | ineffassign . 38 | 39 | .PHONY: predeclared 40 | predeclared: 41 | @go install github.com/nishanths/predeclared@51e8c974458a0f93dc03fe356f91ae1a6d791e6f 42 | predeclared ./... 43 | 44 | .PHONY: golint 45 | golint: 46 | @go install golang.org/x/lint/golint@v0.0.0-20210508222113-6edffad5e616 47 | golint -min_confidence 0.9 -set_exit_status ./... 48 | 49 | .PHONY: errcheck 50 | errcheck: 51 | @go install github.com/kisielk/errcheck@v1.2.0 52 | errcheck ./... 53 | 54 | .PHONY: test 55 | test: deps 56 | go test -race ./... 57 | 58 | .PHONY: install 59 | install: 60 | go install -ldflags '-X "main.version=dev build $(dev_build_version)"' ./... 61 | 62 | .PHONY: release 63 | release: 64 | @go install github.com/goreleaser/goreleaser@v1.21.0 65 | goreleaser release --clean 66 | 67 | .PHONY: docker 68 | docker: 69 | @echo $(dev_build_version) > VERSION 70 | docker build -t fullstorydev/gcsemulator:$(dev_build_version) . 71 | @rm VERSION 72 | -------------------------------------------------------------------------------- /storage/cmd/gcsemulator/gcsemulator.go: -------------------------------------------------------------------------------- 1 | // gcsemulator launches the Cloud Storage emulator on the given address. 2 | package main 3 | 4 | import ( 5 | "flag" 6 | "fmt" 7 | "log" 8 | 9 | "github.com/fullstorydev/emulators/storage/gcsemu" 10 | ) 11 | 12 | var ( 13 | host = flag.String("host", "localhost", "the address to bind to on the local machine") 14 | port = flag.Int("port", 9000, "the port number to bind to on the local machine") 15 | dir = flag.String("dir", "", "if set, use persistence in the given directory") 16 | verbose = flag.Bool("verbose", true, "log verbosely") 17 | ) 18 | 19 | func main() { 20 | flag.Parse() 21 | opts := gcsemu.Options{ 22 | Verbose: *verbose, 23 | Log: func(err error, fmt string, args ...interface{}) { 24 | if err != nil { 25 | fmt = "ERROR: " + fmt + ": %s" 26 | args = append(args, err) 27 | } 28 | log.Printf(fmt, args...) 29 | }, 30 | } 31 | if *dir != "" { 32 | fmt.Printf("Writing to: %s\n", *dir) 33 | opts.Store = gcsemu.NewFileStore(*dir) 34 | } 35 | 36 | laddr := fmt.Sprintf("%s:%d", *host, *port) 37 | server, err := gcsemu.NewServer(laddr, opts) 38 | if err != nil { 39 | log.Fatalf("failed to start server: %s", err) 40 | } 41 | defer server.Close() 42 | 43 | fmt.Printf("Cloud Storage emulator running on %s\n", server.Addr) 44 | select {} 45 | } 46 | -------------------------------------------------------------------------------- /storage/gcsemu/batch.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "mime/multipart" 9 | "net/http" 10 | "net/http/httptest" 11 | "net/textproto" 12 | ) 13 | 14 | // BatchHandler handles emulated GCS http requests for "storage.googleapis.com/batch/storage/v1". 15 | func (g *GcsEmu) BatchHandler(w http.ResponseWriter, r *http.Request) { 16 | // First parse the entire incoming message. 17 | reader, err := r.MultipartReader() 18 | if err != nil { 19 | g.gapiError(w, httpStatusCodeOf(err), err.Error()) 20 | return 21 | } 22 | 23 | var reqs []*http.Request 24 | var contentIds []string 25 | for i := 0; true; i++ { 26 | part, err := reader.NextPart() 27 | if err == io.EOF { 28 | break // done 29 | } else if err != nil { 30 | g.gapiError(w, http.StatusBadRequest, err.Error()) 31 | return 32 | } 33 | 34 | if ct := part.Header.Get("Content-Type"); ct != "application/http" { 35 | g.gapiError(w, http.StatusBadRequest, fmt.Sprintf("Content-Type: want=application/http, got=%s", ct)) 36 | return 37 | } 38 | 39 | contentId := part.Header.Get("Content-ID") 40 | 41 | content, err := io.ReadAll(part) 42 | _ = part.Close() 43 | if err != nil { 44 | g.gapiError(w, http.StatusBadRequest, fmt.Sprintf("part=%d, Content-ID=%s: read error %v", i, contentId, err)) 45 | return 46 | } 47 | 48 | newReader := bufio.NewReader(bytes.NewReader(content)) 49 | req, err := http.ReadRequest(newReader) 50 | if err != nil { 51 | g.gapiError(w, http.StatusBadRequest, fmt.Sprintf("part=%d, Content-ID=%s: unable to parse request %v", i, contentId, err)) 52 | return 53 | } 54 | // Any remaining bytes are the body. 55 | rem, _ := io.ReadAll(newReader) 56 | if len(rem) > 0 { 57 | req.GetBody = func() (io.ReadCloser, error) { 58 | return io.NopCloser(bytes.NewReader(rem)), nil 59 | } 60 | req.Body, _ = req.GetBody() 61 | } 62 | if cte := part.Header.Get("Content-Transfer-Encoding"); cte != "" { 63 | req.Header.Set("Transfer-Encoding", cte) 64 | } 65 | // encoded requests don't include a host, so patch it up from the incoming request 66 | req.Host = r.Host 67 | reqs = append(reqs, req) 68 | contentIds = append(contentIds, contentId) 69 | } 70 | 71 | // At this point, we can respond with a 200. 72 | mw := multipart.NewWriter(w) 73 | w.Header().Set("Content-Type", "multipart/mixed; boundary="+mw.Boundary()) 74 | w.WriteHeader(http.StatusOK) 75 | 76 | // run each request 77 | for i := range reqs { 78 | req, contentId := reqs[i], contentIds[i] 79 | 80 | rw := httptest.NewRecorder() 81 | g.Handler(rw, req) 82 | rsp := rw.Result() 83 | rsp.ContentLength = int64(rw.Body.Len()) 84 | 85 | partHeaders := textproto.MIMEHeader{} 86 | partHeaders.Set("Content-Type", "application/http") 87 | if contentId != "" { 88 | if contentId[0] == '<' { 89 | contentId = "" 43 | } 44 | return fmt.Sprintf("http error %d: %s", err.code, err.cause) 45 | } 46 | 47 | func (err *httpError) Unwrap() error { 48 | return err.cause 49 | } 50 | -------------------------------------------------------------------------------- /storage/gcsemu/filestore.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | 13 | cloudstorage "cloud.google.com/go/storage" 14 | "google.golang.org/api/storage/v1" 15 | ) 16 | 17 | const ( 18 | metaExtention = ".emumeta" 19 | ) 20 | 21 | type filestore struct { 22 | gcsDir string 23 | } 24 | 25 | var _ Store = (*filestore)(nil) 26 | 27 | // NewFileStore returns a new Store that writes to the given directory. 28 | func NewFileStore(gcsDir string) *filestore { 29 | return &filestore{gcsDir: gcsDir} 30 | } 31 | 32 | type composeObj struct { 33 | filename string 34 | conds cloudstorage.Conditions 35 | } 36 | 37 | func (fs *filestore) CreateBucket(bucket string) error { 38 | bucketDir := filepath.Join(fs.gcsDir, bucket) 39 | return os.MkdirAll(bucketDir, 0777) 40 | } 41 | 42 | func (fs *filestore) GetBucketMeta(baseUrl HttpBaseUrl, bucket string) (*storage.Bucket, error) { 43 | f := fs.filename(bucket, "") 44 | fInfo, err := os.Stat(f) 45 | if err != nil { 46 | if os.IsNotExist(err) { 47 | return nil, nil 48 | } 49 | return nil, fmt.Errorf("stating %s: %w", f, err) 50 | } 51 | 52 | obj := BucketMeta(baseUrl, bucket) 53 | obj.Updated = fInfo.ModTime().UTC().Format(time.RFC3339Nano) 54 | return obj, nil 55 | } 56 | 57 | func (fs *filestore) Get(baseUrl HttpBaseUrl, bucket string, filename string) (*storage.Object, []byte, error) { 58 | obj, err := fs.GetMeta(baseUrl, bucket, filename) 59 | if err != nil { 60 | return nil, nil, err 61 | } 62 | if obj == nil { 63 | return nil, nil, nil 64 | } 65 | 66 | f := fs.filename(bucket, filename) 67 | contents, err := os.ReadFile(f) 68 | if err != nil { 69 | return nil, nil, fmt.Errorf("reading %s: %w", f, err) 70 | } 71 | return obj, contents, nil 72 | } 73 | 74 | func (fs *filestore) GetMeta(baseUrl HttpBaseUrl, bucket string, filename string) (*storage.Object, error) { 75 | f := fs.filename(bucket, filename) 76 | fInfo, err := os.Stat(f) 77 | if err != nil { 78 | if os.IsNotExist(err) { 79 | return nil, nil 80 | } 81 | return nil, fmt.Errorf("stating %s: %w", f, err) 82 | } 83 | 84 | return fs.ReadMeta(baseUrl, bucket, filename, fInfo) 85 | } 86 | 87 | func (fs *filestore) Add(bucket string, filename string, contents []byte, meta *storage.Object) error { 88 | f := fs.filename(bucket, filename) 89 | if err := os.MkdirAll(filepath.Dir(f), 0777); err != nil { 90 | return fmt.Errorf("could not create dirs for: %s: %w", f, err) 91 | } 92 | 93 | if err := os.WriteFile(f, contents, 0666); err != nil { 94 | return fmt.Errorf("could not write: %s: %w", f, err) 95 | } 96 | 97 | // Force a new modification time, since this is what Generation is based on. 98 | now := time.Now().UTC() 99 | _ = os.Chtimes(f, now, now) 100 | 101 | InitScrubbedMeta(meta, filename) 102 | meta.Metageneration = 1 103 | if meta.TimeCreated == "" { 104 | meta.TimeCreated = now.UTC().Format(time.RFC3339Nano) 105 | } 106 | 107 | fMeta := metaFilename(f) 108 | if err := os.WriteFile(fMeta, mustJson(meta), 0666); err != nil { 109 | return fmt.Errorf("could not write metadata file: %s: %w", fMeta, err) 110 | } 111 | 112 | return nil 113 | } 114 | 115 | func (fs *filestore) UpdateMeta(bucket string, filename string, meta *storage.Object, metagen int64) error { 116 | InitScrubbedMeta(meta, filename) 117 | meta.Metageneration = metagen 118 | 119 | fMeta := metaFilename(fs.filename(bucket, filename)) 120 | if err := os.WriteFile(fMeta, mustJson(meta), 0666); err != nil { 121 | return fmt.Errorf("could not write metadata file: %s: %w", fMeta, err) 122 | } 123 | 124 | return nil 125 | } 126 | 127 | func (fs *filestore) Copy(srcBucket string, srcFile string, dstBucket string, dstFile string) (bool, error) { 128 | // Make sure it's there 129 | meta, err := fs.GetMeta(dontNeedUrls, srcBucket, srcFile) 130 | if err != nil { 131 | return false, err 132 | } 133 | // Handle object-not-found 134 | if meta == nil { 135 | return false, nil 136 | } 137 | 138 | // Copy with metadata 139 | f1 := fs.filename(srcBucket, srcFile) 140 | contents, err := os.ReadFile(f1) 141 | if err != nil { 142 | return false, err 143 | } 144 | meta.TimeCreated = "" // reset creation time on the dest file 145 | err = fs.Add(dstBucket, dstFile, contents, meta) 146 | if err != nil { 147 | return false, err 148 | } 149 | 150 | return true, nil 151 | } 152 | 153 | func (fs *filestore) Delete(bucket string, filename string) error { 154 | f := fs.filename(bucket, filename) 155 | 156 | err := func() error { 157 | // Check if the bucket exists 158 | if _, err := os.Stat(f); os.IsNotExist(err) { 159 | return os.ErrNotExist 160 | } 161 | 162 | // Remove the bucket 163 | if filename == "" { 164 | return os.RemoveAll(f) 165 | } 166 | 167 | // Remove just the file and the associated metadata file 168 | if err := os.Remove(f); err != nil { 169 | return err 170 | } 171 | err := os.Remove(metaFilename(f)) 172 | if os.IsNotExist(err) { 173 | // Legacy files do not have an accompanying metadata file. 174 | return nil 175 | } 176 | return err 177 | }() 178 | if err != nil { 179 | if os.IsNotExist(err) { 180 | return err 181 | } 182 | return fmt.Errorf("could not delete %s: %w", f, err) 183 | } 184 | 185 | // Try to delete empty directories 186 | for fp := filepath.Dir(f); len(fp) > len(fs.filename(bucket, "")); fp = filepath.Dir(fp) { 187 | files, err := os.ReadDir(fp) 188 | if err != nil || len(files) > 0 { 189 | // Quit trying to delete the directory 190 | break 191 | } 192 | if err := os.Remove(fp); err != nil { 193 | // If removing fails, quit trying 194 | break 195 | } 196 | } 197 | return nil 198 | } 199 | 200 | func (fs *filestore) ReadMeta(baseUrl HttpBaseUrl, bucket string, filename string, fInfo os.FileInfo) (*storage.Object, error) { 201 | if fInfo.IsDir() { 202 | return nil, nil 203 | } 204 | 205 | f := fs.filename(bucket, filename) 206 | obj := &storage.Object{} 207 | fMeta := metaFilename(f) 208 | buf, err := os.ReadFile(fMeta) 209 | if err != nil { 210 | if !os.IsNotExist(err) { 211 | return nil, fmt.Errorf("could not read metadata file %s: %w", fMeta, err) 212 | } 213 | } 214 | 215 | if len(buf) != 0 { 216 | if err := json.NewDecoder(bytes.NewReader(buf)).Decode(obj); err != nil { 217 | return nil, fmt.Errorf("could not parse file attributes %q for %s: %w", buf, f, err) 218 | } 219 | } 220 | 221 | InitMetaWithUrls(baseUrl, obj, bucket, filename, uint64(fInfo.Size())) 222 | obj.Generation = fInfo.ModTime().UnixNano() // use the mod time as the generation number 223 | obj.Updated = fInfo.ModTime().UTC().Format(time.RFC3339Nano) 224 | return obj, nil 225 | } 226 | 227 | func (fs *filestore) filename(bucket string, filename string) string { 228 | if filename == "" { 229 | return filepath.Join(fs.gcsDir, bucket) 230 | } 231 | return filepath.Join(fs.gcsDir, bucket, filename) 232 | } 233 | 234 | func metaFilename(filename string) string { 235 | return filename + metaExtention 236 | } 237 | 238 | func (fs *filestore) Walk(ctx context.Context, bucket string, cb func(ctx context.Context, filename string, fInfo os.FileInfo) error) error { 239 | root := filepath.Join(fs.gcsDir, bucket) 240 | return filepath.Walk(root, func(path string, fInfo os.FileInfo, err error) error { 241 | if strings.HasSuffix(path, metaExtention) { 242 | // Ignore metadata files 243 | return nil 244 | } 245 | 246 | filename := strings.TrimPrefix(path, root) 247 | filename = strings.TrimPrefix(filename, string(os.PathSeparator)) 248 | if err != nil { 249 | if os.IsNotExist(err) { 250 | return err 251 | } 252 | return fmt.Errorf("walk error at %s: %w", filename, err) 253 | } 254 | 255 | if err := cb(ctx, filename, fInfo); err != nil { 256 | return err 257 | } 258 | return nil 259 | }) 260 | } 261 | -------------------------------------------------------------------------------- /storage/gcsemu/filestore_test.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | "time" 12 | 13 | "gotest.tools/v3/assert" 14 | ) 15 | 16 | func TestFileStore(t *testing.T) { 17 | // Setup an on-disk emulator. 18 | gcsDir := filepath.Join(os.TempDir(), fmt.Sprintf("gcsemu-test-%d", time.Now().Unix())) 19 | gcsEmu := NewGcsEmu(Options{ 20 | Store: NewFileStore(gcsDir), 21 | Verbose: true, 22 | Log: func(err error, fmt string, args ...interface{}) { 23 | t.Helper() 24 | if err != nil { 25 | fmt = "ERROR: " + fmt + ": %s" 26 | args = append(args, err) 27 | } 28 | t.Logf(fmt, args...) 29 | }, 30 | }) 31 | mux := http.NewServeMux() 32 | gcsEmu.Register(mux) 33 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 34 | t.Logf("about to method=%s host=%s u=%s", r.Method, r.Host, r.URL) 35 | mux.ServeHTTP(w, r) 36 | })) 37 | t.Cleanup(svr.Close) 38 | 39 | gcsClient, err := NewTestClientWithHost(context.Background(), svr.URL) 40 | assert.NilError(t, err) 41 | t.Cleanup(func() { 42 | _ = gcsClient.Close() 43 | }) 44 | 45 | bh := BucketHandle{ 46 | Name: "file-bucket", 47 | BucketHandle: gcsClient.Bucket("file-bucket"), 48 | } 49 | initBucket(t, bh) 50 | attrs, err := bh.Attrs(context.Background()) 51 | assert.NilError(t, err) 52 | assert.Equal(t, bh.Name, attrs.Name) 53 | 54 | t.Parallel() 55 | for _, tc := range testCases { 56 | tc := tc 57 | t.Run(tc.name, func(t *testing.T) { 58 | t.Parallel() 59 | tc.f(t, bh) 60 | }) 61 | } 62 | 63 | t.Run("RawHttp", func(t *testing.T) { 64 | t.Parallel() 65 | testRawHttp(t, bh, http.DefaultClient, svr.URL) 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /storage/gcsemu/http_wrappers.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | // DrainRequestHandler wraps the given handler to drain the incoming request body on exit. 10 | func DrainRequestHandler(h http.HandlerFunc) http.HandlerFunc { 11 | return func(w http.ResponseWriter, r *http.Request) { 12 | defer func() { 13 | // Always drain and close the request body to properly free up the connection. 14 | // See https://groups.google.com/forum/#!topic/golang-nuts/pP3zyUlbT00 15 | _, _ = io.Copy(io.Discard, r.Body) 16 | _ = r.Body.Close() 17 | }() 18 | h(w, r) 19 | } 20 | } 21 | 22 | // GzipRequestHandler wraps the given handler to automatically decompress gzipped content. 23 | func GzipRequestHandler(h http.HandlerFunc) http.HandlerFunc { 24 | return func(w http.ResponseWriter, r *http.Request) { 25 | if r.Header.Get("Content-Encoding") == "gzip" { 26 | gzr, err := gzip.NewReader(r.Body) 27 | if err != nil { 28 | http.Error(w, err.Error(), http.StatusBadRequest) 29 | return 30 | } 31 | r.Body = gzr 32 | } 33 | h(w, r) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /storage/gcsemu/memstore.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "sync" 7 | "time" 8 | 9 | "github.com/google/btree" 10 | "google.golang.org/api/storage/v1" 11 | ) 12 | 13 | type memstore struct { 14 | mu sync.RWMutex 15 | buckets map[string]*memBucket 16 | } 17 | 18 | var _ Store = (*memstore)(nil) 19 | 20 | // NewMemStore returns a Store that operates purely in memory. 21 | func NewMemStore() *memstore { 22 | return &memstore{buckets: map[string]*memBucket{}} 23 | } 24 | 25 | type memBucket struct { 26 | created time.Time 27 | 28 | // mutex required (despite lock map in gcsemu), because btree mutations are not structurally safe 29 | mu sync.RWMutex 30 | files *btree.BTree 31 | } 32 | 33 | func (ms *memstore) getBucket(bucket string) *memBucket { 34 | ms.mu.RLock() 35 | defer ms.mu.RUnlock() 36 | return ms.buckets[bucket] 37 | } 38 | 39 | type memFile struct { 40 | meta storage.Object 41 | data []byte 42 | } 43 | 44 | func (mf *memFile) Less(than btree.Item) bool { 45 | // TODO(dragonsinth): is a simple lexical sort ok for Walk? 46 | return mf.meta.Name < than.(*memFile).meta.Name 47 | } 48 | 49 | var _ btree.Item = (*memFile)(nil) 50 | 51 | func (ms *memstore) CreateBucket(bucket string) error { 52 | ms.mu.Lock() 53 | defer ms.mu.Unlock() 54 | if ms.buckets[bucket] == nil { 55 | ms.buckets[bucket] = &memBucket{ 56 | created: time.Now(), 57 | files: btree.New(16), 58 | } 59 | } 60 | return nil 61 | } 62 | 63 | func (ms *memstore) GetBucketMeta(baseUrl HttpBaseUrl, bucket string) (*storage.Bucket, error) { 64 | if b := ms.getBucket(bucket); b != nil { 65 | obj := BucketMeta(baseUrl, bucket) 66 | obj.Updated = b.created.UTC().Format(time.RFC3339Nano) 67 | return obj, nil 68 | } 69 | return nil, nil 70 | } 71 | 72 | func (ms *memstore) Get(baseUrl HttpBaseUrl, bucket string, filename string) (*storage.Object, []byte, error) { 73 | f := ms.find(bucket, filename) 74 | if f != nil { 75 | return &f.meta, f.data, nil 76 | } 77 | return nil, nil, nil 78 | } 79 | 80 | func (ms *memstore) GetMeta(baseUrl HttpBaseUrl, bucket string, filename string) (*storage.Object, error) { 81 | f := ms.find(bucket, filename) 82 | if f != nil { 83 | meta := f.meta 84 | InitMetaWithUrls(baseUrl, &meta, bucket, filename, uint64(len(f.data))) 85 | return &meta, nil 86 | } 87 | return nil, nil 88 | } 89 | 90 | func (ms *memstore) Add(bucket string, filename string, contents []byte, meta *storage.Object) error { 91 | _ = ms.CreateBucket(bucket) 92 | 93 | InitScrubbedMeta(meta, filename) 94 | meta.Metageneration = 1 95 | 96 | // Cannot be overridden by caller 97 | now := time.Now().UTC() 98 | meta.Updated = now.UTC().Format(time.RFC3339Nano) 99 | meta.Generation = now.UnixNano() 100 | if meta.TimeCreated == "" { 101 | meta.TimeCreated = meta.Updated 102 | } 103 | 104 | b := ms.getBucket(bucket) 105 | b.mu.Lock() 106 | defer b.mu.Unlock() 107 | b.files.ReplaceOrInsert(&memFile{ 108 | meta: *meta, 109 | data: contents, 110 | }) 111 | return nil 112 | } 113 | 114 | func (ms *memstore) UpdateMeta(bucket string, filename string, meta *storage.Object, metagen int64) error { 115 | f := ms.find(bucket, filename) 116 | if f == nil { 117 | return os.ErrNotExist 118 | } 119 | 120 | InitScrubbedMeta(meta, filename) 121 | meta.Metageneration = metagen 122 | 123 | b := ms.getBucket(bucket) 124 | b.mu.Lock() 125 | defer b.mu.Unlock() 126 | b.files.ReplaceOrInsert(&memFile{ 127 | meta: *meta, 128 | data: f.data, 129 | }) 130 | return nil 131 | } 132 | 133 | func (ms *memstore) Copy(srcBucket string, srcFile string, dstBucket string, dstFile string) (bool, error) { 134 | src := ms.find(srcBucket, srcFile) 135 | if src == nil { 136 | return false, nil 137 | } 138 | 139 | // Copy with metadata 140 | meta := src.meta 141 | meta.TimeCreated = "" // reset creation time on the dest file 142 | err := ms.Add(dstBucket, dstFile, src.data, &meta) 143 | if err != nil { 144 | return false, err 145 | } 146 | 147 | return true, nil 148 | } 149 | 150 | func (ms *memstore) Delete(bucket string, filename string) error { 151 | if filename == "" { 152 | // Remove the bucket 153 | ms.mu.Lock() 154 | defer ms.mu.Unlock() 155 | if _, ok := ms.buckets[bucket]; !ok { 156 | return os.ErrNotExist 157 | } 158 | 159 | delete(ms.buckets, bucket) 160 | } else if b := ms.getBucket(bucket); b != nil { 161 | // Remove just the file 162 | b.mu.Lock() 163 | defer b.mu.Unlock() 164 | if b.files.Delete(ms.key(filename)) == nil { 165 | // case file does not exist 166 | return os.ErrNotExist 167 | } 168 | } else { 169 | return os.ErrNotExist 170 | } 171 | 172 | return nil 173 | } 174 | 175 | func (ms *memstore) ReadMeta(baseUrl HttpBaseUrl, bucket string, filename string, _ os.FileInfo) (*storage.Object, error) { 176 | return ms.GetMeta(baseUrl, bucket, filename) 177 | } 178 | 179 | func (ms *memstore) Walk(ctx context.Context, bucket string, cb func(ctx context.Context, filename string, fInfo os.FileInfo) error) error { 180 | if b := ms.getBucket(bucket); b != nil { 181 | var err error 182 | b.mu.RLock() 183 | defer b.mu.RUnlock() 184 | b.files.Ascend(func(i btree.Item) bool { 185 | mf := i.(*memFile) 186 | err = cb(ctx, mf.meta.Name, nil) 187 | return err == nil 188 | }) 189 | return nil 190 | } 191 | return os.ErrNotExist 192 | } 193 | 194 | func (ms *memstore) key(filename string) btree.Item { 195 | return &memFile{ 196 | meta: storage.Object{ 197 | Name: filename, 198 | }, 199 | } 200 | } 201 | 202 | func (ms *memstore) find(bucket string, filename string) *memFile { 203 | if b := ms.getBucket(bucket); b != nil { 204 | b.mu.Lock() 205 | defer b.mu.Unlock() 206 | f := b.files.Get(ms.key(filename)) 207 | if f != nil { 208 | return f.(*memFile) 209 | } 210 | } 211 | return nil 212 | } 213 | -------------------------------------------------------------------------------- /storage/gcsemu/memstore_test.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "gotest.tools/v3/assert" 10 | ) 11 | 12 | func TestMemStore(t *testing.T) { 13 | // Setup an in-memory emulator. 14 | gcsEmu := NewGcsEmu(Options{ 15 | Verbose: true, 16 | Log: func(err error, fmt string, args ...interface{}) { 17 | t.Helper() 18 | if err != nil { 19 | fmt = "ERROR: " + fmt + ": %s" 20 | args = append(args, err) 21 | } 22 | t.Logf(fmt, args...) 23 | }, 24 | }) 25 | mux := http.NewServeMux() 26 | gcsEmu.Register(mux) 27 | svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 28 | t.Logf("about to method=%s host=%s u=%s", r.Method, r.Host, r.URL) 29 | mux.ServeHTTP(w, r) 30 | })) 31 | t.Cleanup(svr.Close) 32 | 33 | gcsClient, err := NewTestClientWithHost(context.Background(), svr.URL) 34 | assert.NilError(t, err) 35 | t.Cleanup(func() { 36 | _ = gcsClient.Close() 37 | }) 38 | 39 | bh := BucketHandle{ 40 | Name: "mem-bucket", 41 | BucketHandle: gcsClient.Bucket("mem-bucket"), 42 | } 43 | initBucket(t, bh) 44 | attrs, err := bh.Attrs(context.Background()) 45 | assert.NilError(t, err) 46 | assert.Equal(t, bh.Name, attrs.Name) 47 | 48 | t.Parallel() 49 | for _, tc := range testCases { 50 | tc := tc 51 | t.Run(tc.name, func(t *testing.T) { 52 | t.Parallel() 53 | tc.f(t, bh) 54 | }) 55 | } 56 | 57 | t.Run("RawHttp", func(t *testing.T) { 58 | t.Parallel() 59 | testRawHttp(t, bh, http.DefaultClient, svr.URL) 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /storage/gcsemu/meta.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "fmt" 5 | "mime" 6 | "strings" 7 | 8 | "google.golang.org/api/storage/v1" 9 | ) 10 | 11 | // BucketMeta returns a default bucket metadata for the given name and base url. 12 | func BucketMeta(baseUrl HttpBaseUrl, bucket string) *storage.Bucket { 13 | return &storage.Bucket{ 14 | Kind: "storage#bucket", 15 | Name: bucket, 16 | SelfLink: BucketUrl(baseUrl, bucket), 17 | StorageClass: "STANDARD", 18 | } 19 | } 20 | 21 | // InitScrubbedMeta "bakes" metadata with intrinsic values and removes fields that are intrinsic / computed. 22 | func InitScrubbedMeta(meta *storage.Object, filename string) { 23 | parts := strings.Split(filename, ".") 24 | ext := parts[len(parts)-1] 25 | 26 | if meta.ContentType == "" { 27 | meta.ContentType = mime.TypeByExtension(ext) 28 | } 29 | meta.Name = filename 30 | ScrubMeta(meta) 31 | } 32 | 33 | // InitMetaWithUrls "bakes" metadata with intrinsic values, including computed links. 34 | func InitMetaWithUrls(baseUrl HttpBaseUrl, meta *storage.Object, bucket string, filename string, size uint64) { 35 | parts := strings.Split(filename, ".") 36 | ext := parts[len(parts)-1] 37 | 38 | meta.Bucket = bucket 39 | if meta.ContentType == "" { 40 | meta.ContentType = mime.TypeByExtension(ext) 41 | } 42 | meta.Kind = "storage#object" 43 | meta.MediaLink = ObjectUrl(baseUrl, bucket, filename) + "?alt=media" 44 | meta.Name = filename 45 | meta.SelfLink = ObjectUrl(baseUrl, bucket, filename) 46 | meta.Size = size 47 | meta.StorageClass = "STANDARD" 48 | } 49 | 50 | // ScrubMeta removes fields that are intrinsic / computed for minimal storage. 51 | func ScrubMeta(meta *storage.Object) { 52 | meta.Bucket = "" 53 | meta.Kind = "" 54 | meta.MediaLink = "" 55 | meta.SelfLink = "" 56 | meta.Size = 0 57 | meta.StorageClass = "" 58 | } 59 | 60 | // BucketUrl returns the URL for a bucket. 61 | func BucketUrl(baseUrl HttpBaseUrl, bucket string) string { 62 | return fmt.Sprintf("%sstorage/v1/b/%s", normalizeBaseUrl(baseUrl), bucket) 63 | } 64 | 65 | // ObjectUrl returns the URL for a file. 66 | func ObjectUrl(baseUrl HttpBaseUrl, bucket string, filepath string) string { 67 | return fmt.Sprintf("%sstorage/v1/b/%s/o/%s", normalizeBaseUrl(baseUrl), bucket, filepath) 68 | } 69 | 70 | // HttpBaseUrl represents the emulator base URL, including trailing slash; e.g. https://www.googleapis.com/ 71 | type HttpBaseUrl string 72 | 73 | // when the caller doesn't really care about the object meta URLs 74 | const dontNeedUrls = HttpBaseUrl("") 75 | 76 | func normalizeBaseUrl(baseUrl HttpBaseUrl) HttpBaseUrl { 77 | if baseUrl == dontNeedUrls || baseUrl == "https://storage.googleapis.com/" { 78 | return "https://www.googleapis.com/" 79 | } else if baseUrl == "http://storage.googleapis.com/" { 80 | return "http://www.googleapis.com/" 81 | } else { 82 | return baseUrl 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /storage/gcsemu/multipart.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "mime" 8 | "mime/multipart" 9 | "net/http" 10 | 11 | "google.golang.org/api/storage/v1" 12 | ) 13 | 14 | func readMultipartInsert(r *http.Request) (*storage.Object, []byte, error) { 15 | v := r.Header.Get("Content-Type") 16 | if v == "" { 17 | return nil, nil, fmt.Errorf("failed to parse Content-Type header: %q", v) 18 | } 19 | d, params, err := mime.ParseMediaType(v) 20 | if err != nil || d != "multipart/related" { 21 | return nil, nil, fmt.Errorf("failed to parse Content-Type header: %q", v) 22 | } 23 | boundary, ok := params["boundary"] 24 | if !ok { 25 | return nil, nil, fmt.Errorf("Content-Type header is missing boundary: %q", v) 26 | } 27 | 28 | reader := multipart.NewReader(r.Body, boundary) 29 | 30 | readPart := func() ([]byte, error) { 31 | part, err := reader.NextPart() 32 | if err != nil { 33 | return nil, fmt.Errorf("failed to get multipart: %w", err) 34 | } 35 | 36 | b, err := io.ReadAll(part) 37 | if err != nil { 38 | return nil, fmt.Errorf("failed to get read multipart: %w", err) 39 | } 40 | 41 | return b, nil 42 | } 43 | 44 | // read the first part to get the storage.Object (in json) 45 | b, err := readPart() 46 | if err != nil { 47 | return nil, nil, fmt.Errorf("failed to read first part of body: %w", err) 48 | } 49 | 50 | var obj storage.Object 51 | err = json.Unmarshal(b, &obj) 52 | if err != nil { 53 | return nil, nil, fmt.Errorf("failed to parse body as json: %w", err) 54 | } 55 | 56 | // read the next part to get the file contents 57 | contents, err := readPart() 58 | if err != nil { 59 | return nil, nil, fmt.Errorf("failed to read second part of body: %w", err) 60 | } 61 | 62 | obj.Size = uint64(len(contents)) 63 | 64 | return &obj, contents, nil 65 | } 66 | -------------------------------------------------------------------------------- /storage/gcsemu/parse.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "net/url" 5 | "regexp" 6 | ) 7 | 8 | const ( 9 | // example: "/storage/v1/b/my-bucket/o/2013-tax-returns.pdf" (for a file) or "/storage/v1/b/my-bucket/o" (for a bucket) 10 | gcsObjectPathPattern = "/storage/v1/b/([^\\/]+)/o(?:/(.+))?" 11 | // example: "//b/my-bucket/o/2013-tax-returns.pdf" (for a file) or "/b/my-bucket/o" (for a bucket) 12 | gcsObjectPathPattern2 = "/b/([^\\/]+)/o(?:/(.+))?" 13 | // example: "/storage/v1/b/my-bucket 14 | gcsBucketPathPattern = "/storage/v1/b(?:/([^\\/]+))?" 15 | // example: "/my-bucket/2013-tax-returns.pdf" (for a file) 16 | gcsStoragePathPattern = "/([^\\/]+)/(.+)" 17 | ) 18 | 19 | var ( 20 | gcsObjectPathRegex = regexp.MustCompile(gcsObjectPathPattern) 21 | gcsObjectPathRegex2 = regexp.MustCompile(gcsObjectPathPattern2) 22 | gcsBucketPathRegex = regexp.MustCompile(gcsBucketPathPattern) 23 | gcsStoragePathRegex = regexp.MustCompile(gcsStoragePathPattern) 24 | ) 25 | 26 | // GcsParams represent a parsed GCS url. 27 | type GcsParams struct { 28 | Bucket string 29 | Object string 30 | IsPublic bool 31 | } 32 | 33 | // ParseGcsUrl parses a GCS url. 34 | func ParseGcsUrl(u *url.URL) (*GcsParams, bool) { 35 | if g, ok := parseGcsUrl(gcsObjectPathRegex, u); ok { 36 | return g, true 37 | } 38 | if g, ok := parseGcsUrl(gcsBucketPathRegex, u); ok { 39 | return g, true 40 | } 41 | if g, ok := parseGcsUrl(gcsObjectPathRegex2, u); ok { 42 | return g, true 43 | } 44 | if g, ok := parseGcsUrl(gcsStoragePathRegex, u); ok { 45 | g.IsPublic = true 46 | return g, true 47 | } 48 | return nil, false 49 | } 50 | 51 | func parseGcsUrl(re *regexp.Regexp, u *url.URL) (*GcsParams, bool) { 52 | submatches := re.FindStringSubmatch(u.Path) 53 | if submatches == nil { 54 | return nil, false 55 | } 56 | 57 | g := &GcsParams{} 58 | if len(submatches) > 1 { 59 | g.Bucket = submatches[1] 60 | } 61 | if len(submatches) > 2 { 62 | g.Object = submatches[2] 63 | } 64 | return g, true 65 | } 66 | -------------------------------------------------------------------------------- /storage/gcsemu/range.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | ) 7 | 8 | type byteRange struct { 9 | lo, hi, sz int64 10 | } 11 | 12 | func parseByteRange(in string) *byteRange { 13 | var err error 14 | if !strings.HasPrefix(in, "bytes ") { 15 | return nil 16 | } 17 | in = strings.TrimPrefix(in, "bytes ") 18 | parts := strings.Split(in, "/") 19 | if len(parts) != 2 { 20 | return nil 21 | } 22 | 23 | ret := byteRange{ 24 | lo: -1, 25 | hi: -1, 26 | sz: -1, 27 | } 28 | 29 | if parts[0] != "*" { 30 | parts := strings.Split(parts[0], "-") 31 | if len(parts) != 2 { 32 | return nil 33 | } 34 | ret.lo, err = strconv.ParseInt(parts[0], 10, 64) 35 | if err != nil { 36 | return nil 37 | } 38 | ret.hi, err = strconv.ParseInt(parts[1], 10, 64) 39 | if err != nil { 40 | return nil 41 | } 42 | } 43 | 44 | if parts[1] != "*" { 45 | ret.sz, err = strconv.ParseInt(parts[1], 10, 64) 46 | if err != nil { 47 | return nil 48 | } 49 | } 50 | 51 | return &ret 52 | } 53 | -------------------------------------------------------------------------------- /storage/gcsemu/range_test.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/v3/assert" 7 | ) 8 | 9 | func TestParseByteRange(t *testing.T) { 10 | tcs := []struct { 11 | in string 12 | expect byteRange 13 | }{ 14 | {in: "bytes 0-8388607/*", expect: byteRange{lo: 0, hi: 8388607, sz: -1}}, 15 | {in: "bytes 8388608-10485759/10485760", expect: byteRange{lo: 8388608, hi: 10485759, sz: 10485760}}, 16 | {in: "bytes */10485760", expect: byteRange{lo: -1, hi: -1, sz: 10485760}}, 17 | } 18 | 19 | for _, tc := range tcs { 20 | t.Logf("test case: %s", tc.in) 21 | assert.Equal(t, tc.expect, *parseByteRange(tc.in)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /storage/gcsemu/raw_http_test.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | api "google.golang.org/api/storage/v1" 10 | "gotest.tools/v3/assert" 11 | "io" 12 | "mime" 13 | "mime/multipart" 14 | "net/http" 15 | "net/http/httputil" 16 | "net/textproto" 17 | "strings" 18 | "testing" 19 | ) 20 | 21 | func testRawHttp(t *testing.T, bh BucketHandle, httpClient *http.Client, url string) { 22 | const name = "gscemu-test3.txt" 23 | const name2 = "gscemu-test4.txt" 24 | const delName = "gscemu-test-deletion.txt" // used for successful deletion 25 | const delName2 = "gscemu-test-deletion-2.txt" // used for not found deletion 26 | 27 | expectMetaGen := int64(1) 28 | tcs := []struct { 29 | name string 30 | makeRequest func(*testing.T) *http.Request 31 | checkResponse func(*testing.T, *http.Response) 32 | }{ 33 | { 34 | name: "rawGetObject", 35 | makeRequest: func(t *testing.T) *http.Request { 36 | u := fmt.Sprintf("%s/download/storage/v1/b/%s/o/%s?alt=media", url, bh.Name, name) 37 | t.Log(u) 38 | req, err := http.NewRequest("GET", u, nil) 39 | assert.NilError(t, err) 40 | return req 41 | }, 42 | checkResponse: func(t *testing.T, rsp *http.Response) { 43 | body, err := io.ReadAll(rsp.Body) 44 | assert.NilError(t, err) 45 | assert.Equal(t, http.StatusOK, rsp.StatusCode) 46 | assert.Equal(t, v1, string(body)) 47 | }, 48 | }, 49 | { 50 | name: "rawGetMeta", 51 | makeRequest: func(t *testing.T) *http.Request { 52 | u := fmt.Sprintf("%s/storage/v1/b/%s/o/%s", url, bh.Name, name) 53 | t.Log(u) 54 | req, err := http.NewRequest("GET", u, nil) 55 | assert.NilError(t, err) 56 | return req 57 | }, 58 | checkResponse: func(t *testing.T, rsp *http.Response) { 59 | body, err := io.ReadAll(rsp.Body) 60 | assert.NilError(t, err) 61 | assert.Equal(t, http.StatusOK, rsp.StatusCode) 62 | 63 | var attrs api.Object 64 | err = json.NewDecoder(bytes.NewReader(body)).Decode(&attrs) 65 | assert.NilError(t, err) 66 | assert.Equal(t, name, attrs.Name) 67 | assert.Equal(t, bh.Name, attrs.Bucket) 68 | assert.Equal(t, uint64(len(v1)), attrs.Size) 69 | assert.Equal(t, expectMetaGen, attrs.Metageneration) 70 | }, 71 | }, 72 | { 73 | name: "rawPatchMeta", 74 | makeRequest: func(t *testing.T) *http.Request { 75 | u := fmt.Sprintf("%s/storage/v1/b/%s/o/%s", url, bh.Name, name) 76 | t.Log(u) 77 | req, err := http.NewRequest("PATCH", u, strings.NewReader(`{"metadata": {"type": "tabby"}}`)) 78 | assert.NilError(t, err) 79 | req.Header.Set("Content-Type", "application/json") 80 | return req 81 | }, 82 | checkResponse: func(t *testing.T, rsp *http.Response) { 83 | body, err := io.ReadAll(rsp.Body) 84 | assert.NilError(t, err) 85 | assert.Equal(t, http.StatusOK, rsp.StatusCode) 86 | 87 | expectMetaGen++ 88 | 89 | var attrs api.Object 90 | err = json.NewDecoder(bytes.NewReader(body)).Decode(&attrs) 91 | assert.NilError(t, err) 92 | assert.Equal(t, name, attrs.Name) 93 | assert.Equal(t, bh.Name, attrs.Bucket) 94 | assert.Equal(t, uint64(len(v1)), attrs.Size) 95 | assert.Equal(t, expectMetaGen, attrs.Metageneration) 96 | assert.Equal(t, "tabby", attrs.Metadata["type"]) 97 | }, 98 | }, 99 | { 100 | name: "rawDeleteObject-Success", 101 | makeRequest: func(t *testing.T) *http.Request { 102 | u := fmt.Sprintf("%s/storage/v1/b/%s/o/%s", url, bh.Name, delName) 103 | t.Log(u) 104 | req, err := http.NewRequest("DELETE", u, nil) 105 | assert.NilError(t, err) 106 | req.Header.Set("Content-Type", "text/plain") 107 | return req 108 | }, 109 | checkResponse: func(t *testing.T, rsp *http.Response) { 110 | assert.Equal(t, http.StatusNoContent, rsp.StatusCode) 111 | }, 112 | }, 113 | { 114 | name: "rawDeleteObject-ObjectNotFound", 115 | makeRequest: func(t *testing.T) *http.Request { 116 | u := fmt.Sprintf("%s/storage/v1/b/%s/o/%s", url, bh.Name, delName2) 117 | t.Log(u) 118 | req, err := http.NewRequest("DELETE", u, nil) 119 | assert.NilError(t, err) 120 | req.Header.Set("Content-Type", "text/plain") 121 | return req 122 | }, 123 | checkResponse: func(t *testing.T, rsp *http.Response) { 124 | assert.Equal(t, http.StatusNotFound, rsp.StatusCode) 125 | }, 126 | }, 127 | { 128 | name: "rawDeleteObject-BucketNotFound", 129 | makeRequest: func(t *testing.T) *http.Request { 130 | u := fmt.Sprintf("%s/storage/v1/b/%s/o/%s", url, invalidBucketName, delName) 131 | t.Log(u) 132 | req, err := http.NewRequest("DELETE", u, nil) 133 | assert.NilError(t, err) 134 | req.Header.Set("Content-Type", "text/plain") 135 | return req 136 | }, 137 | checkResponse: func(t *testing.T, rsp *http.Response) { 138 | assert.Equal(t, http.StatusNotFound, rsp.StatusCode) 139 | }, 140 | }, 141 | { 142 | name: "rawDeleteBucket-BucketNotFound", 143 | makeRequest: func(t *testing.T) *http.Request { 144 | u := fmt.Sprintf("%s/storage/v1/b/%s", url, invalidBucketName) 145 | t.Log(u) 146 | req, err := http.NewRequest("DELETE", u, nil) 147 | assert.NilError(t, err) 148 | req.Header.Set("Content-Type", "text/plain") 149 | return req 150 | }, 151 | checkResponse: func(t *testing.T, rsp *http.Response) { 152 | assert.Equal(t, http.StatusNotFound, rsp.StatusCode) 153 | }, 154 | }, 155 | { 156 | name: "rawUpload", 157 | makeRequest: func(t *testing.T) *http.Request { 158 | u := fmt.Sprintf("%s/upload/storage/v1/b/%s/o?uploadType=media&name=%s", url, bh.Name, name2) 159 | t.Log(u) 160 | req, err := http.NewRequest("POST", u, strings.NewReader(v2)) 161 | assert.NilError(t, err) 162 | req.Header.Set("Content-Type", "text/plain") 163 | return req 164 | }, 165 | checkResponse: func(t *testing.T, rsp *http.Response) { 166 | body, err := io.ReadAll(rsp.Body) 167 | assert.NilError(t, err) 168 | assert.Equal(t, http.StatusOK, rsp.StatusCode) 169 | 170 | var attrs api.Object 171 | err = json.NewDecoder(bytes.NewReader(body)).Decode(&attrs) 172 | assert.NilError(t, err) 173 | assert.Equal(t, name2, attrs.Name) 174 | assert.Equal(t, bh.Name, attrs.Bucket) 175 | assert.Equal(t, uint64(len(v2)), attrs.Size) 176 | assert.Equal(t, int64(1), attrs.Metageneration) 177 | }, 178 | }, 179 | { 180 | name: "publicUrl", 181 | makeRequest: func(t *testing.T) *http.Request { 182 | u := fmt.Sprintf("%s/%s/%s?alt=media", url, bh.Name, name) 183 | t.Log(u) 184 | req, err := http.NewRequest("GET", u, nil) 185 | assert.NilError(t, err) 186 | return req 187 | }, 188 | checkResponse: func(t *testing.T, rsp *http.Response) { 189 | body, err := io.ReadAll(rsp.Body) 190 | assert.NilError(t, err) 191 | assert.Equal(t, http.StatusOK, rsp.StatusCode) 192 | assert.Equal(t, v1, string(body)) 193 | }, 194 | }, 195 | } 196 | 197 | ctx := context.Background() 198 | oh := bh.Object(name) 199 | 200 | // Create the object 1. 201 | w := oh.NewWriter(ctx) 202 | assert.NilError(t, write(w, v1)) 203 | 204 | // Make sure object 2 is not there. 205 | _ = bh.Object(name2).Delete(ctx) 206 | 207 | // batch setup 208 | // Create the object for successful deletion. 209 | w = bh.Object(delName).NewWriter(ctx) 210 | assert.NilError(t, write(w, v1)) 211 | // Make sure object for not found deletion is not there. 212 | _ = bh.Object(delName2).Delete(ctx) 213 | 214 | // Run each test individually. 215 | for _, tc := range tcs { 216 | tc := tc 217 | t.Run(tc.name, func(t *testing.T) { 218 | req := tc.makeRequest(t) 219 | rsp, err := httpClient.Do(req) 220 | assert.NilError(t, err) 221 | body, err := httputil.DumpResponse(rsp, true) 222 | assert.NilError(t, err) 223 | t.Log(string(body)) 224 | tc.checkResponse(t, rsp) 225 | }) 226 | } 227 | 228 | // batch setup again for batch deletion step 229 | // Create the object for successful deletion. 230 | w = bh.Object(delName).NewWriter(ctx) 231 | assert.NilError(t, write(w, v1)) 232 | // Make sure object for not found deletion is not there. 233 | _ = bh.Object(delName2).Delete(ctx) 234 | 235 | // Batch requests don't support upload and download, only metadata stuff. 236 | t.Run("batch", func(t *testing.T) { 237 | var buf bytes.Buffer 238 | w := multipart.NewWriter(&buf) 239 | 240 | // Only use the [second, fifth] requests. 241 | batchTcs := tcs[1:6] 242 | for i, tc := range batchTcs { 243 | req := tc.makeRequest(t) 244 | req.Host = "" 245 | req.URL.Host = "" 246 | 247 | p, _ := w.CreatePart(textproto.MIMEHeader{ 248 | "Content-Type": []string{"application/http"}, 249 | "Content-Transfer-Encoding": []string{"binary"}, 250 | "Content-ID": []string{fmt.Sprintf("", i)}, 251 | }) 252 | buf, err := httputil.DumpRequest(req, true) 253 | assert.NilError(t, err) 254 | _, _ = p.Write(buf) 255 | } 256 | _ = w.Close() 257 | 258 | // Compile the request 259 | req, err := http.NewRequest("POST", fmt.Sprintf("%s/batch/storage/v1", url), &buf) 260 | assert.NilError(t, err) 261 | req.Header.Set("Content-Type", "multipart/mixed; boundary="+w.Boundary()) 262 | 263 | body, err := httputil.DumpRequest(req, true) 264 | assert.NilError(t, err) 265 | t.Log(string(body)) 266 | 267 | rsp, err := httpClient.Do(req) 268 | assert.NilError(t, err) 269 | assert.Equal(t, http.StatusOK, rsp.StatusCode) 270 | 271 | body, err = httputil.DumpResponse(rsp, true) 272 | assert.NilError(t, err) 273 | t.Log(string(body)) 274 | 275 | // decode the multipart response 276 | v := rsp.Header.Get("Content-type") 277 | assert.Check(t, v != "") 278 | d, params, err := mime.ParseMediaType(v) 279 | assert.NilError(t, err) 280 | assert.Equal(t, "multipart/mixed", d) 281 | boundary, ok := params["boundary"] 282 | assert.Check(t, ok) 283 | 284 | r := multipart.NewReader(rsp.Body, boundary) 285 | for i, tc := range batchTcs { 286 | part, err := r.NextPart() 287 | assert.NilError(t, err) 288 | assert.Equal(t, "application/http", part.Header.Get("Content-Type")) 289 | assert.Equal(t, fmt.Sprintf("", i), part.Header.Get("Content-ID")) 290 | b, err := io.ReadAll(part) 291 | assert.NilError(t, err) 292 | 293 | // Decode the buffer into an http.Response 294 | rsp, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(b)), nil) 295 | assert.NilError(t, err) 296 | tc.checkResponse(t, rsp) 297 | } 298 | }) 299 | } 300 | -------------------------------------------------------------------------------- /storage/gcsemu/remote_test.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "cloud.google.com/go/storage" 9 | "golang.org/x/oauth2/google" 10 | "gotest.tools/v3/assert" 11 | ) 12 | 13 | func TestRealStore(t *testing.T) { 14 | bucket := os.Getenv("BUCKET_ID") 15 | if bucket == "" { 16 | t.Skip("BUCKET_ID must be set to run this") 17 | } 18 | 19 | ctx := context.Background() 20 | gcsClient, err := storage.NewClient(ctx) 21 | assert.NilError(t, err) 22 | t.Cleanup(func() { 23 | _ = gcsClient.Close() 24 | }) 25 | 26 | bh := BucketHandle{ 27 | Name: bucket, 28 | BucketHandle: gcsClient.Bucket(bucket), 29 | } 30 | 31 | // Instead of `initBucket`, just check that it exists. 32 | _, err = bh.Attrs(ctx) 33 | assert.NilError(t, err) 34 | 35 | t.Parallel() 36 | for _, tc := range testCases { 37 | tc := tc 38 | t.Run(tc.name, func(t *testing.T) { 39 | t.Parallel() 40 | tc.f(t, bh) 41 | }) 42 | } 43 | 44 | httpClient, err := google.DefaultClient(context.Background()) 45 | assert.NilError(t, err) 46 | t.Run("RawHttp", func(t *testing.T) { 47 | t.Parallel() 48 | testRawHttp(t, bh, httpClient, "https://storage.googleapis.com") 49 | }) 50 | } 51 | -------------------------------------------------------------------------------- /storage/gcsemu/server.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "net/http/httptest" 8 | "strings" 9 | ) 10 | 11 | // Server is an in-memory Cloud Storage emulator; it is unauthenticated, and only a rough approximation. 12 | type Server struct { 13 | Addr string 14 | *httptest.Server 15 | *GcsEmu 16 | } 17 | 18 | // NewServer creates a new Server with the given options. 19 | // The Server will be listening for HTTP connections, without TLS, 20 | // on the provided address. The resolved address is named by the Addr field. 21 | // An address with a port of 0 will bind to an open port on the system. 22 | // 23 | // For running a full in-process setup (e.g. unit tests), initialize 24 | // os.Setenv("GCS_EMULATOR_HOST", srv.Addr) so that subsequent calls to NewClient() 25 | // will return an in-process targeted storage client. 26 | func NewServer(laddr string, opts Options) (*Server, error) { 27 | gcsEmu := NewGcsEmu(opts) 28 | mux := http.NewServeMux() 29 | gcsEmu.Register(mux) 30 | 31 | srv := httptest.NewUnstartedServer(mux) 32 | l, err := net.Listen("tcp", laddr) 33 | if err != nil { 34 | return nil, fmt.Errorf("failed to listen on addr %s: %w", laddr, err) 35 | } 36 | srv.Listener = l 37 | srv.Start() 38 | 39 | return &Server{ 40 | Addr: strings.TrimPrefix(srv.URL, "http://"), 41 | Server: srv, 42 | GcsEmu: gcsEmu, 43 | }, nil 44 | } 45 | -------------------------------------------------------------------------------- /storage/gcsemu/store.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "google.golang.org/api/storage/v1" 8 | ) 9 | 10 | // Store is an interface to either on-disk or in-mem storage 11 | type Store interface { 12 | // CreateBucket creates a bucket; no error if the bucket already exists. 13 | CreateBucket(bucket string) error 14 | 15 | // Get returns a bucket's metadata. 16 | GetBucketMeta(baseUrl HttpBaseUrl, bucket string) (*storage.Bucket, error) 17 | 18 | // Get returns a file's contents and metadata. 19 | Get(url HttpBaseUrl, bucket string, filename string) (*storage.Object, []byte, error) 20 | 21 | // GetMeta returns a file's metadata. 22 | GetMeta(url HttpBaseUrl, bucket string, filename string) (*storage.Object, error) 23 | 24 | // Add creates the specified file. 25 | Add(bucket string, filename string, contents []byte, meta *storage.Object) error 26 | 27 | // UpdateMeta updates the given file's metadata. 28 | UpdateMeta(bucket string, filename string, meta *storage.Object, metagen int64) error 29 | 30 | // Copy copies the file 31 | Copy(srcBucket string, srcFile string, dstBucket string, dstFile string) (bool, error) 32 | 33 | // Delete deletes the file. 34 | Delete(bucket string, filename string) error 35 | 36 | // ReadMeta reads the GCS metadata for a file, when you already have file info. 37 | ReadMeta(url HttpBaseUrl, bucket string, filename string, fInfo os.FileInfo) (*storage.Object, error) 38 | 39 | // Walks the given bucket. 40 | Walk(ctx context.Context, bucket string, cb func(ctx context.Context, filename string, fInfo os.FileInfo) error) error 41 | } 42 | -------------------------------------------------------------------------------- /storage/gcsemu/util.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "net/http" 7 | "regexp" 8 | "strings" 9 | 10 | "google.golang.org/api/googleapi" 11 | ) 12 | 13 | // jsonRespond json-encodes rsp and writes it to w. If an error occurs, then it is logged and a 500 error is written to w. 14 | func (g *GcsEmu) jsonRespond(w http.ResponseWriter, rsp interface{}) { 15 | // do NOT write a http status since OK will be the default and this allows the caller to use their own if they want 16 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 17 | 18 | encoder := json.NewEncoder(w) 19 | if err := encoder.Encode(rsp); err != nil { 20 | g.log(err, "failed to send response") 21 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 22 | } 23 | } 24 | 25 | type gapiErrorPartial struct { 26 | // Code is the HTTP response status code and will always be populated. 27 | Code int `json:"code"` 28 | 29 | // Message is the server response message and is only populated when 30 | // explicitly referenced by the JSON server response. 31 | Message string `json:"message"` 32 | 33 | Errors []googleapi.ErrorItem `json:"errors,omitempty"` 34 | } 35 | 36 | // gapiError responds to the client with a GAPI error 37 | func (g *GcsEmu) gapiError(w http.ResponseWriter, code int, message string) { 38 | if code == 0 { 39 | code = http.StatusInternalServerError 40 | } 41 | if code != http.StatusNotFound { 42 | g.log(errors.New(message), "responding with HTTP %d", code) 43 | } 44 | if message == "" { 45 | message = http.StatusText(code) 46 | } 47 | 48 | // format copied from errorReply struct in google.golang.org/api/googleapi 49 | rsp := struct { 50 | Error gapiErrorPartial `json:"error"` 51 | }{ 52 | Error: gapiErrorPartial{ 53 | Code: code, 54 | Message: message, 55 | }, 56 | } 57 | 58 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 59 | w.WriteHeader(code) 60 | enc := json.NewEncoder(w) 61 | enc.SetIndent("", " ") 62 | _ = enc.Encode(&rsp) 63 | } 64 | 65 | // mustJson serializes the given value to json, panicking on failure 66 | func mustJson(val interface{}) []byte { 67 | if val == nil { 68 | return []byte("null") 69 | } 70 | 71 | b, err := json.MarshalIndent(val, "", " ") 72 | if err != nil { 73 | panic(err) 74 | } 75 | return b 76 | } 77 | 78 | // requestHost returns the host from an http.Request, respecting proxy headers. Works locally with devproxy 79 | // and gulp proxies as well as in AppEngine (both real GAE and the dev_appserver). 80 | func requestHost(req *http.Request) string { 81 | // proxies like gulp are supposed to accumulate original host, next-step-host, etc in order from 82 | // client-most to server-most in X-ForwardedHost; return the first entry from that if any are listed 83 | if proxyHost := req.Header.Get("X-Forwarded-Host"); proxyHost != "" { 84 | // Use the first (closest to client) host 85 | splits := strings.SplitN(proxyHost, ",", 2) 86 | return splits[0] 87 | } 88 | 89 | // Forwarded is the standardized version of X-Forwarded-Host. 90 | f := parseForwardedHeader(req.Header.Get("Forwarded")) 91 | if len(f.Host) > 0 && len(f.Host[0]) > 0 { 92 | return f.Host[0] 93 | } 94 | 95 | // Clients that generate HTTP/2 requests should use the :authority header instead 96 | // of Host. See http://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.3 97 | host := req.Header.Get("Authority") 98 | if len(host) > 0 { 99 | return host 100 | } 101 | 102 | // Fall back to the host line. 103 | return req.Host 104 | } 105 | 106 | // forwarded represents the values of a Forwarded HTTP header. 107 | // 108 | // For more details, see the RFC: https://tools.ietf.org/html/rfc7239 and 109 | // MDN: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded. 110 | type forwarded struct { 111 | By []string 112 | For []string 113 | Host []string 114 | Proto []string 115 | } 116 | 117 | var ( 118 | forwardedHostRx = regexp.MustCompile(`(?i)host=(.*?)(?:[,;\s]|$)`) 119 | ) 120 | 121 | func removeDoubleQuotes(s string) string { 122 | return strings.TrimSuffix(strings.TrimPrefix(s, `"`), `"`) 123 | } 124 | 125 | // Note: this currently only supports the forwarded.Host field. 126 | func parseForwardedHeader(s string) forwarded { 127 | var f forwarded 128 | 129 | if s == "" { 130 | return f 131 | } 132 | 133 | matches := forwardedHostRx.FindAllStringSubmatch(s, -1) 134 | for _, m := range matches { 135 | if len(m) > 0 { 136 | f.Host = append(f.Host, removeDoubleQuotes(m[1])) 137 | } 138 | } 139 | 140 | return f 141 | } 142 | -------------------------------------------------------------------------------- /storage/gcsemu/walk.go: -------------------------------------------------------------------------------- 1 | package gcsemu 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/fullstorydev/emulators/storage/gcsutil" 13 | "google.golang.org/api/storage/v1" 14 | ) 15 | 16 | // Iterate over the file system to serve a GCS list-bucket request. 17 | func (g *GcsEmu) makeBucketListResults(ctx context.Context, baseUrl HttpBaseUrl, w http.ResponseWriter, delimiter string, cursor string, prefix string, bucket string, maxResults int) { 18 | var errAbort = errors.New("sentinel error to abort walk") 19 | 20 | type item struct { 21 | filename string 22 | fInfo os.FileInfo 23 | } 24 | var found []item 25 | var prefixes []string 26 | seenPrefixes := make(map[string]bool) 27 | 28 | dbgWalk := func(fmt string, args ...interface{}) { 29 | if g.verbose { 30 | g.log(nil, fmt, args...) 31 | } 32 | } 33 | 34 | moreResults := false 35 | count := 0 36 | err := g.store.Walk(ctx, bucket, func(ctx context.Context, filename string, fInfo os.FileInfo) error { 37 | dbgWalk("walk: %s", filename) 38 | 39 | // If we're beyond the prefix, we're completely done. 40 | if greaterThanPrefix(filename, prefix) { 41 | dbgWalk("%q > prefix=%q aborting", filename, prefix) 42 | return errAbort 43 | } 44 | 45 | // In the filesystem implementation, skip any directories strictly less than the cursor or prefix. 46 | if fInfo != nil && fInfo.IsDir() { 47 | if lessThanPrefix(filename, cursor) { 48 | dbgWalk("%q < cursor=%q skip dir", filename, cursor) 49 | return filepath.SkipDir 50 | } 51 | if lessThanPrefix(filename, prefix) { 52 | dbgWalk("%q < prefix=%q skip dir", filename, prefix) 53 | return filepath.SkipDir 54 | } 55 | return nil // keep going 56 | } 57 | 58 | // If the file is <= cursor, or < prefix, skip. 59 | if filename <= cursor { 60 | dbgWalk("%q <= cursor=%q skipping", filename, cursor) 61 | return nil 62 | } 63 | if !strings.HasPrefix(filename, prefix) { 64 | dbgWalk("%q < prefix=%q skipping", filename, prefix) 65 | return nil 66 | } 67 | 68 | if count >= maxResults { 69 | moreResults = true 70 | return errAbort 71 | } 72 | count++ 73 | 74 | if delimiter != "" { 75 | // See if the filename (beyond the prefix) contains delimiter, if it does, don't record the item, 76 | // instead record the prefix (including the delimiter). 77 | withoutPrefix := strings.TrimPrefix(filename, prefix) 78 | delimiterPos := strings.Index(withoutPrefix, delimiter) 79 | if delimiterPos >= 0 { 80 | // Got a hit, reconstruct the item's prefix, including the trailing delimiter 81 | itemPrefix := filename[:len(prefix)+delimiterPos+len(delimiter)] 82 | if !seenPrefixes[itemPrefix] { 83 | seenPrefixes[itemPrefix] = true 84 | prefixes = append(prefixes, itemPrefix) 85 | } 86 | return nil 87 | } 88 | } 89 | 90 | found = append(found, item{ 91 | filename: filename, 92 | fInfo: fInfo, 93 | }) 94 | return nil 95 | }) 96 | // Sentinel error is not an error 97 | if err == errAbort { 98 | err = nil 99 | } 100 | if err != nil { 101 | if len(found) == 0 { 102 | if os.IsNotExist(err) { 103 | g.gapiError(w, http.StatusNotFound, fmt.Sprintf("%s not found", bucket)) 104 | } else { 105 | g.gapiError(w, http.StatusInternalServerError, "failed to iterate: "+err.Error()) 106 | } 107 | return 108 | } 109 | // return our partial results + the cursor so that the client can retry from this point 110 | g.log(nil, "failed to iterate") 111 | } 112 | 113 | // Resolve the found items. 114 | var items []*storage.Object 115 | for _, item := range found { 116 | if obj, err := g.store.ReadMeta(baseUrl, bucket, item.filename, item.fInfo); err != nil { 117 | // return our partial results + the cursor so that the client can retry from this point 118 | g.log(nil, "failed to resolve: %s", item.filename) 119 | break 120 | } else { 121 | items = append(items, obj) 122 | } 123 | } 124 | 125 | var nextPageToken = "" 126 | if moreResults && len(items) > 0 { 127 | lastItemName := items[len(items)-1].Name 128 | nextPageToken = gcsutil.EncodePageToken(lastItemName) 129 | } 130 | 131 | rsp := storage.Objects{ 132 | Kind: "storage#objects", 133 | NextPageToken: nextPageToken, 134 | Items: items, 135 | Prefixes: prefixes, 136 | } 137 | 138 | g.jsonRespond(w, &rsp) 139 | } 140 | -------------------------------------------------------------------------------- /storage/gcsutil/counted_lock.go: -------------------------------------------------------------------------------- 1 | package gcsutil 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type countedLock struct { 8 | // a 1 element channel; empty == unlocked, full == locked 9 | // push an element into the channel to lock, remove the element to unlock 10 | ch chan struct{} 11 | 12 | // should only be accessed while the outer _map_ lock is held (not this key lock) 13 | refcount int64 14 | } 15 | 16 | func newCountedLock() *countedLock { 17 | return &countedLock{ 18 | ch: make(chan struct{}, 1), 19 | refcount: 0, 20 | } 21 | } 22 | 23 | func (m *countedLock) Lock(ctx context.Context) bool { 24 | // If the context is already cancelled don't even try to lock. 25 | if ctx.Err() != nil { 26 | return false 27 | } 28 | select { 29 | case m.ch <- struct{}{}: 30 | return true 31 | case <-ctx.Done(): 32 | return false 33 | } 34 | } 35 | 36 | func (m *countedLock) Unlock() { 37 | select { 38 | case <-m.ch: 39 | return 40 | default: 41 | panic("BUG: lock not held") 42 | } 43 | } 44 | 45 | func (m *countedLock) Run(ctx context.Context, f func(ctx context.Context) error) error { 46 | if !m.Lock(ctx) { 47 | return ctx.Err() 48 | } 49 | defer m.Unlock() 50 | return f(ctx) 51 | } 52 | -------------------------------------------------------------------------------- /storage/gcsutil/doc.go: -------------------------------------------------------------------------------- 1 | // Package gcsutil contains some generic utilities to support gcsemu. 2 | // TODO(dragonsinth): consider open sourcing these separately, or finding open source replacements. 3 | package gcsutil 4 | -------------------------------------------------------------------------------- /storage/gcsutil/gcspagetoken.go: -------------------------------------------------------------------------------- 1 | package gcsutil 2 | 3 | //go:generate protoc --go_out=. --go_opt=paths=source_relative gcspagetoken.proto 4 | 5 | import ( 6 | "encoding/base64" 7 | "fmt" 8 | 9 | "google.golang.org/protobuf/proto" 10 | ) 11 | 12 | // EncodePageToken returns a synthetic page token to find files greater than the given string. 13 | // If this is part of a prefix query, the token should fall within the prefixed range. 14 | // BRITTLE: relies on a reverse-engineered internal GCS token format, which may be subject to change. 15 | func EncodePageToken(greaterThan string) string { 16 | bytes, err := proto.Marshal(&GcsPageToken{ 17 | LastFile: greaterThan, 18 | }) 19 | if err != nil { 20 | panic("could not encode gcsPageToken:" + err.Error()) 21 | } 22 | return base64.StdEncoding.EncodeToString(bytes) 23 | } 24 | 25 | // DecodePageToken decodes a GCS pageToken to the name of the last file returned. 26 | func DecodePageToken(pageToken string) (string, error) { 27 | bytes, err := base64.StdEncoding.DecodeString(pageToken) 28 | if err != nil { 29 | return "", fmt.Errorf("could not base64 decode pageToken %s: %w", pageToken, err) 30 | } 31 | var message GcsPageToken 32 | if err := proto.Unmarshal(bytes, &message); err != nil { 33 | return "", fmt.Errorf("could not unmarshal proto: %w", err) 34 | } 35 | 36 | return message.LastFile, nil 37 | } 38 | -------------------------------------------------------------------------------- /storage/gcsutil/gcspagetoken.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.26.0 4 | // protoc v3.17.3 5 | // source: gcspagetoken.proto 6 | 7 | package gcsutil 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type GcsPageToken struct { 24 | state protoimpl.MessageState 25 | sizeCache protoimpl.SizeCache 26 | unknownFields protoimpl.UnknownFields 27 | 28 | // The full name of the last result file, when returned from the server. 29 | // When sent as a cursor, interpreted as "return files greater than this value". 30 | LastFile string `protobuf:"bytes,1,opt,name=LastFile,proto3" json:"LastFile,omitempty"` 31 | } 32 | 33 | func (x *GcsPageToken) Reset() { 34 | *x = GcsPageToken{} 35 | if protoimpl.UnsafeEnabled { 36 | mi := &file_gcspagetoken_proto_msgTypes[0] 37 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 38 | ms.StoreMessageInfo(mi) 39 | } 40 | } 41 | 42 | func (x *GcsPageToken) String() string { 43 | return protoimpl.X.MessageStringOf(x) 44 | } 45 | 46 | func (*GcsPageToken) ProtoMessage() {} 47 | 48 | func (x *GcsPageToken) ProtoReflect() protoreflect.Message { 49 | mi := &file_gcspagetoken_proto_msgTypes[0] 50 | if protoimpl.UnsafeEnabled && x != nil { 51 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 52 | if ms.LoadMessageInfo() == nil { 53 | ms.StoreMessageInfo(mi) 54 | } 55 | return ms 56 | } 57 | return mi.MessageOf(x) 58 | } 59 | 60 | // Deprecated: Use GcsPageToken.ProtoReflect.Descriptor instead. 61 | func (*GcsPageToken) Descriptor() ([]byte, []int) { 62 | return file_gcspagetoken_proto_rawDescGZIP(), []int{0} 63 | } 64 | 65 | func (x *GcsPageToken) GetLastFile() string { 66 | if x != nil { 67 | return x.LastFile 68 | } 69 | return "" 70 | } 71 | 72 | var File_gcspagetoken_proto protoreflect.FileDescriptor 73 | 74 | var file_gcspagetoken_proto_rawDesc = []byte{ 75 | 0x0a, 0x12, 0x67, 0x63, 0x73, 0x70, 0x61, 0x67, 0x65, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x2e, 0x70, 76 | 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x67, 0x63, 0x73, 0x75, 0x74, 0x69, 0x6c, 0x22, 0x2a, 0x0a, 77 | 0x0c, 0x47, 0x63, 0x73, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1a, 0x0a, 78 | 0x08, 0x4c, 0x61, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 79 | 0x08, 0x4c, 0x61, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 80 | 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x75, 0x6c, 0x6c, 0x73, 0x74, 0x6f, 0x72, 81 | 0x79, 0x64, 0x65, 0x76, 0x2f, 0x65, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x6f, 0x72, 0x73, 0x2f, 0x73, 82 | 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2f, 0x67, 0x63, 0x73, 0x75, 0x74, 0x69, 0x6c, 0x62, 0x06, 83 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 84 | } 85 | 86 | var ( 87 | file_gcspagetoken_proto_rawDescOnce sync.Once 88 | file_gcspagetoken_proto_rawDescData = file_gcspagetoken_proto_rawDesc 89 | ) 90 | 91 | func file_gcspagetoken_proto_rawDescGZIP() []byte { 92 | file_gcspagetoken_proto_rawDescOnce.Do(func() { 93 | file_gcspagetoken_proto_rawDescData = protoimpl.X.CompressGZIP(file_gcspagetoken_proto_rawDescData) 94 | }) 95 | return file_gcspagetoken_proto_rawDescData 96 | } 97 | 98 | var file_gcspagetoken_proto_msgTypes = make([]protoimpl.MessageInfo, 1) 99 | var file_gcspagetoken_proto_goTypes = []interface{}{ 100 | (*GcsPageToken)(nil), // 0: gcsutil.GcsPageToken 101 | } 102 | var file_gcspagetoken_proto_depIdxs = []int32{ 103 | 0, // [0:0] is the sub-list for method output_type 104 | 0, // [0:0] is the sub-list for method input_type 105 | 0, // [0:0] is the sub-list for extension type_name 106 | 0, // [0:0] is the sub-list for extension extendee 107 | 0, // [0:0] is the sub-list for field type_name 108 | } 109 | 110 | func init() { file_gcspagetoken_proto_init() } 111 | func file_gcspagetoken_proto_init() { 112 | if File_gcspagetoken_proto != nil { 113 | return 114 | } 115 | if !protoimpl.UnsafeEnabled { 116 | //lint:ignore SA1019 generated 117 | file_gcspagetoken_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 118 | switch v := v.(*GcsPageToken); i { 119 | case 0: 120 | return &v.state 121 | case 1: 122 | return &v.sizeCache 123 | case 2: 124 | return &v.unknownFields 125 | default: 126 | return nil 127 | } 128 | } 129 | } 130 | type x struct{} 131 | out := protoimpl.TypeBuilder{ 132 | File: protoimpl.DescBuilder{ 133 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 134 | RawDescriptor: file_gcspagetoken_proto_rawDesc, 135 | NumEnums: 0, 136 | NumMessages: 1, 137 | NumExtensions: 0, 138 | NumServices: 0, 139 | }, 140 | GoTypes: file_gcspagetoken_proto_goTypes, 141 | DependencyIndexes: file_gcspagetoken_proto_depIdxs, 142 | MessageInfos: file_gcspagetoken_proto_msgTypes, 143 | }.Build() 144 | File_gcspagetoken_proto = out.File 145 | file_gcspagetoken_proto_rawDesc = nil 146 | file_gcspagetoken_proto_goTypes = nil 147 | file_gcspagetoken_proto_depIdxs = nil 148 | } 149 | -------------------------------------------------------------------------------- /storage/gcsutil/gcspagetoken.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package gcsutil; 4 | 5 | option go_package = "github.com/fullstorydev/emulators/storage/gcsutil"; 6 | 7 | message GcsPageToken{ 8 | // The full name of the last result file, when returned from the server. 9 | // When sent as a cursor, interpreted as "return files greater than this value". 10 | string LastFile = 1; 11 | } 12 | -------------------------------------------------------------------------------- /storage/gcsutil/gcspagetoken_test.go: -------------------------------------------------------------------------------- 1 | package gcsutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/v3/assert" 7 | ) 8 | 9 | // TODO(dragonsinth): it would be nice to have an integration test that hits real GCS with known data stored. 10 | 11 | // TestGcsTokenGen tests that we produce expected GCS tokens that match real data we collected. 12 | func TestGcsTokenGen(t *testing.T) { 13 | tcs := []struct { 14 | lastFile string 15 | cursor string 16 | }{ 17 | { 18 | lastFile: "containers/images/4dcc5142000d12f1a0f67c1e95df4035ca0ebba70117cc04101e53422d391d61/json", 19 | cursor: "Cldjb250YWluZXJzL2ltYWdlcy80ZGNjNTE0MjAwMGQxMmYxYTBmNjdjMWU5NWRmNDAzNWNhMGViYmE3MDExN2NjMDQxMDFlNTM0MjJkMzkxZDYxL2pzb24=", 20 | }, 21 | { 22 | lastFile: "containers/images/sha256:0e89fc4aeb48f92acff2dddaf610b2ceea5d76a93a44d4c20b31e69d1ed68c10", 23 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6MGU4OWZjNGFlYjQ4ZjkyYWNmZjJkZGRhZjYxMGIyY2VlYTVkNzZhOTNhNDRkNGMyMGIzMWU2OWQxZWQ2OGMxMA==", 24 | }, 25 | { 26 | lastFile: "containers/images/sha256:2072bc4567e1f13081af323d046f39453f010471701fa11fc50b786b60512e99", 27 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6MjA3MmJjNDU2N2UxZjEzMDgxYWYzMjNkMDQ2ZjM5NDUzZjAxMDQ3MTcwMWZhMTFmYzUwYjc4NmI2MDUxMmU5OQ==", 28 | }, 29 | { 30 | lastFile: "containers/images/sha256:43ecade58b2ddff87a696fada3970491de28c8ea1dca09c988b447b5c5a56412", 31 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6NDNlY2FkZTU4YjJkZGZmODdhNjk2ZmFkYTM5NzA0OTFkZTI4YzhlYTFkY2EwOWM5ODhiNDQ3YjVjNWE1NjQxMg==", 32 | }, 33 | { 34 | lastFile: "containers/images/sha256:57076bf87737c2f448e75324ceba121b3b90372ab913f604f928a40da4ebddc7", 35 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6NTcwNzZiZjg3NzM3YzJmNDQ4ZTc1MzI0Y2ViYTEyMWIzYjkwMzcyYWI5MTNmNjA0ZjkyOGE0MGRhNGViZGRjNw==", 36 | }, 37 | { 38 | lastFile: "containers/images/sha256:6f73a9b0052f169d8296382660a31050004b691f3e6252008545a3dcb7371a49", 39 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6NmY3M2E5YjAwNTJmMTY5ZDgyOTYzODI2NjBhMzEwNTAwMDRiNjkxZjNlNjI1MjAwODU0NWEzZGNiNzM3MWE0OQ==", 40 | }, 41 | { 42 | lastFile: "containers/images/sha256:765b6a129bd04f06c876f3d5b5a346e71a72fae522b230215f67375ccb659a11", 43 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6NzY1YjZhMTI5YmQwNGYwNmM4NzZmM2Q1YjVhMzQ2ZTcxYTcyZmFlNTIyYjIzMDIxNWY2NzM3NWNjYjY1OWExMQ==", 44 | }, 45 | { 46 | lastFile: "containers/images/sha256:973d921f391393e65d20a6e990e8ad5aa1129681ab8c54bf59c9192b809594c4", 47 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6OTczZDkyMWYzOTEzOTNlNjVkMjBhNmU5OTBlOGFkNWFhMTEyOTY4MWFiOGM1NGJmNTljOTE5MmI4MDk1OTRjNA==", 48 | }, 49 | { 50 | lastFile: "containers/images/sha256:a9d5802ef798d88c0f4f9dc0094249db5e26d8a8a18ba4c2194aab4a44983d2f", 51 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6YTlkNTgwMmVmNzk4ZDg4YzBmNGY5ZGMwMDk0MjQ5ZGI1ZTI2ZDhhOGExOGJhNGMyMTk0YWFiNGE0NDk4M2QyZg==", 52 | }, 53 | { 54 | lastFile: "containers/images/sha256:b5714cf3ed3a6b5f1fbc736ef0d5673c5637ccb14d53a23b8728dd828f21a22d", 55 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6YjU3MTRjZjNlZDNhNmI1ZjFmYmM3MzZlZjBkNTY3M2M1NjM3Y2NiMTRkNTNhMjNiODcyOGRkODI4ZjIxYTIyZA==", 56 | }, 57 | { 58 | lastFile: "containers/images/sha256:bfdef622d405cb35c466aa29ea7411fd594e7985127bec4e8080572d7ef45cfd", 59 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6YmZkZWY2MjJkNDA1Y2IzNWM0NjZhYTI5ZWE3NDExZmQ1OTRlNzk4NTEyN2JlYzRlODA4MDU3MmQ3ZWY0NWNmZA==", 60 | }, 61 | { 62 | lastFile: "containers/images/sha256:da543d0747020a528b8eee057912f6bd07e28f9c006606da28c82c70dac962e2", 63 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6ZGE1NDNkMDc0NzAyMGE1MjhiOGVlZTA1NzkxMmY2YmQwN2UyOGY5YzAwNjYwNmRhMjhjODJjNzBkYWM5NjJlMg==", 64 | }, 65 | { 66 | lastFile: "containers/images/sha256:ee73b32b0f0a9dafabd9445a3380094f984858e79c85eafde56c2e037c039c6a", 67 | cursor: "Clljb250YWluZXJzL2ltYWdlcy9zaGEyNTY6ZWU3M2IzMmIwZjBhOWRhZmFiZDk0NDVhMzM4MDA5NGY5ODQ4NThlNzljODVlYWZkZTU2YzJlMDM3YzAzOWM2YQ==", 68 | }, 69 | { 70 | lastFile: "containers/repositories/library/cpu-test/tag_v1", 71 | cursor: "Ci9jb250YWluZXJzL3JlcG9zaXRvcmllcy9saWJyYXJ5L2NwdS10ZXN0L3RhZ192MQ==", 72 | }, 73 | { 74 | lastFile: "containers/repositories/library/dns-test/manifest_sha256:9759da4d052f154ba5d0ea32bf0404f442082e3dbad3f3cf6dc6529c6575aec2", 75 | cursor: "Cnljb250YWluZXJzL3JlcG9zaXRvcmllcy9saWJyYXJ5L2Rucy10ZXN0L21hbmlmZXN0X3NoYTI1Njo5NzU5ZGE0ZDA1MmYxNTRiYTVkMGVhMzJiZjA0MDRmNDQyMDgyZTNkYmFkM2YzY2Y2ZGM2NTI5YzY1NzVhZWMy", 76 | }, 77 | { 78 | lastFile: "containers/repositories/library/dns-test/manifest_sha256:ee49d4935c33260d84499e38dbd5ec3f426c9a2a7e100a901267c712874e4c1d", 79 | cursor: "Cnljb250YWluZXJzL3JlcG9zaXRvcmllcy9saWJyYXJ5L2Rucy10ZXN0L21hbmlmZXN0X3NoYTI1NjplZTQ5ZDQ5MzVjMzMyNjBkODQ0OTllMzhkYmQ1ZWMzZjQyNmM5YTJhN2UxMDBhOTAxMjY3YzcxMjg3NGU0YzFk", 80 | }, 81 | { 82 | lastFile: "containers/repositories/library/dns-test/tag_v2", 83 | cursor: "Ci9jb250YWluZXJzL3JlcG9zaXRvcmllcy9saWJyYXJ5L2Rucy10ZXN0L3RhZ192Mg==", 84 | }, 85 | { 86 | lastFile: "containers/repositories/library/kapi-test/manifest_sha256:ca6d9c13fba12363760e6d2495811081b1d2e6fcbf974e551605b65cb5b0a94e", 87 | cursor: "Cnpjb250YWluZXJzL3JlcG9zaXRvcmllcy9saWJyYXJ5L2thcGktdGVzdC9tYW5pZmVzdF9zaGEyNTY6Y2E2ZDljMTNmYmExMjM2Mzc2MGU2ZDI0OTU4MTEwODFiMWQyZTZmY2JmOTc0ZTU1MTYwNWI2NWNiNWIwYTk0ZQ==", 88 | }, 89 | { 90 | lastFile: "containers/repositories/library/memclient-test/manifest_sha256:7b7979b351c9a019062446c1f033b2f8491868cf943758f006ae219eca231e01", 91 | cursor: "Cn9jb250YWluZXJzL3JlcG9zaXRvcmllcy9saWJyYXJ5L21lbWNsaWVudC10ZXN0L21hbmlmZXN0X3NoYTI1Njo3Yjc5NzliMzUxYzlhMDE5MDYyNDQ2YzFmMDMzYjJmODQ5MTg2OGNmOTQzNzU4ZjAwNmFlMjE5ZWNhMjMxZTAx", 92 | }, 93 | } 94 | 95 | for i, tc := range tcs { 96 | actualCursor := EncodePageToken(tc.lastFile) 97 | assert.Equal(t, tc.cursor, actualCursor, "case %d", i) 98 | 99 | lastFile, err := DecodePageToken(actualCursor) 100 | assert.NilError(t, err, "case %i: failed to decode", i) 101 | assert.Equal(t, tc.lastFile, lastFile, "case %d", i) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /storage/gcsutil/transient_lock_map.go: -------------------------------------------------------------------------------- 1 | package gcsutil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | // TransientLockMap is a map of mutexes that is safe for concurrent access. It does not bother to save mutexes after 10 | // they have been unlocked, and thus this data structure is best for situations where the space of keys is very large. 11 | // If the space of keys is small then it may be inefficient to constantly recreate mutexes whenever they are needed. 12 | type TransientLockMap struct { 13 | mu sync.Mutex // the mutex that locks the map 14 | locks map[string]*countedLock // all locks that are currently held 15 | } 16 | 17 | // NewTransientLockMap returns a new TransientLockMap. 18 | func NewTransientLockMap() *TransientLockMap { 19 | return &TransientLockMap{ 20 | locks: make(map[string]*countedLock), 21 | } 22 | } 23 | 24 | // Lock acquires the lock for the specified key and returns true, unless the context finishes before the lock could be 25 | // acquired, in which case false is returned. 26 | func (l *TransientLockMap) Lock(ctx context.Context, key string) bool { 27 | lock := func() *countedLock { 28 | // If there is high lock contention, we could use a readonly lock to check if the lock is already in the map (and 29 | // thus no map writes are necessary), but this is complicated enough as it is so we skip that optimization for now. 30 | l.mu.Lock() 31 | defer l.mu.Unlock() 32 | 33 | // Check if there is already a lock for this key. 34 | lock, ok := l.locks[key] 35 | if !ok { 36 | // no lock yet, so make one and add it to the map 37 | lock = newCountedLock() 38 | l.locks[key] = lock 39 | } 40 | 41 | // Order is very important here. First we have to increment the refcount while we still have the map locked; this 42 | // will prevent anyone else from evicting this lock after we unlock the map but before we lock the key. Second we 43 | // have to unlock the map _before_ we start trying to lock the key (because locking the key could take a long time 44 | // and we don't want to keep the map locked that whole time). 45 | lock.refcount++ // incremented while holding _map_ lock 46 | return lock 47 | }() 48 | 49 | if !lock.Lock(ctx) { 50 | l.returnLockObj(key, lock) 51 | return false 52 | } 53 | return true 54 | } 55 | 56 | // Unlock unlocks the lock for the specified key. Panics if the lock is not currently held. 57 | func (l *TransientLockMap) Unlock(key string) { 58 | lock := func() *countedLock { 59 | l.mu.Lock() 60 | defer l.mu.Unlock() 61 | 62 | lock, ok := l.locks[key] 63 | if !ok { 64 | panic(fmt.Sprintf("lock not held for key %s", key)) 65 | } 66 | return lock 67 | }() 68 | 69 | lock.Unlock() 70 | l.returnLockObj(key, lock) 71 | } 72 | 73 | // Run runs the given callback while holding the lock, unless the context finishes before the lock could be 74 | // acquired, in which case the context error is returned. 75 | func (l *TransientLockMap) Run(ctx context.Context, key string, f func(ctx context.Context) error) error { 76 | if !l.Lock(ctx, key) { 77 | return ctx.Err() 78 | } 79 | defer l.Unlock(key) 80 | return f(ctx) 81 | } 82 | 83 | func (l *TransientLockMap) returnLockObj(key string, lock *countedLock) { 84 | l.mu.Lock() 85 | defer l.mu.Unlock() 86 | 87 | lock.refcount-- 88 | if lock.refcount < 0 { 89 | panic(fmt.Sprintf("BUG: somehow the lock.refcount for %q dropped to %d", key, lock.refcount)) 90 | } 91 | if lock.refcount == 0 { 92 | delete(l.locks, key) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /storage/gcsutil/transient_lock_map_test.go: -------------------------------------------------------------------------------- 1 | package gcsutil 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "sync/atomic" 7 | "testing" 8 | "time" 9 | 10 | "gotest.tools/v3/assert" 11 | ) 12 | 13 | func (l *TransientLockMap) len() int { 14 | l.mu.Lock() 15 | defer l.mu.Unlock() 16 | return len(l.locks) 17 | } 18 | 19 | func TestTransientLockMapBasics(t *testing.T) { 20 | m := NewTransientLockMap() 21 | upstream := make(chan int, 1) 22 | downstream := make(chan int, 1) 23 | 24 | waitFor := func(c <-chan int, expected int) { 25 | val := <-c 26 | assert.Equal(t, expected, val, "got an unexpected value from a channel") 27 | } 28 | 29 | readSignalNow := func(c <-chan int, expected int, msg string) { 30 | select { 31 | case val := <-c: 32 | assert.Equal(t, expected, val, "got an unexpected value from a channel") 33 | default: 34 | t.Fatal(msg) 35 | } 36 | } 37 | 38 | // wrt upstream and downstream, this goroutine is "above" the main thread, so it writes to downstream and reads from 39 | // upstream 40 | go func() { 41 | lockWithin(t, m, "foo", 10*time.Millisecond) 42 | downstream <- 1 43 | waitFor(upstream, 2) 44 | downstream <- 3 45 | m.Unlock("foo") 46 | }() 47 | 48 | // wait until the other goroutine has locked "foo" 49 | waitFor(downstream, 1) 50 | 51 | // prove that we can lock other keys without a problem 52 | lockWithin(t, m, "bar", 10*time.Millisecond) 53 | lockWithin(t, m, "baz", 10*time.Millisecond) 54 | 55 | assert.Equal(t, 3, m.len(), "wrong number of internal locks") 56 | 57 | // start trying to lock "foo" which will block until we signal the other goroutine to unlock it 58 | time.AfterFunc(20*time.Millisecond, func() { upstream <- 2 }) 59 | lockWithin(t, m, "foo", 200*time.Millisecond) 60 | 61 | // there better be a 3 already queued up in the downstream, otherwise we locked too fast 62 | readSignalNow(downstream, 3, "locked \"foo\" before the other goroutine unlocked it...") 63 | 64 | assert.Equal(t, 3, m.len(), "wrong number of internal locks") 65 | 66 | // we can unlock out of order, and locks go away as we unlock them 67 | m.Unlock("bar") 68 | assert.Equal(t, 2, m.len(), "wrong number of internal locks") 69 | 70 | m.Unlock("baz") 71 | assert.Equal(t, 1, m.len(), "wrong number of internal locks") 72 | 73 | m.Unlock("foo") 74 | assert.Equal(t, 0, m.len(), "wrong number of internal locks") 75 | } 76 | 77 | func TestTransientLockMapBadUnlock(t *testing.T) { 78 | // call a function, and fail the test if the function doesn't panic 79 | index := 0 80 | shouldPanic := func(f func()) { 81 | index++ 82 | defer func() { 83 | if recovered := recover(); recovered != nil { 84 | // ok, all is well 85 | } else { 86 | // we were supposed to panic but didn't - fail the test! 87 | t.Fatalf("test #%d did not panic as expected...", index) 88 | } 89 | }() 90 | f() 91 | } 92 | 93 | // unlocking a key that has never been referenced 94 | m := NewTransientLockMap() 95 | shouldPanic(func() { 96 | m.Unlock("foo") 97 | }) 98 | assert.Equal(t, 0, m.len(), "wrong number of internal locks") 99 | 100 | // double-unlocking a key in the same goroutine 101 | m = NewTransientLockMap() 102 | shouldPanic(func() { 103 | assertLock(t, m, "foo") 104 | m.Unlock("foo") 105 | m.Unlock("foo") 106 | }) 107 | assert.Equal(t, 0, m.len(), "wrong number of internal locks") 108 | 109 | // double-unlocking a key across 2 goroutines 110 | m = NewTransientLockMap() 111 | shouldPanic(func() { 112 | signal := make(chan struct{}) 113 | 114 | go func() { 115 | assertLock(t, m, "foo") 116 | m.Unlock("foo") 117 | close(signal) 118 | }() 119 | 120 | <-signal 121 | m.Unlock("foo") 122 | }) 123 | assert.Equal(t, 0, m.len(), "wrong number of internal locks") 124 | } 125 | 126 | func TestTransientLockMapSequence(t *testing.T) { 127 | m := NewTransientLockMap() 128 | 129 | // signals 130 | partnerAboutToLock := make(chan struct{}) 131 | partnerGotLock := make(chan struct{}) 132 | 133 | assertLock(t, m, "foo") 134 | go func() { 135 | close(partnerAboutToLock) 136 | assertLock(t, m, "foo") 137 | close(partnerGotLock) 138 | time.Sleep(125 * time.Millisecond) 139 | m.Unlock("foo") 140 | }() 141 | 142 | <-partnerAboutToLock 143 | time.Sleep(100 * time.Millisecond) // give our partner time to actually call Lock() 144 | m.Unlock("foo") 145 | 146 | <-partnerGotLock 147 | start := time.Now() 148 | assertLock(t, m, "foo") 149 | 150 | // ensure that the prior Lock() call actually blocked and waited for a while, as intended 151 | if d := time.Since(start); d < 100*time.Millisecond { 152 | t.Fatalf("Lock acquired too fast (%s)", d) 153 | } 154 | m.Unlock("foo") 155 | assert.Equal(t, 0, m.len(), "wrong number of internal locks") 156 | } 157 | 158 | func TestTransientLockMapContention(t *testing.T) { 159 | m := NewTransientLockMap() 160 | 161 | var wg sync.WaitGroup 162 | 163 | assertLock(t, m, "foo") 164 | assertLock(t, m, "bar") 165 | 166 | for i := 0; i < 100; i++ { 167 | wg.Add(1) 168 | go func() { 169 | defer wg.Done() 170 | assertLock(t, m, "foo") 171 | m.Unlock("foo") 172 | assertLock(t, m, "bar") 173 | m.Unlock("bar") 174 | }() 175 | } 176 | 177 | time.Sleep(30 * time.Millisecond) 178 | m.Unlock("bar") 179 | m.Unlock("foo") 180 | 181 | wg.Wait() 182 | 183 | assert.Equal(t, 0, m.len(), "wrong number of internal locks") 184 | } 185 | 186 | func TestTransientLockMapTimeout(t *testing.T) { 187 | m := NewTransientLockMap() 188 | 189 | assertLock(t, m, "foo") 190 | 191 | ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) 192 | defer cancel() 193 | 194 | res := m.Lock(ctx, "foo") 195 | assert.Equal(t, false, res, "lock foo (2)") 196 | 197 | m.Unlock("foo") 198 | lockWithin(t, m, "foo", 10*time.Millisecond) // should be able to lock near instantly 199 | 200 | ctx2, cancel := context.WithCancel(context.Background()) 201 | done := make(chan struct{}) 202 | go func() { 203 | res := m.Lock(ctx2, "foo") 204 | assert.Equal(t, false, res, "lock foo (4)") 205 | close(done) 206 | }() 207 | 208 | time.Sleep(25 * time.Millisecond) 209 | 210 | // goroutine should not be done yet (i.e. Lock() should still be blocked) 211 | select { 212 | case <-done: 213 | t.Fatal("done should not be closed yet!") 214 | default: 215 | } 216 | 217 | cancel() 218 | 219 | select { 220 | case <-time.After(25 * time.Millisecond): 221 | t.Fatal("timeout waiting for done channel to close") 222 | case <-done: 223 | // ok 224 | } 225 | 226 | // finally, verify that we can unlock and lock still 227 | m.Unlock("foo") 228 | lockWithin(t, m, "foo", 10*time.Millisecond) // should be able to lock near instantly 229 | m.Unlock("foo") 230 | 231 | assert.Equal(t, 0, m.len(), "wrong number of internal locks") 232 | 233 | // We should not be able to lock with an already-cancelled context. 234 | for i := 0; i < 10; i++ { 235 | assert.Assert(t, !m.Lock(ctx2, "foo"), "lock foo (final) expected canceled") 236 | } 237 | } 238 | 239 | func TestTransientLockMapRun(t *testing.T) { 240 | m := NewTransientLockMap() 241 | 242 | start := make(chan struct{}) 243 | var wgSucc sync.WaitGroup 244 | var wgFail sync.WaitGroup 245 | var nFail int32 246 | var nSucc int32 247 | ctx, cancel := context.WithCancel(context.Background()) 248 | defer cancel() 249 | for i := 0; i < 10; i++ { 250 | if i == 0 { 251 | wgSucc.Add(1) 252 | } else { 253 | wgFail.Add(1) 254 | } 255 | go func() { 256 | <-start 257 | err := m.Run(ctx, "foo", func(ctx context.Context) error { 258 | // Whoever got the lock should cancel everyone else. 259 | cancel() 260 | wgFail.Wait() 261 | return nil 262 | }) 263 | if err == nil { 264 | defer wgSucc.Done() 265 | atomic.AddInt32(&nSucc, 1) 266 | } else { 267 | defer wgFail.Done() 268 | if err == context.Canceled { 269 | atomic.AddInt32(&nFail, 1) 270 | } else { 271 | t.Errorf("expected canceled, got %T: %s", err, err) 272 | } 273 | } 274 | }() 275 | } 276 | 277 | close(start) 278 | wgSucc.Wait() 279 | 280 | assert.Equal(t, int32(1), nSucc, "wrong # success") 281 | assert.Equal(t, int32(9), nFail, "wrong # failures") 282 | } 283 | 284 | func assertLock(t *testing.T, m *TransientLockMap, key string) { 285 | assert.Assert(t, m.Lock(context.Background(), key), "should have locked") 286 | } 287 | 288 | // grabs a lock, panicking if this takes longer than expected 289 | func lockWithin(t *testing.T, m *TransientLockMap, key string, timeout time.Duration) { 290 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 291 | defer cancel() 292 | assert.Assert(t, m.Lock(ctx, key), "should have locked") 293 | } 294 | -------------------------------------------------------------------------------- /storage/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/fullstorydev/emulators/storage 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | cloud.google.com/go/storage v1.54.0 9 | github.com/bluele/gcache v0.0.2 10 | github.com/google/btree v1.1.3 11 | golang.org/x/oauth2 v0.30.0 12 | google.golang.org/api v0.235.0 13 | google.golang.org/protobuf v1.36.6 14 | gotest.tools/v3 v3.5.2 15 | ) 16 | 17 | require ( 18 | cel.dev/expr v0.20.0 // indirect 19 | cloud.google.com/go v0.121.0 // indirect 20 | cloud.google.com/go/auth v0.16.1 // indirect 21 | cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect 22 | cloud.google.com/go/compute/metadata v0.7.0 // indirect 23 | cloud.google.com/go/iam v1.5.2 // indirect 24 | cloud.google.com/go/monitoring v1.24.2 // indirect 25 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect 26 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect 27 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect 28 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 29 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect 30 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect 31 | github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect 32 | github.com/felixge/httpsnoop v1.0.4 // indirect 33 | github.com/go-jose/go-jose/v4 v4.0.5 // indirect 34 | github.com/go-logr/logr v1.4.2 // indirect 35 | github.com/go-logr/stdr v1.2.2 // indirect 36 | github.com/google/go-cmp v0.7.0 // indirect 37 | github.com/google/s2a-go v0.1.9 // indirect 38 | github.com/google/uuid v1.6.0 // indirect 39 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect 40 | github.com/googleapis/gax-go/v2 v2.14.2 // indirect 41 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 42 | github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect 43 | github.com/zeebo/errs v1.4.0 // indirect 44 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 45 | go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect 46 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect 47 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect 48 | go.opentelemetry.io/otel v1.35.0 // indirect 49 | go.opentelemetry.io/otel/metric v1.35.0 // indirect 50 | go.opentelemetry.io/otel/sdk v1.35.0 // indirect 51 | go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect 52 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 53 | golang.org/x/crypto v0.38.0 // indirect 54 | golang.org/x/net v0.40.0 // indirect 55 | golang.org/x/sync v0.14.0 // indirect 56 | golang.org/x/sys v0.33.0 // indirect 57 | golang.org/x/text v0.25.0 // indirect 58 | golang.org/x/time v0.11.0 // indirect 59 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect 60 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect 61 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 // indirect 62 | google.golang.org/grpc v1.72.1 // indirect 63 | ) 64 | -------------------------------------------------------------------------------- /storage/go.sum: -------------------------------------------------------------------------------- 1 | cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= 2 | cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= 3 | cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg= 4 | cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q= 5 | cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= 6 | cloud.google.com/go/auth v0.16.1/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= 7 | cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= 8 | cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= 9 | cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= 10 | cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= 11 | cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= 12 | cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= 13 | cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= 14 | cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= 15 | cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= 16 | cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= 17 | cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= 18 | cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= 19 | cloud.google.com/go/storage v1.54.0 h1:Du3XEyliAiftfyW0bwfdppm2MMLdpVAfiIg4T2nAI+0= 20 | cloud.google.com/go/storage v1.54.0/go.mod h1:hIi9Boe8cHxTyaeqh7KMMwKg088VblFK46C2x/BWaZE= 21 | cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= 22 | cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= 23 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= 24 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= 25 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= 26 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= 27 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= 28 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= 29 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= 30 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= 31 | github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= 32 | github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= 33 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 34 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 35 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= 36 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= 37 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 38 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 39 | github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= 40 | github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= 41 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= 42 | github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= 43 | github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= 44 | github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= 45 | github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= 46 | github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= 47 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 48 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 49 | github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= 50 | github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= 51 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 52 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 53 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 54 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 55 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 56 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 57 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 58 | github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= 59 | github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= 60 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 61 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 62 | github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= 63 | github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= 64 | github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= 65 | github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= 66 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 67 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 68 | github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= 69 | github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= 70 | github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= 71 | github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= 72 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= 73 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= 74 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 75 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 76 | github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= 77 | github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= 78 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 79 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 80 | github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= 81 | github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= 82 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 83 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 84 | go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= 85 | go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= 86 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= 87 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= 88 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= 89 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= 90 | go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= 91 | go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= 92 | go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= 93 | go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= 94 | go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= 95 | go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= 96 | go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= 97 | go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= 98 | go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= 99 | go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= 100 | go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= 101 | go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= 102 | golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= 103 | golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= 104 | golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= 105 | golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= 106 | golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= 107 | golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= 108 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 109 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 110 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 111 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 112 | golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= 113 | golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= 114 | golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= 115 | golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= 116 | google.golang.org/api v0.235.0 h1:C3MkpQSRxS1Jy6AkzTGKKrpSCOd2WOGrezZ+icKSkKo= 117 | google.golang.org/api v0.235.0/go.mod h1:QpeJkemzkFKe5VCE/PMv7GsUfn9ZF+u+q1Q7w6ckxTg= 118 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= 119 | google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= 120 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 h1:vPV0tzlsK6EzEDHNNH5sa7Hs9bd7iXR7B1tSiPepkV0= 121 | google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:pKLAc5OolXC3ViWGI62vvC0n10CpwAtRcTNCFwTKBEw= 122 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9 h1:IkAfh6J/yllPtpYFU0zZN1hUPYdT0ogkBT/9hMxHjvg= 123 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250512202823-5a2f75b736a9/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= 124 | google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= 125 | google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= 126 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 127 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 128 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 129 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 130 | gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= 131 | gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= 132 | -------------------------------------------------------------------------------- /storage/releasing/README.md: -------------------------------------------------------------------------------- 1 | # Releases of emulators/storage 2 | 3 | This document provides instructions for building a release of `emulators/storage`. 4 | 5 | The release process consists of a handful of tasks: 6 | 1. Drop a release tag in git. 7 | 2. Build binaries for various platforms. This is done using the local go tool and uses GOOS and GOARCH environment variables to cross-compile for supported platforms. 8 | 3. Creates a release in GitHub, uploads the binaries, and creates provisional release notes (in the form of a change log). 9 | 4. Build a docker image for the new release. 10 | 5. Push the docker image to Docker Hub, with both a version tag and the "latest" tag. 11 | 12 | Most of this is automated via a script in this same directory. The main thing you will need is a GitHub personal access token, which will be used for creating the release in GitHub (so you need write access to the fullstorydev/emulators repo). 13 | 14 | ## Creating a new release 15 | 16 | So, to actually create a new release, just run the script in this directory. 17 | 18 | First, you need a version number for the new release, following sem-ver format: `v..`. Second, you need a personal access token for GitHub. 19 | 20 | ```sh 21 | # from the root of the package 22 | GITHUB_TOKEN= ./releasing/do-release.sh v.. 23 | ``` 24 | 25 | Wasn't that easy! There is one last step: update the release notes in GitHub. By default, the script just records a change log of commit descriptions. Use that log (and, if necessary, drill into individual PRs included in the release) to flesh out notes in the format of the `RELEASE_NOTES.md` file _in this directory_. Then login to GitHub, go to the new release, edit the notes, and paste in the markdown you just wrote. 26 | 27 | That should be all there is to it! 28 | -------------------------------------------------------------------------------- /storage/releasing/RELEASE_NOTES.md: -------------------------------------------------------------------------------- 1 | ## Changes 2 | 3 | ### Go package "github.com/fullstorydev/emulators/bigtable" 4 | 5 | * _In this list, describe the changes in this repo: "github.com/fullstorydev/emulators/bigtable"._ 6 | * _Use one bullet per change. Include both bug-fixes and improvements._ 7 | -------------------------------------------------------------------------------- /storage/releasing/do-release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # strict mode 4 | set -euo pipefail 5 | IFS=$'\n\t' 6 | 7 | if [[ -z ${DRY_RUN:-} ]]; then 8 | PREFIX="" 9 | else 10 | PREFIX="echo" 11 | fi 12 | 13 | # input validation 14 | if [[ -z ${GITHUB_TOKEN:-} ]]; then 15 | echo "GITHUB_TOKEN environment variable must be set before running." >&2 16 | exit 1 17 | fi 18 | if [[ $# -ne 1 || $1 == "" ]]; then 19 | echo "This program requires one argument: the version number, in 'vM.N.P' format." >&2 20 | exit 1 21 | fi 22 | VERSION=$1 23 | 24 | # Change to root of the repo 25 | cd "$(dirname "$0")/.." 26 | 27 | # Docker release 28 | 29 | # make sure credentials are valid for later push steps; this might 30 | # be interactive since this will prompt for username and password 31 | # if there are no valid current credentials. 32 | $PREFIX docker login 33 | echo "$VERSION" > VERSION 34 | # Docker Buildx support is included in Docker 19.03 35 | # Below step installs emulators for different architectures on the host 36 | # This enables running and building containers for below architectures mentioned using --platforms 37 | $PREFIX docker run --privileged --rm tonistiigi/binfmt:qemu-v6.1.0 --install all 38 | # Create a new builder instance 39 | export DOCKER_CLI_EXPERIMENTAL=enabled 40 | $PREFIX docker buildx create --use --name multiarch-builder --node multiarch-builder0 41 | # push to docker hub, both the given version as a tag and for "latest" tag 42 | $PREFIX docker buildx build --target=gcsemulator --platform linux/amd64,linux/s390x,linux/arm64,linux/ppc64le --tag fullstorydev/gcsemulator:${VERSION} --tag fullstorydev/gcsemulator:latest --push --progress plain --no-cache . 43 | rm VERSION 44 | --------------------------------------------------------------------------------