├── .circleci └── config.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd └── handy-spanner │ └── main.go ├── fake ├── example_test.go ├── integration_test.go ├── server.go └── testdata │ └── schema.sql ├── go.mod ├── go.sum ├── img ├── abort1.png ├── abort2.png └── read_block.png └── server ├── database.go ├── database_function.go ├── database_json.go ├── database_nonjson.go ├── database_query_test.go ├── database_test.go ├── debug.go ├── keyset.go ├── meta_schema.go ├── query.go ├── query_test.go ├── server.go ├── server_test.go ├── server_transaction_test.go ├── session.go ├── spanner_error.go ├── table.go ├── transaction.go ├── value.go └── value_test.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | test: 5 | docker: 6 | - image: golang:1.22-bookworm 7 | environment: 8 | GO111MODULE: "on" 9 | working_directory: /go/src/handy-spanner 10 | steps: 11 | - checkout 12 | - restore_cache: 13 | key: go-mod-{{ checksum "go.sum" }} 14 | - run: 15 | name: Install dependencies 16 | command: | 17 | if [ ! -d 'vendor' ]; then 18 | go mod download 19 | fi 20 | - run: 21 | name: build 22 | command: | 23 | make build 24 | 25 | - run: 26 | name: run tests 27 | command: | 28 | make test 29 | 30 | workflows: 31 | version: 2 32 | build-workflow: 33 | jobs: 34 | - test 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /handy-spanner 2 | /vendor/ 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.18-alpine3.16 AS builder 2 | 3 | RUN set -eux \ 4 | && apk --no-cache add \ 5 | g++ \ 6 | gcc \ 7 | git \ 8 | make \ 9 | musl-dev 10 | 11 | COPY . /go/src/handy-spanner 12 | WORKDIR /go/src/handy-spanner 13 | 14 | RUN make build 15 | 16 | FROM alpine:3.16.1 17 | 18 | COPY --from=builder /go/src/handy-spanner/handy-spanner /usr/local/bin/handy-spanner 19 | 20 | RUN apk --no-cache add \ 21 | ca-certificates tzdata 22 | 23 | USER nobody 24 | EXPOSE 9999 25 | ENTRYPOINT ["/usr/local/bin/handy-spanner"] 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2019 Masahiro Sano 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: test build 2 | 3 | export GO111MODULE=on 4 | 5 | .PHONY: build 6 | build: 7 | go build -v --tags "json1" ./cmd/handy-spanner 8 | 9 | .PHONY: test 10 | test: 11 | go test -v --tags "json1" -race ./... 12 | 13 | docker-build: 14 | docker build . -t handy-spanner 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # An emulator for Cloud Spanner 2 | 3 | ## Install 4 | 5 | ``` 6 | go get github.com/gcpug/handy-spanner/cmd/handy-spanner 7 | ``` 8 | 9 | NOTE: If you want to use some features (e.g. array literal `[]` in DML), require "json1" build tag. 10 | The Spanner emulator uses sqlite3 internally. You may need to build go-sqlite3 explicitly. 11 | It also requires cgo to use sqlite3. 12 | 13 | ``` 14 | go get -u github.com/mattn/go-sqlite3 15 | go install github.com/mattn/go-sqlite3 16 | ``` 17 | 18 | ## Usage 19 | 20 | ### Run as an independent process 21 | 22 | ``` 23 | ./handy-spanner 24 | ``` 25 | 26 | or 27 | 28 | ``` 29 | docker run --rm -it -p 9999:9999 handy-spanner 30 | ``` 31 | 32 | It runs a hand-spanner server as a process. It serves spanner gRPC server by port 9999 by default. 33 | 34 | #### Access to the server 35 | 36 | The google-cloud-go, the official Spanner SDK, supports to access an emulator server. 37 | Set the address to an emulator server to environment variable `SPANNER_EMULATOR_HOST`, then google-cloud-go transparently use the server in the client. 38 | 39 | So if you want to replace spanner server with the handy-spanner you run, just do: 40 | 41 | ``` 42 | export SPANNER_EMULATOR_HOST=localhost:9999 43 | ``` 44 | 45 | #### Schema setup 46 | 47 | The handy-spanner server has no databases nor tables by default. You need to create them by yourself. You can prepare them at startup. 48 | 49 | handy-spanner accepts some command-line arguments to prepare schema. 50 | 51 | * `schema`: Path to a DDL file which handy-spanner creates at startup 52 | * `project`: The project the DDL is applied. 53 | * `instance`: The instance the DDL is applied. 54 | * `database`: The database the DDL is applied. 55 | 56 | If you prepare schema at startup, these 4 arguments are required. 57 | 58 | #### Implicit creation of databases 59 | 60 | Without schema setup, databases are created automatically by your database access. 61 | 62 | handy-spanner creates a database when a session to the database is created. In detail, when `CreateSession` or `BatchCreateSessions` is called, the accessed database is created. 63 | 64 | #### Database operations 65 | 66 | You can also operate databases by your applications. If you need additional databases or database alterations, you need to follow this section. 67 | 68 | As Cloud Spanner supports, there are dedicated gRPC service for instance and database operations for Spanner. 69 | 70 | * Instance operations 71 | * [Document](https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.admin.instance.v1) 72 | * [Protobuf](https://github.com/googleapis/googleapis/blob/master/google/spanner/admin/instance/v1/spanner_instance_admin.proto) 73 | * Database operations 74 | * [Document](https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.admin.database.v1) 75 | * [Protobuf](https://github.com/googleapis/googleapis/blob/master/google/spanner/admin/database/v1/spanner_database_admin.proto) 76 | 77 | handy-spanner also supports these gRPC services so that you can operate databases. The gRPC services are provided in the same address to the normal gRPC service. It means it is provided in `localhost:9999` by default. 78 | 79 | It seems most clients for each language for these gRPC services provided Google Cloud Platform dont support `SPANNER_EMULATOR_HOST` to connect an emulator server, so you need to setup a client to connect an emulator server explicitly. For Go, you can refer [examples](./fake/example_test.go) to connect an emulator server. 80 | 81 | ### Run as a buillt-in server in Go 82 | 83 | If you use a handy-spanner server in tests in Go, it's easier to run it in a process. 84 | See an [example](https://github.com/gcpug/handy-spanner/blob/master/fake/example_test.go) for the details. 85 | 86 | Note that the tests highly depend on handhy-spanner, which means you cannot switch the backend depending on the situation. If you want to test on both Cloud Spanner and handy-spanner, it's better to use a handy-spanner server as an independent process. 87 | 88 | #### Database operations 89 | 90 | handy-spanner provides utility functions to operate databases. For the detail, please refer [examples](./fake/example_test.go). 91 | 92 | * `ParseAndApplyDDL` 93 | * You can pass `io.Reader` as a schema file to apply DDL to a database. 94 | * `ApplyDDL` 95 | * You can pass `ast.DDL` as an already parsed definition to apply DDL to a database. 96 | 97 | You can also operate databases or instances as Cloud Spanner supports. Please also refer _Run as an independent process_ section. 98 | 99 | ## Can and Cannot 100 | 101 | ### Supported features 102 | 103 | * Read 104 | * Keys and KeyRange as KeySet 105 | * Secondary index 106 | * STORING columns for secondary index 107 | * Respect column orders for index 108 | * Query 109 | * Select result set by column name and * 110 | * Most operators in WHERE clause: IN, BETWEEN, IS NULL 111 | * Conditions in WHERE clause: =, !=, >, <, AND, OR 112 | * Order By keyword with ASC, DESC 113 | * Group By and Having statement 114 | * LIMIT OFFSET 115 | * SELECT alias 116 | * Query Parameters 117 | * Literals 118 | * JOINs: COMMMA, CROSS, INNER, LEFT/RIGHT/FULL OUTER 119 | * FULL OUTER has some limitations 120 | * Subquery 121 | * SET operations: UNION, INTERSECT, EXCEPT 122 | * UNNEST: IN UNNEST, FROM UNNEST 123 | * Functions (partially) 124 | * Arithmetic operations 125 | * Mutation 126 | * All mutation types: Insert, Update, InsertOrUpdate, Replace, Delete 127 | * Commit timestamp 128 | * Transaction 129 | * Isolation level: SERIALIZABLE 130 | * DML 131 | * Insert, Update, Delete 132 | * Batch DML 133 | * DDL 134 | * CreateTable, CreateIndex only 135 | * Respect INTERLEAVE 136 | * Data Types 137 | * Int, Float, String, Bool, Byte, Date, Timestamp, Array, Struct 138 | * [Information Schema](https://cloud.google.com/spanner/docs/information-schema) 139 | * partially supported 140 | 141 | ### Not supported features 142 | 143 | * Transaction 144 | * Timestamp bound read 145 | * Query 146 | * Strict type checking 147 | * More functions 148 | * Partionan Query 149 | * EXCEPT ALL and INTERSECT ALL (because of sqlite) 150 | * FULL OUTER JOIN 151 | * Not support table alias. Use `*` for now 152 | * Not support ON condition. Use USING condition for now 153 | * Merging INT64 and FLOAT64 in SET operations 154 | * Array operations 155 | * DDL 156 | * Alter Table, Drop Table, Drop Index 157 | * Database management 158 | * Long running operations 159 | * Replace 160 | * wrong behavior on conflict 161 | * [Change Stream Schema](https://cloud.google.com/spanner/docs/change-streams) 162 | * only can parse in [Database operations](#Database-operations) with `schema` flag. 163 | 164 | ## Implementation 165 | 166 | ### Transaction simulation 167 | 168 | handy-spanner uses sqlite3 in [Shared-Cache Mode](https://www.sqlite.org/sharedcache.html). There is a characteristic in the trasactions. 169 | 170 | * Only one transaction can hold write lock per database to write database tables. 171 | * Other transactions still can hold read lock. 172 | * Write transaction holds write lock against database tables while writing the tables. 173 | * Other read transactions cannot read the table while locked 174 | * Read transaction holds read lock against database tables while reading the tables. 175 | * Other read transactions can read the table by holding read lock 176 | * Write transaction cannot write the table while read-locked 177 | 178 | If we simply use the transactions, dead lock should happen in read and write locks. To simulate spanner transactions correctly as possible, handy-spanner manages sqlite3 transactions inside. 179 | 180 | * Each spanner transaction starts own sqlite3 transaction. 181 | * Only one spanner transaction can hold write lock per database. 182 | * While a transaction holds write lock, other spanner transactions cannot newly get read or write lock. 183 | * When write transaction tries to write a table, it forces transactions that hold read lock to the table to release the lock. 184 | * The transactions become "aborted" 185 | * The aborted transactions are expected to be retried by the client. 186 | 187 | ![abort](img/abort1.png) ![abort](img/abort2.png) 188 | 189 | ### DML 190 | 191 | Because of transaction limitations, DML also has limitations. 192 | 193 | When a transaction(A) updates a table, other transactions cannot read/write the table until the transaction(A) commits. This limitation may become an inconsistency to the Cloud Spanner. Other limitations are same to mutations with commit. 194 | 195 | ![block](img/read_block.png) 196 | 197 | ## Feature Request and Bug Report 198 | 199 | Feature requests and any bug reports are welcome. 200 | 201 | There are many things to implement features and make better compatibility to Cloud Spanner. Now I'm prioritizing features by on-demand from users and me. Please create an issue if you find lack of a feature to use handy-spanner or imcompatibility to Cloud Spanner. 202 | 203 | #### Feature request 204 | 205 | Please describe what feature you want. It's the best to refer a Cloud Spanner's document for the feature. If the feature is partiality implemented or wrong behavior you expected, please also see a Bug Report section. 206 | 207 | #### Bug report 208 | 209 | Please describe a reproducable conditions. If you describe more information about the bug, it makes more easir to fix the bug. 210 | 211 | * Schema 212 | * Initial Data 213 | * Query or any requests you run 214 | * Actual behavior 215 | * Expected behavior 216 | * Document in Cloud Spanner 217 | 218 | ## Copyright 219 | 220 | * Author: Masahiro Sano ([@kazegusuri](https://github.com/kazegusuri)) 221 | * Copyright: 2019 Masahiro Sano 222 | * License: Apache License, Version 2.0 223 | -------------------------------------------------------------------------------- /cmd/handy-spanner/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "flag" 20 | "fmt" 21 | "log" 22 | "net" 23 | "os" 24 | "os/signal" 25 | "syscall" 26 | 27 | "github.com/gcpug/handy-spanner/fake" 28 | "golang.org/x/sync/errgroup" 29 | ) 30 | 31 | var ( 32 | addr = flag.String("addr", "0.0.0.0:9999", "address to listen") 33 | project = flag.String("project", "fake", "project to apply DDL at startup") 34 | database = flag.String("database", "fake", "database to apply DDL at startup") 35 | instance = flag.String("instance", "fake", "instance to apply DDL at startup") 36 | schema = flag.String("schema", "", "path to a DDL file") 37 | ) 38 | 39 | func main() { 40 | flag.CommandLine.SetOutput(os.Stdout) 41 | flag.Parse() 42 | 43 | if err := runMain(); err != nil { 44 | fmt.Fprintf(os.Stderr, "%s\n", err) 45 | os.Exit(1) 46 | 47 | } 48 | } 49 | 50 | func runMain() error { 51 | if *addr == "" { 52 | return fmt.Errorf("addr must be specified") 53 | } 54 | 55 | var file *os.File 56 | if *schema != "" { 57 | f, err := os.Open(*schema) 58 | if err != nil { 59 | return fmt.Errorf("failed to open schema file: %v", err) 60 | } 61 | file = f 62 | defer file.Close() 63 | } 64 | 65 | // root context notifies server shutdown by SIGINT or SIGTERM 66 | ctx, cancel := context.WithCancel(context.Background()) 67 | defer cancel() 68 | 69 | lis, err := net.Listen("tcp", *addr) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | server, err := fake.New(lis) 75 | if err != nil { 76 | return fmt.Errorf("failed to setup fake server: %v", err) 77 | } 78 | 79 | wg, ctx := errgroup.WithContext(ctx) 80 | wg.Go(func() error { return server.Start() }) 81 | 82 | if file != nil { 83 | dbName := fmt.Sprintf("projects/%s/instances/%s/databases/%s", *project, *instance, *database) 84 | if err := server.ParseAndApplyDDL(ctx, dbName, file); err != nil { 85 | server.Stop() 86 | return fmt.Errorf("failed apply DDL: %v", err) 87 | } 88 | } 89 | 90 | log.Print("spanner server is ready") 91 | 92 | // Waiting for SIGTERM or Interrupt signal 93 | sigCh := make(chan os.Signal, 1) 94 | signal.Notify(sigCh, syscall.SIGTERM, os.Interrupt) 95 | select { 96 | case <-sigCh: 97 | log.Print("received SIGTERM, exiting server") 98 | case <-ctx.Done(): 99 | } 100 | 101 | // stop server 102 | server.Stop() 103 | 104 | return wg.Wait() 105 | } 106 | -------------------------------------------------------------------------------- /fake/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fake_test 16 | 17 | import ( 18 | "context" 19 | "log" 20 | "strings" 21 | 22 | "cloud.google.com/go/spanner" 23 | admindatabasev1 "cloud.google.com/go/spanner/admin/database/apiv1" 24 | databasepb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb" 25 | "github.com/gcpug/handy-spanner/fake" 26 | "google.golang.org/api/option" 27 | ) 28 | 29 | func ExampleSpannerClient() { 30 | ctx := context.Background() 31 | dbName := "projects/fake/instances/fake/databases/fake" 32 | 33 | // Run fake server 34 | srv, conn, err := fake.Run() 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | defer srv.Stop() 39 | defer conn.Close() 40 | 41 | // Prepare spanner client 42 | client, err := spanner.NewClient(ctx, dbName, option.WithGRPCConn(conn)) 43 | if err != nil { 44 | log.Fatalf("failed to connect fake spanner server: %v", err) 45 | } 46 | 47 | // Use the client 48 | _, _ = client.Apply(ctx, []*spanner.Mutation{ 49 | spanner.Insert("Test", 50 | []string{"ColA", "ColB"}, 51 | []interface{}{"foo", 100}, 52 | ), 53 | }) 54 | 55 | // output: 56 | } 57 | 58 | func ExampleAdminClient() { 59 | ctx := context.Background() 60 | dbName := "projects/fake/instances/fake/databases/fake" 61 | stmts := []string{ 62 | `CREATE TABLE Table1 ( Id STRING(MAX) NOT NULL ) PRIMARY KEY (Id)`, 63 | `CREATE TABLE Table2 ( Id TIMESTAMP NOT NULL ) PRIMARY KEY (Id)`, 64 | } 65 | 66 | // Run fake server 67 | srv, conn, err := fake.Run() 68 | if err != nil { 69 | log.Fatal(err) 70 | } 71 | defer srv.Stop() 72 | 73 | // Prepare spanner client 74 | adminclient, err := admindatabasev1.NewDatabaseAdminClient(ctx, option.WithGRPCConn(conn)) 75 | if err != nil { 76 | log.Fatalf("failed to connect fake spanner server: %v", err) 77 | } 78 | 79 | // Use the client 80 | _, err = adminclient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ 81 | Database: dbName, 82 | Statements: stmts, 83 | }) 84 | if err != nil { 85 | log.Fatal(err) 86 | } 87 | 88 | // output: 89 | } 90 | 91 | func ExampleApplyDDL() { 92 | ctx := context.Background() 93 | dbName := "projects/fake/instances/fake/databases/fake" 94 | schema := ` 95 | CREATE TABLE Table1 ( Id STRING(MAX) NOT NULL ) PRIMARY KEY (Id); 96 | CREATE TABLE Table2 ( Id TIMESTAMP NOT NULL ) PRIMARY KEY (Id); 97 | ` 98 | 99 | // Run fake server 100 | srv, _, err := fake.Run() 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | defer srv.Stop() 105 | 106 | err = srv.ParseAndApplyDDL(ctx, dbName, strings.NewReader(schema)) 107 | if err != nil { 108 | log.Fatal(err) 109 | } 110 | 111 | // output: 112 | } 113 | -------------------------------------------------------------------------------- /fake/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fake 16 | 17 | import ( 18 | "context" 19 | "io" 20 | "io/ioutil" 21 | "net" 22 | "time" 23 | 24 | lropb "cloud.google.com/go/longrunning/autogen/longrunningpb" 25 | adminv1pb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb" 26 | spannerpb "cloud.google.com/go/spanner/apiv1/spannerpb" 27 | "github.com/cloudspannerecosystem/memefish" 28 | "github.com/cloudspannerecosystem/memefish/ast" 29 | "github.com/cloudspannerecosystem/memefish/token" 30 | "github.com/gcpug/handy-spanner/server" 31 | "google.golang.org/grpc" 32 | channelzsvc "google.golang.org/grpc/channelz/service" 33 | "google.golang.org/grpc/reflection" 34 | ) 35 | 36 | type Server struct { 37 | addr string 38 | lis net.Listener 39 | grpcServer *grpc.Server 40 | srv server.FakeSpannerServer 41 | } 42 | 43 | func Run() (*Server, *grpc.ClientConn, error) { 44 | l, err := net.Listen("tcp", "127.0.0.1:0") 45 | if err != nil { 46 | if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { 47 | return nil, nil, err 48 | } 49 | } 50 | 51 | srv, err := New(l) 52 | if err != nil { 53 | return nil, nil, err 54 | } 55 | go func() { _ = srv.Start() }() 56 | 57 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 58 | defer cancel() 59 | 60 | conn, err := grpc.DialContext(ctx, srv.Addr(), grpc.WithInsecure(), grpc.WithBlock()) 61 | if err != nil { 62 | srv.Stop() 63 | return nil, nil, err 64 | } 65 | 66 | return srv, conn, nil 67 | } 68 | 69 | // New returns Server for fake spanner. 70 | func New(lis net.Listener) (*Server, error) { 71 | s := &Server{ 72 | addr: lis.Addr().String(), 73 | lis: lis, 74 | grpcServer: grpc.NewServer(), 75 | srv: server.NewFakeServer(), 76 | } 77 | 78 | adminv1pb.RegisterDatabaseAdminServer(s.grpcServer, s.srv) 79 | spannerpb.RegisterSpannerServer(s.grpcServer, s.srv) 80 | lropb.RegisterOperationsServer(s.grpcServer, s.srv) 81 | 82 | reflection.Register(s.grpcServer) 83 | channelzsvc.RegisterChannelzServiceToServer(s.grpcServer) 84 | 85 | return s, nil 86 | } 87 | 88 | func (s *Server) Addr() string { 89 | return s.addr 90 | } 91 | 92 | func (s *Server) Start() error { 93 | return s.grpcServer.Serve(s.lis) 94 | } 95 | 96 | func (s *Server) Stop() { 97 | s.grpcServer.Stop() 98 | _ = s.lis.Close() 99 | } 100 | 101 | func (s *Server) ApplyDDL(ctx context.Context, databaseName string, ddl []ast.DDL) error { 102 | for _, stmt := range ddl { 103 | if err := s.srv.ApplyDDL(ctx, databaseName, stmt); err != nil { 104 | return err 105 | } 106 | } 107 | return nil 108 | } 109 | 110 | func (s *Server) ParseAndApplyDDL(ctx context.Context, databaseName string, r io.Reader) error { 111 | b, err := ioutil.ReadAll(r) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | ddl, err := (&memefish.Parser{ 117 | Lexer: &memefish.Lexer{ 118 | File: &token.File{FilePath: "", Buffer: string(b)}, 119 | }, 120 | }).ParseDDLs() 121 | if err != nil { 122 | return err 123 | } 124 | 125 | for _, stmt := range ddl { 126 | if err := s.srv.ApplyDDL(ctx, databaseName, stmt); err != nil { 127 | return err 128 | } 129 | } 130 | 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /fake/testdata/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE Simple ( 2 | Id INT64 NOT NULL, 3 | Value STRING(MAX) NOT NULL, 4 | ) PRIMARY KEY(Id); 5 | 6 | CREATE TABLE CompositePrimaryKeys ( 7 | Id INT64 NOT NULL, 8 | PKey1 STRING(32) NOT NULL, 9 | PKey2 INT64 NOT NULL, 10 | Error INT64 NOT NULL, 11 | X STRING(32) NOT NULL, 12 | Y STRING(32) NOT NULL, 13 | Z STRING(32) NOT NULL, 14 | ) PRIMARY KEY(PKey1, PKey2); 15 | 16 | CREATE INDEX CompositePrimaryKeysByXY ON CompositePrimaryKeys(X, Y); 17 | CREATE INDEX CompositePrimaryKeysByError ON CompositePrimaryKeys(Error); 18 | 19 | CREATE TABLE FullTypes ( 20 | PKey STRING(32) NOT NULL, 21 | FTString STRING(32) NOT NULL, 22 | FTStringNull STRING(32), 23 | FTBool BOOL NOT NULL, 24 | FTBoolNull BOOL, 25 | FTBytes BYTES(32) NOT NULL, 26 | FTBytesNull BYTES(32), 27 | FTTimestamp TIMESTAMP NOT NULL, 28 | FTTimestampNull TIMESTAMP, 29 | FTInt INT64 NOT NULL, 30 | FTIntNull INT64, 31 | FTFloat FLOAT64 NOT NULL, 32 | FTFloatNull FLOAT64, 33 | FTDate DATE NOT NULL, 34 | FTDateNull DATE, 35 | ) PRIMARY KEY(PKey); 36 | 37 | CREATE UNIQUE INDEX FullTypesByFTString ON FullTypes(FTString); 38 | CREATE INDEX FullTypesByIntDate ON FullTypes(FTInt, FTDate); 39 | CREATE INDEX FullTypesByIntTimestamp ON FullTypes(FTInt, FTTimestamp); 40 | CREATE INDEX FullTypesByIntTimestampReverse ON FullTypes(FTInt, FTTimestamp DESC); 41 | CREATE INDEX FullTypesByTimestamp ON FullTypes(FTTimestamp); 42 | 43 | CREATE TABLE ArrayTypes ( 44 | Id INT64 NOT NULL, 45 | ArrayString ARRAY, 46 | ArrayBool ARRAY, 47 | ArrayBytes ARRAY, 48 | ArrayTimestamp ARRAY, 49 | ArrayInt ARRAY, 50 | ArrayFloat ARRAY, 51 | ArrayDate ARRAY, 52 | ) PRIMARY KEY(Id); 53 | 54 | CREATE CHANGE STREAM EverythingStream 55 | FOR ALL; 56 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gcpug/handy-spanner 2 | 3 | go 1.22.7 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | cloud.google.com/go v0.118.2 9 | cloud.google.com/go/iam v1.3.1 10 | cloud.google.com/go/longrunning v0.6.4 11 | cloud.google.com/go/spanner v1.75.0 12 | github.com/cloudspannerecosystem/memefish v0.0.0-20231128072053-0a1141e8eb65 13 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 14 | github.com/google/go-cmp v0.6.0 15 | github.com/google/uuid v1.6.0 16 | github.com/mattn/go-sqlite3 v1.14.14 17 | golang.org/x/sync v0.11.0 18 | google.golang.org/api v0.220.0 19 | google.golang.org/genproto v0.0.0-20250207221924-e9438ea467c6 20 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 21 | google.golang.org/grpc v1.70.0 22 | google.golang.org/protobuf v1.36.5 23 | ) 24 | 25 | require ( 26 | cel.dev/expr v0.19.2 // indirect 27 | cloud.google.com/go/auth v0.14.1 // indirect 28 | cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect 29 | cloud.google.com/go/compute/metadata v0.6.0 // indirect 30 | cloud.google.com/go/monitoring v1.24.0 // indirect 31 | github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.2 // indirect 32 | github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.26.0 // indirect 33 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 34 | github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect 35 | github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect 36 | github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect 37 | github.com/felixge/httpsnoop v1.0.4 // indirect 38 | github.com/go-logr/logr v1.4.2 // indirect 39 | github.com/go-logr/stdr v1.2.2 // indirect 40 | github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect 41 | github.com/google/s2a-go v0.1.9 // indirect 42 | github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect 43 | github.com/googleapis/gax-go/v2 v2.14.1 // indirect 44 | github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect 45 | go.opencensus.io v0.24.0 // indirect 46 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 47 | go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect 48 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect 49 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect 50 | go.opentelemetry.io/otel v1.34.0 // indirect 51 | go.opentelemetry.io/otel/metric v1.34.0 // indirect 52 | go.opentelemetry.io/otel/sdk v1.34.0 // indirect 53 | go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect 54 | go.opentelemetry.io/otel/trace v1.34.0 // indirect 55 | golang.org/x/crypto v0.33.0 // indirect 56 | golang.org/x/net v0.35.0 // indirect 57 | golang.org/x/oauth2 v0.26.0 // indirect 58 | golang.org/x/sys v0.30.0 // indirect 59 | golang.org/x/text v0.22.0 // indirect 60 | golang.org/x/time v0.10.0 // indirect 61 | google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /img/abort1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcpug/handy-spanner/41a19dd9c8f918ee556c80ef4a4bd85f55a1503c/img/abort1.png -------------------------------------------------------------------------------- /img/abort2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcpug/handy-spanner/41a19dd9c8f918ee556c80ef4a4bd85f55a1503c/img/abort2.png -------------------------------------------------------------------------------- /img/read_block.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcpug/handy-spanner/41a19dd9c8f918ee556c80ef4a4bd85f55a1503c/img/read_block.png -------------------------------------------------------------------------------- /server/database_function.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "database/sql" 19 | "fmt" 20 | "math" 21 | "strconv" 22 | "strings" 23 | "time" 24 | 25 | "github.com/dgryski/go-farm" 26 | sqlite "github.com/mattn/go-sqlite3" 27 | ) 28 | 29 | func init() { 30 | sql.Register("sqlite3_spanner", &sqlite.SQLiteDriver{ 31 | ConnectHook: func(conn *sqlite.SQLiteConn) error { 32 | for name, fn := range customFunctions { 33 | name = strings.ToLower(name) 34 | if fn.Func == nil { 35 | continue 36 | } 37 | if err := conn.RegisterFunc(name, fn.Func, true); err != nil { 38 | return err 39 | } 40 | } 41 | return nil 42 | }, 43 | }) 44 | } 45 | 46 | type CustomFunction struct { 47 | Name string 48 | Func interface{} 49 | NArgs int 50 | ArgTypes func([]ValueType) bool 51 | ReturnType func([]ValueType) ValueType 52 | } 53 | 54 | func exactMatchArgumentTypes(vts ...ValueType) func([]ValueType) bool { 55 | return func(vts2 []ValueType) bool { 56 | if len(vts) != len(vts2) { 57 | return false 58 | } 59 | for i := range vts { 60 | ok := compareValueType(vts[i], vts2[i]) 61 | if !ok { 62 | return false 63 | } 64 | } 65 | 66 | return true 67 | } 68 | } 69 | 70 | func staticReturnType(vt ValueType) func([]ValueType) ValueType { 71 | return func([]ValueType) ValueType { 72 | return vt 73 | } 74 | } 75 | 76 | const SqliteArgumentRuntimeErrorPrefix = "_sqlite_argument_runtime_error_: " 77 | const SqliteOutOfRangeRuntimeErrorPrefix = "_sqlite_ooo_runtime_error_: " 78 | 79 | // sqliteArgumentRuntimeError is runtime error to run query in sqlite. 80 | // Runtime error only can be returned as string of error in sqlite. 81 | // To distinguish error as InvalidArgument or InternalError, the error message 82 | // by this has a specific prefix. RowsIterator checks the prefix and returns the error 83 | // as InvalidArgument. 84 | type sqliteArgumentRuntimeError struct { 85 | msg string 86 | } 87 | 88 | func (e *sqliteArgumentRuntimeError) Error() string { 89 | return SqliteArgumentRuntimeErrorPrefix + e.msg 90 | } 91 | 92 | type sqliteOutOfRangeRuntimeError struct { 93 | msg string 94 | } 95 | 96 | func (e *sqliteOutOfRangeRuntimeError) Error() string { 97 | return SqliteOutOfRangeRuntimeErrorPrefix + e.msg 98 | } 99 | 100 | // customFunctionNamesMap is naming map from Spanner's function to sqlite's custom function. 101 | // This is used to avoid conflict of reserved name in sqlite. 102 | var customFunctionNamesMap map[string]string = map[string]string{ 103 | "CURRENT_TIMESTAMP": "___CURRENT_TIMESTAMP", 104 | } 105 | 106 | // customFunctions is functions for spanner. 107 | // 108 | // sqlite cannot register function which returns interface{}. 109 | // If spanner function may return different type value, we need to register multiple functions 110 | // for each returned type. 111 | var customFunctions map[string]CustomFunction = map[string]CustomFunction{ 112 | "SIGN": { 113 | Func: sqlite3FnSign, 114 | NArgs: 1, 115 | ArgTypes: exactMatchArgumentTypes( 116 | ValueType{Code: TCInt64}, 117 | ), 118 | ReturnType: staticReturnType(ValueType{ 119 | Code: TCInt64, 120 | }), 121 | }, 122 | "STARTS_WITH": { 123 | Func: sqlite3FnStartsWith, 124 | NArgs: 2, 125 | ArgTypes: exactMatchArgumentTypes( 126 | ValueType{Code: TCString}, 127 | ValueType{Code: TCString}, 128 | ), 129 | ReturnType: staticReturnType(ValueType{ 130 | Code: TCBool, 131 | }), 132 | }, 133 | "ENDS_WITH": { 134 | Func: sqlite3FnEndsWith, 135 | NArgs: 2, 136 | ArgTypes: exactMatchArgumentTypes( 137 | ValueType{Code: TCString}, 138 | ValueType{Code: TCString}, 139 | ), 140 | ReturnType: staticReturnType(ValueType{ 141 | Code: TCBool, 142 | }), 143 | }, 144 | "CONCAT": { 145 | Func: sqlite3FnConcat, 146 | NArgs: -1, 147 | ArgTypes: func(vts []ValueType) bool { 148 | if len(vts) == 0 { 149 | return false 150 | } 151 | vt := vts[0] 152 | for i := range vts { 153 | if !compareValueType(vt, vts[i]) { 154 | return false 155 | } 156 | } 157 | if vt.Code != TCString && vt.Code != TCBytes { 158 | return false 159 | } 160 | return true 161 | }, 162 | ReturnType: func(vts []ValueType) ValueType { 163 | return vts[0] 164 | }, 165 | }, 166 | 167 | "COUNT": { 168 | Func: nil, 169 | NArgs: 1, 170 | ArgTypes: func(vts []ValueType) bool { 171 | return true 172 | }, 173 | ReturnType: staticReturnType(ValueType{ 174 | Code: TCInt64, 175 | }), 176 | }, 177 | "MAX": { 178 | Func: nil, 179 | NArgs: 1, 180 | ArgTypes: func(vts []ValueType) bool { 181 | code := vts[0].Code 182 | if code == TCArray || code == TCStruct { 183 | return false 184 | } 185 | return true 186 | }, 187 | ReturnType: func(vts []ValueType) ValueType { 188 | return vts[0] 189 | }, 190 | }, 191 | "MIN": { 192 | Func: nil, 193 | NArgs: 1, 194 | ArgTypes: func(vts []ValueType) bool { 195 | code := vts[0].Code 196 | if code == TCArray || code == TCStruct { 197 | return false 198 | } 199 | return true 200 | }, 201 | ReturnType: func(vts []ValueType) ValueType { 202 | return vts[0] 203 | }, 204 | }, 205 | "AVG": { 206 | Func: nil, 207 | NArgs: -1, 208 | ArgTypes: func(vts []ValueType) bool { 209 | for _, vt := range vts { 210 | if vt.Code != TCInt64 && vt.Code == TCFloat64 { 211 | return false 212 | } 213 | } 214 | return true 215 | }, 216 | ReturnType: staticReturnType(ValueType{ 217 | Code: TCFloat64, 218 | }), 219 | }, 220 | "SUM": { 221 | Func: nil, 222 | NArgs: -1, 223 | ArgTypes: func(vts []ValueType) bool { 224 | for _, vt := range vts { 225 | if vt.Code != TCInt64 && vt.Code == TCFloat64 { 226 | return false 227 | } 228 | } 229 | return true 230 | }, 231 | ReturnType: func(vts []ValueType) ValueType { 232 | for _, vt := range vts { 233 | if vt.Code == TCFloat64 { 234 | return ValueType{Code: TCFloat64} 235 | } 236 | } 237 | return ValueType{Code: TCInt64} 238 | }, 239 | }, 240 | 241 | "___EXTRACT_FROM_TIMESTAMP": { 242 | Func: sqlite3FnExtractFromTimestamp, 243 | NArgs: 3, 244 | ArgTypes: func(vts []ValueType) bool { 245 | return vts[0].Code == TCString && 246 | vts[1].Code == TCTimestamp && 247 | vts[2].Code == TCString 248 | }, 249 | ReturnType: func(vts []ValueType) ValueType { 250 | return ValueType{Code: TCInt64} 251 | }, 252 | }, 253 | "___EXTRACT_FROM_DATE": { 254 | Func: sqlite3FnExtractFromDate, 255 | NArgs: 2, 256 | ArgTypes: func(vts []ValueType) bool { 257 | return vts[0].Code == TCString && vts[1].Code == TCDate 258 | }, 259 | ReturnType: func(vts []ValueType) ValueType { 260 | return ValueType{Code: TCInt64} 261 | }, 262 | }, 263 | 264 | "___CAST_INT64_TO_BOOL": { 265 | Func: sqlite3FnCastInt64ToBool, 266 | NArgs: 1, 267 | ArgTypes: func(vts []ValueType) bool { 268 | return vts[0].Code == TCInt64 269 | }, 270 | ReturnType: func(vts []ValueType) ValueType { 271 | return ValueType{Code: TCBool} 272 | }, 273 | }, 274 | "___CAST_INT64_TO_STRING": { 275 | Func: sqlite3FnCastInt64ToString, 276 | NArgs: 1, 277 | ArgTypes: func(vts []ValueType) bool { 278 | return vts[0].Code == TCInt64 279 | }, 280 | ReturnType: func(vts []ValueType) ValueType { 281 | return ValueType{Code: TCString} 282 | }, 283 | }, 284 | "___CAST_INT64_TO_FLOAT64": { 285 | Func: sqlite3FnCastInt64ToFloat64, 286 | NArgs: 1, 287 | ArgTypes: func(vts []ValueType) bool { 288 | return vts[0].Code == TCInt64 289 | }, 290 | ReturnType: func(vts []ValueType) ValueType { 291 | return ValueType{Code: TCFloat64} 292 | }, 293 | }, 294 | "___CAST_FLOAT64_TO_STRING": { 295 | Func: sqlite3FnCastFloat64ToString, 296 | NArgs: 1, 297 | ArgTypes: func(vts []ValueType) bool { 298 | return vts[0].Code == TCFloat64 299 | }, 300 | ReturnType: func(vts []ValueType) ValueType { 301 | return ValueType{Code: TCString} 302 | }, 303 | }, 304 | "___CAST_FLOAT64_TO_INT64": { 305 | Func: sqlite3FnCastFloat64ToInt64, 306 | NArgs: 1, 307 | ArgTypes: func(vts []ValueType) bool { 308 | return vts[0].Code == TCFloat64 309 | }, 310 | ReturnType: func(vts []ValueType) ValueType { 311 | return ValueType{Code: TCInt64} 312 | }, 313 | }, 314 | "___CAST_BOOL_TO_STRING": { 315 | Func: sqlite3FnCastBoolToString, 316 | NArgs: 1, 317 | ArgTypes: func(vts []ValueType) bool { 318 | return vts[0].Code == TCBool 319 | }, 320 | ReturnType: func(vts []ValueType) ValueType { 321 | return ValueType{Code: TCString} 322 | }, 323 | }, 324 | "___CAST_BOOL_TO_INT64": { 325 | Func: sqlite3FnCastBoolToInt64, 326 | NArgs: 1, 327 | ArgTypes: func(vts []ValueType) bool { 328 | return vts[0].Code == TCBool 329 | }, 330 | ReturnType: func(vts []ValueType) ValueType { 331 | return ValueType{Code: TCInt64} 332 | }, 333 | }, 334 | "___CAST_STRING_TO_BOOL": { 335 | Func: sqlite3FnCastStringToBool, 336 | NArgs: 1, 337 | ArgTypes: func(vts []ValueType) bool { 338 | return vts[0].Code == TCString 339 | }, 340 | ReturnType: func(vts []ValueType) ValueType { 341 | return ValueType{Code: TCBool} 342 | }, 343 | }, 344 | "___CAST_STRING_TO_INT64": { 345 | Func: sqlite3FnCastStringToInt64, 346 | NArgs: 1, 347 | ArgTypes: func(vts []ValueType) bool { 348 | return vts[0].Code == TCString 349 | }, 350 | ReturnType: func(vts []ValueType) ValueType { 351 | return ValueType{Code: TCInt64} 352 | }, 353 | }, 354 | "___CAST_STRING_TO_FLOAT64": { 355 | Func: sqlite3FnCastStringToFloat64, 356 | NArgs: 1, 357 | ArgTypes: func(vts []ValueType) bool { 358 | return vts[0].Code == TCString 359 | }, 360 | ReturnType: func(vts []ValueType) ValueType { 361 | return ValueType{Code: TCFloat64} 362 | }, 363 | }, 364 | "___CAST_STRING_TO_DATE": { 365 | Func: sqlite3FnCastStringToDate, 366 | NArgs: 1, 367 | ArgTypes: func(vts []ValueType) bool { 368 | return vts[0].Code == TCString 369 | }, 370 | ReturnType: func(vts []ValueType) ValueType { 371 | return ValueType{Code: TCDate} 372 | }, 373 | }, 374 | "___CAST_STRING_TO_TIMESTAMP": { 375 | Func: sqlite3FnCastStringToTimestamp, 376 | NArgs: 1, 377 | ArgTypes: func(vts []ValueType) bool { 378 | return vts[0].Code == TCString 379 | }, 380 | ReturnType: func(vts []ValueType) ValueType { 381 | return ValueType{Code: TCTimestamp} 382 | }, 383 | }, 384 | "___CAST_DATE_TO_STRING": { 385 | Func: sqlite3FnCastDateToString, 386 | NArgs: 1, 387 | ArgTypes: func(vts []ValueType) bool { 388 | return vts[0].Code == TCDate 389 | }, 390 | ReturnType: func(vts []ValueType) ValueType { 391 | return ValueType{Code: TCString} 392 | }, 393 | }, 394 | "___CAST_DATE_TO_TIMESTAMP": { 395 | Func: sqlite3FnCastDateToTimestamp, 396 | NArgs: 1, 397 | ArgTypes: func(vts []ValueType) bool { 398 | return vts[0].Code == TCDate 399 | }, 400 | ReturnType: func(vts []ValueType) ValueType { 401 | return ValueType{Code: TCTimestamp} 402 | }, 403 | }, 404 | "___CAST_TIMESTAMP_TO_STRING": { 405 | Func: sqlite3FnCastTimestampToString, 406 | NArgs: 1, 407 | ArgTypes: func(vts []ValueType) bool { 408 | return vts[0].Code == TCTimestamp 409 | }, 410 | ReturnType: func(vts []ValueType) ValueType { 411 | return ValueType{Code: TCString} 412 | }, 413 | }, 414 | "___CAST_TIMESTAMP_TO_DATE": { 415 | Func: sqlite3FnCastTimestampToDate, 416 | NArgs: 1, 417 | ArgTypes: func(vts []ValueType) bool { 418 | return vts[0].Code == TCTimestamp 419 | }, 420 | ReturnType: func(vts []ValueType) ValueType { 421 | return ValueType{Code: TCDate} 422 | }, 423 | }, 424 | "PENDING_COMMIT_TIMESTAMP": getCustomFunctionForCurrentTime(), 425 | "___CURRENT_TIMESTAMP": getCustomFunctionForCurrentTime(), 426 | "IFNULL": { 427 | Func: nil, 428 | NArgs: 2, 429 | ArgTypes: func(vts []ValueType) bool { 430 | if len(vts) == 0 { 431 | return false 432 | } 433 | vt := vts[0] 434 | for i := range vts { 435 | if !compareValueType(vt, vts[i]) { 436 | return false 437 | } 438 | } 439 | return true 440 | }, 441 | ReturnType: func(vts []ValueType) ValueType { 442 | return vts[0] 443 | }, 444 | }, 445 | "NULLIF": { 446 | Func: nil, 447 | NArgs: 2, 448 | ArgTypes: func(vts []ValueType) bool { 449 | if len(vts) == 0 { 450 | return false 451 | } 452 | vt := vts[0] 453 | for i := range vts { 454 | if !compareValueType(vt, vts[i]) { 455 | return false 456 | } 457 | } 458 | return true 459 | }, 460 | ReturnType: func(vts []ValueType) ValueType { 461 | return vts[0] 462 | }, 463 | }, 464 | "FARM_FINGERPRINT": { 465 | Func: sqlite3FnFarmFingerprint, 466 | NArgs: 1, 467 | ArgTypes: func(vts []ValueType) bool { 468 | return vts[0].Code == TCString 469 | }, 470 | ReturnType: func(vts []ValueType) ValueType { 471 | return ValueType{Code: TCInt64} 472 | }, 473 | }, 474 | "MOD": { 475 | Func: sqlite3FnMod, 476 | NArgs: 2, 477 | ArgTypes: exactMatchArgumentTypes( 478 | ValueType{Code: TCInt64}, 479 | ValueType{Code: TCInt64}, 480 | ), 481 | ReturnType: func(vts []ValueType) ValueType { 482 | return ValueType{Code: TCInt64} 483 | }, 484 | }, 485 | "GENERATE_ARRAY": { 486 | Func: sqlite3FnGenerateArray, 487 | NArgs: -2, 488 | ArgTypes: func(vts []ValueType) bool { 489 | if len(vts) < 2 || len(vts) > 3 { 490 | return false 491 | } 492 | if vts[0].Code == TCInt64 && vts[1].Code == TCInt64 { 493 | if len(vts) == 3 { 494 | return vts[2].Code == TCInt64 495 | } 496 | return true 497 | } 498 | return false 499 | }, 500 | ReturnType: func(vts []ValueType) ValueType { 501 | return ValueType{ 502 | Code: TCArray, 503 | ArrayType: &ValueType{Code: TCInt64}, 504 | } 505 | }, 506 | }, 507 | } 508 | 509 | func sqlite3FnGenerateArray(xs ...int64) []byte { 510 | step := int64(1) 511 | if len(xs) == 3 { 512 | step = xs[2] 513 | } 514 | a, b := xs[0], xs[1] 515 | res := make([]byte, 0, b-a) 516 | for i := a; i <= b; i += step { 517 | res = append(res, byte(i)) 518 | } 519 | return res 520 | } 521 | 522 | func sqlite3FnMod(a, b int64) int64 { 523 | return a % b 524 | } 525 | 526 | func sqlite3FnFarmFingerprint(s string) int64 { 527 | return int64(farm.Fingerprint64([]byte(s))) 528 | } 529 | 530 | func sqlite3FnSign(x int64) int64 { 531 | if x > 0 { 532 | return 1 533 | } 534 | if x < 0 { 535 | return -1 536 | } 537 | return 0 538 | } 539 | 540 | func sqlite3FnStartsWith(a, b string) bool { 541 | return strings.HasPrefix(a, b) 542 | } 543 | 544 | func sqlite3FnEndsWith(a, b string) bool { 545 | return strings.HasSuffix(a, b) 546 | } 547 | 548 | func sqlite3FnConcat(xs ...string) string { 549 | return strings.Join(xs, "") 550 | } 551 | 552 | // sqlite3FnExtractFromTimestamp is simulation function of EXTRACT. 553 | // It supports except DATE part. 554 | func sqlite3FnExtractFromTimestamp(part string, timestamp string, tz string) (int64, error) { 555 | tzErr := &sqliteOutOfRangeRuntimeError{msg: fmt.Sprintf("Invalid time zone: %s", tz)} 556 | if len(tz) == 0 { 557 | return 0, tzErr 558 | } 559 | 560 | var location *time.Location 561 | 562 | // (+|-)H[H][:M[M]] 563 | if tz[0] == '-' || tz[0] == '+' { 564 | s := tz 565 | var neg bool 566 | if s[0] == '-' { 567 | neg = true 568 | } 569 | 570 | s = s[1:] 571 | 572 | var colonFound bool 573 | hour := -1 574 | min := 0 575 | for _, c := range s { 576 | if c == ':' { 577 | min = -1 578 | colonFound = true 579 | continue 580 | } 581 | if c < '0' && '9' < c { 582 | return 0, tzErr 583 | } 584 | 585 | n := int(c - '0') 586 | 587 | if colonFound { 588 | if min == -1 { 589 | min = n 590 | } else { 591 | min = 10*min + n 592 | } 593 | } else { 594 | if hour == -1 { 595 | hour = n 596 | } else { 597 | hour = 10*hour + n 598 | } 599 | } 600 | } 601 | 602 | if hour == -1 || min == -1 { 603 | return 0, tzErr 604 | } 605 | if hour > 24 || min > 60 { 606 | return 0, tzErr 607 | } 608 | 609 | offset := hour*3600 + min*60 610 | if neg { 611 | offset = -offset 612 | } 613 | location = time.FixedZone(tz, offset) 614 | } else { 615 | loc, err := time.LoadLocation(tz) 616 | if err != nil { 617 | return 0, tzErr 618 | } 619 | location = loc 620 | } 621 | 622 | t, err := time.Parse(time.RFC3339Nano, timestamp) 623 | if err != nil { 624 | // Must not happen 625 | return 0, fmt.Errorf("___EXTRACT_FROM_TIMESTAMP: unexpected format %q as timestamp: %v", timestamp, err) 626 | } 627 | t = t.In(location) 628 | 629 | switch strings.ToUpper(part) { 630 | case "NANOSECOND": 631 | return int64(t.Nanosecond()), nil 632 | case "MICROSECOND": 633 | return int64(t.Nanosecond() / 1000), nil 634 | case "MILLISECOND": 635 | return int64(t.Nanosecond() / 1000000), nil 636 | case "SECOND": 637 | return int64(t.Second()), nil 638 | case "MINUTE": 639 | return int64(t.Minute()), nil 640 | case "HOUR": 641 | return int64(t.Hour()), nil 642 | case "DAYOFWEEK": 643 | return int64(t.Weekday()), nil 644 | case "DAY": 645 | return int64(t.Day()), nil 646 | case "DAYOFYEAR": 647 | return int64(t.YearDay()), nil 648 | case "WEEK": 649 | // TODO: calculate week from timestamp 650 | return 0, fmt.Errorf("___EXTRACT_FROM_TIMESTAMP: WEEK not supported for now") 651 | case "ISOWEEK": 652 | _, w := t.ISOWeek() 653 | return int64(w), nil 654 | case "MONTH": 655 | return int64(t.Month()), nil 656 | case "QUARTER": 657 | // 1 for Jan-Mar, 2 for Apr-Jun, 3 for Jul-Sep, 4 for Oct-Dec 658 | m := t.Month() 659 | return int64(m/4 + 1), nil 660 | case "YEAR": 661 | return int64(t.Year()), nil 662 | case "ISOYEAR": 663 | y, _ := t.ISOWeek() 664 | return int64(y), nil 665 | default: 666 | // Must not happen 667 | return 0, fmt.Errorf("___EXTRACT_FROM_TIMESTAMP: unexpected part: %s", part) 668 | } 669 | } 670 | 671 | // sqlite3FnExtractFromDate is simulation function of EXTRACT. 672 | func sqlite3FnExtractFromDate(part string, date string) (int64, error) { 673 | t, err := time.Parse("2006-01-02", date) 674 | if err != nil { 675 | // Must not happen 676 | return 0, fmt.Errorf("___EXTRACT_FROM_DATE: unexpected format %q as date: %v", date, err) 677 | } 678 | 679 | switch strings.ToUpper(part) { 680 | case "DAYOFWEEK": 681 | return int64(t.Weekday()), nil 682 | case "DAY": 683 | return int64(t.Day()), nil 684 | case "DAYOFYEAR": 685 | return int64(t.YearDay()), nil 686 | case "WEEK": 687 | // TODO: calculate week from timestamp 688 | return 0, fmt.Errorf("___EXTRACT_FROM_DATE: WEEK not supported for now") 689 | case "ISOWEEK": 690 | _, w := t.ISOWeek() 691 | return int64(w), nil 692 | case "MONTH": 693 | return int64(t.Month()), nil 694 | case "QUARTER": 695 | // 1 for Jan-Mar, 2 for Apr-Jun, 3 for Jul-Sep, 4 for Oct-Dec 696 | m := t.Month() 697 | return int64(m/4 + 1), nil 698 | case "YEAR": 699 | return int64(t.Year()), nil 700 | case "ISOYEAR": 701 | y, _ := t.ISOWeek() 702 | return int64(y), nil 703 | default: 704 | // Must not happen 705 | return 0, fmt.Errorf("___EXTRACT_FROM_DATE: unexpected part: %s", part) 706 | } 707 | } 708 | 709 | func sqlite3FnCastInt64ToBool(i int64) bool { 710 | return i != 0 711 | } 712 | 713 | func sqlite3FnCastInt64ToString(i int64) string { 714 | return strconv.FormatInt(i, 10) 715 | } 716 | 717 | func sqlite3FnCastInt64ToFloat64(i int64) float64 { 718 | return float64(i) 719 | } 720 | 721 | func sqlite3FnCastFloat64ToString(f float64) string { 722 | if math.IsInf(f, 0) { 723 | if f < 0 { 724 | return "-inf" 725 | } 726 | return "inf" 727 | } 728 | 729 | return strconv.FormatFloat(f, 'g', -1, 64) 730 | } 731 | 732 | func sqlite3FnCastFloat64ToInt64(f float64) (int64, error) { 733 | if math.IsNaN(f) { 734 | // OutOfRange error 735 | msg := "Illegal conversion of non-finite floating point number to an integer: nan" 736 | return 0, &sqliteOutOfRangeRuntimeError{msg: msg} 737 | } 738 | if math.IsInf(f, 0) { 739 | // OutOfRange error 740 | inf := "inf" 741 | if f < 0 { 742 | inf = "-inf" 743 | } 744 | msg := "Illegal conversion of non-finite floating point number to an integer: " + inf 745 | return 0, &sqliteOutOfRangeRuntimeError{msg: msg} 746 | } 747 | 748 | if f < 0 { 749 | return int64(f - 0.5), nil 750 | } 751 | return int64(f + 0.5), nil 752 | } 753 | 754 | func sqlite3FnCastBoolToString(b bool) string { 755 | if b { 756 | return "TRUE" 757 | } 758 | return "FALSE" 759 | } 760 | 761 | func sqlite3FnCastBoolToInt64(b bool) int64 { 762 | if b { 763 | return 1 764 | } 765 | return 0 766 | } 767 | 768 | func sqlite3FnCastStringToBool(s string) (bool, error) { 769 | ss := strings.ToUpper(s) 770 | if ss == "TRUE" { 771 | return true, nil 772 | } 773 | if ss == "FALSE" { 774 | return false, nil 775 | } 776 | 777 | // OutOfRange error 778 | return false, &sqliteOutOfRangeRuntimeError{msg: fmt.Sprintf("Bad bool value: %s", s)} 779 | } 780 | 781 | func sqlite3FnCastStringToInt64(s string) (int64, error) { 782 | if s == "" { 783 | // OutOfRange error 784 | return 0, &sqliteOutOfRangeRuntimeError{msg: fmt.Sprintf("Bad int64 value: %s", s)} 785 | } 786 | 787 | orig := s 788 | var neg bool 789 | if s[0] == '+' { 790 | s = s[1:] 791 | } else if s[0] == '-' { 792 | neg = true 793 | s = s[1:] 794 | } 795 | 796 | // Base is available only for 10 or 16 in spanner. 797 | base := 10 798 | if len(s) > 2 && s[0] == '0' && s[1] == 'x' { 799 | base = 16 800 | s = s[2:] 801 | } 802 | 803 | n, err := strconv.ParseInt(s, base, 64) 804 | if err != nil { 805 | // OutOfRange error 806 | return 0, &sqliteOutOfRangeRuntimeError{msg: fmt.Sprintf("Bad int64 value: %s", orig)} 807 | } 808 | 809 | if n < 0 { 810 | // OutOfRange error 811 | return 0, &sqliteOutOfRangeRuntimeError{msg: fmt.Sprintf("Bad int64 value: %s", orig)} 812 | } 813 | 814 | if neg { 815 | n = -n 816 | } 817 | 818 | return n, nil 819 | } 820 | 821 | func sqlite3FnCastStringToFloat64(s string) (float64, error) { 822 | n, err := strconv.ParseFloat(s, 64) 823 | if err != nil { 824 | // OutOfRange error 825 | return 0, &sqliteOutOfRangeRuntimeError{msg: fmt.Sprintf("Bad double value: %s", s)} 826 | } 827 | 828 | return n, nil 829 | } 830 | 831 | func sqlite3FnCastStringToDate(s string) (string, error) { 832 | t, ok := parseDateLiteral(s) 833 | if !ok { 834 | // InvalidArgument error 835 | return "", &sqliteArgumentRuntimeError{msg: fmt.Sprintf("Could not cast literal %q to type DATE", s)} 836 | } 837 | 838 | return t.Format("2006-01-02"), nil 839 | } 840 | 841 | func sqlite3FnCastStringToTimestamp(s string) (string, error) { 842 | t, ok := parseTimestampLiteral(s) 843 | if !ok { 844 | // InvalidArgument error 845 | return "", &sqliteArgumentRuntimeError{msg: fmt.Sprintf("Could not cast literal %q to type TIMESTAMP", s)} 846 | } 847 | 848 | return t.Format(time.RFC3339Nano), nil 849 | } 850 | 851 | func sqlite3FnCastDateToString(s string) (string, error) { 852 | t, ok := parseDateLiteral(s) 853 | if !ok { 854 | // InvalidArgument error 855 | return "", &sqliteArgumentRuntimeError{msg: fmt.Sprintf("Could not cast literal %q to type STRING", s)} 856 | } 857 | 858 | return t.Format("2006-01-02"), nil 859 | } 860 | 861 | func sqlite3FnCastDateToTimestamp(s string) (string, error) { 862 | t, ok := parseDateLiteral(s) 863 | if !ok { 864 | // InvalidArgument error 865 | return "", &sqliteArgumentRuntimeError{msg: fmt.Sprintf("Could not cast literal %q to type TIMESTAMP", s)} 866 | } 867 | 868 | return t.UTC().Format(time.RFC3339Nano), nil 869 | } 870 | 871 | func sqlite3FnCastTimestampToString(s string) (string, error) { 872 | t, ok := parseTimestampLiteral(s) 873 | if !ok { 874 | // InvalidArgument error 875 | return "", &sqliteArgumentRuntimeError{msg: fmt.Sprintf("Could not cast literal %q to type STRING", s)} 876 | } 877 | 878 | return t.In(parseLocation).Format("2006-01-02 15:04:05.999999999-07"), nil 879 | } 880 | 881 | func sqlite3FnCastTimestampToDate(s string) (string, error) { 882 | t, ok := parseTimestampLiteral(s) 883 | if !ok { 884 | // InvalidArgument error 885 | return "", &sqliteArgumentRuntimeError{msg: fmt.Sprintf("Could not cast literal %q to type DATE", s)} 886 | } 887 | 888 | return t.In(parseLocation).Format("2006-01-02"), nil 889 | } 890 | 891 | func sqlite3FnCurrentTimestamp() string { 892 | return time.Now().UTC().Format(time.RFC3339Nano) 893 | } 894 | 895 | func getCustomFunctionForCurrentTime() CustomFunction { 896 | return CustomFunction{ 897 | Func: sqlite3FnCurrentTimestamp, 898 | NArgs: 0, 899 | ArgTypes: func(vts []ValueType) bool { 900 | return true 901 | }, 902 | ReturnType: func(vts []ValueType) ValueType { 903 | return ValueType{Code: TCTimestamp} 904 | }, 905 | } 906 | } 907 | -------------------------------------------------------------------------------- /server/database_json.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build json1 16 | 17 | package server 18 | 19 | func useSqliteJSON() { return } 20 | -------------------------------------------------------------------------------- /server/database_nonjson.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build !json1 16 | 17 | package server 18 | 19 | func useSqliteJSON() { panic(`Require "json1" build tag to use some features`) } 20 | -------------------------------------------------------------------------------- /server/debug.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | func Debugf(format string, args ...interface{}) { 22 | if IsDebug() { 23 | fmt.Printf(format, args...) 24 | } 25 | } 26 | 27 | func IsDebug() bool { 28 | return false 29 | } 30 | 31 | func DebugStartEnd(format string, args ...interface{}) func() { 32 | fmt.Printf(format+" start\n", args...) 33 | return func() { 34 | fmt.Printf(format+" end\n", args...) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /server/keyset.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | spannerpb "google.golang.org/genproto/googleapis/spanner/v1" 22 | "google.golang.org/grpc/codes" 23 | "google.golang.org/grpc/status" 24 | structpb "google.golang.org/protobuf/types/known/structpb" 25 | ) 26 | 27 | type KeySet struct { 28 | All bool 29 | Keys []*structpb.ListValue 30 | Ranges []*KeyRange 31 | } 32 | 33 | type KeyRange struct { 34 | start *structpb.ListValue 35 | end *structpb.ListValue 36 | startClosed bool 37 | endClosed bool 38 | } 39 | 40 | func makeKeySet(set *spannerpb.KeySet) *KeySet { 41 | ranges := make([]*KeyRange, 0, len(set.Ranges)) 42 | for _, r := range set.Ranges { 43 | ranges = append(ranges, makeKeyRange(r)) 44 | } 45 | return &KeySet{ 46 | All: set.All, 47 | Keys: set.Keys, 48 | Ranges: ranges, 49 | } 50 | } 51 | 52 | func makeKeyRange(r *spannerpb.KeyRange) *KeyRange { 53 | var kr KeyRange 54 | switch s := r.StartKeyType.(type) { 55 | case *spannerpb.KeyRange_StartClosed: 56 | kr.start = s.StartClosed 57 | kr.startClosed = true 58 | case *spannerpb.KeyRange_StartOpen: 59 | kr.start = s.StartOpen 60 | } 61 | switch e := r.EndKeyType.(type) { 62 | case *spannerpb.KeyRange_EndClosed: 63 | kr.end = e.EndClosed 64 | kr.endClosed = true 65 | case *spannerpb.KeyRange_EndOpen: 66 | kr.end = e.EndOpen 67 | } 68 | return &kr 69 | } 70 | 71 | func buildWhereClauseFromKeySet(keyset *KeySet, indexColumnsName string, indexColumns []*Column, indexColumnDirs []string) (string, []interface{}, error) { 72 | if keyset.All { 73 | return "", nil, nil 74 | } 75 | 76 | var conditions []string 77 | var subargs [][]interface{} 78 | 79 | if len(keyset.Keys) != 0 { 80 | q, a, err := buildKeySetQuery(indexColumnsName, indexColumns, keyset.Keys) 81 | if err != nil { 82 | return "", nil, err 83 | } 84 | conditions = append(conditions, q) 85 | subargs = append(subargs, a) 86 | } 87 | 88 | if len(keyset.Ranges) != 0 { 89 | for _, keyrange := range keyset.Ranges { 90 | q, a, err := buildKeyRangeQuery(indexColumnsName, indexColumns, indexColumnDirs, keyrange) 91 | if err != nil { 92 | return "", nil, err 93 | } 94 | 95 | conditions = append(conditions, q) 96 | subargs = append(subargs, a) 97 | } 98 | } 99 | 100 | cond := strings.Join(conditions, " OR ") 101 | var args []interface{} 102 | for i := range subargs { 103 | args = append(args, subargs[i]...) 104 | } 105 | 106 | return fmt.Sprintf("WHERE %s", cond), args, nil 107 | } 108 | 109 | func buildKeySetQuery(pkeysName string, pkeyColumns []*Column, keys []*structpb.ListValue) (string, []interface{}, error) { 110 | numPKeys := len(pkeyColumns) 111 | args := make([]interface{}, 0, len(keys)*numPKeys) 112 | 113 | for _, key := range keys { 114 | if len(key.Values) != numPKeys { 115 | return "", nil, status.Errorf(codes.InvalidArgument, "TODO: invalid keys") 116 | } 117 | 118 | values, err := convertToDatabaseValues(key, pkeyColumns) 119 | if err != nil { 120 | return "", nil, err 121 | } 122 | 123 | args = append(args, values...) 124 | } 125 | 126 | // build placeholders for values e.g. (?, ?, ?) 127 | valuesPlaceholder := "(?" + strings.Repeat(", ?", numPKeys-1) + ")" 128 | 129 | // repeat placeholders for the number of keys e.g. (?, ?, ?), (?, ?, ?), (?, ?, ?) 130 | valuesExpr := valuesPlaceholder + strings.Repeat(", "+valuesPlaceholder, len(keys)-1) 131 | 132 | // e.g. WHERE (key1, key2, key3) IN ( VALUES (?, ?, ?), (?, ?, ?) ) 133 | whereCondition := fmt.Sprintf("(%s) IN ( VALUES %s )", pkeysName, valuesExpr) 134 | 135 | return whereCondition, args, nil 136 | } 137 | 138 | func buildKeyRangeQuery(pkeysName string, pkeyColumns []*Column, indexColumnDirs []string, keyrange *KeyRange) (string, []interface{}, error) { 139 | numPKeys := len(pkeyColumns) 140 | if numPKeys < len(keyrange.start.Values) { 141 | return "", nil, status.Errorf(codes.InvalidArgument, "TODO: invalid start range key") 142 | } 143 | if numPKeys < len(keyrange.end.Values) { 144 | return "", nil, status.Errorf(codes.InvalidArgument, "TODO: invalid end range key") 145 | } 146 | 147 | startKeyValues, err := convertToDatabaseValues(keyrange.start, pkeyColumns) 148 | if err != nil { 149 | return "", nil, err 150 | } 151 | endKeyValues, err := convertToDatabaseValues(keyrange.end, pkeyColumns) 152 | if err != nil { 153 | return "", nil, err 154 | } 155 | 156 | whereClause := make([]string, 0, len(startKeyValues)+len(endKeyValues)) 157 | args := make([]interface{}, 0, len(whereClause)) 158 | 159 | var maxLen int 160 | if len(startKeyValues) > len(endKeyValues) { 161 | maxLen = len(startKeyValues) 162 | } else { 163 | maxLen = len(endKeyValues) 164 | } 165 | for i := 0; i < maxLen; i++ { 166 | pk := QuoteString(pkeyColumns[i].Name()) 167 | if len(startKeyValues) > i && len(endKeyValues) > i && startKeyValues[i] == endKeyValues[i] { 168 | whereClause = append(whereClause, fmt.Sprintf("%s = ?", pk)) 169 | args = append(args, startKeyValues[i]) 170 | continue 171 | } 172 | if len(startKeyValues) > i { 173 | if indexColumnDirs[i] == "DESC" { 174 | if keyrange.startClosed { 175 | whereClause = append(whereClause, fmt.Sprintf("%s <= ?", pk)) 176 | } else { 177 | whereClause = append(whereClause, fmt.Sprintf("%s < ?", pk)) 178 | } 179 | } else { 180 | if keyrange.startClosed { 181 | whereClause = append(whereClause, fmt.Sprintf("%s >= ?", pk)) 182 | } else { 183 | whereClause = append(whereClause, fmt.Sprintf("%s > ?", pk)) 184 | } 185 | } 186 | args = append(args, startKeyValues[i]) 187 | } 188 | if len(endKeyValues) > i { 189 | if indexColumnDirs[i] == "DESC" { 190 | if keyrange.endClosed { 191 | whereClause = append(whereClause, fmt.Sprintf("%s >= ?", pk)) 192 | } else { 193 | whereClause = append(whereClause, fmt.Sprintf("%s > ?", pk)) 194 | } 195 | } else { 196 | if keyrange.endClosed { 197 | whereClause = append(whereClause, fmt.Sprintf("%s <= ?", pk)) 198 | } else { 199 | whereClause = append(whereClause, fmt.Sprintf("%s < ?", pk)) 200 | } 201 | } 202 | args = append(args, endKeyValues[i]) 203 | } 204 | } 205 | 206 | return fmt.Sprintf("(%s)", strings.Join(whereClause, " AND ")), args, nil 207 | } 208 | -------------------------------------------------------------------------------- /server/meta_schema.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "github.com/cloudspannerecosystem/memefish/ast" 19 | ) 20 | 21 | var metaTablesMap = map[string]string{ 22 | "INFORMATION_SCHEMA.SCHEMATA": "__INFORMATION_SCHEMA__SCHEMATA", 23 | "INFORMATION_SCHEMA.TABLES": "__INFORMATION_SCHEMA__TABLES", 24 | "INFORMATION_SCHEMA.COLUMNS": "__INFORMATION_SCHEMA__COLUMNS", 25 | "INFORMATION_SCHEMA.INDEXES": "__INFORMATION_SCHEMA__INDEXES", 26 | "INFORMATION_SCHEMA.INDEX_COLUMNS": "__INFORMATION_SCHEMA__INDEX_COLUMNS", 27 | "INFORMATION_SCHEMA.COLUMN_OPTIONS": "__INFORMATION_SCHEMA__COLUMN_OPTIONS", 28 | "INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS": "__INFORMATION_SCHEMA__REFERENTIAL_CONSTRAINTS", 29 | } 30 | 31 | var metaTablesReverseMap = map[string][]string{ 32 | "__INFORMATION_SCHEMA__SCHEMATA": {"INFORMATION_SCHEMA", "SCHEMATA"}, 33 | "__INFORMATION_SCHEMA__TABLES": {"INFORMATION_SCHEMA", "TABLES"}, 34 | "__INFORMATION_SCHEMA__COLUMNS": {"INFORMATION_SCHEMA", "COLUMNS"}, 35 | "__INFORMATION_SCHEMA__INDEXES": {"INFORMATION_SCHEMA", "INDEXES"}, 36 | "__INFORMATION_SCHEMA__INDEX_COLUMNS": {"INFORMATION_SCHEMA", "INDEX_COLUMNS"}, 37 | "__INFORMATION_SCHEMA__COLUMN_OPTIONS": {"INFORMATION_SCHEMA", "COLUMN_OPTIONS"}, 38 | "__INFORMATION_SCHEMA__REFERENTIAL_CONSTRAINTS": {"INFORMATION_SCHEMA", "REFERENTIAL_CONSTRAINTS"}, 39 | } 40 | 41 | var metaTables = []*ast.CreateTable{ 42 | { 43 | Name: &ast.Ident{Name: "__INFORMATION_SCHEMA__SCHEMATA"}, 44 | Columns: []*ast.ColumnDef{ 45 | { 46 | Name: &ast.Ident{Name: "CATALOG_NAME"}, 47 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 48 | NotNull: true, 49 | }, 50 | { 51 | Name: &ast.Ident{Name: "SCHEMA_NAME"}, 52 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 53 | NotNull: true, 54 | }, 55 | { 56 | Name: &ast.Ident{Name: "EFFECTIVE_TIMESTAMP"}, 57 | Type: &ast.ScalarSchemaType{Name: ast.Int64TypeName}, 58 | NotNull: false, 59 | }, 60 | }, 61 | PrimaryKeys: []*ast.IndexKey{ 62 | { 63 | Name: &ast.Ident{Name: "CATALOG_NAME"}, 64 | Dir: ast.DirectionAsc, 65 | }, 66 | { 67 | Name: &ast.Ident{Name: "SCHEMA_NAME"}, 68 | Dir: ast.DirectionAsc, 69 | }, 70 | }, 71 | }, 72 | { 73 | Name: &ast.Ident{Name: "__INFORMATION_SCHEMA__TABLES"}, 74 | Columns: []*ast.ColumnDef{ 75 | { 76 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 77 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 78 | NotNull: true, 79 | }, 80 | { 81 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 82 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 83 | NotNull: true, 84 | }, 85 | { 86 | Name: &ast.Ident{Name: "TABLE_NAME"}, 87 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 88 | NotNull: true, 89 | }, 90 | { 91 | Name: &ast.Ident{Name: "PARENT_TABLE_NAME"}, 92 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 93 | NotNull: false, 94 | }, 95 | { 96 | Name: &ast.Ident{Name: "ON_DELETE_ACTION"}, 97 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 98 | NotNull: false, 99 | }, 100 | { 101 | Name: &ast.Ident{Name: "SPANNER_STATE"}, 102 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 103 | NotNull: false, 104 | }, 105 | }, 106 | PrimaryKeys: []*ast.IndexKey{ 107 | { 108 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 109 | Dir: ast.DirectionAsc, 110 | }, 111 | { 112 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 113 | Dir: ast.DirectionAsc, 114 | }, 115 | { 116 | Name: &ast.Ident{Name: "TABLE_NAME"}, 117 | Dir: ast.DirectionAsc, 118 | }, 119 | }, 120 | }, 121 | { 122 | Name: &ast.Ident{Name: "__INFORMATION_SCHEMA__COLUMNS"}, 123 | Columns: []*ast.ColumnDef{ 124 | { 125 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 126 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 127 | NotNull: true, 128 | }, 129 | { 130 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 131 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 132 | NotNull: true, 133 | }, 134 | { 135 | Name: &ast.Ident{Name: "TABLE_NAME"}, 136 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 137 | NotNull: true, 138 | }, 139 | { 140 | Name: &ast.Ident{Name: "COLUMN_NAME"}, 141 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 142 | NotNull: true, 143 | }, 144 | { 145 | Name: &ast.Ident{Name: "ORDINAL_POSITION"}, 146 | Type: &ast.ScalarSchemaType{Name: ast.Int64TypeName}, 147 | NotNull: true, 148 | }, 149 | { 150 | Name: &ast.Ident{Name: "COLUMN_DEFAULT"}, 151 | Type: &ast.SizedSchemaType{Name: ast.BytesTypeName, Max: true}, 152 | NotNull: false, 153 | }, 154 | { 155 | Name: &ast.Ident{Name: "DATA_TYPE"}, 156 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 157 | NotNull: false, 158 | }, 159 | { 160 | Name: &ast.Ident{Name: "IS_NULLABLE"}, 161 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 162 | NotNull: false, 163 | }, 164 | { 165 | Name: &ast.Ident{Name: "SPANNER_TYPE"}, 166 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 167 | NotNull: false, 168 | }, 169 | }, 170 | PrimaryKeys: []*ast.IndexKey{ 171 | { 172 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 173 | Dir: ast.DirectionAsc, 174 | }, 175 | { 176 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 177 | Dir: ast.DirectionAsc, 178 | }, 179 | { 180 | Name: &ast.Ident{Name: "TABLE_NAME"}, 181 | Dir: ast.DirectionAsc, 182 | }, 183 | { 184 | Name: &ast.Ident{Name: "COLUMN_NAME"}, 185 | Dir: ast.DirectionAsc, 186 | }, 187 | }, 188 | }, 189 | { 190 | Name: &ast.Ident{Name: "__INFORMATION_SCHEMA__INDEXES"}, 191 | Columns: []*ast.ColumnDef{ 192 | { 193 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 194 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 195 | NotNull: true, 196 | }, 197 | { 198 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 199 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 200 | NotNull: true, 201 | }, 202 | { 203 | Name: &ast.Ident{Name: "TABLE_NAME"}, 204 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 205 | NotNull: true, 206 | }, 207 | { 208 | Name: &ast.Ident{Name: "INDEX_NAME"}, 209 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 210 | NotNull: true, 211 | }, 212 | { 213 | Name: &ast.Ident{Name: "INDEX_TYPE"}, 214 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 215 | NotNull: true, 216 | }, 217 | { 218 | Name: &ast.Ident{Name: "PARENT_TABLE_NAME"}, 219 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 220 | NotNull: true, 221 | }, 222 | { 223 | Name: &ast.Ident{Name: "IS_UNIQUE"}, 224 | Type: &ast.ScalarSchemaType{Name: ast.BoolTypeName}, 225 | NotNull: true, 226 | }, 227 | { 228 | Name: &ast.Ident{Name: "IS_NULL_FILTERED"}, 229 | Type: &ast.ScalarSchemaType{Name: ast.BoolTypeName}, 230 | NotNull: true, 231 | }, 232 | { 233 | Name: &ast.Ident{Name: "INDEX_STATE"}, 234 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Size: &ast.IntLiteral{Value: "100", Base: 10}}, 235 | // NotNull: true, 236 | NotNull: false, // INFORMATION_SCHEMA.COLUMNS reports this column is non-nullable, but it returns NULL... 237 | }, 238 | { 239 | Name: &ast.Ident{Name: "SPANNER_IS_MANAGED"}, 240 | Type: &ast.ScalarSchemaType{Name: ast.BoolTypeName}, 241 | NotNull: true, 242 | }, 243 | }, 244 | PrimaryKeys: []*ast.IndexKey{ 245 | { 246 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 247 | Dir: ast.DirectionAsc, 248 | }, 249 | { 250 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 251 | Dir: ast.DirectionAsc, 252 | }, 253 | { 254 | Name: &ast.Ident{Name: "TABLE_NAME"}, 255 | Dir: ast.DirectionAsc, 256 | }, 257 | { 258 | Name: &ast.Ident{Name: "INDEX_NAME"}, 259 | Dir: ast.DirectionAsc, 260 | }, 261 | { 262 | Name: &ast.Ident{Name: "INDEX_TYPE"}, 263 | Dir: ast.DirectionAsc, 264 | }, 265 | }, 266 | }, 267 | { 268 | Name: &ast.Ident{Name: "__INFORMATION_SCHEMA__INDEX_COLUMNS"}, 269 | Columns: []*ast.ColumnDef{ 270 | { 271 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 272 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 273 | NotNull: true, 274 | }, 275 | { 276 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 277 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 278 | NotNull: true, 279 | }, 280 | { 281 | Name: &ast.Ident{Name: "TABLE_NAME"}, 282 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 283 | NotNull: true, 284 | }, 285 | { 286 | Name: &ast.Ident{Name: "INDEX_NAME"}, 287 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 288 | NotNull: true, 289 | }, 290 | { 291 | Name: &ast.Ident{Name: "INDEX_TYPE"}, 292 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 293 | NotNull: true, 294 | }, 295 | { 296 | Name: &ast.Ident{Name: "COLUMN_NAME"}, 297 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 298 | NotNull: true, 299 | }, 300 | { 301 | Name: &ast.Ident{Name: "ORDINAL_POSITION"}, 302 | Type: &ast.ScalarSchemaType{Name: ast.Int64TypeName}, 303 | NotNull: false, 304 | }, 305 | { 306 | Name: &ast.Ident{Name: "COLUMN_ORDERING"}, 307 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 308 | NotNull: false, 309 | }, 310 | { 311 | Name: &ast.Ident{Name: "IS_NULLABLE"}, 312 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 313 | NotNull: false, 314 | }, 315 | { 316 | Name: &ast.Ident{Name: "SPANNER_TYPE"}, 317 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 318 | NotNull: false, 319 | }, 320 | }, 321 | PrimaryKeys: []*ast.IndexKey{ 322 | { 323 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 324 | Dir: ast.DirectionAsc, 325 | }, 326 | { 327 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 328 | Dir: ast.DirectionAsc, 329 | }, 330 | { 331 | Name: &ast.Ident{Name: "TABLE_NAME"}, 332 | Dir: ast.DirectionAsc, 333 | }, 334 | { 335 | Name: &ast.Ident{Name: "INDEX_NAME"}, 336 | Dir: ast.DirectionAsc, 337 | }, 338 | { 339 | Name: &ast.Ident{Name: "INDEX_TYPE"}, 340 | Dir: ast.DirectionAsc, 341 | }, 342 | { 343 | Name: &ast.Ident{Name: "COLUMN_NAME"}, 344 | Dir: ast.DirectionAsc, 345 | }, 346 | }, 347 | }, 348 | 349 | { 350 | Name: &ast.Ident{Name: "__INFORMATION_SCHEMA__COLUMN_OPTIONS"}, 351 | Columns: []*ast.ColumnDef{ 352 | { 353 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 354 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 355 | NotNull: true, 356 | }, 357 | { 358 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 359 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 360 | NotNull: true, 361 | }, 362 | { 363 | Name: &ast.Ident{Name: "TABLE_NAME"}, 364 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 365 | NotNull: true, 366 | }, 367 | { 368 | Name: &ast.Ident{Name: "COLUMN_NAME"}, 369 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 370 | NotNull: true, 371 | }, 372 | { 373 | Name: &ast.Ident{Name: "OPTION_NAME"}, 374 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 375 | NotNull: true, 376 | }, 377 | { 378 | Name: &ast.Ident{Name: "OPTION_TYPE"}, 379 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 380 | NotNull: true, 381 | }, 382 | { 383 | Name: &ast.Ident{Name: "OPTION_VALUE"}, 384 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 385 | NotNull: true, 386 | }, 387 | }, 388 | PrimaryKeys: []*ast.IndexKey{ 389 | { 390 | Name: &ast.Ident{Name: "TABLE_CATALOG"}, 391 | Dir: ast.DirectionAsc, 392 | }, 393 | { 394 | Name: &ast.Ident{Name: "TABLE_SCHEMA"}, 395 | Dir: ast.DirectionAsc, 396 | }, 397 | { 398 | Name: &ast.Ident{Name: "TABLE_NAME"}, 399 | Dir: ast.DirectionAsc, 400 | }, 401 | { 402 | Name: &ast.Ident{Name: "COLUMN_NAME"}, 403 | Dir: ast.DirectionAsc, 404 | }, 405 | { 406 | Name: &ast.Ident{Name: "OPTION_NAME"}, 407 | Dir: ast.DirectionAsc, 408 | }, 409 | }, 410 | }, 411 | 412 | { 413 | Name: &ast.Ident{Name: "__INFORMATION_SCHEMA__REFERENTIAL_CONSTRAINTS"}, 414 | Columns: []*ast.ColumnDef{ 415 | { 416 | Name: &ast.Ident{Name: "CONSTRAINT_CATALOG"}, 417 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 418 | NotNull: true, 419 | }, 420 | { 421 | Name: &ast.Ident{Name: "CONSTRAINT_SCHEMA"}, 422 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 423 | NotNull: true, 424 | }, 425 | { 426 | Name: &ast.Ident{Name: "CONSTRAINT_NAME"}, 427 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 428 | NotNull: true, 429 | }, 430 | { 431 | Name: &ast.Ident{Name: "UNIQUE_CONSTRAINT_CATALOG"}, 432 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 433 | NotNull: false, 434 | }, 435 | { 436 | Name: &ast.Ident{Name: "UNIQUE_CONSTRAINT_SCHEMA"}, 437 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 438 | NotNull: false, 439 | }, 440 | { 441 | Name: &ast.Ident{Name: "UNIQUE_CONSTRAINT_NAME"}, 442 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 443 | NotNull: false, 444 | }, 445 | { 446 | Name: &ast.Ident{Name: "MATCH_OPTION"}, 447 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 448 | NotNull: true, 449 | }, 450 | { 451 | Name: &ast.Ident{Name: "UPDATE_RULE"}, 452 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 453 | NotNull: true, 454 | }, 455 | { 456 | Name: &ast.Ident{Name: "DELETE_RULE"}, 457 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 458 | NotNull: true, 459 | }, 460 | { 461 | Name: &ast.Ident{Name: "SPANNER_STATE"}, 462 | Type: &ast.SizedSchemaType{Name: ast.StringTypeName, Max: true}, 463 | NotNull: true, 464 | }, 465 | }, 466 | PrimaryKeys: []*ast.IndexKey{ 467 | { 468 | Name: &ast.Ident{Name: "CONSTRAINT_CATALOG"}, 469 | Dir: ast.DirectionAsc, 470 | }, 471 | { 472 | Name: &ast.Ident{Name: "CONSTRAINT_SCHEMA"}, 473 | Dir: ast.DirectionAsc, 474 | }, 475 | { 476 | Name: &ast.Ident{Name: "CONSTRAINT_NAME"}, 477 | Dir: ast.DirectionAsc, 478 | }, 479 | }, 480 | }, 481 | } 482 | -------------------------------------------------------------------------------- /server/query_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "testing" 19 | 20 | cmp "github.com/google/go-cmp/cmp" 21 | ) 22 | 23 | func newTestQueryBuilder() *QueryBuilder { 24 | return &QueryBuilder{} 25 | } 26 | 27 | func TestQueryBuilder_ExpandParamByPlaceholders(t *testing.T) { 28 | b := newTestQueryBuilder() 29 | 30 | table := []struct { 31 | v Value 32 | ph string 33 | args []interface{} 34 | }{ 35 | { 36 | v: Value{ 37 | Data: []bool{true, false}, 38 | }, 39 | ph: "JSON_ARRAY((?), (?))", 40 | args: []interface{}{true, false}, 41 | }, 42 | { 43 | v: Value{ 44 | Data: []int64{(100), int64(101)}, 45 | }, 46 | ph: "JSON_ARRAY((?), (?))", 47 | args: []interface{}{int64(100), int64(101)}, 48 | }, 49 | { 50 | v: Value{ 51 | Data: []float64{float64(1.1), float64(1.2)}, 52 | }, 53 | ph: "JSON_ARRAY((?), (?))", 54 | args: []interface{}{float64(1.1), float64(1.2)}, 55 | }, 56 | { 57 | v: Value{ 58 | Data: []string{"aa", "bb", "cc"}, 59 | }, 60 | ph: "JSON_ARRAY((?), (?), (?))", 61 | args: []interface{}{"aa", "bb", "cc"}, 62 | }, 63 | { 64 | v: Value{ 65 | Data: [][]byte{[]byte("aa"), []byte("bb"), []byte("cc")}, 66 | }, 67 | ph: "JSON_ARRAY((?), (?), (?))", 68 | args: []interface{}{[]byte("aa"), []byte("bb"), []byte("cc")}, 69 | }, 70 | } 71 | 72 | for _, tc := range table { 73 | s, err := b.expandParamByPlaceholders(tc.v) 74 | if err != nil { 75 | t.Fatalf("unexpected error: %v", err) 76 | } 77 | 78 | if s.Raw != tc.ph { 79 | t.Errorf("expect placeholder %q, but got %q", tc.ph, s.Raw) 80 | } 81 | 82 | if diff := cmp.Diff(tc.args, s.Args); diff != "" { 83 | t.Errorf("(-got, +want)\n%s", diff) 84 | } 85 | 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /server/server_transaction_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "strconv" 21 | "sync" 22 | "testing" 23 | "time" 24 | 25 | spannerpb "google.golang.org/genproto/googleapis/spanner/v1" 26 | "google.golang.org/grpc/codes" 27 | "google.golang.org/grpc/status" 28 | structpb "google.golang.org/protobuf/types/known/structpb" 29 | ) 30 | 31 | func TestReadAndWriteTransaction_Aborted(t *testing.T) { 32 | if testing.Short() { 33 | t.Skip("skip long test") 34 | } 35 | 36 | ctx := context.Background() 37 | 38 | s := newTestServer() 39 | 40 | preareDBAndSession := func(t *testing.T) *spannerpb.Session { 41 | session, dbName := testCreateSession(t, s) 42 | db, ok := s.db[dbName] 43 | if !ok { 44 | t.Fatalf("database not found") 45 | } 46 | for _, s := range allSchema { 47 | ddls := parseDDL(t, s) 48 | for _, ddl := range ddls { 49 | db.ApplyDDL(ctx, ddl) 50 | } 51 | } 52 | 53 | for _, query := range []string{ 54 | `INSERT INTO Simple VALUES(100, "xxx")`, 55 | } { 56 | if _, err := db.db.ExecContext(ctx, query); err != nil { 57 | t.Fatalf("Insert failed: %v", err) 58 | } 59 | } 60 | 61 | return session 62 | } 63 | 64 | session := preareDBAndSession(t) 65 | 66 | begin := func() *spannerpb.Transaction { 67 | tx, err := s.BeginTransaction(ctx, &spannerpb.BeginTransactionRequest{ 68 | Session: session.Name, 69 | Options: &spannerpb.TransactionOptions{ 70 | Mode: &spannerpb.TransactionOptions_ReadWrite_{ 71 | ReadWrite: &spannerpb.TransactionOptions_ReadWrite{}, 72 | }, 73 | }, 74 | }) 75 | if err != nil { 76 | t.Fatalf("unexpected error: %v", err) 77 | } 78 | return tx 79 | } 80 | rollback := func(tx *spannerpb.Transaction) { 81 | _, _ = s.Rollback(ctx, &spannerpb.RollbackRequest{ 82 | Session: session.Name, 83 | TransactionId: tx.Id, 84 | }) 85 | // ignore error 86 | } 87 | readSimple := func(tx *spannerpb.Transaction) error { 88 | fake := NewFakeExecuteStreamingSqlServer(ctx) 89 | err := s.StreamingRead(&spannerpb.ReadRequest{ 90 | Session: session.Name, 91 | Transaction: &spannerpb.TransactionSelector{ 92 | Selector: &spannerpb.TransactionSelector_Id{ 93 | Id: tx.Id, 94 | }, 95 | }, 96 | Table: "Simple", 97 | Columns: []string{"Id"}, 98 | KeySet: &spannerpb.KeySet{All: true}, 99 | }, fake) 100 | return err 101 | } 102 | readFT := func(tx *spannerpb.Transaction) error { 103 | fake := NewFakeExecuteStreamingSqlServer(ctx) 104 | err := s.StreamingRead(&spannerpb.ReadRequest{ 105 | Session: session.Name, 106 | Transaction: &spannerpb.TransactionSelector{ 107 | Selector: &spannerpb.TransactionSelector_Id{ 108 | Id: tx.Id, 109 | }, 110 | }, 111 | Table: "FullTypes", 112 | Columns: []string{"PKey"}, 113 | KeySet: &spannerpb.KeySet{All: true}, 114 | }, fake) 115 | return err 116 | } 117 | querySimple := func(tx *spannerpb.Transaction) error { 118 | fake := NewFakeExecuteStreamingSqlServer(ctx) 119 | err := s.ExecuteStreamingSql(&spannerpb.ExecuteSqlRequest{ 120 | Session: session.Name, 121 | Transaction: &spannerpb.TransactionSelector{ 122 | Selector: &spannerpb.TransactionSelector_Id{ 123 | Id: tx.Id, 124 | }, 125 | }, 126 | Sql: `SELECT Id FROM Simple`, 127 | }, fake) 128 | return err 129 | } 130 | queryFT := func(tx *spannerpb.Transaction) error { 131 | fake := NewFakeExecuteStreamingSqlServer(ctx) 132 | err := s.ExecuteStreamingSql(&spannerpb.ExecuteSqlRequest{ 133 | Session: session.Name, 134 | Transaction: &spannerpb.TransactionSelector{ 135 | Selector: &spannerpb.TransactionSelector_Id{ 136 | Id: tx.Id, 137 | }, 138 | }, 139 | Sql: `SELECT PKey FROM FullTypes`, 140 | }, fake) 141 | return err 142 | } 143 | commitSimple := func(tx *spannerpb.Transaction) error { 144 | _, err := s.Commit(ctx, &spannerpb.CommitRequest{ 145 | Session: session.Name, 146 | Transaction: &spannerpb.CommitRequest_TransactionId{ 147 | TransactionId: tx.Id, 148 | }, 149 | Mutations: []*spannerpb.Mutation{ 150 | { 151 | Operation: &spannerpb.Mutation_InsertOrUpdate{ 152 | InsertOrUpdate: &spannerpb.Mutation_Write{ 153 | Table: "Simple", 154 | Columns: []string{"Id", "Value"}, 155 | Values: []*structpb.ListValue{ 156 | { 157 | Values: []*structpb.Value{ 158 | makeStringValue("300"), 159 | makeStringValue("ccc2"), 160 | }, 161 | }, 162 | }, 163 | }, 164 | }, 165 | }, 166 | }, 167 | }) 168 | return err 169 | } 170 | commitFT := func(tx *spannerpb.Transaction) error { 171 | _, err := s.Commit(ctx, &spannerpb.CommitRequest{ 172 | Session: session.Name, 173 | Transaction: &spannerpb.CommitRequest_TransactionId{ 174 | TransactionId: tx.Id, 175 | }, 176 | Mutations: []*spannerpb.Mutation{ 177 | { 178 | Operation: &spannerpb.Mutation_InsertOrUpdate{ 179 | InsertOrUpdate: &spannerpb.Mutation_Write{ 180 | Table: "FullTypes", 181 | Columns: fullTypesKeys, 182 | Values: []*structpb.ListValue{ 183 | { 184 | Values: []*structpb.Value{ 185 | makeStringValue("xxx"), // PKey STRING(32) NOT NULL, 186 | makeStringValue("xxx"), // FTString STRING(32) NOT NULL, 187 | makeNullValue(), // FTStringNull STRING(32), 188 | makeBoolValue(true), // FTBool BOOL NOT NULL, 189 | makeNullValue(), // FTBoolNull BOOL, 190 | makeStringValue("eHh4"), // FTBytes BYTES(32) NOT NULL, 191 | makeNullValue(), // FTBytesNull BYTES(32), 192 | makeStringValue("2012-03-04T12:34:56.123456789Z"), // FTTimestamp TIMESTAMP NOT NULL, 193 | makeNullValue(), // FTTimestampNull TIMESTAMP, 194 | makeStringValue("100"), // FTInt INT64 NOT NULL, 195 | makeNullValue(), // FTIntNull INT64, 196 | makeNumberValue(0.5), // FTFloat FLOAT64 NOT NULL, 197 | makeNullValue(), // FTFloatNull FLOAT64, 198 | makeStringValue("2012-03-04"), // FTDate DATE NOT NULL, 199 | makeNullValue(), // FTDateNull DATE, 200 | }, 201 | }, 202 | }, 203 | }, 204 | }, 205 | }, 206 | }, 207 | }) 208 | return err 209 | } 210 | 211 | readPattern := []struct { 212 | name string 213 | readSimple func(tx *spannerpb.Transaction) error 214 | readFT func(tx *spannerpb.Transaction) error 215 | }{ 216 | {"Read", readSimple, readFT}, 217 | {"Query", querySimple, queryFT}, 218 | } 219 | 220 | t.Run("ReadLockAborted", func(t *testing.T) { 221 | for _, tc := range readPattern { 222 | t.Run(tc.name, func(t *testing.T) { 223 | tx1 := begin() 224 | tx2 := begin() 225 | defer rollback(tx1) 226 | defer rollback(tx2) 227 | assertStatusCode(t, tc.readSimple(tx1), codes.OK) 228 | assertStatusCode(t, commitSimple(tx2), codes.OK) 229 | assertStatusCode(t, tc.readSimple(tx1), codes.Aborted) 230 | }) 231 | } 232 | }) 233 | 234 | t.Run("ReadLock_DifferentTable", func(t *testing.T) { 235 | for _, tc := range readPattern { 236 | t.Run(tc.name, func(t *testing.T) { 237 | tx1 := begin() 238 | tx2 := begin() 239 | tx3 := begin() 240 | defer rollback(tx1) 241 | defer rollback(tx2) 242 | defer rollback(tx3) 243 | assertStatusCode(t, tc.readSimple(tx1), codes.OK) 244 | assertStatusCode(t, tc.readFT(tx2), codes.OK) 245 | assertStatusCode(t, commitSimple(tx3), codes.OK) 246 | assertStatusCode(t, tc.readSimple(tx1), codes.Aborted) 247 | assertStatusCode(t, tc.readFT(tx2), codes.OK) 248 | }) 249 | } 250 | }) 251 | 252 | t.Run("WriteTest", func(t *testing.T) { 253 | for _, tc := range readPattern { 254 | t.Run(tc.name, func(t *testing.T) { 255 | tx1 := begin() 256 | tx2 := begin() 257 | defer rollback(tx1) 258 | defer rollback(tx2) 259 | assertStatusCode(t, tc.readSimple(tx1), codes.OK) 260 | assertStatusCode(t, tc.readFT(tx2), codes.OK) 261 | assertStatusCode(t, commitSimple(tx1), codes.OK) 262 | assertStatusCode(t, commitFT(tx2), codes.OK) 263 | }) 264 | } 265 | }) 266 | 267 | t.Run("ReadAbortedWhileReading", func(t *testing.T) { 268 | for _, tc := range readPattern { 269 | t.Run(tc.name, func(t *testing.T) { 270 | for i := 0; i < 100; i++ { 271 | tx1 := begin() 272 | tx2 := begin() 273 | done := make(chan struct{}) 274 | errCh := make(chan error, 2) 275 | 276 | assertStatusCode(t, tc.readSimple(tx1), codes.OK) // read lock first 277 | go func() { 278 | for { 279 | time.Sleep(100 * time.Microsecond) 280 | 281 | // try to happen aborted while reading 282 | err := tc.readSimple(tx1) 283 | code := status.Code(err) 284 | if code == codes.OK { 285 | continue 286 | } 287 | if code == codes.Aborted { 288 | close(done) 289 | return 290 | } 291 | 292 | errCh <- err 293 | close(done) 294 | return 295 | } 296 | }() 297 | time.Sleep(time.Duration(i/10+1) * time.Millisecond) 298 | assertStatusCode(t, commitSimple(tx2), codes.OK) 299 | <-done 300 | select { 301 | case err := <-errCh: 302 | if err != nil { 303 | t.Fatalf("error: %v", err) 304 | } 305 | default: 306 | } 307 | } 308 | }) 309 | } 310 | }) 311 | } 312 | 313 | func TestReadAndWriteTransaction_AtomicUpdate(t *testing.T) { 314 | if testing.Short() { 315 | t.Skip("skip long test") 316 | } 317 | 318 | ctx, cancel := context.WithCancel(context.Background()) 319 | defer cancel() 320 | 321 | s := newTestServer() 322 | 323 | preareDB := func(t *testing.T) string { 324 | _, dbName := testCreateSession(t, s) 325 | db, ok := s.db[dbName] 326 | if !ok { 327 | t.Fatalf("database not found") 328 | } 329 | for _, s := range allSchema { 330 | ddls := parseDDL(t, s) 331 | for _, ddl := range ddls { 332 | db.ApplyDDL(ctx, ddl) 333 | } 334 | } 335 | 336 | return dbName 337 | } 338 | 339 | createSession := func(t *testing.T, dbName string) *spannerpb.Session { 340 | session, err := s.CreateSession(context.Background(), &spannerpb.CreateSessionRequest{ 341 | Database: dbName, 342 | }) 343 | if err != nil { 344 | t.Fatalf("failed to create session: %v", err) 345 | } 346 | return session 347 | } 348 | 349 | begin := func(session *spannerpb.Session) (*spannerpb.Transaction, error) { 350 | return s.BeginTransaction(ctx, &spannerpb.BeginTransactionRequest{ 351 | Session: session.Name, 352 | Options: &spannerpb.TransactionOptions{ 353 | Mode: &spannerpb.TransactionOptions_ReadWrite_{ 354 | ReadWrite: &spannerpb.TransactionOptions_ReadWrite{}, 355 | }, 356 | }, 357 | }) 358 | } 359 | read := func(session *spannerpb.Session, tx *spannerpb.Transaction, pKey string, sql bool) (int, error) { 360 | var txSel *spannerpb.TransactionSelector 361 | if tx == nil { 362 | txSel = &spannerpb.TransactionSelector{ 363 | Selector: &spannerpb.TransactionSelector_SingleUse{ 364 | SingleUse: &spannerpb.TransactionOptions{ 365 | Mode: &spannerpb.TransactionOptions_ReadWrite_{ 366 | ReadWrite: &spannerpb.TransactionOptions_ReadWrite{}, 367 | }, 368 | }, 369 | }, 370 | } 371 | } else { 372 | txSel = &spannerpb.TransactionSelector{ 373 | Selector: &spannerpb.TransactionSelector_Id{ 374 | Id: tx.Id, 375 | }, 376 | } 377 | } 378 | 379 | fake := NewFakeExecuteStreamingSqlServer(ctx) 380 | if sql { 381 | err := s.ExecuteStreamingSql(&spannerpb.ExecuteSqlRequest{ 382 | Session: session.Name, 383 | Transaction: txSel, 384 | Sql: `SELECT Value FROM Simple WHERE Id = @key`, 385 | ParamTypes: map[string]*spannerpb.Type{ 386 | "key": &spannerpb.Type{ 387 | Code: spannerpb.TypeCode_INT64, 388 | }, 389 | }, 390 | Params: &structpb.Struct{ 391 | Fields: map[string]*structpb.Value{ 392 | "key": makeStringValue(pKey), 393 | }, 394 | }, 395 | }, fake) 396 | if err != nil { 397 | return 0, err 398 | } 399 | } else { 400 | err := s.StreamingRead(&spannerpb.ReadRequest{ 401 | Session: session.Name, 402 | Transaction: txSel, 403 | Table: "Simple", 404 | Columns: []string{"Value"}, 405 | KeySet: &spannerpb.KeySet{ 406 | Keys: []*structpb.ListValue{ 407 | { 408 | Values: []*structpb.Value{ 409 | makeStringValue(pKey), 410 | }, 411 | }, 412 | }, 413 | }, 414 | }, fake) 415 | if err != nil { 416 | return 0, err 417 | } 418 | } 419 | 420 | var results [][]*structpb.Value 421 | for _, set := range fake.sets { 422 | results = append(results, set.Values) 423 | } 424 | 425 | if len(results) != 1 { 426 | return 0, fmt.Errorf("results should be 1 record but got %v", len(results)) 427 | } 428 | 429 | v := results[0][0].Kind.(*structpb.Value_StringValue).StringValue 430 | n, _ := strconv.Atoi(v) 431 | return n, nil 432 | } 433 | 434 | commit := func(session *spannerpb.Session, tx *spannerpb.Transaction, pKey string, sql bool, next int) error { 435 | var mus []*spannerpb.Mutation 436 | if sql { 437 | _, err := s.ExecuteSql(ctx, &spannerpb.ExecuteSqlRequest{ 438 | Session: session.Name, 439 | Transaction: &spannerpb.TransactionSelector{ 440 | Selector: &spannerpb.TransactionSelector_Id{ 441 | Id: tx.Id, 442 | }, 443 | }, 444 | Sql: fmt.Sprintf(`UPDATE Simple Set Value = "%d" WHERE Id = %s`, next, pKey), 445 | }) 446 | if err != nil { 447 | return err 448 | } 449 | } else { 450 | mus = []*spannerpb.Mutation{ 451 | { 452 | Operation: &spannerpb.Mutation_Update{ 453 | Update: &spannerpb.Mutation_Write{ 454 | Table: "Simple", 455 | Columns: []string{"Id", "Value"}, 456 | Values: []*structpb.ListValue{ 457 | { 458 | Values: []*structpb.Value{ 459 | makeStringValue(pKey), 460 | makeStringValue(fmt.Sprint(next)), 461 | }, 462 | }, 463 | }, 464 | }, 465 | }, 466 | }, 467 | } 468 | } 469 | 470 | _, err := s.Commit(ctx, &spannerpb.CommitRequest{ 471 | Session: session.Name, 472 | Transaction: &spannerpb.CommitRequest_TransactionId{ 473 | TransactionId: tx.Id, 474 | }, 475 | Mutations: mus, 476 | }) 477 | 478 | return err 479 | } 480 | 481 | dbname := preareDB(t) 482 | session := createSession(t, dbname) 483 | 484 | initialValue := 100 485 | 486 | table := []struct { 487 | pKey string 488 | concurrency int // should be under 32 489 | tries int 490 | }{ 491 | {pKey: "1000", concurrency: 5, tries: 3}, 492 | {pKey: "1001", concurrency: 10, tries: 10}, 493 | {pKey: "1002", concurrency: 20, tries: 10}, 494 | {pKey: "1003", concurrency: 50, tries: 5}, 495 | // {pKey: "1004", concurrency: 100, tries: 30}, 496 | } 497 | 498 | for _, tc := range table { 499 | pKey := fmt.Sprint(tc.pKey) 500 | concurrency := tc.concurrency 501 | tries := tc.tries 502 | t.Run(fmt.Sprintf("KEY%s_Con%d_Try%d", pKey, concurrency, tries), func(t *testing.T) { 503 | _, err := s.Commit(ctx, &spannerpb.CommitRequest{ 504 | Session: session.Name, 505 | Transaction: &spannerpb.CommitRequest_SingleUseTransaction{ 506 | SingleUseTransaction: &spannerpb.TransactionOptions{ 507 | Mode: &spannerpb.TransactionOptions_ReadWrite_{ 508 | ReadWrite: &spannerpb.TransactionOptions_ReadWrite{}, 509 | }, 510 | }, 511 | }, 512 | Mutations: []*spannerpb.Mutation{ 513 | { 514 | Operation: &spannerpb.Mutation_Insert{ 515 | Insert: &spannerpb.Mutation_Write{ 516 | Table: "Simple", 517 | Columns: []string{"Id", "Value"}, 518 | Values: []*structpb.ListValue{ 519 | { 520 | Values: []*structpb.Value{ 521 | makeStringValue(pKey), 522 | makeStringValue(fmt.Sprint(initialValue)), 523 | }, 524 | }, 525 | }, 526 | }, 527 | }, 528 | }, 529 | }, 530 | }) 531 | if err != nil { 532 | t.Fatalf("unexpected error: %v", err) 533 | } 534 | 535 | errCh := make(chan error, concurrency*tries) 536 | wg := &sync.WaitGroup{} 537 | readerDone := make(chan struct{}) 538 | for i := 0; i < concurrency; i++ { 539 | wg.Add(1) 540 | go func(me int) { 541 | defer wg.Done() 542 | 543 | // create own session 544 | session := createSession(t, dbname) 545 | 546 | err := func() error { 547 | try := 0 548 | for { 549 | select { 550 | case <-ctx.Done(): 551 | return ctx.Err() 552 | default: 553 | } 554 | 555 | // begin transaction 556 | tx, err := begin(session) 557 | if err != nil { 558 | return fmt.Errorf("begin: %v", err) 559 | } 560 | 561 | // read the current value 562 | useSql := (me % 2) == 0 563 | n, err := read(session, tx, pKey, useSql) 564 | if err != nil { 565 | if code := status.Code(err); code == codes.Aborted { 566 | continue 567 | } 568 | return fmt.Errorf("[%s] read: %v", string(tx.Id), err) 569 | } 570 | next := n + 1 571 | 572 | // write +1 value 573 | if err := commit(session, tx, pKey, useSql, next); err != nil { 574 | if code := status.Code(err); code == codes.Aborted { 575 | continue 576 | } 577 | return fmt.Errorf("[%s] commit: %v", string(tx.Id), err) 578 | } 579 | 580 | try++ 581 | if try == tries { 582 | return nil 583 | } 584 | } 585 | }() 586 | if err != nil { 587 | errCh <- err 588 | cancel() 589 | } 590 | }(i) 591 | } 592 | 593 | go func() { 594 | // create own session 595 | session := createSession(t, dbname) 596 | readLoop: 597 | for { 598 | select { 599 | case <-readerDone: 600 | break readLoop 601 | default: 602 | } 603 | if _, err := read(session, nil, pKey, false); err != nil { 604 | if code := status.Code(err); code == codes.Aborted { 605 | continue 606 | } 607 | errCh <- err 608 | cancel() 609 | break 610 | } 611 | } 612 | }() 613 | 614 | wg.Wait() 615 | 616 | close(readerDone) 617 | 618 | var errs []error 619 | L: 620 | for { 621 | select { 622 | case err := <-errCh: 623 | if err != nil { 624 | errs = append(errs, err) 625 | } 626 | cancel() 627 | default: 628 | break L 629 | } 630 | } 631 | 632 | if len(errs) != 0 { 633 | for _, err := range errs { 634 | t.Errorf("error %v", err) 635 | } 636 | t.FailNow() 637 | } 638 | 639 | n, err := read(session, nil, pKey, false) 640 | if err != nil { 641 | t.Fatalf("read error %v", err) 642 | } 643 | 644 | expected := initialValue + tries*concurrency 645 | if n != expected { 646 | t.Errorf("expect n to be %v, but got %v", expected, n) 647 | } 648 | }) 649 | } 650 | } 651 | -------------------------------------------------------------------------------- /server/session.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | "sync" 21 | "time" 22 | 23 | uuidpkg "github.com/google/uuid" 24 | spannerpb "google.golang.org/genproto/googleapis/spanner/v1" 25 | "google.golang.org/grpc/codes" 26 | "google.golang.org/grpc/status" 27 | "google.golang.org/protobuf/types/known/timestamppb" 28 | ) 29 | 30 | func validateSessionName(sessionName string) bool { 31 | parts := strings.Split(sessionName, "/") 32 | if len(parts) != 8 { 33 | return false 34 | } 35 | if parts[0] != "projects" { 36 | return false 37 | } 38 | if parts[2] != "instances" { 39 | return false 40 | } 41 | if parts[4] != "databases" { 42 | return false 43 | } 44 | if parts[6] != "sessions" { 45 | return false 46 | } 47 | if parts[1] == "" || parts[3] == "" || parts[5] == "" || parts[7] == "" { 48 | return false 49 | } 50 | return true 51 | } 52 | 53 | type session struct { 54 | id string 55 | dbName string 56 | database *database 57 | name string 58 | createdAt time.Time 59 | lastUse time.Time 60 | 61 | mu sync.Mutex 62 | transactions map[string]*transaction 63 | activeRWTx []*transaction 64 | } 65 | 66 | func (s *session) Name() string { 67 | return s.name 68 | } 69 | 70 | func (s *session) Proto() *spannerpb.Session { 71 | ctime := timestamppb.New(s.createdAt) 72 | last := timestamppb.New(s.lastUse) 73 | return &spannerpb.Session{ 74 | Name: s.name, 75 | CreateTime: ctime, 76 | ApproximateLastUseTime: last, 77 | } 78 | } 79 | 80 | func (s *session) createTransaction(txMode transactionMode, single bool) (*transaction, error) { 81 | s.mu.Lock() 82 | defer s.mu.Unlock() 83 | 84 | for i := 0; i < 3; i++ { 85 | tx := newTransaction(s, txMode, single) 86 | if _, ok := s.transactions[tx.Name()]; ok { 87 | continue 88 | } 89 | s.transactions[tx.Name()] = tx 90 | 91 | if err := s.database.BeginTransaction(tx); err != nil { 92 | return nil, err // TODO 93 | } 94 | 95 | // read write transactions are kept only 32 in a session 96 | // it seems no limitation for read only transactions 97 | if txMode == txReadWrite { 98 | s.activeRWTx = append(s.activeRWTx, tx) 99 | if len(s.activeRWTx) > 32 { 100 | oldtx := s.activeRWTx[0] 101 | s.activeRWTx = s.activeRWTx[1:] 102 | oldtx.Done(TransactionInvalidated) 103 | } 104 | } 105 | return tx, nil 106 | } 107 | 108 | return nil, fmt.Errorf("failed to create transaction") 109 | } 110 | 111 | func (s *session) GetTransaction(id []byte) (*transaction, bool) { 112 | s.mu.Lock() 113 | tx, ok := s.transactions[string(id)] 114 | s.mu.Unlock() 115 | return tx, ok 116 | } 117 | 118 | // GetTransactionBySelector returns a transaction by selector from session. 119 | // The second return value means a new transaction is created or not. 120 | // When the selector specifies Begin, true is returned. Otherwise false even if SingleUse option is specified. 121 | func (s *session) GetTransactionBySelector(txsel *spannerpb.TransactionSelector) (*transaction, bool, error) { 122 | switch sel := txsel.GetSelector().(type) { 123 | case nil: 124 | // From documents: If none is provided, the default is a 125 | // single use transaction with strong concurrency. 126 | tx, err := s.createTransaction(txReadWrite, true) 127 | return tx, false, err 128 | case *spannerpb.TransactionSelector_SingleUse: 129 | tx, err := s.createTransaction(txReadWrite, true) 130 | return tx, false, err 131 | case *spannerpb.TransactionSelector_Id: 132 | tx, ok := s.GetTransaction(sel.Id) 133 | if !ok { 134 | return nil, false, status.Errorf(codes.InvalidArgument, "Transaction was started in a different session") 135 | } 136 | return tx, false, nil 137 | case *spannerpb.TransactionSelector_Begin: 138 | tx, err := s.BeginTransaction(sel.Begin) 139 | return tx, true, err 140 | default: 141 | return nil, false, fmt.Errorf("unknown transaction selector: %v", sel) 142 | } 143 | } 144 | 145 | // GetTransactionForCommit returns a transaction by selector from session. 146 | // The argument is expected to be spannerpb.isCommitRequest_Transaction. It is not exported so interface{} 147 | // is used instead. 148 | func (s *session) GetTransactionForCommit(txsel interface{}) (*transaction, error) { 149 | switch v := txsel.(type) { 150 | case *spannerpb.CommitRequest_TransactionId: 151 | tx, ok := s.GetTransaction(v.TransactionId) 152 | if !ok { 153 | return nil, status.Errorf(codes.InvalidArgument, "Transaction was started in a different session") 154 | } 155 | return tx, nil 156 | case *spannerpb.CommitRequest_SingleUseTransaction: 157 | return s.beginTransaction(v.SingleUseTransaction, true) 158 | default: 159 | return nil, status.Errorf(codes.Unknown, "unknown transaction: %v", v) 160 | } 161 | } 162 | 163 | // BeginTransaction creates a new transaction for a session. 164 | func (s *session) BeginTransaction(opt *spannerpb.TransactionOptions) (*transaction, error) { 165 | return s.beginTransaction(opt, false) 166 | } 167 | 168 | func (s *session) beginTransaction(opt *spannerpb.TransactionOptions, single bool) (*transaction, error) { 169 | var txMode transactionMode 170 | switch v := opt.GetMode().(type) { 171 | case *spannerpb.TransactionOptions_ReadWrite_: 172 | txMode = txReadWrite 173 | case *spannerpb.TransactionOptions_ReadOnly_: 174 | txMode = txReadOnly 175 | case *spannerpb.TransactionOptions_PartitionedDml_: 176 | txMode = txPartitionedDML 177 | case nil: 178 | // TransactionOptions is required 179 | return nil, status.Errorf(codes.InvalidArgument, "Invalid BeginTransaction request") 180 | default: 181 | return nil, status.Errorf(codes.Unknown, "unknown transaction mode: %v", v) 182 | } 183 | 184 | tx, err := s.createTransaction(txMode, single) 185 | if err != nil { 186 | return nil, err 187 | } 188 | 189 | return tx, nil 190 | } 191 | 192 | func newSession(db *database, dbName string) *session { 193 | id := uuidpkg.New().String() 194 | return &session{ 195 | id: id, 196 | database: db, 197 | dbName: dbName, 198 | name: fmt.Sprintf("%s/sessions/%s", dbName, id), 199 | createdAt: time.Now(), 200 | transactions: make(map[string]*transaction), 201 | activeRWTx: make([]*transaction, 0, 8), 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /server/spanner_error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "fmt" 19 | 20 | "google.golang.org/genproto/googleapis/rpc/errdetails" 21 | "google.golang.org/grpc/codes" 22 | "google.golang.org/grpc/status" 23 | ) 24 | 25 | func newSpannerDatabaseNotFoundError(name string) error { 26 | detail := &errdetails.ResourceInfo{ 27 | ResourceType: "type.googleapis.com/google.spanner.admin.database.v1.Database", 28 | ResourceName: name, 29 | Description: "Database does not exist.", 30 | } 31 | st, err := status.Newf(codes.NotFound, "Database not found: %s", name).WithDetails(detail) 32 | if err != nil { 33 | return status.Errorf(codes.Internal, "failed to build status error") 34 | } 35 | 36 | return st.Err() 37 | } 38 | 39 | func newSpannerSessionNotFoundError(name string) error { 40 | detail := &errdetails.ResourceInfo{ 41 | ResourceType: "type.googleapis.com/google.spanner.v1.Session", 42 | ResourceName: name, 43 | Description: "Session does not exist.", 44 | } 45 | st, err := status.Newf(codes.NotFound, "Session not found: %s", name).WithDetails(detail) 46 | if err != nil { 47 | return status.Errorf(codes.Internal, "failed to build status error") 48 | } 49 | 50 | return st.Err() 51 | } 52 | 53 | func newSpannerTableNotFoundError(name string) error { 54 | detail := &errdetails.ResourceInfo{ 55 | ResourceType: "spanner.googleapis.com/Table", 56 | ResourceName: name, 57 | Description: "Table not found", 58 | } 59 | st, err := status.Newf(codes.NotFound, "Table not found: %s", name).WithDetails(detail) 60 | if err != nil { 61 | return status.Errorf(codes.Internal, "failed to build status error") 62 | } 63 | 64 | return st.Err() 65 | } 66 | 67 | func newSpannerColumnNotFoundError(table, name string) error { 68 | detail := &errdetails.ResourceInfo{ 69 | ResourceType: "spanner.googleapis.com/Column", 70 | ResourceName: name, 71 | // no description in real spanner 72 | } 73 | st, err := status.Newf(codes.NotFound, "Column not found in table %s: %s", table, name).WithDetails(detail) 74 | if err != nil { 75 | return status.Errorf(codes.Internal, "failed to build status error") 76 | } 77 | 78 | return st.Err() 79 | } 80 | 81 | func newSpannerIndexnNotFoundError(table, name string) error { 82 | detail := &errdetails.ResourceInfo{ 83 | ResourceType: "spanner.googleapis.com/Index", 84 | ResourceName: name, 85 | Description: fmt.Sprintf("Index not found on table %s", table), 86 | } 87 | st, err := status.Newf(codes.NotFound, "Index not found on table %s: %s", table, name).WithDetails(detail) 88 | if err != nil { 89 | return status.Errorf(codes.Internal, "failed to build status error") 90 | } 91 | 92 | return st.Err() 93 | } 94 | -------------------------------------------------------------------------------- /server/table.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | 21 | "github.com/cloudspannerecosystem/memefish/ast" 22 | ) 23 | 24 | type Table struct { 25 | ast *ast.CreateTable 26 | Name string 27 | 28 | columns []*Column 29 | columnsMap map[string]*Column 30 | 31 | primaryKey *TableIndex 32 | index []*TableIndex 33 | } 34 | 35 | func newTable() *Table { 36 | return &Table{ 37 | columnsMap: make(map[string]*Column), 38 | } 39 | } 40 | 41 | func (t *Table) TableIndex(idx string) (*TableIndex, bool) { 42 | if idx == "" { 43 | return t.primaryKey, true 44 | } 45 | 46 | for _, index := range t.index { 47 | if index.Name() == idx { 48 | return index, true 49 | } 50 | } 51 | 52 | return nil, false 53 | } 54 | 55 | func (t *Table) TableView() *TableView { 56 | return createTableViewFromTable(t, "") 57 | } 58 | 59 | func (t *Table) TableViewWithAlias(alias string) *TableView { 60 | return createTableViewFromTable(t, alias) 61 | } 62 | 63 | // NonNullableAndNonGeneratedColumnsExist checks non nullable columns exist in the spciefied columns. 64 | // It returns true and the columns if non nullable and non generated columns exist. 65 | func (t *Table) NonNullableAndNonGeneratedColumnsExist(columns []string) (bool, []string) { 66 | usedColumns := make(map[string]struct{}, len(columns)) 67 | for _, name := range columns { 68 | usedColumns[name] = struct{}{} 69 | } 70 | 71 | var noExistNonNullableColumns []string 72 | for _, c := range t.columns { 73 | if c.nullable { 74 | continue 75 | } 76 | if c.ast != nil && c.ast.GeneratedExpr != nil { 77 | continue 78 | } 79 | 80 | n := c.Name() 81 | if _, ok := usedColumns[n]; !ok { 82 | noExistNonNullableColumns = append(noExistNonNullableColumns, n) 83 | } 84 | } 85 | 86 | if len(noExistNonNullableColumns) > 0 { 87 | return true, noExistNonNullableColumns 88 | } 89 | 90 | return false, nil 91 | } 92 | 93 | func createTableFromAST(stmt *ast.CreateTable) (*Table, error) { 94 | t := newTable() 95 | t.Name = stmt.Name.Name 96 | t.ast = stmt 97 | 98 | for _, col := range stmt.Columns { 99 | t.addColumn(col) 100 | } 101 | t.reorderColumnPosition() 102 | 103 | if err := t.setPrimaryKeys(stmt.PrimaryKeys); err != nil { 104 | return nil, err 105 | } 106 | 107 | return t, nil 108 | } 109 | 110 | func (t *Table) addColumn(col *ast.ColumnDef) { 111 | column := newColumn(col) 112 | t.columns = append(t.columns, column) 113 | t.columnsMap[column.Name()] = column 114 | } 115 | 116 | func (t *Table) reorderColumnPosition() { 117 | for i := range t.columns { 118 | t.columns[i].setPosition(i + 1) 119 | } 120 | } 121 | 122 | func (t *Table) setPrimaryKeys(pkey []*ast.IndexKey) error { 123 | index, err := createPrimaryKey(t, pkey) 124 | if err != nil { 125 | return err 126 | } 127 | t.primaryKey = index 128 | 129 | for i, col := range index.IndexColumns() { 130 | col.markPrimaryKey(i + 1) 131 | } 132 | 133 | return nil 134 | } 135 | 136 | func (t *Table) createIndex(stmt *ast.CreateIndex) (*TableIndex, error) { 137 | idx, err := createTableIndexFromAST(t, stmt) 138 | if err != nil { 139 | return nil, err 140 | } 141 | t.index = append(t.index, idx) 142 | return idx, nil 143 | } 144 | 145 | func (t *Table) getColumn(name string) (*Column, error) { 146 | c, ok := t.columnsMap[name] 147 | if !ok { 148 | return nil, newSpannerColumnNotFoundError(t.Name, name) 149 | } 150 | return c, nil 151 | } 152 | 153 | func (t *Table) getColumnsByName(names []string) ([]*Column, error) { 154 | columns := make([]*Column, len(names)) 155 | for i, name := range names { 156 | c, err := t.getColumn(name) 157 | if err != nil { 158 | return nil, err 159 | } 160 | columns[i] = c 161 | } 162 | 163 | return columns, nil 164 | } 165 | 166 | type Column struct { 167 | ast *ast.ColumnDef 168 | pos int 169 | 170 | alias string 171 | 172 | valueType ValueType 173 | dbDataType dbDataType 174 | 175 | nullable bool 176 | isArray bool 177 | isSized bool 178 | isMax bool 179 | size int64 180 | 181 | allowCommitTimestamp bool 182 | 183 | isPrimaryKey bool 184 | primaryKeyPos int 185 | } 186 | 187 | func (c *Column) Name() string { 188 | return c.ast.Name.Name 189 | } 190 | 191 | func (c *Column) Alias() string { 192 | if c.alias != "" { 193 | return c.alias 194 | } 195 | return c.ast.Name.Name 196 | } 197 | 198 | func (c *Column) setPosition(pos int) { 199 | c.pos = pos 200 | } 201 | 202 | func (c *Column) markPrimaryKey(pos int) { 203 | c.isPrimaryKey = true 204 | c.primaryKeyPos = pos 205 | } 206 | 207 | type columnType struct { 208 | dataType ast.ScalarTypeName 209 | 210 | isArray bool 211 | isSized bool 212 | isMax bool 213 | size int64 214 | } 215 | 216 | type dbDataType string 217 | 218 | const ( 219 | DBDTInteger dbDataType = "INTEGER" 220 | DBDTReal dbDataType = "REAL" 221 | DBDTText dbDataType = "TEXT" 222 | DBDTBlob dbDataType = "BLOB" 223 | DBDTJson dbDataType = "JSON" 224 | ) 225 | 226 | func (ct columnType) SqliteDataType() string { 227 | switch ct.dataType { 228 | case ast.BoolTypeName: 229 | return "INTEGER" 230 | case ast.Int64TypeName: 231 | return "INTEGER" 232 | case ast.Float64TypeName: 233 | return "REAL" 234 | case ast.StringTypeName: 235 | return "TEXT" 236 | case ast.BytesTypeName: 237 | return "BLOB" 238 | case ast.DateTypeName: 239 | return "TEXT" 240 | case ast.TimestampTypeName: 241 | return "TEXT" 242 | } 243 | 244 | panic(fmt.Sprintf("unknown data type: %s", ct.dataType)) 245 | } 246 | 247 | func newColumn(def *ast.ColumnDef) *Column { 248 | ct := toColumnType(def.Type) 249 | vt := toValueType(def.Type) 250 | 251 | var dbdt dbDataType 252 | switch vt.Code { 253 | default: 254 | panic(fmt.Sprintf("unknown value type %#v", vt)) 255 | case TCBool: 256 | dbdt = DBDTInteger 257 | case TCInt64: 258 | dbdt = DBDTInteger 259 | case TCFloat64: 260 | dbdt = DBDTReal 261 | case TCString: 262 | dbdt = DBDTText 263 | case TCBytes: 264 | dbdt = DBDTBlob 265 | case TCDate: 266 | dbdt = DBDTText 267 | case TCTimestamp: 268 | dbdt = DBDTText 269 | case TCArray: 270 | dbdt = DBDTJson 271 | case TCJson: 272 | dbdt = DBDTJson 273 | } 274 | 275 | var allowCommitTimestamp bool 276 | if def.Options != nil { 277 | allowCommitTimestamp = def.Options.AllowCommitTimestamp 278 | } 279 | 280 | return &Column{ 281 | ast: def, 282 | 283 | valueType: vt, 284 | dbDataType: dbdt, 285 | 286 | nullable: !def.NotNull, 287 | isArray: ct.isArray, 288 | isSized: ct.isSized, 289 | isMax: ct.isMax, 290 | size: ct.size, 291 | 292 | allowCommitTimestamp: allowCommitTimestamp, 293 | } 294 | } 295 | 296 | func astTypeToTypeCode(astTypeName ast.ScalarTypeName) TypeCode { 297 | switch astTypeName { 298 | case ast.BoolTypeName: 299 | return TCBool 300 | case ast.Int64TypeName: 301 | return TCInt64 302 | case ast.Float64TypeName: 303 | return TCFloat64 304 | case ast.StringTypeName: 305 | return TCString 306 | case ast.BytesTypeName: 307 | return TCBytes 308 | case ast.DateTypeName: 309 | return TCDate 310 | case ast.TimestampTypeName: 311 | return TCTimestamp 312 | case ast.ScalarTypeName("JSON"): 313 | return TCJson 314 | default: 315 | panic("unknown type") 316 | } 317 | } 318 | 319 | func toValueType(t ast.SchemaType) ValueType { 320 | switch v := t.(type) { 321 | case *ast.ScalarSchemaType: 322 | return ValueType{Code: astTypeToTypeCode(v.Name)} 323 | 324 | case *ast.SizedSchemaType: 325 | return ValueType{Code: astTypeToTypeCode(v.Name)} 326 | 327 | case *ast.ArraySchemaType: 328 | arrType := toValueType(v.Item) 329 | return ValueType{ 330 | Code: TCArray, 331 | ArrayType: &arrType, 332 | } 333 | default: 334 | panic(fmt.Sprintf("unknow type %v", t)) 335 | } 336 | } 337 | 338 | func schemaTypetoTypString(t ast.SchemaType) string { 339 | switch v := t.(type) { 340 | case *ast.ScalarSchemaType: 341 | return astTypeToTypeCode(v.Name).String() 342 | 343 | case *ast.SizedSchemaType: 344 | typ := astTypeToTypeCode(v.Name).String() 345 | size := "MAX" 346 | if !v.Max { 347 | intLit := v.Size.(*ast.IntLiteral) 348 | size = intLit.Value // TODO: respect base? 349 | } 350 | return fmt.Sprintf("%s(%s)", typ, size) 351 | 352 | case *ast.ArraySchemaType: 353 | arrType := schemaTypetoTypString(v.Item) 354 | return fmt.Sprintf("ARRAY<%s>", arrType) 355 | default: 356 | panic(fmt.Sprintf("unknow type %v", t)) 357 | } 358 | } 359 | 360 | func toColumnType(t ast.SchemaType) columnType { 361 | switch v := t.(type) { 362 | case *ast.ScalarSchemaType: 363 | return columnType{ 364 | dataType: v.Name, 365 | } 366 | case *ast.SizedSchemaType: 367 | if v.Max { 368 | return columnType{ 369 | dataType: v.Name, 370 | isSized: true, 371 | isMax: true, 372 | } 373 | } 374 | 375 | intLit, ok := v.Size.(*ast.IntLiteral) 376 | if !ok { 377 | panic(fmt.Sprintf("expected IntLiteral but %v", v.Size)) 378 | } 379 | 380 | n, err := strconv.ParseInt(intLit.Value, intLit.Base, 64) 381 | if err != nil { 382 | panic(fmt.Sprintf("cannot parse IntLiteral: %v", intLit)) 383 | } 384 | 385 | return columnType{ 386 | dataType: v.Name, 387 | isSized: true, 388 | isMax: false, 389 | size: n, 390 | } 391 | 392 | case *ast.ArraySchemaType: 393 | ct := toColumnType(v.Item) 394 | ct.isArray = true 395 | return ct 396 | 397 | default: 398 | panic(fmt.Sprintf("unknow type %v", t)) 399 | } 400 | 401 | } 402 | 403 | type TableIndex struct { 404 | ast *ast.CreateIndex 405 | astIndexKeys []*ast.IndexKey 406 | 407 | name string 408 | table *Table 409 | 410 | unique bool 411 | nullFiltered bool 412 | 413 | columnsRef []*Column 414 | columnNames []string 415 | columnDirctions []string 416 | storedColumns map[string]struct{} 417 | } 418 | 419 | func createTableIndexFromAST(table *Table, stmt *ast.CreateIndex) (*TableIndex, error) { 420 | return createTableIndex(table, stmt.Keys, stmt) 421 | } 422 | 423 | func createPrimaryKey(table *Table, pkeys []*ast.IndexKey) (*TableIndex, error) { 424 | return createTableIndex(table, pkeys, nil) 425 | } 426 | 427 | func createTableIndex(table *Table, keys []*ast.IndexKey, secondaryIdx *ast.CreateIndex) (*TableIndex, error) { 428 | columns := make([]*Column, len(keys)) 429 | columnNames := make([]string, len(keys)) 430 | columnDirctions := make([]string, len(keys)) 431 | for i, key := range keys { 432 | col, ok := table.columnsMap[key.Name.Name] 433 | if !ok { 434 | return nil, fmt.Errorf("primary key not found: %s", key.Name.Name) 435 | } 436 | 437 | columns[i] = col 438 | columnNames[i] = col.Name() 439 | dir := string(key.Dir) 440 | if dir == "" { // work around 441 | dir = "ASC" 442 | } 443 | columnDirctions[i] = dir 444 | } 445 | 446 | name := "PRIMARY_KEY" 447 | unique := true 448 | nullFiltered := false 449 | storedColumns := make(map[string]struct{}) 450 | 451 | if secondaryIdx != nil { 452 | name = secondaryIdx.Name.Name 453 | unique = secondaryIdx.Unique 454 | nullFiltered = secondaryIdx.NullFiltered 455 | 456 | // columns for Index Keys 457 | for _, c := range columns { 458 | storedColumns[c.Name()] = struct{}{} 459 | } 460 | 461 | // secondary index also has primary key columns by default 462 | for _, name := range table.primaryKey.IndexColumnNames() { 463 | storedColumns[name] = struct{}{} 464 | } 465 | 466 | // storing columns 467 | if secondaryIdx.Storing != nil { 468 | for _, c := range secondaryIdx.Storing.Columns { 469 | storedColumns[c.Name] = struct{}{} 470 | } 471 | } 472 | } else { 473 | // Primry Keys have all columns 474 | for _, c := range table.columns { 475 | storedColumns[c.Name()] = struct{}{} 476 | } 477 | } 478 | 479 | return &TableIndex{ 480 | ast: secondaryIdx, 481 | astIndexKeys: keys, 482 | 483 | name: name, 484 | table: table, 485 | 486 | unique: unique, 487 | nullFiltered: nullFiltered, 488 | 489 | columnsRef: columns, 490 | columnNames: columnNames, 491 | storedColumns: storedColumns, 492 | columnDirctions: columnDirctions, 493 | }, nil 494 | } 495 | 496 | func (i *TableIndex) Name() string { 497 | return i.name 498 | } 499 | 500 | func (i *TableIndex) IndexColumns() []*Column { 501 | return i.columnsRef 502 | } 503 | 504 | func (i *TableIndex) IndexColumnNames() []string { 505 | return i.columnNames 506 | } 507 | 508 | func (i *TableIndex) IndexColumnDirections() []string { 509 | return i.columnDirctions 510 | } 511 | 512 | func (i *TableIndex) HasColumn(c string) bool { 513 | _, ok := i.storedColumns[c] 514 | return ok 515 | } 516 | 517 | type TableView struct { 518 | ResultItems []ResultItem 519 | ResultItemsMap map[string]ResultItem 520 | ambiguous map[string]struct{} 521 | } 522 | 523 | func (v *TableView) AllItems() []ResultItem { 524 | return v.ResultItems 525 | } 526 | 527 | func (v *TableView) Get(id string) (ResultItem, bool, bool) { 528 | if _, ok := v.ambiguous[id]; ok { 529 | return ResultItem{}, true, false 530 | } 531 | item, ok := v.ResultItemsMap[id] 532 | if !ok { 533 | return ResultItem{}, false, true 534 | } 535 | return item, false, false 536 | } 537 | 538 | func (v *TableView) ToStruct() *StructType { 539 | names := make([]string, len(v.ResultItems)) 540 | vts := make([]*ValueType, len(v.ResultItems)) 541 | for i := range v.ResultItems { 542 | names[i] = v.ResultItems[i].Name 543 | vts[i] = &v.ResultItems[i].ValueType 544 | } 545 | 546 | return &StructType{ 547 | FieldNames: names, 548 | FieldTypes: vts, 549 | IsTable: true, 550 | } 551 | } 552 | 553 | func createTableViewFromItems(items1 []ResultItem, items2 []ResultItem) *TableView { 554 | newItems := make([]ResultItem, 0, len(items1)+len(items2)) 555 | newItemsMap := make(map[string]ResultItem, len(items1)+len(items2)) 556 | ambiguous := make(map[string]struct{}) 557 | 558 | for _, items := range [][]ResultItem{items1, items2} { 559 | for _, item := range items { 560 | newItems = append(newItems, item) 561 | if item.Name != "" { 562 | _, ok := newItemsMap[item.Name] 563 | if ok { 564 | ambiguous[item.Name] = struct{}{} 565 | } else { 566 | newItemsMap[item.Name] = item 567 | } 568 | } 569 | } 570 | } 571 | 572 | return &TableView{ 573 | ResultItems: newItems, 574 | ResultItemsMap: newItemsMap, 575 | ambiguous: ambiguous, 576 | } 577 | } 578 | 579 | func createTableViewFromTable(table *Table, alias string) *TableView { 580 | items := make([]ResultItem, 0, len(table.columns)) 581 | itemsMap := make(map[string]ResultItem, len(table.columns)) 582 | for _, column := range table.columns { 583 | item := createResultItemFromColumn(column) 584 | // if alias specified, add the alias to Expr 585 | if alias != "" { 586 | item.Expr.Raw = fmt.Sprintf("%s.%s", QuoteString(alias), item.Expr.Raw) 587 | } 588 | items = append(items, item) 589 | itemsMap[column.Name()] = item 590 | } 591 | return &TableView{ 592 | ResultItems: items, 593 | ResultItemsMap: itemsMap, 594 | } 595 | } 596 | 597 | type ResultItem struct { 598 | Name string 599 | ValueType ValueType 600 | 601 | Expr Expr 602 | } 603 | 604 | func createResultItemFromColumn(column *Column) ResultItem { 605 | return ResultItem{ 606 | Name: column.Name(), 607 | ValueType: column.valueType, 608 | Expr: Expr{ 609 | Raw: QuoteString(column.Name()), 610 | ValueType: column.valueType, 611 | }, 612 | } 613 | } 614 | -------------------------------------------------------------------------------- /server/transaction.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "database/sql" 21 | "fmt" 22 | "sync" 23 | "sync/atomic" 24 | "time" 25 | 26 | uuidpkg "github.com/google/uuid" 27 | spannerpb "google.golang.org/genproto/googleapis/spanner/v1" 28 | ) 29 | 30 | type databaseReader interface { 31 | QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) 32 | } 33 | 34 | type databaseWriter interface { 35 | ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) 36 | Commit() error 37 | Rollback() error 38 | } 39 | 40 | var ( 41 | ErrNotStartedTransaction = fmt.Errorf("transaction is not started") 42 | ErrNotAvailableTransaction = fmt.Errorf("transaction is not available") 43 | ErrInvalidatedTransaction = fmt.Errorf("transaction is invalidated") 44 | ) 45 | 46 | type transactionMode int 47 | 48 | const ( 49 | txReadOnly transactionMode = 1 50 | txReadWrite transactionMode = 2 51 | txPartitionedDML transactionMode = 3 52 | ) 53 | 54 | type TransactionStatus int 55 | 56 | const ( 57 | TransactionActive TransactionStatus = 1 58 | TransactionInvalidated TransactionStatus = 2 59 | TransactionCommited TransactionStatus = 3 60 | TransactionRollbacked TransactionStatus = 4 61 | TransactionAborted TransactionStatus = 5 62 | ) 63 | 64 | func (s TransactionStatus) String() string { 65 | switch s { 66 | case TransactionActive: 67 | return "ACTIVE" 68 | case TransactionInvalidated: 69 | return "INVALIDATED" 70 | case TransactionCommited: 71 | return "COMMITED" 72 | case TransactionRollbacked: 73 | return "ROLLBACKED" 74 | case TransactionAborted: 75 | return "ABORTED" 76 | default: 77 | return "UNKNOWN" 78 | } 79 | } 80 | 81 | type transaction struct { 82 | id []byte 83 | name string 84 | session *session 85 | single bool 86 | mode transactionMode 87 | status int32 // for atomic operation 88 | createdAt time.Time 89 | 90 | ctx context.Context 91 | cancel func() 92 | 93 | mu sync.RWMutex 94 | tx *sql.Tx 95 | close func(*transaction, *sql.Tx) // database.endTransaction() 96 | } 97 | 98 | func (tx *transaction) Context() context.Context { 99 | return tx.ctx 100 | } 101 | 102 | func (tx *transaction) Equals(t *transaction) bool { 103 | if tx == nil || t == nil { 104 | return false 105 | } 106 | 107 | if bytes.Equal(t.ID(), tx.ID()) { 108 | return true 109 | } 110 | 111 | return false 112 | } 113 | 114 | func (tx *transaction) ID() []byte { 115 | return tx.id 116 | } 117 | 118 | func (tx *transaction) Name() string { 119 | if tx == nil { 120 | return "" 121 | } 122 | return tx.name 123 | } 124 | 125 | func (tx *transaction) Proto() *spannerpb.Transaction { 126 | return &spannerpb.Transaction{ 127 | Id: tx.id, 128 | } 129 | } 130 | 131 | func (tx *transaction) Status() TransactionStatus { 132 | return TransactionStatus(atomic.LoadInt32(&tx.status)) 133 | } 134 | 135 | func (tx *transaction) ReadWrite() bool { 136 | return tx.mode == txReadWrite 137 | } 138 | 139 | func (tx *transaction) Available() bool { 140 | return tx.Status() == TransactionActive 141 | } 142 | 143 | func (tx *transaction) Invalidated() bool { 144 | st := tx.Status() 145 | return st == TransactionInvalidated || st == TransactionAborted 146 | } 147 | 148 | func (tx *transaction) SingleUse() bool { 149 | return tx.single 150 | } 151 | 152 | func (tx *transaction) Done(status TransactionStatus) { 153 | tx.mu.Lock() 154 | defer tx.mu.Unlock() 155 | 156 | // skip if the transaction is done already 157 | if tx.Status() != TransactionActive { 158 | return 159 | } 160 | 161 | Debugf("[%s] transaction.Done %s\n", tx.Name(), status) 162 | atomic.StoreInt32(&tx.status, int32(status)) 163 | 164 | if tx.close != nil { 165 | tx.close(tx, tx.tx) 166 | tx.close = nil 167 | } 168 | 169 | // Cancling transaction context is very unstable. 170 | // Call cancel after explicitly stop the transaction. 171 | tx.cancel() 172 | } 173 | 174 | func (tx *transaction) SetTransaction(dbtx *sql.Tx, closer func(*transaction, *sql.Tx)) error { 175 | tx.mu.Lock() 176 | defer tx.mu.Unlock() 177 | 178 | if tx.tx != nil { 179 | return fmt.Errorf("transaction already started") 180 | } 181 | 182 | tx.tx = dbtx 183 | tx.close = closer 184 | 185 | return nil 186 | } 187 | 188 | func (tx *transaction) WriteTransaction(fn func(databaseWriter) error) error { 189 | if IsDebug() { 190 | defer DebugStartEnd("[%s] transaction.WriteTransaction", tx.Name())() 191 | } 192 | 193 | tx.mu.Lock() 194 | defer tx.mu.Unlock() 195 | 196 | if !tx.Available() { 197 | return ErrNotAvailableTransaction 198 | } 199 | 200 | if tx.tx == nil { 201 | return ErrNotStartedTransaction 202 | } 203 | 204 | return fn(tx.tx) 205 | } 206 | 207 | func (tx *transaction) ReadTransaction(fn func(context.Context, databaseReader) error) error { 208 | if IsDebug() { 209 | defer DebugStartEnd("[%s] transaction.ReadTransaction", tx.Name())() 210 | } 211 | 212 | tx.mu.Lock() 213 | defer tx.mu.Unlock() 214 | 215 | if !tx.Available() { 216 | return ErrNotAvailableTransaction 217 | } 218 | 219 | if tx.tx == nil { 220 | return ErrNotStartedTransaction 221 | } 222 | 223 | return fn(tx.ctx, tx.tx) 224 | } 225 | 226 | func newTransaction(s *session, mode transactionMode, single bool) *transaction { 227 | id := uuidpkg.New().String() 228 | ctx, cancel := context.WithCancel(context.Background()) 229 | return &transaction{ 230 | id: []byte(id), 231 | name: id, 232 | session: s, 233 | single: single, 234 | mode: mode, 235 | status: int32(TransactionActive), 236 | createdAt: time.Now(), 237 | 238 | ctx: ctx, 239 | cancel: cancel, 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /server/value.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "database/sql" 19 | "database/sql/driver" 20 | "encoding/base64" 21 | "encoding/json" 22 | "fmt" 23 | "reflect" 24 | "strconv" 25 | "strings" 26 | "time" 27 | 28 | spannerpb "google.golang.org/genproto/googleapis/spanner/v1" 29 | "google.golang.org/grpc/codes" 30 | "google.golang.org/grpc/status" 31 | structpb "google.golang.org/protobuf/types/known/structpb" 32 | ) 33 | 34 | var NullExpr = Expr{} 35 | var NullValue = Value{} 36 | 37 | type Expr struct { 38 | Raw string 39 | ValueType ValueType 40 | Args []interface{} 41 | } 42 | 43 | type RowIterator interface { 44 | ResultSet() []ResultItem 45 | 46 | Do(func([]interface{}) error) error 47 | } 48 | 49 | type Value struct { 50 | Data interface{} 51 | Type ValueType 52 | } 53 | 54 | type ValueType struct { 55 | Code TypeCode 56 | ArrayType *ValueType 57 | StructType *StructType 58 | } 59 | 60 | func (t ValueType) IsArray() bool { 61 | return t.Code == TCArray 62 | } 63 | 64 | func (t ValueType) IsStruct() bool { 65 | return t.Code == TCStruct 66 | } 67 | 68 | func (t ValueType) String() string { 69 | switch t.Code { 70 | case TCBool, TCInt64, TCFloat64, TCTimestamp, TCDate, TCString, TCBytes: 71 | return t.Code.String() 72 | case TCArray: 73 | return fmt.Sprintf("ARRAY<%s>", t.ArrayType.String()) 74 | case TCStruct: 75 | n := len(t.StructType.FieldTypes) 76 | ss := make([]string, n) 77 | for i := 0; i < n; i++ { 78 | name := t.StructType.FieldNames[i] 79 | vt := t.StructType.FieldTypes[i] 80 | if name == "" { 81 | ss[i] = vt.String() 82 | } else { 83 | ss[i] = name + " " + vt.String() 84 | 85 | } 86 | } 87 | return fmt.Sprintf("STRUCT<%s>", strings.Join(ss, ", ")) 88 | } 89 | return "(unknown type)" 90 | } 91 | 92 | func compareValueType(a, b ValueType) bool { 93 | if a.Code != b.Code { 94 | return false 95 | } 96 | 97 | if a.Code == TCStruct && b.Code == TCStruct { 98 | aStr := a.StructType 99 | bStr := b.StructType 100 | 101 | if len(aStr.FieldTypes) != len(bStr.FieldTypes) { 102 | return false 103 | } 104 | for i := 0; i < len(aStr.FieldTypes); i++ { 105 | b := compareValueType(*aStr.FieldTypes[i], *bStr.FieldTypes[i]) 106 | if !b { 107 | return false 108 | } 109 | } 110 | 111 | return true 112 | } 113 | 114 | if a.Code == TCArray && b.Code == TCArray { 115 | return compareValueType(*a.ArrayType, *b.ArrayType) 116 | } 117 | 118 | return true 119 | } 120 | 121 | func compatibleValueType(a, b ValueType) (ValueType, bool) { 122 | if a.Code == TCInt64 && b.Code == TCFloat64 { 123 | return b, true 124 | } 125 | if b.Code == TCInt64 && a.Code == TCFloat64 { 126 | return a, true 127 | } 128 | return a, a == b 129 | } 130 | 131 | func decideArrayElementsValueType(vts ...ValueType) (ValueType, error) { 132 | vt := ValueType{Code: TCInt64} 133 | if len(vts) > 0 { 134 | vt = vts[0] 135 | } 136 | 137 | used := map[string]struct{}{} 138 | 139 | for i := range vts { 140 | used[vts[i].String()] = struct{}{} 141 | } 142 | 143 | for i := range vts { 144 | var ok bool 145 | // TODO: if ValueType is changed, types of all values also need changing 146 | // vt, ok = compatibleValueType(vt, vts[i]) 147 | ok = compareValueType(vt, vts[i]) 148 | if !ok { 149 | var typ string 150 | first := true 151 | for n := range used { 152 | if !first { 153 | typ += ", " 154 | } 155 | typ += n 156 | first = false 157 | } 158 | 159 | return ValueType{}, fmt.Errorf("Array elements of types {%s} do not have a common supertype", typ) 160 | } 161 | } 162 | 163 | return vt, nil 164 | } 165 | 166 | type StructType struct { 167 | FieldNames []string 168 | FieldTypes []*ValueType 169 | 170 | // Table can be struct but it behaves differently. 171 | // So a struct created from table should be marked. 172 | IsTable bool 173 | } 174 | 175 | func (s *StructType) AllItems() []ResultItem { 176 | n := len(s.FieldTypes) 177 | items := make([]ResultItem, n) 178 | for i := 0; i < n; i++ { 179 | name := s.FieldNames[i] 180 | vt := s.FieldTypes[i] 181 | items[i] = ResultItem{ 182 | Name: name, 183 | ValueType: *vt, 184 | Expr: Expr{ 185 | Raw: name, 186 | ValueType: *vt, 187 | }, 188 | } 189 | } 190 | 191 | return items 192 | } 193 | 194 | type TypeCode int32 195 | 196 | func (c TypeCode) String() string { 197 | switch c { 198 | case TCBool: 199 | return "BOOL" 200 | case TCInt64: 201 | return "INT64" 202 | case TCFloat64: 203 | return "FLOAT64" 204 | case TCTimestamp: 205 | return "TIMESTAMP" 206 | case TCDate: 207 | return "DATE" 208 | case TCString: 209 | return "STRING" 210 | case TCBytes: 211 | return "BYTES" 212 | case TCArray: 213 | return "ARRAY" 214 | case TCStruct: 215 | return "STRUCT" 216 | default: 217 | return "(unknown)" 218 | } 219 | } 220 | 221 | const ( 222 | TCBool TypeCode = iota + 1 223 | TCInt64 224 | TCFloat64 225 | TCTimestamp 226 | TCDate 227 | TCString 228 | TCBytes 229 | TCArray 230 | TCStruct 231 | TCJson 232 | ) 233 | 234 | type ArrayValue interface { 235 | Elements() interface{} 236 | } 237 | 238 | type ArrayValueEncoder struct { 239 | Values interface{} 240 | Invalid bool 241 | } 242 | 243 | func (a *ArrayValueEncoder) Value() (driver.Value, error) { 244 | if a.Invalid { 245 | return nil, fmt.Errorf("cannot use invalid value") 246 | } 247 | 248 | b, err := json.Marshal(a.Values) 249 | if err != nil { 250 | return nil, fmt.Errorf("json.Marshal failed in %T: %v", a, err) 251 | } 252 | 253 | return driver.Value(string(b)), nil 254 | } 255 | 256 | func (a *ArrayValueEncoder) Elements() interface{} { 257 | return a.Values 258 | } 259 | 260 | type BoolDecoder struct { 261 | Bool *bool 262 | } 263 | 264 | func (b *BoolDecoder) UnmarshalJSON(data []byte) error { 265 | if len(data) == 1 { 266 | var v bool 267 | if data[0] == '0' { 268 | v = false 269 | b.Bool = &v 270 | return nil 271 | } else if data[0] == '1' { 272 | v = true 273 | b.Bool = &v 274 | return nil 275 | } 276 | } 277 | 278 | if err := json.Unmarshal(data, &b.Bool); err != nil { 279 | return fmt.Errorf("json.Unmarshal failed for bool: %v", err) 280 | } 281 | 282 | return nil 283 | } 284 | 285 | type ArrayValueDecoder struct { 286 | Values interface{} 287 | Type ValueType 288 | Invalid bool `json:"-"` 289 | } 290 | 291 | func (a *ArrayValueDecoder) Value() interface{} { 292 | return a.Values 293 | } 294 | 295 | func (a *ArrayValueDecoder) Scan(src interface{}) error { 296 | if src == nil { 297 | a.Invalid = true 298 | return nil 299 | } 300 | 301 | switch v := src.(type) { 302 | case string: 303 | return a.UnmarshalJSON([]byte(v)) 304 | case []uint8: 305 | res := make([]int64, len(v)) 306 | for i, _ := range v { 307 | res[i] = int64(v[i]) 308 | } 309 | a.Values = res 310 | default: 311 | return fmt.Errorf("unexpected type %T for %T", src, a) 312 | } 313 | return nil 314 | } 315 | 316 | func (a *ArrayValueDecoder) UnmarshalJSON(b []byte) error { 317 | if a.Type.ArrayType.Code == TCStruct { 318 | var arr []json.RawMessage 319 | if err := json.Unmarshal(b, &arr); err != nil { 320 | return fmt.Errorf("json.Unmarshal failed in %T: %v", a, err) 321 | } 322 | 323 | var svs []*StructValue 324 | for _, b2 := range arr { 325 | vv, err := a.decodeStruct(b2, a.Type.ArrayType.StructType) 326 | if err != nil { 327 | return err 328 | } 329 | 330 | svs = append(svs, vv) 331 | } 332 | 333 | a.Values = svs 334 | } else { 335 | v, err := a.decodeValue(b, a.Type) 336 | if err != nil { 337 | return err 338 | } 339 | 340 | a.Values = v 341 | } 342 | 343 | return nil 344 | } 345 | 346 | func (a *ArrayValueDecoder) decodeStruct(b []byte, typ *StructType) (*StructValue, error) { 347 | var vv struct { 348 | Keys []string `json:"keys"` 349 | Values []json.RawMessage `json:"values"` 350 | } 351 | 352 | if err := json.Unmarshal(b, &vv); err != nil { 353 | return nil, fmt.Errorf("json.Unmarshal failed in %T: %v", a, err) 354 | } 355 | 356 | var values []interface{} 357 | for i, value := range vv.Values { 358 | typ := typ.FieldTypes[i] 359 | vvv, err := a.decodeValue(value, *typ) 360 | if err != nil { 361 | return nil, err 362 | } 363 | 364 | values = append(values, vvv) 365 | } 366 | 367 | return &StructValue{ 368 | Keys: vv.Keys, 369 | Values: values, 370 | }, nil 371 | } 372 | 373 | func (a *ArrayValueDecoder) decodeValue(b []byte, typ ValueType) (interface{}, error) { 374 | var rv reflect.Value 375 | switch typ.Code { 376 | case TCBool: 377 | rv = reflect.New(reflect.TypeOf(BoolDecoder{})) 378 | case TCInt64: 379 | rv = reflect.New(reflect.TypeOf(int64(0))) 380 | case TCFloat64: 381 | rv = reflect.New(reflect.TypeOf(float64(0))) 382 | case TCTimestamp, TCDate, TCString: 383 | rv = reflect.New(reflect.TypeOf(string(""))) 384 | case TCBytes: 385 | rv = reflect.New(reflect.TypeOf([]byte{})) 386 | case TCArray: 387 | switch typ.ArrayType.Code { 388 | case TCBool: 389 | rv = reflect.New(reflect.TypeOf([]*BoolDecoder{})) 390 | case TCInt64: 391 | rv = reflect.New(reflect.TypeOf([]*int64{})) 392 | case TCFloat64: 393 | rv = reflect.New(reflect.TypeOf([]*float64{})) 394 | case TCTimestamp, TCDate, TCString: 395 | rv = reflect.New(reflect.TypeOf([]*string{})) 396 | case TCBytes: 397 | rv = reflect.New(reflect.TypeOf([][]byte{})) 398 | case TCStruct: 399 | v := reflect.New(reflect.TypeOf(ArrayValueDecoder{})) 400 | reflect.Indirect(v).FieldByName("Type").Set(reflect.ValueOf(typ)) 401 | rv = v 402 | 403 | default: 404 | return nil, fmt.Errorf("unknownn supported type for Array: %v", typ.ArrayType.Code) 405 | } 406 | case TCStruct: 407 | return nil, fmt.Errorf("unknown supported type: %v", typ.Code) 408 | } 409 | 410 | rvv := rv.Interface() 411 | if err := json.Unmarshal([]byte(b), rvv); err != nil { 412 | return nil, fmt.Errorf("json.Unmarshalll failed for %T in %T: %v", rvv, a, err) 413 | } 414 | 415 | var value interface{} 416 | switch vv := rv.Interface().(type) { 417 | case *BoolDecoder: 418 | value = *vv.Bool 419 | case *[]*BoolDecoder: 420 | if vv == nil { 421 | value = nil 422 | } else { 423 | vv := *vv 424 | vs := make([]*bool, len(vv)) 425 | for i := 0; i < len(vv); i++ { 426 | if vv[i] == nil { 427 | vs[i] = nil 428 | } else { 429 | vs[i] = vv[i].Bool 430 | } 431 | } 432 | value = vs 433 | } 434 | case *ArrayValueDecoder: 435 | value = vv.Value() 436 | default: 437 | value = reflect.Indirect(rv).Interface() 438 | } 439 | 440 | return value, nil 441 | } 442 | 443 | type StructValue struct { 444 | Keys []string `json:"keys"` 445 | Values []interface{} `json:"values"` 446 | } 447 | 448 | type rows struct { 449 | rows *sql.Rows 450 | resultItems []ResultItem 451 | transaction *transaction 452 | 453 | lastErr error 454 | } 455 | 456 | func (r *rows) ResultSet() []ResultItem { 457 | return r.resultItems 458 | } 459 | 460 | func (it *rows) Do(fn func([]interface{}) error) error { 461 | var lastErr error 462 | var rows []interface{} 463 | for { 464 | row, ok := it.next() 465 | if !ok { 466 | break 467 | } 468 | rows = append(rows, row) 469 | if err := fn(row); err != nil { 470 | lastErr = err 471 | break 472 | } 473 | } 474 | if it.lastErr != nil { 475 | if lastErr == nil { 476 | lastErr = it.lastErr 477 | } 478 | } 479 | 480 | if err := it.rows.Err(); err != nil { 481 | if lastErr == nil { 482 | lastErr = err 483 | } 484 | } 485 | if err := it.rows.Close(); err != nil { 486 | if lastErr == nil { 487 | lastErr = err 488 | } 489 | } 490 | 491 | // convert sqlite runtime error as InvalidArgument error if it is SqliteArgumentRuntimeError. 492 | if lastErr != nil { 493 | msg := lastErr.Error() 494 | if strings.HasPrefix(msg, SqliteArgumentRuntimeErrorPrefix) { 495 | msg = strings.TrimPrefix(msg, SqliteArgumentRuntimeErrorPrefix) 496 | return status.Errorf(codes.InvalidArgument, "%s", msg) 497 | } 498 | if strings.HasPrefix(msg, SqliteOutOfRangeRuntimeErrorPrefix) { 499 | msg = strings.TrimPrefix(msg, SqliteOutOfRangeRuntimeErrorPrefix) 500 | return status.Errorf(codes.OutOfRange, "%s", msg) 501 | } 502 | } 503 | 504 | // database/sql has a possible bug that cannot read any data without error. 505 | // It may happen context is canceled in bad timing. 506 | // Here checks the transaction is available or not and if it's not available return aborted error. 507 | if !it.transaction.Available() { 508 | lastErr = status.Errorf(codes.Aborted, "transaction aborted") 509 | } 510 | 511 | return lastErr 512 | } 513 | 514 | func (it *rows) next() ([]interface{}, bool) { 515 | ok := it.rows.Next() 516 | if !ok { 517 | return nil, false 518 | } 519 | 520 | values := make([]reflect.Value, len(it.resultItems)) 521 | ptrs := make([]interface{}, len(it.resultItems)) 522 | for i, item := range it.resultItems { 523 | switch item.ValueType.Code { 524 | case TCBool: 525 | values[i] = reflect.New(reflect.TypeOf(sql.NullBool{})) 526 | case TCInt64: 527 | values[i] = reflect.New(reflect.TypeOf(sql.NullInt64{})) 528 | case TCFloat64: 529 | values[i] = reflect.New(reflect.TypeOf(sql.NullFloat64{})) 530 | case TCTimestamp, TCDate, TCString, TCJson: 531 | values[i] = reflect.New(reflect.TypeOf(sql.NullString{})) 532 | case TCBytes: 533 | values[i] = reflect.New(reflect.TypeOf(&[]byte{})) 534 | case TCArray: 535 | v := reflect.New(reflect.TypeOf(ArrayValueDecoder{})) 536 | reflect.Indirect(v).FieldByName("Type").Set(reflect.ValueOf(item.ValueType)) 537 | values[i] = v 538 | case TCStruct: 539 | it.lastErr = fmt.Errorf("unknown supported type: %v", item.ValueType.Code) 540 | return nil, false 541 | } 542 | ptrs[i] = values[i].Interface() 543 | } 544 | 545 | if err := it.rows.Scan(ptrs...); err != nil { 546 | it.lastErr = err 547 | return nil, false 548 | } 549 | 550 | data := make([]interface{}, len(it.resultItems)) 551 | 552 | for i := range values { 553 | v := reflect.Indirect(values[i]).Interface() 554 | switch vv := v.(type) { 555 | case sql.NullBool: 556 | if !vv.Valid { 557 | data[i] = nil 558 | } else { 559 | data[i] = vv.Bool 560 | } 561 | 562 | case sql.NullString: 563 | if !vv.Valid { 564 | data[i] = nil 565 | } else { 566 | data[i] = vv.String 567 | } 568 | case sql.NullInt64: 569 | if !vv.Valid { 570 | data[i] = nil 571 | } else { 572 | data[i] = vv.Int64 573 | } 574 | case sql.NullFloat64: 575 | if !vv.Valid { 576 | data[i] = nil 577 | } else { 578 | data[i] = vv.Float64 579 | } 580 | case *[]byte: 581 | if vv == nil { 582 | data[i] = nil 583 | } else { 584 | data[i] = *vv 585 | } 586 | case ArrayValueDecoder: 587 | if vv.Invalid { 588 | data[i] = nil 589 | } else { 590 | data[i] = vv.Value() 591 | } 592 | default: 593 | data[i] = v 594 | } 595 | } 596 | 597 | return data, true 598 | } 599 | 600 | func convertToDatabaseValues(lv *structpb.ListValue, columns []*Column) ([]interface{}, error) { 601 | values := make([]interface{}, 0, len(columns)) 602 | for i, v := range lv.Values { 603 | column := columns[i] 604 | vv, err := spannerValue2DatabaseValue(v, *column) 605 | if err != nil { 606 | return nil, status.Errorf(codes.InvalidArgument, "%v", err) 607 | } 608 | values = append(values, vv) 609 | } 610 | return values, nil 611 | } 612 | 613 | func spannerValue2DatabaseValue(v *structpb.Value, col Column) (interface{}, error) { 614 | // special handling of commit_stamp 615 | // It needs to be checked if the column allows to use commit_timestamp 616 | if col.valueType.Code == TCTimestamp { 617 | if vv, ok := v.Kind.(*structpb.Value_StringValue); ok { 618 | s := vv.StringValue 619 | if s == "spanner.commit_timestamp()" { 620 | if !col.allowCommitTimestamp { 621 | msg := "Cannot write commit timestamp because the allow_commit_timestamp column option is not set to true for column %s, or for all corresponding shared key columns in this table's interleaved table hierarchy." 622 | return nil, fmt.Errorf(msg, col.Name) // TODO: return FailedPrecondition 623 | } 624 | now := time.Now().UTC() 625 | vv.StringValue = now.Format(time.RFC3339Nano) 626 | } 627 | } 628 | } 629 | 630 | vv, err := makeDataFromSpannerValue(col.Name(), v, col.valueType) 631 | if err != nil { 632 | return nil, err 633 | } 634 | 635 | // sqlite doesn not support nil with type like []string(nil) 636 | // explicitly convert those values to nil to store as null value 637 | rv := reflect.ValueOf(vv) 638 | if rv.Kind() == reflect.Slice && rv.IsNil() { 639 | return nil, nil 640 | } 641 | 642 | return vv, nil 643 | } 644 | 645 | func encodeBase64(b []byte) string { 646 | return base64.StdEncoding.EncodeToString(b) 647 | } 648 | 649 | func decodeBase64(s string) ([]byte, error) { 650 | b, err := base64.StdEncoding.DecodeString(s) 651 | if err != nil { 652 | // seems Spanner tries to use both padding and no-padding 653 | return base64.RawStdEncoding.DecodeString(s) 654 | } 655 | return b, nil 656 | } 657 | 658 | func makeSpannerTypeFromValueType(typ ValueType) *spannerpb.Type { 659 | var code spannerpb.TypeCode 660 | switch typ.Code { 661 | case TCBool: 662 | code = spannerpb.TypeCode_BOOL 663 | case TCInt64: 664 | code = spannerpb.TypeCode_INT64 665 | case TCFloat64: 666 | code = spannerpb.TypeCode_FLOAT64 667 | case TCTimestamp: 668 | code = spannerpb.TypeCode_TIMESTAMP 669 | case TCDate: 670 | code = spannerpb.TypeCode_DATE 671 | case TCString: 672 | code = spannerpb.TypeCode_STRING 673 | case TCBytes: 674 | code = spannerpb.TypeCode_BYTES 675 | case TCArray: 676 | code = spannerpb.TypeCode_ARRAY 677 | case TCStruct: 678 | code = spannerpb.TypeCode_STRUCT 679 | case TCJson: 680 | code = spannerpb.TypeCode_JSON 681 | } 682 | 683 | st := &spannerpb.Type{Code: code} 684 | if code == spannerpb.TypeCode_ARRAY { 685 | st = &spannerpb.Type{ 686 | Code: code, 687 | ArrayElementType: makeSpannerTypeFromValueType(*typ.ArrayType), 688 | } 689 | } 690 | if code == spannerpb.TypeCode_STRUCT { 691 | n := len(typ.StructType.FieldTypes) 692 | fields := make([]*spannerpb.StructType_Field, n) 693 | for i := 0; i < n; i++ { 694 | fields[i] = &spannerpb.StructType_Field{ 695 | Name: typ.StructType.FieldNames[i], 696 | Type: makeSpannerTypeFromValueType(*typ.StructType.FieldTypes[i]), 697 | } 698 | } 699 | 700 | st = &spannerpb.Type{ 701 | Code: code, 702 | StructType: &spannerpb.StructType{ 703 | Fields: fields, 704 | }, 705 | } 706 | } 707 | return st 708 | } 709 | 710 | func makeValueTypeFromSpannerType(typ *spannerpb.Type) (ValueType, error) { 711 | switch typ.Code { 712 | case spannerpb.TypeCode_BOOL: 713 | return ValueType{ 714 | Code: TCBool, 715 | }, nil 716 | case spannerpb.TypeCode_INT64: 717 | return ValueType{ 718 | Code: TCInt64, 719 | }, nil 720 | case spannerpb.TypeCode_FLOAT64: 721 | return ValueType{ 722 | Code: TCFloat64, 723 | }, nil 724 | case spannerpb.TypeCode_TIMESTAMP: 725 | return ValueType{ 726 | Code: TCTimestamp, 727 | }, nil 728 | case spannerpb.TypeCode_DATE: 729 | return ValueType{ 730 | Code: TCDate, 731 | }, nil 732 | case spannerpb.TypeCode_STRING: 733 | return ValueType{ 734 | Code: TCString, 735 | }, nil 736 | case spannerpb.TypeCode_BYTES: 737 | return ValueType{ 738 | Code: TCBytes, 739 | }, nil 740 | case spannerpb.TypeCode_ARRAY: 741 | var array *ValueType 742 | if typ.ArrayElementType != nil { 743 | vt, err := makeValueTypeFromSpannerType(typ.ArrayElementType) 744 | if err != nil { 745 | return ValueType{}, err 746 | } 747 | array = &vt 748 | } 749 | return ValueType{ 750 | Code: TCArray, 751 | ArrayType: array, 752 | }, nil 753 | case spannerpb.TypeCode_STRUCT: 754 | fields := typ.GetStructType().GetFields() 755 | names := make([]string, len(fields)) 756 | types := make([]*ValueType, len(fields)) 757 | for i, field := range fields { 758 | vt, err := makeValueTypeFromSpannerType(field.Type) 759 | if err != nil { 760 | return ValueType{}, err 761 | } 762 | names[i] = field.Name 763 | types[i] = &vt 764 | } 765 | return ValueType{ 766 | Code: TCStruct, 767 | StructType: &StructType{ 768 | FieldNames: names, 769 | FieldTypes: types, 770 | }, 771 | }, nil 772 | case spannerpb.TypeCode_JSON: 773 | return ValueType{ 774 | Code: TCJson, 775 | }, nil 776 | } 777 | 778 | return ValueType{}, fmt.Errorf("unknown code for spanner.Type: %v", typ.Code) 779 | } 780 | 781 | func spannerValueFromValue(x interface{}) (*structpb.Value, error) { 782 | switch x := x.(type) { 783 | case nil: 784 | return &structpb.Value{Kind: &structpb.Value_NullValue{}}, nil 785 | 786 | case bool: 787 | return &structpb.Value{Kind: &structpb.Value_BoolValue{x}}, nil 788 | case int64: 789 | // The Spanner int64 is actually a decimal string. 790 | s := strconv.FormatInt(x, 10) 791 | return &structpb.Value{Kind: &structpb.Value_StringValue{s}}, nil 792 | case float64: 793 | return &structpb.Value{Kind: &structpb.Value_NumberValue{x}}, nil 794 | case string: 795 | return &structpb.Value{Kind: &structpb.Value_StringValue{x}}, nil 796 | case []byte: 797 | if x == nil { 798 | return &structpb.Value{Kind: &structpb.Value_NullValue{}}, nil 799 | } else { 800 | return &structpb.Value{Kind: &structpb.Value_StringValue{encodeBase64(x)}}, nil 801 | } 802 | case StructValue: 803 | n := len(x.Values) 804 | fields := make(map[string]*structpb.Value) 805 | for i := 0; i < n; i++ { 806 | elem := x.Values[i] 807 | v, err := spannerValueFromValue(elem) 808 | if err != nil { 809 | return nil, err 810 | } 811 | fields[x.Keys[i]] = v 812 | } 813 | return &structpb.Value{Kind: &structpb.Value_StructValue{ 814 | StructValue: &structpb.Struct{ 815 | Fields: fields, 816 | }, 817 | }}, nil 818 | 819 | case []*bool, []*int64, []*float64, []*string, []*StructValue, [][]byte: 820 | rv := reflect.ValueOf(x) 821 | n := rv.Len() 822 | vs := make([]*structpb.Value, n) 823 | for i := 0; i < n; i++ { 824 | var elem interface{} 825 | 826 | rvv := rv.Index(i) 827 | if !rvv.IsNil() { 828 | elem = reflect.Indirect(rv.Index(i)).Interface() 829 | } 830 | 831 | v, err := spannerValueFromValue(elem) 832 | if err != nil { 833 | return nil, err 834 | } 835 | vs[i] = v 836 | } 837 | return &structpb.Value{Kind: &structpb.Value_ListValue{ 838 | &structpb.ListValue{Values: vs}, 839 | }}, nil 840 | 841 | default: 842 | return nil, fmt.Errorf("unknown database value type %T", x) 843 | } 844 | } 845 | 846 | func makeDataFromSpannerValue(key string, v *structpb.Value, typ ValueType) (interface{}, error) { 847 | if typ.StructType != nil { 848 | return nil, newBindingErrorf(key, typ, "Struct type is not supported yet") 849 | } 850 | 851 | if typ.Code == TCArray { 852 | if typ.ArrayType == nil { 853 | return nil, status.Error(codes.InvalidArgument, "The array_element_type field is required for ARRAYs") 854 | } 855 | 856 | if _, ok := v.Kind.(*structpb.Value_NullValue); ok { 857 | switch typ.ArrayType.Code { 858 | case TCBool: 859 | return []bool(nil), nil 860 | case TCInt64: 861 | return []int64(nil), nil 862 | case TCFloat64: 863 | return []float64(nil), nil 864 | case TCTimestamp, TCDate, TCString: 865 | return []string(nil), nil 866 | case TCBytes: 867 | return [][]byte(nil), nil 868 | case TCArray, TCStruct: 869 | // this should not be error actually but no reason to support. 870 | return nil, newBindingErrorf(key, typ, "nested Array or Struct for Array is not supported yet") 871 | default: 872 | return nil, fmt.Errorf("unexpected type %d for Null value as Array", typ.ArrayType.Code) 873 | } 874 | } 875 | 876 | vv, ok := v.Kind.(*structpb.Value_ListValue) 877 | if !ok { 878 | return nil, newBindingErrorf(key, typ, "unexpected value %T and type %s as Array", v.Kind, typ) 879 | } 880 | 881 | n := len(vv.ListValue.Values) 882 | switch typ.ArrayType.Code { 883 | case TCBool: 884 | ret := make([]*bool, n) 885 | for i, vv := range vv.ListValue.Values { 886 | elemKey := fmt.Sprintf("%s[%d]", key, i) 887 | vvv, err := makeDataFromSpannerValue(elemKey, vv, *typ.ArrayType) 888 | if err != nil { 889 | return nil, err 890 | } 891 | if vvv == nil { 892 | ret[i] = nil 893 | } else { 894 | vvvv, ok := vvv.(bool) 895 | if !ok { 896 | panic(fmt.Sprintf("unexpected value type: %T", vvv)) 897 | } 898 | ret[i] = &vvvv 899 | } 900 | } 901 | return &ArrayValueEncoder{Values: ret}, nil 902 | case TCInt64: 903 | ret := make([]*int64, n) 904 | for i, vv := range vv.ListValue.Values { 905 | elemKey := fmt.Sprintf("%s[%d]", key, i) 906 | vvv, err := makeDataFromSpannerValue(elemKey, vv, *typ.ArrayType) 907 | if err != nil { 908 | return nil, err 909 | } 910 | if vvv == nil { 911 | ret[i] = nil 912 | } else { 913 | vvvv, ok := vvv.(int64) 914 | if !ok { 915 | panic(fmt.Sprintf("unexpected value type: %T", vvv)) 916 | } 917 | ret[i] = &vvvv 918 | } 919 | } 920 | return &ArrayValueEncoder{Values: ret}, nil 921 | case TCFloat64: 922 | ret := make([]*float64, n) 923 | for i, vv := range vv.ListValue.Values { 924 | elemKey := fmt.Sprintf("%s[%d]", key, i) 925 | vvv, err := makeDataFromSpannerValue(elemKey, vv, *typ.ArrayType) 926 | if err != nil { 927 | return nil, err 928 | } 929 | if vvv == nil { 930 | ret[i] = nil 931 | } else { 932 | vvvv, ok := vvv.(float64) 933 | if !ok { 934 | panic(fmt.Sprintf("unexpected value type: %T", vvv)) 935 | } 936 | ret[i] = &vvvv 937 | } 938 | } 939 | return &ArrayValueEncoder{Values: ret}, nil 940 | case TCTimestamp, TCDate, TCString: 941 | ret := make([]*string, n) 942 | for i, vv := range vv.ListValue.Values { 943 | elemKey := fmt.Sprintf("%s[%d]", key, i) 944 | vvv, err := makeDataFromSpannerValue(elemKey, vv, *typ.ArrayType) 945 | if err != nil { 946 | return nil, err 947 | } 948 | if vvv == nil { 949 | ret[i] = nil 950 | } else { 951 | vvvv, ok := vvv.(string) 952 | if !ok { 953 | panic(fmt.Sprintf("unexpected value type: %T", vvv)) 954 | } 955 | ret[i] = &vvvv 956 | } 957 | } 958 | return &ArrayValueEncoder{Values: ret}, nil 959 | case TCBytes: 960 | ret := make([][]byte, n) 961 | for i, vv := range vv.ListValue.Values { 962 | elemKey := fmt.Sprintf("%s[%d]", key, i) 963 | vvv, err := makeDataFromSpannerValue(elemKey, vv, *typ.ArrayType) 964 | if err != nil { 965 | return nil, err 966 | } 967 | if vvv == nil { 968 | ret[i] = nil 969 | } else { 970 | vvvv, ok := vvv.([]byte) 971 | if !ok { 972 | panic(fmt.Sprintf("unexpected value type: %T", vvv)) 973 | } 974 | ret[i] = vvvv 975 | } 976 | } 977 | return &ArrayValueEncoder{Values: ret}, nil 978 | case TCArray, TCStruct: 979 | // just visit elements for appropriate error handling 980 | for i, vv := range vv.ListValue.Values { 981 | elemKey := fmt.Sprintf("%s[%d]", key, i) 982 | _, err := makeDataFromSpannerValue(elemKey, vv, *typ.ArrayType) 983 | if err != nil { 984 | return nil, err 985 | } 986 | } 987 | 988 | // must be unreachable 989 | panic("array of array or array of struct is not supported") 990 | 991 | default: 992 | return nil, fmt.Errorf("unknown TypeCode for ArrayElement %v", typ.Code) 993 | } 994 | } 995 | 996 | if _, ok := v.Kind.(*structpb.Value_NullValue); ok { 997 | return nil, nil 998 | } 999 | 1000 | switch typ.Code { 1001 | case TCBool: 1002 | switch vv := v.Kind.(type) { 1003 | case *structpb.Value_BoolValue: 1004 | return vv.BoolValue, nil 1005 | } 1006 | case TCInt64: 1007 | switch vv := v.Kind.(type) { 1008 | case *structpb.Value_StringValue: 1009 | // base is always 10 1010 | n, err := strconv.ParseInt(vv.StringValue, 10, 64) 1011 | if err != nil { 1012 | return nil, newBindingErrorf(key, typ, "unexpected format %q as int64: %v", vv.StringValue, err) 1013 | } 1014 | return n, nil 1015 | } 1016 | 1017 | case TCFloat64: 1018 | switch vv := v.Kind.(type) { 1019 | case *structpb.Value_NumberValue: 1020 | return vv.NumberValue, nil 1021 | } 1022 | 1023 | case TCTimestamp: 1024 | switch vv := v.Kind.(type) { 1025 | case *structpb.Value_StringValue: 1026 | s := vv.StringValue 1027 | if _, err := time.Parse(time.RFC3339Nano, s); err != nil { 1028 | return nil, newBindingErrorf(key, typ, "unexpected format %q as timestamp: %v", s, err) 1029 | } 1030 | return s, nil 1031 | } 1032 | 1033 | case TCDate: 1034 | switch vv := v.Kind.(type) { 1035 | case *structpb.Value_StringValue: 1036 | s := vv.StringValue 1037 | if _, err := time.Parse("2006-01-02", s); err != nil { 1038 | return nil, newBindingErrorf(key, typ, "unexpected format for %q as date: %v", s, err) 1039 | } 1040 | return s, nil 1041 | } 1042 | 1043 | case TCString, TCJson: 1044 | switch vv := v.Kind.(type) { 1045 | case *structpb.Value_StringValue: 1046 | return vv.StringValue, nil 1047 | } 1048 | case TCBytes: 1049 | switch vv := v.Kind.(type) { 1050 | case *structpb.Value_StringValue: 1051 | b, err := decodeBase64(vv.StringValue) 1052 | if err != nil { 1053 | return nil, newBindingErrorf(key, typ, "decoding base64 failed: %v", err) 1054 | } 1055 | return b, nil 1056 | } 1057 | default: 1058 | return nil, fmt.Errorf("unknown Type %s", typ) 1059 | } 1060 | 1061 | return nil, newBindingErrorf(key, typ, "unexpected value %T and type %s", v.Kind, typ) 1062 | } 1063 | 1064 | func makeValueFromSpannerValue(key string, v *structpb.Value, typ *spannerpb.Type) (Value, error) { 1065 | vt, err := makeValueTypeFromSpannerType(typ) 1066 | if err != nil { 1067 | return Value{}, err 1068 | } 1069 | 1070 | data, err := makeDataFromSpannerValue(key, v, vt) 1071 | if err != nil { 1072 | return Value{}, err 1073 | } 1074 | 1075 | return Value{ 1076 | Data: data, 1077 | Type: vt, 1078 | }, nil 1079 | } 1080 | 1081 | type bindingError struct { 1082 | key string 1083 | msg string 1084 | expected ValueType 1085 | } 1086 | 1087 | func (e *bindingError) Error() string { 1088 | return e.msg 1089 | } 1090 | 1091 | func (e *bindingError) GRPCStatus() *status.Status { 1092 | return status.Newf(codes.InvalidArgument, "Invalid value for bind parameter %s: Expected %s.", e.key, e.expected) 1093 | } 1094 | 1095 | func newBindingErrorf(key string, expected ValueType, format string, a ...interface{}) error { 1096 | return &bindingError{ 1097 | key: key, 1098 | expected: expected, 1099 | msg: fmt.Sprintf(format, a...), 1100 | } 1101 | } 1102 | -------------------------------------------------------------------------------- /server/value_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Masahiro Sano 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "regexp" 21 | "testing" 22 | "time" 23 | 24 | "database/sql" 25 | 26 | "github.com/cloudspannerecosystem/memefish/ast" 27 | cmp "github.com/google/go-cmp/cmp" 28 | uuidpkg "github.com/google/uuid" 29 | spannerpb "google.golang.org/genproto/googleapis/spanner/v1" 30 | "google.golang.org/grpc/codes" 31 | "google.golang.org/grpc/status" 32 | structpb "google.golang.org/protobuf/types/known/structpb" 33 | ) 34 | 35 | func TestCompareValueType(t *testing.T) { 36 | table := []struct { 37 | a ValueType 38 | b ValueType 39 | result bool 40 | }{ 41 | { 42 | a: ValueType{Code: TCInt64}, 43 | b: ValueType{Code: TCInt64}, 44 | result: true, 45 | }, 46 | { 47 | a: ValueType{Code: TCString}, 48 | b: ValueType{Code: TCString}, 49 | result: true, 50 | }, 51 | { 52 | a: ValueType{Code: TCInt64}, 53 | b: ValueType{Code: TCString}, 54 | result: false, 55 | }, 56 | { 57 | a: ValueType{ 58 | Code: TCArray, 59 | ArrayType: &ValueType{Code: TCInt64}, 60 | }, 61 | b: ValueType{ 62 | Code: TCArray, 63 | ArrayType: &ValueType{Code: TCInt64}, 64 | }, 65 | result: true, 66 | }, 67 | { 68 | a: ValueType{ 69 | Code: TCArray, 70 | ArrayType: &ValueType{Code: TCString}, 71 | }, 72 | b: ValueType{ 73 | Code: TCArray, 74 | ArrayType: &ValueType{Code: TCString}, 75 | }, 76 | result: true, 77 | }, 78 | { 79 | a: ValueType{ 80 | Code: TCArray, 81 | ArrayType: &ValueType{Code: TCString}, 82 | }, 83 | b: ValueType{ 84 | Code: TCArray, 85 | ArrayType: &ValueType{Code: TCInt64}, 86 | }, 87 | result: false, 88 | }, 89 | { 90 | a: ValueType{Code: TCInt64}, 91 | b: ValueType{ 92 | Code: TCArray, 93 | ArrayType: &ValueType{Code: TCInt64}, 94 | }, 95 | result: false, 96 | }, 97 | { 98 | a: ValueType{ 99 | Code: TCStruct, 100 | StructType: &StructType{ 101 | FieldNames: []string{"", "", ""}, 102 | FieldTypes: []*ValueType{ 103 | &ValueType{Code: TCInt64}, 104 | &ValueType{Code: TCString}, 105 | }, 106 | IsTable: false, 107 | }, 108 | }, 109 | b: ValueType{ 110 | Code: TCStruct, 111 | StructType: &StructType{ 112 | FieldNames: []string{"", "", ""}, 113 | FieldTypes: []*ValueType{ 114 | &ValueType{Code: TCInt64}, 115 | &ValueType{Code: TCString}, 116 | }, 117 | IsTable: false, 118 | }, 119 | }, 120 | result: true, 121 | }, 122 | { 123 | a: ValueType{ 124 | Code: TCStruct, 125 | StructType: &StructType{ 126 | FieldNames: []string{"", "", ""}, 127 | FieldTypes: []*ValueType{ 128 | &ValueType{Code: TCInt64}, 129 | &ValueType{Code: TCString}, 130 | }, 131 | IsTable: false, 132 | }, 133 | }, 134 | b: ValueType{ 135 | Code: TCStruct, 136 | StructType: &StructType{ 137 | FieldNames: []string{"a", "b", "c"}, 138 | FieldTypes: []*ValueType{ 139 | &ValueType{Code: TCInt64}, 140 | &ValueType{Code: TCString}, 141 | }, 142 | IsTable: true, 143 | }, 144 | }, 145 | result: true, 146 | }, 147 | { 148 | a: ValueType{ 149 | Code: TCStruct, 150 | StructType: &StructType{ 151 | FieldNames: []string{"", "", ""}, 152 | FieldTypes: []*ValueType{ 153 | &ValueType{Code: TCInt64}, 154 | &ValueType{Code: TCString}, 155 | }, 156 | IsTable: false, 157 | }, 158 | }, 159 | b: ValueType{ 160 | Code: TCStruct, 161 | StructType: &StructType{ 162 | FieldNames: []string{"", "", ""}, 163 | FieldTypes: []*ValueType{ 164 | &ValueType{Code: TCString}, 165 | &ValueType{Code: TCInt64}, 166 | }, 167 | IsTable: false, 168 | }, 169 | }, 170 | result: false, 171 | }, 172 | { 173 | a: ValueType{ 174 | Code: TCStruct, 175 | StructType: &StructType{ 176 | FieldNames: []string{"", "", ""}, 177 | FieldTypes: []*ValueType{ 178 | &ValueType{Code: TCInt64}, 179 | &ValueType{ 180 | Code: TCArray, 181 | ArrayType: &ValueType{Code: TCInt64}, 182 | }, 183 | }, 184 | IsTable: false, 185 | }, 186 | }, 187 | b: ValueType{ 188 | Code: TCStruct, 189 | StructType: &StructType{ 190 | FieldNames: []string{"", "", ""}, 191 | FieldTypes: []*ValueType{ 192 | &ValueType{Code: TCInt64}, 193 | &ValueType{ 194 | Code: TCArray, 195 | ArrayType: &ValueType{Code: TCInt64}, 196 | }, 197 | }, 198 | IsTable: false, 199 | }, 200 | }, 201 | result: true, 202 | }, 203 | } 204 | 205 | for _, tc := range table { 206 | r := compareValueType(tc.a, tc.b) 207 | if r != tc.result { 208 | t.Errorf("expect result %v, but got %v", tc.result, r) 209 | } 210 | } 211 | } 212 | 213 | func TestDatabaseEncDec(t *testing.T) { 214 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 215 | defer cancel() 216 | 217 | table := map[string]struct { 218 | value *structpb.Value 219 | typ ValueType 220 | expected interface{} 221 | }{ 222 | "Bool": { 223 | value: makeBoolValue(true), 224 | typ: ValueType{Code: TCBool}, 225 | expected: true, 226 | }, 227 | "Int": { 228 | value: makeStringValue("100"), 229 | typ: ValueType{Code: TCInt64}, 230 | expected: int64(100), 231 | }, 232 | "Float": { 233 | value: makeNumberValue(0.5), 234 | typ: ValueType{Code: TCFloat64}, 235 | expected: float64(0.5), 236 | }, 237 | "Bytes": { 238 | value: makeStringValue("eHh4"), // xxx 239 | typ: ValueType{Code: TCBytes}, 240 | expected: []byte("xxx"), 241 | }, 242 | "Timestamp": { 243 | value: makeStringValue("2012-03-04T00:00:00.123456789Z"), 244 | typ: ValueType{Code: TCTimestamp}, 245 | expected: "2012-03-04T00:00:00.123456789Z", 246 | }, 247 | "ArrayBool": { 248 | value: makeListValueAsValue(makeListValue( 249 | makeBoolValue(true), 250 | makeBoolValue(false), 251 | )), 252 | typ: ValueType{ 253 | Code: TCArray, 254 | ArrayType: &ValueType{Code: TCBool}, 255 | }, 256 | expected: makeTestArray(TCBool, true, false), 257 | }, 258 | "ArrayString": { 259 | value: makeListValueAsValue(makeListValue( 260 | makeStringValue("xxx"), 261 | makeStringValue("yyy"), 262 | )), 263 | typ: ValueType{ 264 | Code: TCArray, 265 | ArrayType: &ValueType{Code: TCString}, 266 | }, 267 | expected: makeTestArray(TCString, "xxx", "yyy"), 268 | }, 269 | "ArrayInt": { 270 | value: makeListValueAsValue(makeListValue( 271 | makeStringValue("100"), 272 | makeStringValue("200"), 273 | )), 274 | typ: ValueType{ 275 | Code: TCArray, 276 | ArrayType: &ValueType{Code: TCInt64}, 277 | }, 278 | expected: makeTestArray(TCInt64, 100, 200), 279 | }, 280 | "ArrayFloat": { 281 | value: makeListValueAsValue(makeListValue( 282 | makeNumberValue(0.1), 283 | makeNumberValue(0.2), 284 | )), 285 | typ: ValueType{ 286 | Code: TCArray, 287 | ArrayType: &ValueType{Code: TCFloat64}, 288 | }, 289 | expected: makeTestArray(TCFloat64, float64(0.1), float64(0.2)), 290 | }, 291 | "ArrayBytes": { 292 | value: makeListValueAsValue(makeListValue( 293 | makeStringValue("eHl6"), // xyz 294 | makeStringValue("eHh4"), // xxx 295 | )), 296 | typ: ValueType{ 297 | Code: TCArray, 298 | ArrayType: &ValueType{Code: TCBytes}, 299 | }, 300 | expected: makeTestArray(TCBytes, []byte("xyz"), []byte("xxx")), 301 | }, 302 | "JSON": { 303 | value: makeStringValue(`{"a": 1, "b": 2}`), 304 | typ: ValueType{ 305 | Code: TCJson, 306 | }, 307 | expected: `{"a": 1, "b": 2}`, 308 | }, 309 | } 310 | 311 | uuid := uuidpkg.New().String() 312 | db, err := sql.Open("sqlite3_spanner", fmt.Sprintf("file:%s.db?cache=shared&mode=memory", uuid)) 313 | if err != nil { 314 | t.Fatal(err) 315 | } 316 | defer db.Close() 317 | 318 | if _, err := db.ExecContext(ctx, "CREATE TABLE test (js JSON)"); err != nil { 319 | t.Fatal(err) 320 | } 321 | 322 | for name, tc := range table { 323 | t.Run(name, func(t *testing.T) { 324 | defer func() { 325 | if _, err := db.ExecContext(ctx, "DELETE FROM test"); err != nil { 326 | t.Fatalf("delete failed: %v", err) 327 | } 328 | }() 329 | 330 | column := Column{ 331 | ast: &ast.ColumnDef{Name: &ast.Ident{Name: "Id"}}, 332 | valueType: tc.typ, 333 | dbDataType: DBDTJson, 334 | } 335 | 336 | v, err := spannerValue2DatabaseValue(tc.value, column) 337 | if err != nil { 338 | t.Fatalf("spannerValue2DatabaseValue failed: %v", err) 339 | } 340 | 341 | if _, err := db.ExecContext(ctx, "INSERT INTO test VALUES(?)", v); err != nil { 342 | t.Fatalf("insert failed: %v", err) 343 | } 344 | 345 | r, err := db.QueryContext(ctx, "SELECT js FROM test") 346 | if err != nil { 347 | t.Fatalf("select failed: %v", err) 348 | } 349 | defer r.Close() 350 | 351 | item := createResultItemFromColumn(&column) 352 | iter := rows{rows: r, resultItems: []ResultItem{item}, transaction: &transaction{status: 1}} 353 | 354 | var rows [][]interface{} 355 | err = iter.Do(func(row []interface{}) error { 356 | rows = append(rows, row) 357 | return nil 358 | }) 359 | if err != nil { 360 | t.Fatalf("unexpected error in iteration: %v", err) 361 | } 362 | 363 | if len(rows) != 1 { 364 | t.Errorf("there should be only 1 row") 365 | } 366 | if diff := cmp.Diff(tc.expected, rows[0][0]); diff != "" { 367 | t.Errorf("(-got, +want)\n%s", diff) 368 | } 369 | }) 370 | } 371 | } 372 | 373 | func TestMakeValueFromSpannerValue(t *testing.T) { 374 | table := map[string]struct { 375 | value *structpb.Value 376 | typ *spannerpb.Type 377 | expected Value 378 | }{ 379 | "Null": { 380 | value: makeNullValue(), 381 | typ: &spannerpb.Type{ 382 | Code: spannerpb.TypeCode_INT64, 383 | }, 384 | expected: Value{ 385 | Data: nil, 386 | Type: ValueType{ 387 | Code: TCInt64, 388 | }, 389 | }, 390 | }, 391 | "Int": { 392 | value: makeStringValue("100"), 393 | typ: &spannerpb.Type{ 394 | Code: spannerpb.TypeCode_INT64, 395 | }, 396 | expected: Value{ 397 | Data: int64(100), 398 | Type: ValueType{ 399 | Code: TCInt64, 400 | }, 401 | }, 402 | }, 403 | "String": { 404 | value: makeStringValue("xx"), 405 | typ: &spannerpb.Type{ 406 | Code: spannerpb.TypeCode_STRING, 407 | }, 408 | expected: Value{ 409 | Data: "xx", 410 | Type: ValueType{ 411 | Code: TCString, 412 | }, 413 | }, 414 | }, 415 | "Bool": { 416 | value: makeBoolValue(true), 417 | typ: &spannerpb.Type{ 418 | Code: spannerpb.TypeCode_BOOL, 419 | }, 420 | expected: Value{ 421 | Data: true, 422 | Type: ValueType{ 423 | Code: TCBool, 424 | }, 425 | }, 426 | }, 427 | "Number": { 428 | value: makeNumberValue(0.123), 429 | typ: &spannerpb.Type{ 430 | Code: spannerpb.TypeCode_FLOAT64, 431 | }, 432 | expected: Value{ 433 | Data: 0.123, 434 | Type: ValueType{ 435 | Code: TCFloat64, 436 | }, 437 | }, 438 | }, 439 | "Timestamp": { 440 | value: makeStringValue("2012-03-04T00:00:00.123456789Z"), 441 | typ: &spannerpb.Type{ 442 | Code: spannerpb.TypeCode_TIMESTAMP, 443 | }, 444 | expected: Value{ 445 | Data: "2012-03-04T00:00:00.123456789Z", 446 | Type: ValueType{ 447 | Code: TCTimestamp, 448 | }, 449 | }, 450 | }, 451 | "Date": { 452 | value: makeStringValue("2012-03-04"), 453 | typ: &spannerpb.Type{ 454 | Code: spannerpb.TypeCode_DATE, 455 | }, 456 | expected: Value{ 457 | Data: "2012-03-04", 458 | Type: ValueType{ 459 | Code: TCDate, 460 | }, 461 | }, 462 | }, 463 | "Bytes": { 464 | value: makeStringValue("eHh4eHg="), 465 | typ: &spannerpb.Type{ 466 | Code: spannerpb.TypeCode_BYTES, 467 | }, 468 | expected: Value{ 469 | Data: []byte("xxxxx"), 470 | Type: ValueType{ 471 | Code: TCBytes, 472 | }, 473 | }, 474 | }, 475 | "ListInt": { 476 | value: makeListValueAsValue(makeListValue( 477 | makeStringValue("100"), 478 | makeStringValue("101"), 479 | )), 480 | typ: &spannerpb.Type{ 481 | Code: spannerpb.TypeCode_ARRAY, 482 | ArrayElementType: &spannerpb.Type{ 483 | Code: spannerpb.TypeCode_INT64, 484 | }, 485 | }, 486 | expected: Value{ 487 | Data: makeTestWrappedArray(TCInt64, 100, 101), 488 | Type: ValueType{ 489 | Code: TCArray, 490 | ArrayType: &ValueType{ 491 | Code: TCInt64, 492 | }, 493 | }, 494 | }, 495 | }, 496 | "ListString": { 497 | value: makeListValueAsValue(makeListValue( 498 | makeStringValue("xxx"), 499 | makeStringValue("yyy"), 500 | )), 501 | typ: &spannerpb.Type{ 502 | Code: spannerpb.TypeCode_ARRAY, 503 | ArrayElementType: &spannerpb.Type{ 504 | Code: spannerpb.TypeCode_STRING, 505 | }, 506 | }, 507 | expected: Value{ 508 | Data: makeTestWrappedArray(TCString, "xxx", "yyy"), 509 | Type: ValueType{ 510 | Code: TCArray, 511 | ArrayType: &ValueType{ 512 | Code: TCString, 513 | }, 514 | }, 515 | }, 516 | }, 517 | "ListStringNull": { 518 | value: makeNullValue(), 519 | typ: &spannerpb.Type{ 520 | Code: spannerpb.TypeCode_ARRAY, 521 | ArrayElementType: &spannerpb.Type{ 522 | Code: spannerpb.TypeCode_STRING, 523 | }, 524 | }, 525 | expected: Value{ 526 | Data: []string(nil), 527 | Type: ValueType{ 528 | Code: TCArray, 529 | ArrayType: &ValueType{ 530 | Code: TCString, 531 | }, 532 | }, 533 | }, 534 | }, 535 | "ListBool": { 536 | value: makeListValueAsValue(makeListValue( 537 | makeBoolValue(true), 538 | makeBoolValue(false), 539 | )), 540 | typ: &spannerpb.Type{ 541 | Code: spannerpb.TypeCode_ARRAY, 542 | ArrayElementType: &spannerpb.Type{ 543 | Code: spannerpb.TypeCode_BOOL, 544 | }, 545 | }, 546 | expected: Value{ 547 | Data: makeTestWrappedArray(TCBool, true, false), 548 | Type: ValueType{ 549 | Code: TCArray, 550 | ArrayType: &ValueType{ 551 | Code: TCBool, 552 | }, 553 | }, 554 | }, 555 | }, 556 | "ListBoolNull": { 557 | value: makeNullValue(), 558 | typ: &spannerpb.Type{ 559 | Code: spannerpb.TypeCode_ARRAY, 560 | ArrayElementType: &spannerpb.Type{ 561 | Code: spannerpb.TypeCode_BOOL, 562 | }, 563 | }, 564 | expected: Value{ 565 | Data: []bool(nil), 566 | Type: ValueType{ 567 | Code: TCArray, 568 | ArrayType: &ValueType{ 569 | Code: TCBool, 570 | }, 571 | }, 572 | }, 573 | }, 574 | "ListNumber": { 575 | value: makeListValueAsValue(makeListValue( 576 | makeNumberValue(0.123), 577 | makeNumberValue(1.123), 578 | )), 579 | typ: &spannerpb.Type{ 580 | Code: spannerpb.TypeCode_ARRAY, 581 | ArrayElementType: &spannerpb.Type{ 582 | Code: spannerpb.TypeCode_FLOAT64, 583 | }, 584 | }, 585 | expected: Value{ 586 | Data: makeTestWrappedArray(TCFloat64, 0.123, 1.123), 587 | Type: ValueType{ 588 | Code: TCArray, 589 | ArrayType: &ValueType{ 590 | Code: TCFloat64, 591 | }, 592 | }, 593 | }, 594 | }, 595 | "ListNumberNull": { 596 | value: makeNullValue(), 597 | typ: &spannerpb.Type{ 598 | Code: spannerpb.TypeCode_ARRAY, 599 | ArrayElementType: &spannerpb.Type{ 600 | Code: spannerpb.TypeCode_FLOAT64, 601 | }, 602 | }, 603 | expected: Value{ 604 | Data: []float64(nil), 605 | Type: ValueType{ 606 | Code: TCArray, 607 | ArrayType: &ValueType{ 608 | Code: TCFloat64, 609 | }, 610 | }, 611 | }, 612 | }, 613 | "ListTimestamp": { 614 | value: makeListValueAsValue(makeListValue( 615 | makeStringValue("2012-03-04T00:00:00.123456789Z"), 616 | makeStringValue("2012-03-04T00:00:00.000000000Z"), 617 | )), 618 | typ: &spannerpb.Type{ 619 | Code: spannerpb.TypeCode_ARRAY, 620 | ArrayElementType: &spannerpb.Type{ 621 | Code: spannerpb.TypeCode_TIMESTAMP, 622 | }, 623 | }, 624 | expected: Value{ 625 | Data: makeTestWrappedArray(TCString, 626 | "2012-03-04T00:00:00.123456789Z", 627 | "2012-03-04T00:00:00.000000000Z", 628 | ), 629 | Type: ValueType{ 630 | Code: TCArray, 631 | ArrayType: &ValueType{ 632 | Code: TCTimestamp, 633 | }, 634 | }, 635 | }, 636 | }, 637 | "ListTimestampNull": { 638 | value: makeNullValue(), 639 | typ: &spannerpb.Type{ 640 | Code: spannerpb.TypeCode_ARRAY, 641 | ArrayElementType: &spannerpb.Type{ 642 | Code: spannerpb.TypeCode_TIMESTAMP, 643 | }, 644 | }, 645 | expected: Value{ 646 | Data: []string(nil), 647 | Type: ValueType{ 648 | Code: TCArray, 649 | ArrayType: &ValueType{ 650 | Code: TCTimestamp, 651 | }, 652 | }, 653 | }, 654 | }, 655 | "ListDate": { 656 | value: makeListValueAsValue(makeListValue( 657 | makeStringValue("2012-03-04"), 658 | makeStringValue("2012-03-05"), 659 | )), 660 | typ: &spannerpb.Type{ 661 | Code: spannerpb.TypeCode_ARRAY, 662 | ArrayElementType: &spannerpb.Type{ 663 | Code: spannerpb.TypeCode_DATE, 664 | }, 665 | }, 666 | expected: Value{ 667 | Data: makeTestWrappedArray(TCString, "2012-03-04", "2012-03-05"), 668 | Type: ValueType{ 669 | Code: TCArray, 670 | ArrayType: &ValueType{ 671 | Code: TCDate, 672 | }, 673 | }, 674 | }, 675 | }, 676 | "ListDateNull": { 677 | value: makeNullValue(), 678 | typ: &spannerpb.Type{ 679 | Code: spannerpb.TypeCode_ARRAY, 680 | ArrayElementType: &spannerpb.Type{ 681 | Code: spannerpb.TypeCode_DATE, 682 | }, 683 | }, 684 | expected: Value{ 685 | Data: []string(nil), 686 | Type: ValueType{ 687 | Code: TCArray, 688 | ArrayType: &ValueType{ 689 | Code: TCDate, 690 | }, 691 | }, 692 | }, 693 | }, 694 | "ListBytes": { 695 | value: makeListValueAsValue(makeListValue( 696 | makeStringValue("eHh4eHg="), 697 | makeStringValue("eXl5eXk="), 698 | )), 699 | typ: &spannerpb.Type{ 700 | Code: spannerpb.TypeCode_ARRAY, 701 | ArrayElementType: &spannerpb.Type{ 702 | Code: spannerpb.TypeCode_BYTES, 703 | }, 704 | }, 705 | expected: Value{ 706 | Data: makeTestWrappedArray(TCBytes, []byte("xxxxx"), []byte("yyyyy")), 707 | Type: ValueType{ 708 | Code: TCArray, 709 | ArrayType: &ValueType{ 710 | Code: TCBytes, 711 | }, 712 | }, 713 | }, 714 | }, 715 | "ListBytesNull": { 716 | value: makeNullValue(), 717 | typ: &spannerpb.Type{ 718 | Code: spannerpb.TypeCode_ARRAY, 719 | ArrayElementType: &spannerpb.Type{ 720 | Code: spannerpb.TypeCode_BYTES, 721 | }, 722 | }, 723 | expected: Value{ 724 | Data: [][]byte(nil), 725 | Type: ValueType{ 726 | Code: TCArray, 727 | ArrayType: &ValueType{ 728 | Code: TCBytes, 729 | }, 730 | }, 731 | }, 732 | }, 733 | "JSON": { 734 | value: makeStringValue(`{"a": 1, "b": 2}`), 735 | typ: &spannerpb.Type{ 736 | Code: spannerpb.TypeCode_JSON, 737 | }, 738 | expected: Value{ 739 | Data: `{"a": 1, "b": 2}`, 740 | Type: ValueType{ 741 | Code: TCJson, 742 | }, 743 | }, 744 | }, 745 | } 746 | 747 | for name, tc := range table { 748 | t.Run(name, func(t *testing.T) { 749 | res, err := makeValueFromSpannerValue("foo", tc.value, tc.typ) 750 | if err != nil { 751 | t.Fatalf("unexpected error: %v", err) 752 | } 753 | if diff := cmp.Diff(tc.expected, res); diff != "" { 754 | t.Errorf("(-got, +want)\n%s", diff) 755 | } 756 | }) 757 | } 758 | 759 | } 760 | 761 | func TestMakeValueFromSpannerValue_Error(t *testing.T) { 762 | table := map[string]struct { 763 | value *structpb.Value 764 | typ *spannerpb.Type 765 | msg *regexp.Regexp 766 | }{ 767 | "Int_InvalidValue": { 768 | value: makeStringValue("xxx"), 769 | typ: &spannerpb.Type{ 770 | Code: spannerpb.TypeCode_INT64, 771 | }, 772 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected INT64`), 773 | }, 774 | "Int_InvalidType": { 775 | value: makeBoolValue(true), 776 | typ: &spannerpb.Type{ 777 | Code: spannerpb.TypeCode_INT64, 778 | }, 779 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected INT64`), 780 | }, 781 | 782 | "Timestamp_InvalidValue": { 783 | value: makeStringValue("xxx"), 784 | typ: &spannerpb.Type{ 785 | Code: spannerpb.TypeCode_TIMESTAMP, 786 | }, 787 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected TIMESTAMP`), 788 | }, 789 | "Timestamp_InvalidType": { 790 | value: makeBoolValue(true), 791 | typ: &spannerpb.Type{ 792 | Code: spannerpb.TypeCode_TIMESTAMP, 793 | }, 794 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected TIMESTAMP`), 795 | }, 796 | 797 | "Date_InvalidValue": { 798 | value: makeStringValue("xxx"), 799 | typ: &spannerpb.Type{ 800 | Code: spannerpb.TypeCode_DATE, 801 | }, 802 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected DATE`), 803 | }, 804 | "Date_InvalidType": { 805 | value: makeBoolValue(true), 806 | typ: &spannerpb.Type{ 807 | Code: spannerpb.TypeCode_DATE, 808 | }, 809 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected DATE`), 810 | }, 811 | 812 | "Bytes_InvalidValue": { 813 | value: makeStringValue("x"), 814 | typ: &spannerpb.Type{ 815 | Code: spannerpb.TypeCode_BYTES, 816 | }, 817 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected BYTES`), 818 | }, 819 | "Bytes_InvalidType": { 820 | value: makeBoolValue(true), 821 | typ: &spannerpb.Type{ 822 | Code: spannerpb.TypeCode_BYTES, 823 | }, 824 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected BYTES`), 825 | }, 826 | 827 | "Array_InvalidType": { 828 | value: makeStringValue("xxx"), 829 | typ: &spannerpb.Type{ 830 | Code: spannerpb.TypeCode_ARRAY, 831 | ArrayElementType: &spannerpb.Type{ 832 | Code: spannerpb.TypeCode_STRING, 833 | }, 834 | }, 835 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo: Expected ARRAY`), 836 | }, 837 | "Array_InvalidElementValue": { 838 | value: makeListValueAsValue(makeListValue( 839 | makeStringValue("100"), 840 | makeStringValue("xxx"), 841 | )), 842 | typ: &spannerpb.Type{ 843 | Code: spannerpb.TypeCode_ARRAY, 844 | ArrayElementType: &spannerpb.Type{ 845 | Code: spannerpb.TypeCode_INT64, 846 | }, 847 | }, 848 | msg: regexp.MustCompile(`^Invalid value for bind parameter foo\[1\]: Expected INT64`), 849 | }, 850 | } 851 | 852 | for name, tc := range table { 853 | t.Run(name, func(t *testing.T) { 854 | _, err := makeValueFromSpannerValue("foo", tc.value, tc.typ) 855 | if err == nil { 856 | t.Fatalf("unexpected success") 857 | } 858 | st := status.Convert(err) 859 | if st.Code() != codes.InvalidArgument { 860 | t.Errorf("expect code to be %v, but got %v", codes.InvalidArgument, st.Code()) 861 | } 862 | if !tc.msg.MatchString(st.Message()) { 863 | t.Errorf("unexpected error message: \n %q\n expected:\n %q", st.Message(), tc.msg) 864 | } 865 | }) 866 | } 867 | 868 | } 869 | --------------------------------------------------------------------------------