├── .github └── workflows │ ├── full_test.yml │ └── unit_test.yml ├── .gitignore ├── LICENSE ├── README.md ├── auth.go ├── docs └── img │ ├── append_only_mode.png │ ├── default_mode.png │ ├── logo_dark.png │ └── logo_light.png ├── go.mod ├── go.sum ├── google └── auth │ ├── doc.go │ ├── models.go │ ├── oauth2.go │ ├── oauth2_test.go │ ├── service.go │ ├── service_test.go │ ├── utils.go │ └── utils_test.go ├── internal ├── codec │ ├── codec.go │ └── codec_test.go ├── common │ ├── mapstructure.go │ ├── range.go │ ├── range_test.go │ ├── set.go │ ├── time.go │ ├── values.go │ └── values_test.go ├── google │ ├── fixtures │ │ ├── client_secret.json │ │ ├── file.go │ │ ├── service_account.json │ │ └── stored_credentials.json │ ├── sheets │ │ ├── models.go │ │ ├── models_test.go │ │ ├── wrapper.go │ │ ├── wrapper_mock.go │ │ └── wrapper_test.go │ └── store │ │ ├── kv.go │ │ ├── kv_test.go │ │ ├── kv_v2.go │ │ ├── kv_v2_test.go │ │ ├── models.go │ │ ├── models_test.go │ │ ├── row.go │ │ ├── row_test.go │ │ ├── stmt.go │ │ ├── stmt_test.go │ │ └── utils.go └── models │ ├── kv.go │ └── row.go ├── kv.go ├── kv_test.go ├── row.go └── row_test.go /.github/workflows/full_test.yml: -------------------------------------------------------------------------------- 1 | name: Full Test 2 | on: 3 | pull_request: 4 | types: [opened, synchronize] 5 | 6 | env: 7 | INTEGRATION_TEST_SPREADSHEET_ID: ${{ secrets.INTEGRATION_TEST_SPREADSHEET_ID }} 8 | INTEGRATION_TEST_AUTH_JSON: ${{ secrets.INTEGRATION_TEST_AUTH_JSON }} 9 | 10 | jobs: 11 | full_test: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | # Cannot add all, it will create parallel jobs, might trigger Google Sheets API rate limit. 16 | # Technically, we can change this to sequential job, but it will make the YML file longer. 17 | go-version: ['1.23.x'] 18 | 19 | # This essentially means either the PR is just approved or it's edocsss who runs this. 20 | if: github.event.review.state == 'approved' || github.event.pull_request.user.login == 'edocsss' 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | with: 25 | ref: ${{ github.head_ref }} 26 | 27 | - name: Dump GitHub context 28 | env: 29 | GITHUB_CONTEXT: ${{ toJson(github) }} 30 | run: echo "$GITHUB_CONTEXT" 31 | 32 | - name: Setup Go ${{ matrix.go-version }} 33 | uses: actions/setup-go@v3 34 | with: 35 | go-version: ${{ matrix.go-version }} 36 | # As there is only one Golang script being run. 37 | # Faster if we don't cache as the cache is per commit anyway. 38 | cache: false 39 | 40 | - name: Golang version 41 | run: go version 42 | 43 | - name: Full Test 44 | run: | 45 | go test -v -count=1 -cover ./... -coverprofile coverage.out -coverpkg ./... 46 | go tool cover -func coverage.out -o coverage.out 47 | 48 | - name: Go Coverage Badge 49 | uses: tj-actions/coverage-badge-go@v2 50 | with: 51 | green: 80 52 | filename: coverage.out 53 | 54 | # - name: Commit updated readme 55 | # run: | 56 | # - git config user.name "GitHub Bot" 57 | # - git config user.email "github-actions[bot]@users.noreply.github.com" 58 | # - git add ./README.md 59 | # - git commit -m "CI: Update README with test coverage badge" 60 | # - git push origin 61 | 62 | - name: Add Coverage Badge 63 | uses: stefanzweifel/git-auto-commit-action@v4 64 | id: auto-commit-action 65 | with: 66 | commit_message: Apply Code Coverage Badge 67 | skip_fetch: true 68 | skip_checkout: true 69 | file_pattern: ./README.md 70 | 71 | - name: Push Changes 72 | if: steps.auto-commit-action.outputs.changes_detected == 'true' 73 | uses: ad-m/github-push-action@master 74 | with: 75 | github_token: ${{ github.token }} 76 | branch: ${{ github.head_ref }} -------------------------------------------------------------------------------- /.github/workflows/unit_test.yml: -------------------------------------------------------------------------------- 1 | name: Unit Test 2 | on: push 3 | 4 | jobs: 5 | unit_test: 6 | runs-on: ubuntu-latest 7 | strategy: 8 | matrix: 9 | go-version: ['1.18.x', '1.19.x', '1.20.x', '1.21.x', '1.22.x', '1.23.x'] 10 | 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - name: Setup Go ${{ matrix.go-version }} 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: ${{ matrix.go-version }} 18 | # As there is only one Golang script being run. 19 | # Faster if we don't cache as the cache is per commit anyway. 20 | cache: false 21 | 22 | - name: Golang version 23 | run: go version 24 | 25 | - name: Unit test 26 | run: go test -v -cover -count=1 ./... 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .private 2 | examples 3 | 4 | # If you prefer the allow list template instead of the deny list, see community template: 5 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 6 | # 7 | # Binaries for programs and plugins 8 | *.exe 9 | *.exe~ 10 | *.dll 11 | *.so 12 | *.dylib 13 | 14 | # Test binary, built with `go test -c` 15 | *.test 16 | 17 | # Output of the go coverage tool, specifically when used with LiteIDE 18 | *.out 19 | 20 | # Dependency directories (remove the comment below to include it) 21 | vendor/ 22 | 23 | # Go workspace file 24 | go.work 25 | .idea 26 | .aider* 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Edwin Candinegara 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GoFreeDB 2 |
3 | 4 |
5 | 6 | 7 | 8 | 9 |

Ship Faster with Google Sheets as a Database!

10 |
11 | 12 |

13 | GoFreeDB is a Golang library that provides common and simple database abstractions on top of Google Sheets. 14 |

15 | 16 |
17 | 18 |
19 | 20 | ![Unit Test](https://github.com/FreeLeh/GoFreeDB/actions/workflows/unit_test.yml/badge.svg) 21 | ![Integration Test](https://github.com/FreeLeh/GoFreeDB/actions/workflows/full_test.yml/badge.svg) 22 | ![Coverage](https://img.shields.io/badge/Coverage-83.1%25-brightgreen) 23 | [![Go Report Card](https://goreportcard.com/badge/github.com/FreeLeh/GoFreeDB)](https://goreportcard.com/report/github.com/FreeLeh/GoFreeDB) 24 | [![Go Reference](https://pkg.go.dev/badge/github.com/FreeLeh/GoFreeDB.svg)](https://pkg.go.dev/github.com/FreeLeh/GoFreeDB) 25 | 26 |
27 | 28 | ## Features 29 | 30 | 1. Provide a straightforward **key-value** and **row based database** interfaces on top of Google Sheets. 31 | 2. Serve your data **without any server setup** (by leveraging Google Sheets infrastructure). 32 | 3. Support **flexible enough query language** to perform various data queries. 33 | 4. **Manually manipulate data** via the familiar Google Sheets UI (no admin page required). 34 | 35 | > For more details, please read [our analysis](https://github.com/FreeLeh/docs/blob/main/freedb/alternatives.md#why-should-you-choose-freedb) 36 | > on other alternatives and how it compares with `FreeDB`. 37 | 38 | ## Table of Contents 39 | 40 | * [Protocols](#protocols) 41 | * [Getting Started](#getting-started) 42 | * [Installation](#installation) 43 | * [Pre-requisites](#pre-requisites) 44 | * [Row Store](#row-store) 45 | * [Querying Rows](#querying-rows) 46 | * [Counting Rows](#counting-rows) 47 | * [Inserting Rows](#inserting-rows) 48 | * [Updating Rows](#updating-rows) 49 | * [Deleting Rows](#deleting-rows) 50 | * [Struct Field to Column Mapping](#struct-field-to-column-mapping) 51 | * [KV Store](#kv-store) 52 | * [Get Value](#get-value) 53 | * [Set Key](#set-key) 54 | * [Delete Key](#delete-key) 55 | * [Supported Modes](#supported-modes) 56 | * [KV Store V2](#kv-store-v2) 57 | * [Get Value](#get-value-v2) 58 | * [Set Key](#set-key-v2) 59 | * [Delete Key](#delete-key-v2) 60 | * [Supported Modes](#supported-modes-v2) 61 | 62 | ## Protocols 63 | 64 | Clients are strongly encouraged to read through the **[protocols document](https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md)** to see how things work 65 | under the hood and **the limitations**. 66 | 67 | ## Getting Started 68 | 69 | ### Installation 70 | 71 | ``` 72 | go get github.com/FreeLeh/GoFreeDB 73 | ``` 74 | 75 | ### Pre-requisites 76 | 77 | 1. Obtain a Google [OAuth2](https://github.com/FreeLeh/docs/blob/main/google/authentication.md#oauth2-flow) or [Service Account](https://github.com/FreeLeh/docs/blob/main/google/authentication.md#service-account-flow) credentials. 78 | 2. Prepare a Google Sheets spreadsheet where the data will be stored. 79 | 80 | ## Row Store 81 | 82 | Let's assume each row in the table is represented by the `Person` struct. 83 | 84 | ```go 85 | type Person struct { 86 | Name string `db:"name"` 87 | Age int `db:"age"` 88 | } 89 | ``` 90 | 91 | Please read the [struct field to column mapping](#struct-field-to-column-mapping) section 92 | to understand the purpose of the `db` struct field tag. 93 | 94 | ```go 95 | import ( 96 | "github.com/FreeLeh/GoFreeDB" 97 | "github.com/FreeLeh/GoFreeDB/google/auth" 98 | ) 99 | 100 | // If using Google Service Account. 101 | auth, err := auth.NewServiceFromFile( 102 | "", 103 | freedb.FreeDBGoogleAuthScopes, 104 | auth.ServiceConfig{}, 105 | ) 106 | 107 | // If using Google OAuth2 Flow. 108 | auth, err := auth.NewOAuth2FromFile( 109 | "", 110 | "", 111 | freedb.FreeDBGoogleAuthScopes, 112 | auth.OAuth2Config{}, 113 | ) 114 | 115 | store := freedb.NewGoogleSheetRowStore( 116 | auth, 117 | "", 118 | "", 119 | freedb.GoogleSheetRowStoreConfig{Columns: []string{"name", "age"}}, 120 | ) 121 | defer store.Close(context.Background()) 122 | ``` 123 | 124 | ### Querying Rows 125 | 126 | ```go 127 | // Output variable 128 | var output []Person 129 | 130 | // Select all columns for all rows 131 | err := store. 132 | Select(&output). 133 | Exec(context.Background()) 134 | 135 | // Select a few columns for all rows (non-selected struct fields will have default value) 136 | err := store. 137 | Select(&output, "name"). 138 | Exec(context.Background()) 139 | 140 | // Select rows with conditions 141 | err := store. 142 | Select(&output). 143 | Where("name = ? OR age >= ?", "freedb", 10). 144 | Exec(context.Background()) 145 | 146 | // Select rows with sorting/order by 147 | ordering := []freedb.ColumnOrderBy{ 148 | {Column: "name", OrderBy: freedb.OrderByAsc}, 149 | {Column: "age", OrderBy: freedb.OrderByDesc}, 150 | } 151 | err := store. 152 | Select(&output). 153 | OrderBy(ordering). 154 | Exec(context.Background()) 155 | 156 | // Select rows with offset and limit 157 | err := store. 158 | Select(&output). 159 | Offset(10). 160 | Limit(20). 161 | Exec(context.Background()) 162 | ``` 163 | 164 | ### Counting Rows 165 | 166 | ```go 167 | // Count all rows 168 | count, err := store. 169 | Count(). 170 | Exec(context.Background()) 171 | 172 | // Count rows with conditions 173 | count, err := store. 174 | Count(). 175 | Where("name = ? OR age >= ?", "freedb", 10). 176 | Exec(context.Background()) 177 | ``` 178 | 179 | ### Inserting Rows 180 | 181 | ```go 182 | err := store.Insert( 183 | Person{Name: "no_pointer", Age: 10}, 184 | &Person{Name: "with_pointer", Age: 20}, 185 | ).Exec(context.Background()) 186 | ``` 187 | 188 | ### Updating Rows 189 | 190 | ```go 191 | colToUpdate := make(map[string]interface{}) 192 | colToUpdate["name"] = "new_name" 193 | colToUpdate["age"] = 12 194 | 195 | // Update all rows 196 | err := store. 197 | Update(colToUpdate). 198 | Exec(context.Background()) 199 | 200 | // Update rows with conditions 201 | err := store. 202 | Update(colToUpdate). 203 | Where("name = ? OR age >= ?", "freedb", 10). 204 | Exec(context.Background()) 205 | ``` 206 | 207 | ### Deleting Rows 208 | 209 | ```go 210 | // Delete all rows 211 | err := store. 212 | Delete(). 213 | Exec(context.Background()) 214 | 215 | // Delete rows with conditions 216 | err := store. 217 | Delete(). 218 | Where("name = ? OR age >= ?", "freedb", 10). 219 | Exec(context.Background()) 220 | ``` 221 | 222 | ### Struct Field to Column Mapping 223 | 224 | The struct field tag `db` can be used for defining the mapping between the struct field and the column name. 225 | This works just like the `json` tag from [`encoding/json`](https://pkg.go.dev/encoding/json). 226 | 227 | Without `db` tag, the library will use the field name directly (case-sensitive). 228 | 229 | ```go 230 | // This will map to the exact column name of "Name" and "Age". 231 | type NoTagPerson struct { 232 | Name string 233 | Age int 234 | } 235 | 236 | // This will map to the exact column name of "name" and "age" 237 | type WithTagPerson struct { 238 | Name string `db:"name"` 239 | Age int `db:"age"` 240 | } 241 | ``` 242 | 243 | ## KV Store 244 | 245 | > Please use `KV Store V2` as much as possible, especially if you are creating a new storage. 246 | 247 | ```go 248 | import ( 249 | "github.com/FreeLeh/GoFreeDB" 250 | "github.com/FreeLeh/GoFreeDB/google/auth" 251 | ) 252 | 253 | // If using Google Service Account. 254 | auth, err := auth.NewServiceFromFile( 255 | "", 256 | freedb.FreeDBGoogleAuthScopes, 257 | auth.ServiceConfig{}, 258 | ) 259 | 260 | // If using Google OAuth2 Flow. 261 | auth, err := auth.NewOAuth2FromFile( 262 | "", 263 | "", 264 | freedb.FreeDBGoogleAuthScopes, 265 | auth.OAuth2Config{}, 266 | ) 267 | 268 | kv := freedb.NewGoogleSheetKVStore( 269 | auth, 270 | "", 271 | "", 272 | freedb.GoogleSheetKVStoreConfig{Mode: freedb.KVSetModeAppendOnly}, 273 | ) 274 | defer kv.Close(context.Background()) 275 | ``` 276 | 277 | ### Get Value 278 | 279 | If the key is not found, `freedb.ErrKeyNotFound` will be returned. 280 | 281 | ```go 282 | value, err := kv.Get(context.Background(), "k1") 283 | ``` 284 | 285 | ### Set Key 286 | 287 | ```go 288 | err := kv.Set(context.Background(), "k1", []byte("some_value")) 289 | ``` 290 | 291 | ### Delete Key 292 | 293 | ```go 294 | err := kv.Delete(context.Background(), "k1") 295 | ``` 296 | 297 | ### Supported Modes 298 | 299 | > For more details on how the two modes are different, please read the [protocol document](https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md). 300 | 301 | There are 2 different modes supported: 302 | 303 | 1. Default mode. 304 | 2. Append only mode. 305 | 306 | ```go 307 | // Default mode 308 | kv := freedb.NewGoogleSheetKVStore( 309 | auth, 310 | "", 311 | "", 312 | freedb.GoogleSheetKVStoreConfig{Mode: freedb.KVModeDefault}, 313 | ) 314 | 315 | // Append only mode 316 | kv := freedb.NewGoogleSheetKVStore( 317 | auth, 318 | "", 319 | "", 320 | freedb.GoogleSheetKVStoreConfig{Mode: freedb.KVModeAppendOnly}, 321 | ) 322 | ``` 323 | 324 | ## KV Store V2 325 | 326 | The KV Store V2 is implemented internally using the row store. 327 | 328 | > The original `KV Store` was created using more complicated formulas, making it less maintainable. 329 | > You can still use the original `KV Store` implementation, but we strongly suggest using this new `KV Store V2`. 330 | 331 | You cannot use an existing sheet based on `KV Store` with `KV Store V2` as the sheet structure is different. 332 | - If you want to convert an existing sheet, just add an `_rid` column and insert the first key-value row with `1` 333 | and increase it by 1 until the last row. 334 | - Remove the timestamp column as `KV Store V2` does not depend on it anymore. 335 | 336 | ```go 337 | import ( 338 | "github.com/FreeLeh/GoFreeDB" 339 | "github.com/FreeLeh/GoFreeDB/google/auth" 340 | ) 341 | 342 | // If using Google Service Account. 343 | auth, err := auth.NewServiceFromFile( 344 | "", 345 | freedb.FreeDBGoogleAuthScopes, 346 | auth.ServiceConfig{}, 347 | ) 348 | 349 | // If using Google OAuth2 Flow. 350 | auth, err := auth.NewOAuth2FromFile( 351 | "", 352 | "", 353 | freedb.FreeDBGoogleAuthScopes, 354 | auth.OAuth2Config{}, 355 | ) 356 | 357 | kv := freedb.NewGoogleSheetKVStoreV2( 358 | auth, 359 | "", 360 | "", 361 | freedb.GoogleSheetKVStoreV2Config{Mode: freedb.KVSetModeAppendOnly}, 362 | ) 363 | defer kv.Close(context.Background()) 364 | ``` 365 | 366 | ### Get Value V2 367 | 368 | If the key is not found, `freedb.ErrKeyNotFound` will be returned. 369 | 370 | ```go 371 | value, err := kv.Get(context.Background(), "k1") 372 | ``` 373 | 374 | ### Set Key V2 375 | 376 | ```go 377 | err := kv.Set(context.Background(), "k1", []byte("some_value")) 378 | ``` 379 | 380 | ### Delete Key V2 381 | 382 | ```go 383 | err := kv.Delete(context.Background(), "k1") 384 | ``` 385 | 386 | ### Supported Modes V2 387 | 388 | > For more details on how the two modes are different, please read the [protocol document](https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md). 389 | 390 | There are 2 different modes supported: 391 | 392 | 1. Default mode. 393 | 2. Append only mode. 394 | 395 | ```go 396 | // Default mode 397 | kv := freedb.NewGoogleSheetKVStoreV2( 398 | auth, 399 | "", 400 | "", 401 | freedb.GoogleSheetKVStoreV2Config{Mode: freedb.KVModeDefault}, 402 | ) 403 | 404 | // Append only mode 405 | kv := freedb.NewGoogleSheetKVStoreV2( 406 | auth, 407 | "", 408 | "", 409 | freedb.GoogleSheetKVStoreV2Config{Mode: freedb.KVModeAppendOnly}, 410 | ) 411 | ``` 412 | 413 | ## License 414 | 415 | This project is [MIT licensed](https://github.com/FreeLeh/GoFreeDB/blob/main/LICENSE). 416 | -------------------------------------------------------------------------------- /auth.go: -------------------------------------------------------------------------------- 1 | package freedb 2 | 3 | import "github.com/FreeLeh/GoFreeDB/google/auth" 4 | 5 | // GoogleAuthScopes specifies the list of Google Auth scopes required to run FreeDB implementations properly. 6 | var ( 7 | GoogleAuthScopes = auth.GoogleSheetsReadWrite 8 | ) 9 | -------------------------------------------------------------------------------- /docs/img/append_only_mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FreeLeh/GoFreeDB/dc9979a8509908763b6ae7b643935476b948fafc/docs/img/append_only_mode.png -------------------------------------------------------------------------------- /docs/img/default_mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FreeLeh/GoFreeDB/dc9979a8509908763b6ae7b643935476b948fafc/docs/img/default_mode.png -------------------------------------------------------------------------------- /docs/img/logo_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FreeLeh/GoFreeDB/dc9979a8509908763b6ae7b643935476b948fafc/docs/img/logo_dark.png -------------------------------------------------------------------------------- /docs/img/logo_light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FreeLeh/GoFreeDB/dc9979a8509908763b6ae7b643935476b948fafc/docs/img/logo_light.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/FreeLeh/GoFreeDB 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/mitchellh/mapstructure v1.5.0 7 | github.com/stretchr/testify v1.8.0 8 | golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 9 | google.golang.org/api v0.94.0 10 | gopkg.in/h2non/gock.v1 v1.1.2 11 | ) 12 | 13 | require ( 14 | cloud.google.com/go/compute v1.9.0 // indirect 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 17 | github.com/golang/protobuf v1.5.2 // indirect 18 | github.com/google/uuid v1.3.0 // indirect 19 | github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect 20 | github.com/googleapis/gax-go/v2 v2.5.1 // indirect 21 | github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect 22 | github.com/pmezard/go-difflib v1.0.0 // indirect 23 | go.opencensus.io v0.23.0 // indirect 24 | golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect 25 | golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect 26 | golang.org/x/text v0.3.7 // indirect 27 | google.golang.org/appengine v1.6.7 // indirect 28 | google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf // indirect 29 | google.golang.org/grpc v1.49.0 // indirect 30 | google.golang.org/protobuf v1.28.1 // indirect 31 | gopkg.in/yaml.v3 v3.0.1 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /google/auth/doc.go: -------------------------------------------------------------------------------- 1 | // Package auth provides general Google authentication implementation agnostic to what specific Google services or 2 | // resources are used. Implementations in this package generate a https://pkg.go.dev/net/http#Client that can be used 3 | // to access Google REST APIs seamlessly. Authentications will be handled automatically, including refreshing 4 | //the access token when necessary. 5 | package auth 6 | -------------------------------------------------------------------------------- /google/auth/models.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | // Scopes encapsulates a list of Google resources scopes to request during authentication step. 4 | type Scopes []string 5 | 6 | var ( 7 | GoogleSheetsReadOnly Scopes = []string{"https://www.googleapis.com/auth/spreadsheets.readonly"} 8 | GoogleSheetsWriteOnly Scopes = []string{"https://www.googleapis.com/auth/spreadsheets"} 9 | GoogleSheetsReadWrite Scopes = []string{"https://www.googleapis.com/auth/spreadsheets"} 10 | ) 11 | -------------------------------------------------------------------------------- /google/auth/oauth2.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "math/rand" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "strings" 13 | "time" 14 | 15 | "golang.org/x/oauth2" 16 | "golang.org/x/oauth2/google" 17 | ) 18 | 19 | const ( 20 | alphanumeric = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890" 21 | stateLength = 32 22 | ) 23 | 24 | // OAuth2Config defines a list of configurations that can be used to customise how the Google OAuth2 flow works. 25 | type OAuth2Config struct { 26 | // HTTPClient allows the client to customise the HTTP client used to perform the REST API calls. 27 | // This will be useful if you want to have a more granular control over the HTTP client (e.g. using a connection pool). 28 | HTTPClient *http.Client 29 | } 30 | 31 | // OAuth2 takes in OAuth2 relevant information and sets up *http.Client that can be used to access 32 | // Google APIs seamlessly. Authentications will be handled automatically, including refreshing the access token 33 | // when necessary. 34 | type OAuth2 struct { 35 | googleAuthClient *http.Client 36 | } 37 | 38 | // HTTPClient returns a Google OAuth2 authenticated *http.Client that can be used to access Google APIs. 39 | func (o *OAuth2) HTTPClient() *http.Client { 40 | return o.googleAuthClient 41 | } 42 | 43 | // NewOAuth2FromFile creates an OAuth2 instance by reading the OAuth2 related information from a secret file. 44 | // 45 | // The "secretFilePath" is referring to the OAuth2 credentials JSON file that can be obtained by 46 | // creating a new OAuth2 credentials in https://console.cloud.google.com/apis/credentials. 47 | // You can put any link for the redirection URL field. 48 | // 49 | // The "credsFilePath" is referring to a file where the generated access and refresh token will be cached. 50 | // This file will be created automatically once the OAuth2 authentication is successful. 51 | // 52 | // The "scopes" tells Google what your application can do to your spreadsheets. 53 | // 54 | // Note that since this is an OAuth2 server flow, human interaction will be needed for the very first authentication. 55 | // During the OAuth2 flow, you will be asked to click a generated URL in the terminal. 56 | func NewOAuth2FromFile(secretFilePath string, credsFilePath string, scopes Scopes, config OAuth2Config) (*OAuth2, error) { 57 | rawAuthConfig, err := os.ReadFile(secretFilePath) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | if _, err := os.Stat(credsFilePath); err != nil { 63 | return newFromClientSecret(rawAuthConfig, credsFilePath, scopes, config) 64 | } 65 | 66 | rawCreds, err := os.ReadFile(credsFilePath) 67 | if err != nil { 68 | return nil, err 69 | } 70 | return newFromStoredCreds(rawAuthConfig, rawCreds, scopes, config) 71 | } 72 | 73 | func newFromStoredCreds(rawAuthConfig []byte, rawCreds []byte, scopes Scopes, config OAuth2Config) (*OAuth2, error) { 74 | var token oauth2.Token 75 | if err := json.Unmarshal(rawCreds, &token); err != nil { 76 | return nil, err 77 | } 78 | 79 | c, err := google.ConfigFromJSON(rawAuthConfig, scopes...) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | return &OAuth2{ 85 | googleAuthClient: c.Client(getClientCtx(config.HTTPClient), &token), 86 | }, nil 87 | } 88 | 89 | func newFromClientSecret(rawAuthConfig []byte, credsFilePath string, scopes Scopes, config OAuth2Config) (*OAuth2, error) { 90 | c, err := google.ConfigFromJSON(rawAuthConfig, scopes...) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | authCode, err := getAuthCode(c) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | token, err := getToken(c, authCode) 101 | if err != nil { 102 | return nil, err 103 | } 104 | if err := storeCredentials(credsFilePath, token); err != nil { 105 | return nil, err 106 | } 107 | 108 | return &OAuth2{ 109 | googleAuthClient: c.Client(getClientCtx(config.HTTPClient), token), 110 | }, nil 111 | } 112 | 113 | func getAuthCode(c *oauth2.Config) (string, error) { 114 | state := generateState() 115 | authCodeURL := c.AuthCodeURL(state, oauth2.AccessTypeOffline) 116 | 117 | fmt.Printf("Visit the URL for the auth dialog: %v\n", authCodeURL) 118 | fmt.Print("Paste the redirection URL here: ") 119 | 120 | var rawRedirectionURL string 121 | if _, err := fmt.Scan(&rawRedirectionURL); err != nil { 122 | return "", err 123 | } 124 | 125 | redirectionURL, err := url.Parse(rawRedirectionURL) 126 | if err != nil { 127 | return "", err 128 | } 129 | 130 | query := redirectionURL.Query() 131 | if query.Get("state") != state { 132 | return "", errors.New("oauth state does not match") 133 | } 134 | return query.Get("code"), nil 135 | } 136 | 137 | func generateState() string { 138 | sb := strings.Builder{} 139 | randSrc := rand.NewSource(time.Now().UnixMilli()) 140 | 141 | for i := 0; i < stateLength; i++ { 142 | idx := randSrc.Int63() % int64(len(alphanumeric)) 143 | sb.WriteByte(alphanumeric[idx]) 144 | } 145 | 146 | return sb.String() 147 | } 148 | 149 | func getToken(c *oauth2.Config, authCode string) (*oauth2.Token, error) { 150 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) 151 | defer cancel() 152 | 153 | return c.Exchange(ctx, authCode) 154 | } 155 | 156 | func storeCredentials(credsFilePath string, token *oauth2.Token) error { 157 | tokenJSON, err := json.Marshal(token) 158 | if err != nil { 159 | return err 160 | } 161 | return os.WriteFile(credsFilePath, tokenJSON, 0644) 162 | } 163 | -------------------------------------------------------------------------------- /google/auth/oauth2_test.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/FreeLeh/GoFreeDB/internal/google/fixtures" 8 | "github.com/stretchr/testify/assert" 9 | "golang.org/x/oauth2" 10 | ) 11 | 12 | // Note that it is not really possible to test the "without stored credentials" path as 13 | // it requires a real user input to get the auth code and also a mock HTTP server. 14 | func TestNewOAuth2_CheckWrappedTransport_WithStoredCredentials(t *testing.T) { 15 | secretPath := fixtures.PathToFixture("client_secret.json") 16 | credsPath := fixtures.PathToFixture("stored_credentials.json") 17 | 18 | auth, err := NewOAuth2FromFile(secretPath, credsPath, []string{}, OAuth2Config{}) 19 | assert.Nil(t, err, "should not have any error instantiating the OAuth2 wrapper") 20 | 21 | _, ok := auth.HTTPClient().Transport.(*oauth2.Transport) 22 | assert.True(t, ok, "the HTTP client should be using the custom Google OAuth2 HTTP transport") 23 | 24 | _, err = os.Stat(credsPath) 25 | assert.Nil(t, err, "credential file should be created with the token info inside") 26 | } 27 | -------------------------------------------------------------------------------- /google/auth/service.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | 7 | "golang.org/x/oauth2/google" 8 | ) 9 | 10 | // ServiceConfig defines a list of configurations that can be used to customise how the Google 11 | // service account authentication flow works. 12 | type ServiceConfig struct { 13 | // HTTPClient allows the client to customise the HTTP client used to perform the REST API calls. 14 | // This will be useful if you want to have a more granular control over the HTTP client (e.g. using a connection pool). 15 | HTTPClient *http.Client 16 | } 17 | 18 | // Service takes in service account relevant information and sets up *http.Client that can be used to access 19 | // Google APIs seamlessly. Authentications will be handled automatically, including refreshing the access token 20 | // when necessary. 21 | type Service struct { 22 | googleAuthClient *http.Client 23 | } 24 | 25 | // HTTPClient returns a Google OAuth2 authenticated *http.Client that can be used to access Google APIs. 26 | func (s *Service) HTTPClient() *http.Client { 27 | return s.googleAuthClient 28 | } 29 | 30 | // NewServiceFromFile creates a Service instance by reading the Google service account related information from a file. 31 | // 32 | // The "filePath" is referring to the service account JSON file that can be obtained by 33 | // creating a new service account credentials in https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount. 34 | // 35 | // The "scopes" tells Google what your application can do to your spreadsheets. 36 | func NewServiceFromFile(filePath string, scopes Scopes, config ServiceConfig) (*Service, error) { 37 | authConfig, err := os.ReadFile(filePath) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return NewServiceFromJSON(authConfig, scopes, config) 42 | } 43 | 44 | // NewServiceFromJSON works exactly the same as NewServiceFromFile, but instead of reading from a file, the raw content 45 | // of the Google service account JSON file is provided directly. 46 | func NewServiceFromJSON(raw []byte, scopes Scopes, config ServiceConfig) (*Service, error) { 47 | c, err := google.JWTConfigFromJSON(raw, scopes...) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | return &Service{ 53 | googleAuthClient: c.Client(getClientCtx(config.HTTPClient)), 54 | }, nil 55 | } 56 | -------------------------------------------------------------------------------- /google/auth/service_test.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/FreeLeh/GoFreeDB/internal/google/fixtures" 7 | "github.com/stretchr/testify/assert" 8 | "golang.org/x/oauth2" 9 | ) 10 | 11 | func TestService_CheckWrappedTransport(t *testing.T) { 12 | path := fixtures.PathToFixture("service_account.json") 13 | 14 | service, err := NewServiceFromFile(path, []string{}, ServiceConfig{}) 15 | assert.Nil(t, err, "should not have any error instantiating the service account wrapper") 16 | 17 | _, ok := service.HTTPClient().Transport.(*oauth2.Transport) 18 | assert.True(t, ok, "the HTTP client should be using the custom Google OAuth2 HTTP transport") 19 | } 20 | -------------------------------------------------------------------------------- /google/auth/utils.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "golang.org/x/oauth2" 8 | ) 9 | 10 | func getClientCtx(customClient *http.Client) context.Context { 11 | ctx := context.Background() 12 | if customClient != nil { 13 | ctx = context.WithValue(ctx, oauth2.HTTPClient, customClient) 14 | } 15 | return ctx 16 | } 17 | -------------------------------------------------------------------------------- /google/auth/utils_test.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "golang.org/x/oauth2" 9 | ) 10 | 11 | func TestGetClientCtx(t *testing.T) { 12 | t.Run("without_custom_client", func(t *testing.T) { 13 | ctx := getClientCtx(nil) 14 | client := ctx.Value(oauth2.HTTPClient) 15 | assert.Nil(t, client, "client should be nil") 16 | }) 17 | 18 | t.Run("with_custom_client", func(t *testing.T) { 19 | custom := &http.Client{} 20 | ctx := getClientCtx(custom) 21 | client := ctx.Value(oauth2.HTTPClient).(*http.Client) 22 | 23 | assert.NotNil(t, client, "client should not be nil") 24 | assert.True(t, custom == client, "client should be the given custom HTTP client") 25 | }) 26 | } 27 | -------------------------------------------------------------------------------- /internal/codec/codec.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import "errors" 4 | 5 | const basicCodecPrefix string = "!" 6 | 7 | // Basic encodes and decodes any bytes data using a very simple encoding rule. 8 | // A prefix (an exclamation mark "!") will be attached to the raw bytes data. 9 | // 10 | // This allows the library to differentiate between empty raw bytes provided by the client from 11 | // getting an empty data from Google Sheets API. 12 | type Basic struct{} 13 | 14 | // Encode encodes the given raw bytes by using an exclamation mark "!" as a prefix. 15 | func (c *Basic) Encode(value []byte) (string, error) { 16 | return basicCodecPrefix + string(value), nil 17 | } 18 | 19 | // Decode converts the string data read from Google Sheet into raw bytes data after removing the 20 | // exclamation mark "!" prefix. 21 | func (c *Basic) Decode(value string) ([]byte, error) { 22 | if len(value) == 0 { 23 | return nil, errors.New("basic decode fail: empty string") 24 | } 25 | if value[:len(basicCodecPrefix)] != basicCodecPrefix { 26 | return nil, errors.New("basic decode fail: first character is not an empty space") 27 | } 28 | return []byte(value[len(basicCodecPrefix):]), nil 29 | } 30 | 31 | func NewBasic() *Basic { 32 | return &Basic{} 33 | } 34 | -------------------------------------------------------------------------------- /internal/codec/codec_test.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/FreeLeh/GoFreeDB/internal/models" 9 | ) 10 | 11 | func TestBasicCodecEncode(t *testing.T) { 12 | tc := []struct { 13 | name string 14 | input string 15 | expected string 16 | }{ 17 | { 18 | name: "empty_string", 19 | input: "", 20 | expected: "!", 21 | }, 22 | { 23 | name: "non_empty_string", 24 | input: "test", 25 | expected: "!test", 26 | }, 27 | { 28 | name: "emoji", 29 | input: "😀", 30 | expected: "!😀", 31 | }, 32 | { 33 | name: "NA_value", 34 | input: models.NAValue, 35 | expected: "!" + models.NAValue, 36 | }, 37 | } 38 | codec := &Basic{} 39 | 40 | for _, c := range tc { 41 | t.Run(c.name, func(t *testing.T) { 42 | result, err := codec.Encode([]byte(c.input)) 43 | assert.Nil(t, err) 44 | assert.Equal(t, c.expected, result) 45 | }) 46 | } 47 | } 48 | 49 | func TestBasicCodecDecode(t *testing.T) { 50 | tc := []struct { 51 | name string 52 | input string 53 | expected []byte 54 | hasErr bool 55 | }{ 56 | { 57 | name: "empty_string", 58 | input: "", 59 | expected: []byte(nil), 60 | hasErr: true, 61 | }, 62 | { 63 | name: "non_empty_string_no_whitespace", 64 | input: "test", 65 | expected: []byte(nil), 66 | hasErr: true, 67 | }, 68 | { 69 | name: "non_empty_string", 70 | input: "!test", 71 | expected: []byte("test"), 72 | hasErr: false, 73 | }, 74 | { 75 | name: "emoji", 76 | input: "!😀", 77 | expected: []byte("😀"), 78 | hasErr: false, 79 | }, 80 | { 81 | name: "NA_value", 82 | input: "!" + models.NAValue, 83 | expected: []byte(models.NAValue), 84 | hasErr: false, 85 | }, 86 | } 87 | codec := &Basic{} 88 | 89 | for _, c := range tc { 90 | t.Run(c.name, func(t *testing.T) { 91 | result, err := codec.Decode(c.input) 92 | assert.Equal(t, c.hasErr, err != nil) 93 | assert.Equal(t, c.expected, result) 94 | }) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /internal/common/mapstructure.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/mitchellh/mapstructure" 4 | 5 | func MapStructureDecode(input interface{}, output interface{}) error { 6 | config := &mapstructure.DecoderConfig{ 7 | Result: output, 8 | TagName: "db", 9 | } 10 | 11 | decoder, err := mapstructure.NewDecoder(config) 12 | if err != nil { 13 | return err 14 | } 15 | 16 | return decoder.Decode(input) 17 | } 18 | -------------------------------------------------------------------------------- /internal/common/range.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | func GetA1Range(sheetName string, rng string) string { 4 | return sheetName + "!" + rng 5 | } 6 | 7 | type ColIdx struct { 8 | Name string 9 | Idx int 10 | } 11 | 12 | type ColsMapping map[string]ColIdx 13 | 14 | func (m ColsMapping) NameMap() map[string]string { 15 | result := make(map[string]string, 0) 16 | for col, val := range m { 17 | result[col] = val.Name 18 | } 19 | return result 20 | } 21 | 22 | const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 23 | 24 | func GenerateColumnMapping(columns []string) map[string]ColIdx { 25 | mapping := make(map[string]ColIdx, len(columns)) 26 | for n, col := range columns { 27 | mapping[col] = ColIdx{ 28 | Name: GenerateColumnName(n), 29 | Idx: n, 30 | } 31 | } 32 | return mapping 33 | } 34 | 35 | func GenerateColumnName(n int) string { 36 | // This is not purely a Base26 conversion since the second char can start from "A" (or 0) again. 37 | // In a normal Base26 int to string conversion, the second char can only start from "B" (or 1). 38 | // Hence, we need to hack it by checking the first round separately from the subsequent round. 39 | // For the subsequent rounds, we need to subtract by 1 first or else it will always start from 1 (not 0). 40 | col := string(alphabet[n%26]) 41 | n = n / 26 42 | 43 | for { 44 | if n <= 0 { 45 | break 46 | } 47 | 48 | n -= 1 49 | col = string(alphabet[n%26]) + col 50 | n = n / 26 51 | } 52 | 53 | return col 54 | } 55 | -------------------------------------------------------------------------------- /internal/common/range_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestGetA1Range(t *testing.T) { 9 | assert.Equal(t, "sheet!A1:A50", GetA1Range("sheet", "A1:A50")) 10 | assert.Equal(t, "sheet!A1", GetA1Range("sheet", "A1")) 11 | assert.Equal(t, "sheet!A", GetA1Range("sheet", "A")) 12 | } 13 | 14 | func TestGenerateColumnName(t *testing.T) { 15 | tc := []struct { 16 | name string 17 | input int 18 | expected string 19 | }{ 20 | { 21 | name: "zero", 22 | input: 0, 23 | expected: "A", 24 | }, 25 | { 26 | name: "single_character", 27 | input: 15, 28 | expected: "P", 29 | }, 30 | { 31 | name: "single_character_2", 32 | input: 25, 33 | expected: "Z", 34 | }, 35 | { 36 | name: "single_character_3", 37 | input: 5, 38 | expected: "F", 39 | }, 40 | { 41 | name: "double_character", 42 | input: 26, 43 | expected: "AA", 44 | }, 45 | { 46 | name: "double_character_2", 47 | input: 52, 48 | expected: "BA", 49 | }, 50 | { 51 | name: "double_character_2", 52 | input: 89, 53 | expected: "CL", 54 | }, 55 | { 56 | name: "max_column", 57 | input: 18277, 58 | expected: "ZZZ", 59 | }, 60 | } 61 | 62 | for _, c := range tc { 63 | t.Run(c.name, func(t *testing.T) { 64 | assert.Equal(t, c.expected, GenerateColumnName(c.input)) 65 | }) 66 | } 67 | } 68 | 69 | func TestGenerateColumnMapping(t *testing.T) { 70 | tc := []struct { 71 | name string 72 | input []string 73 | expected map[string]ColIdx 74 | }{ 75 | { 76 | name: "single_column", 77 | input: []string{"col1"}, 78 | expected: map[string]ColIdx{ 79 | "col1": {"A", 0}, 80 | }, 81 | }, 82 | { 83 | name: "three_column", 84 | input: []string{"col1", "col2", "col3"}, 85 | expected: map[string]ColIdx{ 86 | "col1": {"A", 0}, 87 | "col2": {"B", 1}, 88 | "col3": {"C", 2}, 89 | }, 90 | }, 91 | { 92 | name: "many_column", 93 | input: []string{ 94 | "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", 95 | "c11", "c12", "c13", "c14", "c15", "c16", "c17", "c18", "c19", "c20", 96 | "c21", "c22", "c23", "c24", "c25", "c26", "c27", "c28", 97 | }, 98 | expected: map[string]ColIdx{ 99 | "c1": {"A", 0}, "c2": {"B", 1}, "c3": {"C", 2}, "c4": {"D", 3}, 100 | "c5": {"E", 4}, "c6": {"F", 5}, "c7": {"G", 6}, "c8": {"H", 7}, 101 | "c9": {"I", 8}, "c10": {"J", 9}, "c11": {"K", 10}, "c12": {"L", 11}, 102 | "c13": {"M", 12}, "c14": {"N", 13}, "c15": {"O", 14}, "c16": {"P", 15}, 103 | "c17": {"Q", 16}, "c18": {"R", 17}, "c19": {"S", 18}, "c20": {"T", 19}, 104 | "c21": {"U", 20}, "c22": {"V", 21}, "c23": {"W", 22}, "c24": {"X", 23}, 105 | "c25": {"Y", 24}, "c26": {"Z", 25}, "c27": {"AA", 26}, "c28": {"AB", 27}, 106 | }, 107 | }, 108 | } 109 | 110 | for _, c := range tc { 111 | t.Run(c.name, func(t *testing.T) { 112 | assert.Equal(t, c.expected, GenerateColumnMapping(c.input)) 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /internal/common/set.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type Set[T comparable] struct { 4 | values map[T]struct{} 5 | } 6 | 7 | func (s *Set[T]) Contains(v T) bool { 8 | _, ok := s.values[v] 9 | return ok 10 | } 11 | 12 | func NewSet[T comparable](values []T) *Set[T] { 13 | s := &Set[T]{ 14 | values: make(map[T]struct{}, len(values)), 15 | } 16 | for _, v := range values { 17 | s.values[v] = struct{}{} 18 | } 19 | return s 20 | } 21 | -------------------------------------------------------------------------------- /internal/common/time.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "time" 4 | 5 | func CurrentTimeMs() int64 { 6 | return time.Now().UnixMilli() 7 | } 8 | -------------------------------------------------------------------------------- /internal/common/values.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | func EscapeValue(value interface{}) interface{} { 9 | // This is to ensure that string value will always be a string representation in Google Sheets. 10 | // Without this, "1" may be converted automatically into an integer. 11 | // "2020-01-01" may be converted into a date format. 12 | switch value.(type) { 13 | case string: 14 | return fmt.Sprintf("'%s", value) 15 | default: 16 | return value 17 | } 18 | } 19 | 20 | func CheckIEEE754SafeInteger(value interface{}) error { 21 | switch converted := value.(type) { 22 | case int: 23 | return isIEEE754SafeInteger(int64(converted)) 24 | case int64: 25 | return isIEEE754SafeInteger(converted) 26 | case uint: 27 | return isIEEE754SafeInteger(int64(converted)) 28 | case uint64: 29 | return isIEEE754SafeInteger(int64(converted)) 30 | default: 31 | return nil 32 | } 33 | } 34 | 35 | func isIEEE754SafeInteger(value int64) error { 36 | if value == int64(float64(value)) { 37 | return nil 38 | } 39 | return errors.New("integer provided is not within the IEEE 754 safe integer boundary of [-(2^53), 2^53], the integer may have a precision lost") 40 | } 41 | -------------------------------------------------------------------------------- /internal/common/values_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestEscapeValue(t *testing.T) { 9 | assert.Equal(t, "'blah", EscapeValue("blah")) 10 | assert.Equal(t, 1, EscapeValue(1)) 11 | assert.Equal(t, true, EscapeValue(true)) 12 | } 13 | 14 | func TestCheckIEEE754SafeInteger(t *testing.T) { 15 | assert.Nil(t, CheckIEEE754SafeInteger(int64(0))) 16 | assert.Nil(t, CheckIEEE754SafeInteger(int(0))) 17 | assert.Nil(t, CheckIEEE754SafeInteger(uint(0))) 18 | 19 | // -(2^53) 20 | assert.Nil(t, CheckIEEE754SafeInteger(int64(-9007199254740992))) 21 | 22 | // (2^53) 23 | assert.Nil(t, CheckIEEE754SafeInteger(int64(9007199254740992))) 24 | assert.Nil(t, CheckIEEE754SafeInteger(uint64(9007199254740992))) 25 | 26 | // Below and above the limit. 27 | assert.NotNil(t, CheckIEEE754SafeInteger(int64(-9007199254740993))) 28 | assert.NotNil(t, CheckIEEE754SafeInteger(int64(9007199254740993))) 29 | assert.NotNil(t, CheckIEEE754SafeInteger(uint64(9007199254740993))) 30 | 31 | // Other types 32 | assert.Nil(t, CheckIEEE754SafeInteger("blah")) 33 | assert.Nil(t, CheckIEEE754SafeInteger(true)) 34 | assert.Nil(t, CheckIEEE754SafeInteger([]byte("something"))) 35 | } 36 | -------------------------------------------------------------------------------- /internal/google/fixtures/client_secret.json: -------------------------------------------------------------------------------- 1 | { 2 | "web": { 3 | "client_id": "client_id", 4 | "project_id": "project_id", 5 | "auth_uri": "auth_uri", 6 | "token_uri": "token_uri", 7 | "auth_provider_x509_cert_url": "auth_provider_x509_cert_url", 8 | "client_secret": "client_secret", 9 | "redirect_uris": [ 10 | "http://localhost:8000" 11 | ], 12 | "javascript_origins": [ 13 | "http://localhost:8080" 14 | ] 15 | } 16 | } -------------------------------------------------------------------------------- /internal/google/fixtures/file.go: -------------------------------------------------------------------------------- 1 | package fixtures 2 | 3 | import ( 4 | "path/filepath" 5 | "runtime" 6 | ) 7 | 8 | // PathToFixture returns the absolute path to the given fixtureFileName. 9 | // This is a helper function to make it easier load some test fixture files. 10 | func PathToFixture(fixtureFileName string) string { 11 | _, currentFile, _, _ := runtime.Caller(0) 12 | return filepath.Join(filepath.Dir(currentFile), fixtureFileName) 13 | } 14 | -------------------------------------------------------------------------------- /internal/google/fixtures/service_account.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "project_id", 4 | "private_key_id": "private_key_id", 5 | "private_key": "private_key", 6 | "client_email": "client_email", 7 | "client_id": "client_id", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://oauth2.googleapis.com/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509" 12 | } -------------------------------------------------------------------------------- /internal/google/fixtures/stored_credentials.json: -------------------------------------------------------------------------------- 1 | { 2 | "access_token": "", 3 | "refresh_token": "", 4 | "token_type": "", 5 | "expiry": "2009-11-10T23:00:00Z" 6 | } -------------------------------------------------------------------------------- /internal/google/sheets/models.go: -------------------------------------------------------------------------------- 1 | package sheets 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type appendMode string 9 | 10 | const ( 11 | majorDimensionRows = "ROWS" 12 | valueInputUserEntered = "USER_ENTERED" 13 | responseValueRenderFormatted = "FORMATTED_VALUE" 14 | appendModeInsert appendMode = "INSERT_ROWS" 15 | appendModeOverwrite appendMode = "OVERWRITE" 16 | 17 | queryRowsURLTemplate = "https://docs.google.com/spreadsheets/d/%s/gviz/tq" 18 | ) 19 | 20 | type A1Range struct { 21 | Original string 22 | SheetName string 23 | FromCell string 24 | ToCell string 25 | } 26 | 27 | func NewA1Range(s string) A1Range { 28 | exclamationIdx := strings.Index(s, "!") 29 | colonIdx := strings.Index(s, ":") 30 | 31 | if exclamationIdx == -1 { 32 | if colonIdx == -1 { 33 | return A1Range{ 34 | Original: s, 35 | SheetName: "", 36 | FromCell: s, 37 | ToCell: s, 38 | } 39 | } else { 40 | return A1Range{ 41 | Original: s, 42 | SheetName: "", 43 | FromCell: s[:colonIdx], 44 | ToCell: s[colonIdx+1:], 45 | } 46 | } 47 | } else { 48 | if colonIdx == -1 { 49 | return A1Range{ 50 | Original: s, 51 | SheetName: s[:exclamationIdx], 52 | FromCell: s[exclamationIdx+1:], 53 | ToCell: s[exclamationIdx+1:], 54 | } 55 | } else { 56 | return A1Range{ 57 | Original: s, 58 | SheetName: s[:exclamationIdx], 59 | FromCell: s[exclamationIdx+1 : colonIdx], 60 | ToCell: s[colonIdx+1:], 61 | } 62 | } 63 | } 64 | } 65 | 66 | type InsertRowsResult struct { 67 | UpdatedRange A1Range 68 | UpdatedRows int64 69 | UpdatedColumns int64 70 | UpdatedCells int64 71 | InsertedValues [][]interface{} 72 | } 73 | 74 | type UpdateRowsResult struct { 75 | UpdatedRange A1Range 76 | UpdatedRows int64 77 | UpdatedColumns int64 78 | UpdatedCells int64 79 | UpdatedValues [][]interface{} 80 | } 81 | 82 | type BatchUpdateRowsRequest struct { 83 | A1Range string 84 | Values [][]interface{} 85 | } 86 | 87 | type BatchUpdateRowsResult []UpdateRowsResult 88 | 89 | /* 90 | { 91 | "version":"0.6", 92 | "reqId":"0", 93 | "status":"ok", 94 | "sig":"141753603", 95 | "table":{ 96 | "cols":[ 97 | {"id":"A","label":"","type":"string"}, 98 | {"id":"B","label":"","type":"number","pattern":"General"} 99 | ], 100 | "rows":[ 101 | {"c":[{"v":"k1"},{"v":103.0,"f":"103"}]}, 102 | {"c":[{"v":"k2"},{"v":111.0,"f":"111"}]}, 103 | {"c":[{"v":"k3"},{"v":123.0,"f":"123"}]} 104 | ], 105 | "parsedNumHeaders":0 106 | } 107 | } 108 | */ 109 | type rawQueryRowsResult struct { 110 | Table rawQueryRowsResultTable `json:"table"` 111 | } 112 | 113 | func (r rawQueryRowsResult) toQueryRowsResult() (QueryRowsResult, error) { 114 | result := QueryRowsResult{ 115 | Rows: make([][]interface{}, len(r.Table.Rows)), 116 | } 117 | 118 | for rowIdx, row := range r.Table.Rows { 119 | result.Rows[rowIdx] = make([]interface{}, len(row.Cells)) 120 | for cellIdx, cell := range row.Cells { 121 | val, err := r.convertRawValue(cellIdx, cell) 122 | if err != nil { 123 | return QueryRowsResult{}, err 124 | } 125 | result.Rows[rowIdx][cellIdx] = val 126 | } 127 | } 128 | 129 | return result, nil 130 | } 131 | 132 | func (r rawQueryRowsResult) convertRawValue(cellIdx int, cell rawQueryRowsResultCell) (interface{}, error) { 133 | col := r.Table.Cols[cellIdx] 134 | switch col.Type { 135 | case "boolean": 136 | return cell.Value, nil 137 | case "number": 138 | return cell.Value, nil 139 | case "string": 140 | // `string` type does not have the raw value 141 | return cell.Value, nil 142 | case "date", "datetime", "timeofday": 143 | return cell.Raw, nil 144 | } 145 | return nil, fmt.Errorf("unsupported cell value: %s", col.Type) 146 | } 147 | 148 | type rawQueryRowsResultTable struct { 149 | Cols []rawQueryRowsResultColumn `json:"cols"` 150 | Rows []rawQueryRowsResultRow `json:"rows"` 151 | } 152 | 153 | type rawQueryRowsResultColumn struct { 154 | ID string `json:"id"` 155 | Type string `json:"type"` 156 | } 157 | 158 | type rawQueryRowsResultRow struct { 159 | Cells []rawQueryRowsResultCell `json:"c"` 160 | } 161 | 162 | type rawQueryRowsResultCell struct { 163 | Value interface{} `json:"v"` 164 | Raw string `json:"f"` 165 | } 166 | 167 | type QueryRowsResult struct { 168 | Rows [][]interface{} 169 | } 170 | -------------------------------------------------------------------------------- /internal/google/sheets/models_test.go: -------------------------------------------------------------------------------- 1 | package sheets 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestA1Range(t *testing.T) { 10 | tc := []struct { 11 | name string 12 | input string 13 | sheetName string 14 | fromCell string 15 | toCell string 16 | }{ 17 | { 18 | name: "no_sheet_name_single_range", 19 | input: "A1", 20 | sheetName: "", 21 | fromCell: "A1", 22 | toCell: "A1", 23 | }, 24 | { 25 | name: "no_sheet_name_multiple_range", 26 | input: "A1:A2", 27 | sheetName: "", 28 | fromCell: "A1", 29 | toCell: "A2", 30 | }, 31 | { 32 | name: "has_sheet_name_single_range", 33 | input: "Sheet1!A1", 34 | sheetName: "Sheet1", 35 | fromCell: "A1", 36 | toCell: "A1", 37 | }, 38 | { 39 | name: "has_sheet_name_multiple_range", 40 | input: "Sheet1!A1:A2", 41 | sheetName: "Sheet1", 42 | fromCell: "A1", 43 | toCell: "A2", 44 | }, 45 | { 46 | name: "empty_input", 47 | input: "", 48 | sheetName: "", 49 | fromCell: "", 50 | toCell: "", 51 | }, 52 | } 53 | 54 | for _, c := range tc { 55 | t.Run(c.name, func(t *testing.T) { 56 | a1 := NewA1Range(c.input) 57 | assert.Equal(t, a1.Original, c.input, "A1Range original should have the same value as the input") 58 | assert.Equal(t, a1.SheetName, c.sheetName, "wrong sheet name") 59 | assert.Equal(t, a1.FromCell, c.fromCell, "wrong from cell") 60 | assert.Equal(t, a1.ToCell, c.toCell, "wrong to cell") 61 | }) 62 | } 63 | } 64 | 65 | func TestRawQueryRowsResult_toQueryRowsResult(t *testing.T) { 66 | t.Run("empty_rows", func(t *testing.T) { 67 | r := rawQueryRowsResult{ 68 | Table: rawQueryRowsResultTable{ 69 | Cols: []rawQueryRowsResultColumn{ 70 | {ID: "A", Type: "number"}, 71 | {ID: "B", Type: "string"}, 72 | }, 73 | Rows: []rawQueryRowsResultRow{}, 74 | }, 75 | } 76 | 77 | expected := QueryRowsResult{Rows: make([][]interface{}, 0)} 78 | 79 | result, err := r.toQueryRowsResult() 80 | assert.Nil(t, err) 81 | assert.Equal(t, expected, result) 82 | }) 83 | 84 | t.Run("few_rows", func(t *testing.T) { 85 | r := rawQueryRowsResult{ 86 | Table: rawQueryRowsResultTable{ 87 | Cols: []rawQueryRowsResultColumn{ 88 | {ID: "A", Type: "number"}, 89 | {ID: "B", Type: "string"}, 90 | {ID: "C", Type: "boolean"}, 91 | }, 92 | Rows: []rawQueryRowsResultRow{ 93 | { 94 | []rawQueryRowsResultCell{ 95 | {Value: 123.0, Raw: "123"}, 96 | {Value: "blah", Raw: "blah"}, 97 | {Value: true, Raw: "true"}, 98 | }, 99 | }, 100 | { 101 | []rawQueryRowsResultCell{ 102 | {Value: 456.0, Raw: "456"}, 103 | {Value: "blah2", Raw: "blah2"}, 104 | {Value: false, Raw: "FALSE"}, 105 | }, 106 | }, 107 | { 108 | []rawQueryRowsResultCell{ 109 | {Value: 123.1, Raw: "123.1"}, 110 | {Value: "blah", Raw: "blah"}, 111 | {Value: true, Raw: "TRUE"}, 112 | }, 113 | }, 114 | }, 115 | }, 116 | } 117 | 118 | expected := QueryRowsResult{ 119 | Rows: [][]interface{}{ 120 | {float64(123), "blah", true}, 121 | {float64(456), "blah2", false}, 122 | {123.1, "blah", true}, 123 | }, 124 | } 125 | 126 | result, err := r.toQueryRowsResult() 127 | assert.Nil(t, err) 128 | assert.Equal(t, expected, result) 129 | }) 130 | 131 | t.Run("unexpected_type", func(t *testing.T) { 132 | r := rawQueryRowsResult{ 133 | Table: rawQueryRowsResultTable{ 134 | Cols: []rawQueryRowsResultColumn{ 135 | {ID: "A", Type: "number"}, 136 | {ID: "B", Type: "string"}, 137 | {ID: "C", Type: "something"}, 138 | }, 139 | Rows: []rawQueryRowsResultRow{ 140 | { 141 | []rawQueryRowsResultCell{ 142 | {Value: 123.0, Raw: "123"}, 143 | {Value: "blah", Raw: "blah"}, 144 | {Value: true, Raw: "true"}, 145 | }, 146 | }, 147 | { 148 | []rawQueryRowsResultCell{ 149 | {Value: 456.0, Raw: "456"}, 150 | {Value: "blah2", Raw: "blah2"}, 151 | {Value: false, Raw: "FALSE"}, 152 | }, 153 | }, 154 | { 155 | []rawQueryRowsResultCell{ 156 | {Value: 123.1, Raw: "123.1"}, 157 | {Value: "blah", Raw: "blah"}, 158 | {Value: true, Raw: "TRUE"}, 159 | }, 160 | }, 161 | }, 162 | }, 163 | } 164 | 165 | result, err := r.toQueryRowsResult() 166 | assert.Equal(t, QueryRowsResult{}, result) 167 | assert.NotNil(t, err) 168 | }) 169 | } 170 | -------------------------------------------------------------------------------- /internal/google/sheets/wrapper.go: -------------------------------------------------------------------------------- 1 | package sheets 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "net/url" 11 | "strconv" 12 | "strings" 13 | 14 | "google.golang.org/api/option" 15 | "google.golang.org/api/sheets/v4" 16 | ) 17 | 18 | type AuthClient interface { 19 | HTTPClient() *http.Client 20 | } 21 | 22 | type Wrapper struct { 23 | service *sheets.Service 24 | rawClient *http.Client 25 | } 26 | 27 | func (w *Wrapper) CreateSpreadsheet(ctx context.Context, title string) (string, error) { 28 | createSpreadsheetReq := w.service.Spreadsheets.Create(&sheets.Spreadsheet{ 29 | Properties: &sheets.SpreadsheetProperties{Title: title}, 30 | }).Context(ctx) 31 | 32 | spreadsheet, err := createSpreadsheetReq.Do() 33 | if err != nil { 34 | return "", err 35 | } 36 | return spreadsheet.SpreadsheetId, nil 37 | } 38 | 39 | func (w *Wrapper) CreateSheet(ctx context.Context, spreadsheetID string, sheetName string) error { 40 | addSheetReq := &sheets.AddSheetRequest{Properties: &sheets.SheetProperties{Title: sheetName}} 41 | requests := []*sheets.Request{ 42 | {AddSheet: addSheetReq}, 43 | } 44 | batchUpdateSpreadsheetReq := w.service.Spreadsheets.BatchUpdate( 45 | spreadsheetID, 46 | &sheets.BatchUpdateSpreadsheetRequest{Requests: requests}, 47 | ).Context(ctx) 48 | 49 | _, err := batchUpdateSpreadsheetReq.Do() 50 | return err 51 | } 52 | 53 | func (w *Wrapper) GetSheetNameToID(ctx context.Context, spreadsheetID string) (map[string]int64, error) { 54 | resp, err := w.service.Spreadsheets.Get(spreadsheetID).Context(ctx).Do() 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | result := make(map[string]int64) 60 | for _, sheet := range resp.Sheets { 61 | if sheet.Properties == nil { 62 | return nil, errors.New("failed getSheetIDByName due to empty sheet properties") 63 | } 64 | result[sheet.Properties.Title] = sheet.Properties.SheetId 65 | } 66 | 67 | return result, nil 68 | } 69 | 70 | func (w *Wrapper) DeleteSheets(ctx context.Context, spreadsheetID string, sheetIDs []int64) error { 71 | requests := make([]*sheets.Request, 0, len(sheetIDs)) 72 | for _, sheetID := range sheetIDs { 73 | deleteSheetReq := &sheets.DeleteSheetRequest{SheetId: sheetID} 74 | requests = append(requests, &sheets.Request{DeleteSheet: deleteSheetReq}) 75 | } 76 | 77 | batchUpdateSpreadsheetReq := w.service.Spreadsheets.BatchUpdate( 78 | spreadsheetID, 79 | &sheets.BatchUpdateSpreadsheetRequest{Requests: requests}, 80 | ).Context(ctx) 81 | 82 | _, err := batchUpdateSpreadsheetReq.Do() 83 | return err 84 | } 85 | 86 | func (w *Wrapper) InsertRows( 87 | ctx context.Context, 88 | spreadsheetID string, 89 | a1Range string, 90 | values [][]interface{}, 91 | ) (InsertRowsResult, error) { 92 | return w.insertRows(ctx, spreadsheetID, a1Range, values, appendModeInsert) 93 | } 94 | 95 | func (w *Wrapper) OverwriteRows( 96 | ctx context.Context, 97 | spreadsheetID string, 98 | a1Range string, 99 | values [][]interface{}, 100 | ) (InsertRowsResult, error) { 101 | return w.insertRows(ctx, spreadsheetID, a1Range, values, appendModeOverwrite) 102 | } 103 | 104 | func (w *Wrapper) insertRows( 105 | ctx context.Context, 106 | spreadsheetID string, 107 | a1Range string, 108 | values [][]interface{}, 109 | mode appendMode, 110 | ) (InsertRowsResult, error) { 111 | valueRange := &sheets.ValueRange{ 112 | MajorDimension: majorDimensionRows, 113 | Range: a1Range, 114 | Values: values, 115 | } 116 | 117 | req := w.service.Spreadsheets.Values.Append(spreadsheetID, a1Range, valueRange). 118 | InsertDataOption(string(mode)). 119 | IncludeValuesInResponse(true). 120 | ResponseValueRenderOption(responseValueRenderFormatted). 121 | ValueInputOption(valueInputUserEntered). 122 | Context(ctx) 123 | 124 | resp, err := req.Do() 125 | if err != nil { 126 | return InsertRowsResult{}, err 127 | } 128 | 129 | return InsertRowsResult{ 130 | UpdatedRange: NewA1Range(resp.Updates.UpdatedRange), 131 | UpdatedRows: resp.Updates.UpdatedRows, 132 | UpdatedColumns: resp.Updates.UpdatedColumns, 133 | UpdatedCells: resp.Updates.UpdatedCells, 134 | InsertedValues: resp.Updates.UpdatedData.Values, 135 | }, nil 136 | } 137 | 138 | func (w *Wrapper) UpdateRows( 139 | ctx context.Context, 140 | spreadsheetID string, 141 | a1Range string, 142 | values [][]interface{}, 143 | ) (UpdateRowsResult, error) { 144 | valueRange := &sheets.ValueRange{ 145 | MajorDimension: majorDimensionRows, 146 | Range: a1Range, 147 | Values: values, 148 | } 149 | 150 | req := w.service.Spreadsheets.Values.Update(spreadsheetID, a1Range, valueRange). 151 | IncludeValuesInResponse(true). 152 | ResponseValueRenderOption(responseValueRenderFormatted). 153 | ValueInputOption(valueInputUserEntered). 154 | Context(ctx) 155 | 156 | resp, err := req.Do() 157 | if err != nil { 158 | return UpdateRowsResult{}, err 159 | } 160 | 161 | return UpdateRowsResult{ 162 | UpdatedRange: NewA1Range(resp.UpdatedRange), 163 | UpdatedRows: resp.UpdatedRows, 164 | UpdatedColumns: resp.UpdatedColumns, 165 | UpdatedCells: resp.UpdatedCells, 166 | UpdatedValues: resp.UpdatedData.Values, 167 | }, nil 168 | } 169 | 170 | func (w *Wrapper) BatchUpdateRows( 171 | ctx context.Context, 172 | spreadsheetID string, 173 | requests []BatchUpdateRowsRequest, 174 | ) (BatchUpdateRowsResult, error) { 175 | valueRanges := make([]*sheets.ValueRange, len(requests)) 176 | for i := range requests { 177 | valueRanges[i] = &sheets.ValueRange{ 178 | MajorDimension: majorDimensionRows, 179 | Range: requests[i].A1Range, 180 | Values: requests[i].Values, 181 | } 182 | } 183 | 184 | batchUpdate := &sheets.BatchUpdateValuesRequest{ 185 | Data: valueRanges, 186 | IncludeValuesInResponse: true, 187 | ResponseValueRenderOption: responseValueRenderFormatted, 188 | ValueInputOption: valueInputUserEntered, 189 | } 190 | 191 | req := w.service.Spreadsheets.Values.BatchUpdate(spreadsheetID, batchUpdate).Context(ctx) 192 | 193 | resp, err := req.Do() 194 | if err != nil { 195 | return BatchUpdateRowsResult{}, err 196 | } 197 | 198 | results := make(BatchUpdateRowsResult, len(requests)) 199 | for i := range resp.Responses { 200 | results[i] = UpdateRowsResult{ 201 | UpdatedRange: NewA1Range(resp.Responses[i].UpdatedRange), 202 | UpdatedRows: resp.Responses[i].UpdatedRows, 203 | UpdatedColumns: resp.Responses[i].UpdatedColumns, 204 | UpdatedCells: resp.Responses[i].UpdatedCells, 205 | UpdatedValues: resp.Responses[i].UpdatedData.Values, 206 | } 207 | } 208 | 209 | return results, nil 210 | } 211 | 212 | func (w *Wrapper) QueryRows( 213 | ctx context.Context, 214 | spreadsheetID string, 215 | sheetName string, 216 | query string, 217 | skipHeader bool, 218 | ) (QueryRowsResult, error) { 219 | rawResult, err := w.execQueryRows(ctx, spreadsheetID, sheetName, query, skipHeader) 220 | if err != nil { 221 | return QueryRowsResult{}, err 222 | } 223 | return rawResult.toQueryRowsResult() 224 | } 225 | 226 | func (w *Wrapper) execQueryRows( 227 | ctx context.Context, 228 | spreadsheetID string, 229 | sheetName string, 230 | query string, 231 | skipHeader bool, 232 | ) (rawQueryRowsResult, error) { 233 | params := url.Values{} 234 | params.Add("sheet", sheetName) 235 | params.Add("tqx", "responseHandler:freedb") 236 | params.Add("tq", query) 237 | 238 | header := 0 239 | if skipHeader { 240 | header = 1 241 | } 242 | params.Add("headers", strconv.FormatInt(int64(header), 10)) 243 | 244 | url := fmt.Sprintf(queryRowsURLTemplate, spreadsheetID) + "?" + params.Encode() 245 | 246 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 247 | if err != nil { 248 | return rawQueryRowsResult{}, err 249 | } 250 | 251 | resp, err := w.rawClient.Do(req) 252 | if err != nil { 253 | return rawQueryRowsResult{}, err 254 | } 255 | defer resp.Body.Close() 256 | 257 | respBytes, err := io.ReadAll(resp.Body) 258 | if err != nil { 259 | return rawQueryRowsResult{}, err 260 | } 261 | respString := string(respBytes) 262 | 263 | firstCurly := strings.Index(respString, "{") 264 | if firstCurly == -1 { 265 | return rawQueryRowsResult{}, fmt.Errorf("opening curly bracket not found: %s", respString) 266 | } 267 | 268 | lastCurly := strings.LastIndex(respString, "}") 269 | if lastCurly == -1 { 270 | return rawQueryRowsResult{}, fmt.Errorf("closing curly bracket not found: %s", respString) 271 | } 272 | 273 | result := rawQueryRowsResult{} 274 | if err := json.Unmarshal([]byte(respString[firstCurly:lastCurly+1]), &result); err != nil { 275 | return rawQueryRowsResult{}, err 276 | } 277 | return result, nil 278 | } 279 | 280 | func (w *Wrapper) Clear(ctx context.Context, spreadsheetID string, ranges []string) ([]string, error) { 281 | req := w.service.Spreadsheets.Values.BatchClear(spreadsheetID, &sheets.BatchClearValuesRequest{Ranges: ranges}). 282 | Context(ctx) 283 | resp, err := req.Do() 284 | if err != nil { 285 | return nil, err 286 | } 287 | return resp.ClearedRanges, nil 288 | } 289 | 290 | func NewWrapper(authClient AuthClient) (*Wrapper, error) { 291 | // The `ctx` provided into `NewService` is not really used for anything in our case. 292 | // Internally it seems it's used for creating a new HTTP client, but we already provide with our 293 | // own auth HTTP client. 294 | service, err := sheets.NewService(context.Background(), option.WithHTTPClient(authClient.HTTPClient())) 295 | if err != nil { 296 | return nil, err 297 | } 298 | return &Wrapper{ 299 | service: service, 300 | rawClient: authClient.HTTPClient(), 301 | }, nil 302 | } 303 | -------------------------------------------------------------------------------- /internal/google/sheets/wrapper_mock.go: -------------------------------------------------------------------------------- 1 | package sheets 2 | 3 | import "context" 4 | 5 | type MockWrapper struct { 6 | CreateSpreadsheetResult string 7 | CreateSpreadsheetError error 8 | 9 | CreateSheetError error 10 | 11 | InsertRowsResult InsertRowsResult 12 | InsertRowsError error 13 | 14 | OverwriteRowsResult InsertRowsResult 15 | OverwriteRowsError error 16 | 17 | UpdateRowsResult UpdateRowsResult 18 | UpdateRowsError error 19 | 20 | BatchUpdateRowsResult BatchUpdateRowsResult 21 | BatchUpdateRowsError error 22 | 23 | QueryRowsResult QueryRowsResult 24 | QueryRowsError error 25 | 26 | ClearResult []string 27 | ClearError error 28 | } 29 | 30 | func (w *MockWrapper) CreateSpreadsheet(ctx context.Context, title string) (string, error) { 31 | return w.CreateSpreadsheetResult, w.CreateSpreadsheetError 32 | } 33 | 34 | func (w *MockWrapper) GetSheetNameToID(ctx context.Context, spreadsheetID string) (map[string]int64, error) { 35 | return nil, nil 36 | } 37 | 38 | func (w *MockWrapper) DeleteSheets(ctx context.Context, spreadsheetID string, sheetIDs []int64) error { 39 | return nil 40 | } 41 | 42 | func (w *MockWrapper) CreateSheet(ctx context.Context, spreadsheetID string, sheetName string) error { 43 | return w.CreateSheetError 44 | } 45 | 46 | func (w *MockWrapper) InsertRows(ctx context.Context, spreadsheetID string, a1Range string, values [][]interface{}) (InsertRowsResult, error) { 47 | return w.InsertRowsResult, w.InsertRowsError 48 | } 49 | 50 | func (w *MockWrapper) OverwriteRows(ctx context.Context, spreadsheetID string, a1Range string, values [][]interface{}) (InsertRowsResult, error) { 51 | return w.OverwriteRowsResult, w.OverwriteRowsError 52 | } 53 | 54 | func (w *MockWrapper) UpdateRows(ctx context.Context, spreadsheetID string, a1Range string, values [][]interface{}) (UpdateRowsResult, error) { 55 | return w.UpdateRowsResult, w.UpdateRowsError 56 | } 57 | 58 | func (w *MockWrapper) BatchUpdateRows(ctx context.Context, spreadsheetID string, requests []BatchUpdateRowsRequest) (BatchUpdateRowsResult, error) { 59 | return w.BatchUpdateRowsResult, w.BatchUpdateRowsError 60 | } 61 | 62 | func (w *MockWrapper) QueryRows(ctx context.Context, spreadsheetID string, sheetName string, query string, skipHeader bool) (QueryRowsResult, error) { 63 | return w.QueryRowsResult, w.QueryRowsError 64 | } 65 | 66 | func (w *MockWrapper) Clear(ctx context.Context, spreadsheetID string, ranges []string) ([]string, error) { 67 | return w.ClearResult, w.ClearError 68 | } 69 | -------------------------------------------------------------------------------- /internal/google/sheets/wrapper_test.go: -------------------------------------------------------------------------------- 1 | package sheets 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/FreeLeh/GoFreeDB/google/auth" 9 | "github.com/FreeLeh/GoFreeDB/internal/google/fixtures" 10 | "github.com/stretchr/testify/assert" 11 | "gopkg.in/h2non/gock.v1" 12 | ) 13 | 14 | func TestCreateSpreadsheet(t *testing.T) { 15 | path := fixtures.PathToFixture("service_account.json") 16 | 17 | auth, err := auth.NewServiceFromFile(path, []string{}, auth.ServiceConfig{}) 18 | assert.Nil(t, err, "should not have any error instantiating a new service account client") 19 | 20 | wrapper, err := NewWrapper(auth) 21 | assert.Nil(t, err, "should not have any error instantiating a new sheets wrapper") 22 | 23 | gock.InterceptClient(auth.HTTPClient()) 24 | 25 | t.Run("successful", func(t *testing.T) { 26 | expectedReqBody := map[string]map[string]string{ 27 | "properties": { 28 | "title": "title", 29 | }, 30 | } 31 | resp := map[string]string{"spreadsheetId": "123"} 32 | 33 | gock.New("https://sheets.googleapis.com"). 34 | Post("/v4/spreadsheets"). 35 | JSON(expectedReqBody). 36 | Reply(http.StatusOK). 37 | JSON(resp) 38 | 39 | sid, err := wrapper.CreateSpreadsheet(context.Background(), "title") 40 | assert.Equal(t, "123", sid, "returned spreadsheetID does not match with the mocked spreadsheetID") 41 | assert.Nil(t, err, "should not have any error creating a new spreadsheet") 42 | }) 43 | 44 | t.Run("http500", func(t *testing.T) { 45 | expectedReqBody := map[string]map[string]string{ 46 | "properties": { 47 | "title": "title", 48 | }, 49 | } 50 | 51 | gock.New("https://sheets.googleapis.com"). 52 | Post("/v4/spreadsheets"). 53 | JSON(expectedReqBody). 54 | Reply(http.StatusInternalServerError) 55 | 56 | sid, err := wrapper.CreateSpreadsheet(context.Background(), "title") 57 | assert.Equal(t, "", sid, "returned spreadsheetID should be empty as there is HTTP error") 58 | assert.NotNil(t, err, "should have an error when creating the spreadsheet as there is HTTP error") 59 | }) 60 | 61 | t.Run("empty_title", func(t *testing.T) { 62 | expectedReqBody := map[string]map[string]string{ 63 | "properties": { 64 | "title": "title", 65 | }, 66 | } 67 | resp := map[string]string{"spreadsheetId": "123"} 68 | 69 | gock.New("https://sheets.googleapis.com"). 70 | Post("/v4/spreadsheets"). 71 | JSON(expectedReqBody). 72 | Reply(http.StatusOK). 73 | JSON(resp) 74 | 75 | sid, err := wrapper.CreateSpreadsheet(context.Background(), "title") 76 | assert.Equal(t, "123", sid, "returned spreadsheetID does not match with the mocked spreadsheetID") 77 | assert.Nil(t, err, "should not have any error creating a new spreadsheet") 78 | }) 79 | } 80 | 81 | func TestCreateSheet(t *testing.T) { 82 | path := fixtures.PathToFixture("service_account.json") 83 | 84 | auth, err := auth.NewServiceFromFile(path, []string{}, auth.ServiceConfig{}) 85 | assert.Nil(t, err, "should not have any error instantiating a new service account client") 86 | 87 | wrapper, err := NewWrapper(auth) 88 | assert.Nil(t, err, "should not have any error instantiating a new sheets wrapper") 89 | 90 | gock.InterceptClient(auth.HTTPClient()) 91 | 92 | t.Run("successful", func(t *testing.T) { 93 | expectedReqBody := map[string][]map[string]map[string]map[string]string{ 94 | "requests": { 95 | { 96 | "addSheet": { 97 | "properties": { 98 | "title": "sheet", 99 | }, 100 | }, 101 | }, 102 | }, 103 | } 104 | resp := map[string]interface{}{ 105 | "spreadsheetId": "123", 106 | "replies": []map[string]map[string]map[string]string{ 107 | { 108 | "addSheet": { 109 | "properties": { 110 | "title": "sheet", 111 | }, 112 | }, 113 | }, 114 | }, 115 | } 116 | 117 | gock.New("https://sheets.googleapis.com"). 118 | Post("/v4/spreadsheets/123:batchUpdate"). 119 | JSON(expectedReqBody). 120 | Reply(http.StatusOK). 121 | JSON(resp) 122 | 123 | err := wrapper.CreateSheet(context.Background(), "123", "sheet") 124 | assert.Nil(t, err, "should not have any error creating a new sheet") 125 | }) 126 | 127 | t.Run("http500", func(t *testing.T) { 128 | expectedReqBody := map[string][]map[string]map[string]map[string]string{ 129 | "requests": { 130 | { 131 | "addSheet": { 132 | "properties": { 133 | "title": "sheet", 134 | }, 135 | }, 136 | }, 137 | }, 138 | } 139 | 140 | gock.New("https://sheets.googleapis.com"). 141 | Post("/v4/spreadsheets/123:batchUpdate"). 142 | JSON(expectedReqBody). 143 | Reply(http.StatusInternalServerError) 144 | 145 | err := wrapper.CreateSheet(context.Background(), "123", "sheet") 146 | assert.NotNil(t, err, "should have an error when creating a sheet as there is HTTP error") 147 | }) 148 | 149 | t.Run("empty_title", func(t *testing.T) { 150 | expectedReqBody := map[string][]map[string]map[string]map[string]string{ 151 | "requests": { 152 | { 153 | "addSheet": {"properties": {}}, 154 | }, 155 | }, 156 | } 157 | resp := map[string]interface{}{ 158 | "spreadsheetId": "123", 159 | "replies": []map[string]map[string]map[string]string{ 160 | { 161 | "addSheet": { 162 | "properties": { 163 | "title": "untitled", 164 | }, 165 | }, 166 | }, 167 | }, 168 | } 169 | 170 | gock.New("https://sheets.googleapis.com"). 171 | Post("/v4/spreadsheets/123:batchUpdate"). 172 | JSON(expectedReqBody). 173 | Reply(http.StatusOK). 174 | JSON(resp) 175 | 176 | err := wrapper.CreateSheet(context.Background(), "123", "") 177 | assert.Nil(t, err, "should not have any error creating a new sheet") 178 | }) 179 | } 180 | 181 | func TestInsertAppendRows(t *testing.T) { 182 | path := fixtures.PathToFixture("service_account.json") 183 | 184 | auth, err := auth.NewServiceFromFile(path, []string{}, auth.ServiceConfig{}) 185 | assert.Nil(t, err, "should not have any error instantiating a new service account client") 186 | 187 | wrapper, err := NewWrapper(auth) 188 | assert.Nil(t, err, "should not have any error instantiating a new sheets wrapper") 189 | 190 | gock.InterceptClient(auth.HTTPClient()) 191 | 192 | t.Run("successful", func(t *testing.T) { 193 | expectedParams := map[string]string{ 194 | "includeValuesInResponse": "true", 195 | "responseValueRenderOption": responseValueRenderFormatted, 196 | "insertDataOption": string(appendModeOverwrite), 197 | "valueInputOption": valueInputUserEntered, 198 | } 199 | expectedReqBody := map[string]interface{}{ 200 | "majorDimension": majorDimensionRows, 201 | "range": "Sheet1!A1:A2", 202 | "values": [][]interface{}{ 203 | {"1", "2"}, 204 | {"3", "4"}, 205 | }, 206 | } 207 | resp := map[string]interface{}{ 208 | "spreadsheetId": "123", 209 | "tableRange": "Sheet1!A1:A2", 210 | "updates": map[string]interface{}{ 211 | "spreadsheetId": "123", 212 | "updatedRange": "Sheet1!A1:B3", 213 | "updatedRows": 2, 214 | "updatedColumns": 2, 215 | "updatedCells": 4, 216 | "updatedData": map[string]interface{}{ 217 | "range": "Sheet1!A1:B3", 218 | "majorDimension": majorDimensionRows, 219 | "values": [][]interface{}{ 220 | {"1", "2"}, 221 | {"3", "4"}, 222 | }, 223 | }, 224 | }, 225 | } 226 | 227 | gock.New("https://sheets.googleapis.com"). 228 | Post("/v4/spreadsheets/123/values/Sheet1!A1:A2:append"). 229 | MatchParams(expectedParams). 230 | JSON(expectedReqBody). 231 | Reply(http.StatusOK). 232 | JSON(resp) 233 | 234 | values := [][]interface{}{{"1", "2"}, {"3", "4"}} 235 | res, err := wrapper.insertRows(context.Background(), "123", "Sheet1!A1:A2", values, appendModeOverwrite) 236 | 237 | assert.Nil(t, err, "should not have any error inserting rows") 238 | assert.Equal(t, NewA1Range("Sheet1!A1:B3"), res.UpdatedRange) 239 | assert.Equal(t, int64(2), res.UpdatedRows) 240 | assert.Equal(t, int64(2), res.UpdatedColumns) 241 | assert.Equal(t, int64(4), res.UpdatedCells) 242 | assert.Equal(t, values, res.InsertedValues) 243 | }) 244 | 245 | t.Run("http500", func(t *testing.T) { 246 | expectedParams := map[string]string{ 247 | "includeValuesInResponse": "true", 248 | "responseValueRenderOption": responseValueRenderFormatted, 249 | "insertDataOption": string(appendModeOverwrite), 250 | "valueInputOption": valueInputUserEntered, 251 | } 252 | expectedReqBody := map[string]interface{}{ 253 | "majorDimension": majorDimensionRows, 254 | "range": "Sheet1!A1:A2", 255 | "values": [][]interface{}{ 256 | {"1", "2"}, 257 | {"3", "4"}, 258 | }, 259 | } 260 | 261 | gock.New("https://sheets.googleapis.com"). 262 | Post("/v4/spreadsheets/123/values/Sheet1!A1:A2:append"). 263 | MatchParams(expectedParams). 264 | JSON(expectedReqBody). 265 | Reply(http.StatusInternalServerError) 266 | 267 | values := [][]interface{}{{"1", "2"}, {"3", "4"}} 268 | res, err := wrapper.insertRows(context.Background(), "123", "Sheet1!A1:A2", values, appendModeOverwrite) 269 | 270 | assert.NotNil(t, err, "should have error inserting a new row") 271 | assert.Equal(t, NewA1Range(""), res.UpdatedRange) 272 | assert.Equal(t, int64(0), res.UpdatedRows) 273 | assert.Equal(t, int64(0), res.UpdatedColumns) 274 | assert.Equal(t, int64(0), res.UpdatedCells) 275 | assert.Equal(t, [][]interface{}(nil), res.InsertedValues) 276 | }) 277 | } 278 | 279 | func TestUpdateRows(t *testing.T) { 280 | path := fixtures.PathToFixture("service_account.json") 281 | 282 | auth, err := auth.NewServiceFromFile(path, []string{}, auth.ServiceConfig{}) 283 | assert.Nil(t, err, "should not have any error instantiating a new service account client") 284 | 285 | wrapper, err := NewWrapper(auth) 286 | assert.Nil(t, err, "should not have any error instantiating a new sheets wrapper") 287 | 288 | gock.InterceptClient(auth.HTTPClient()) 289 | 290 | t.Run("successful", func(t *testing.T) { 291 | expectedParams := map[string]string{ 292 | "includeValuesInResponse": "true", 293 | "responseValueRenderOption": responseValueRenderFormatted, 294 | "valueInputOption": valueInputUserEntered, 295 | } 296 | expectedReqBody := map[string]interface{}{ 297 | "majorDimension": majorDimensionRows, 298 | "range": "Sheet1!A1:A2", 299 | "values": [][]interface{}{ 300 | {"1", "2"}, 301 | {"3", "4"}, 302 | }, 303 | } 304 | resp := map[string]interface{}{ 305 | "spreadsheetId": "123", 306 | "updatedRange": "Sheet1!A1:B3", 307 | "updatedRows": 2, 308 | "updatedColumns": 2, 309 | "updatedCells": 4, 310 | "updatedData": map[string]interface{}{ 311 | "range": "Sheet1!A1:B3", 312 | "majorDimension": majorDimensionRows, 313 | "values": [][]interface{}{ 314 | {"1", "2"}, 315 | {"3", "4"}, 316 | }, 317 | }, 318 | } 319 | 320 | gock.New("https://sheets.googleapis.com"). 321 | Put("/v4/spreadsheets/123/values/Sheet1!A1:A2"). 322 | MatchParams(expectedParams). 323 | JSON(expectedReqBody). 324 | Reply(http.StatusOK). 325 | JSON(resp) 326 | 327 | values := [][]interface{}{{"1", "2"}, {"3", "4"}} 328 | res, err := wrapper.UpdateRows(context.Background(), "123", "Sheet1!A1:A2", values) 329 | 330 | assert.Nil(t, err, "should not have any error inserting rows") 331 | assert.Equal(t, NewA1Range("Sheet1!A1:B3"), res.UpdatedRange) 332 | assert.Equal(t, int64(2), res.UpdatedRows) 333 | assert.Equal(t, int64(2), res.UpdatedColumns) 334 | assert.Equal(t, int64(4), res.UpdatedCells) 335 | assert.Equal(t, values, res.UpdatedValues) 336 | }) 337 | 338 | t.Run("http500", func(t *testing.T) { 339 | expectedParams := map[string]string{ 340 | "includeValuesInResponse": "true", 341 | "responseValueRenderOption": responseValueRenderFormatted, 342 | "valueInputOption": valueInputUserEntered, 343 | } 344 | expectedReqBody := map[string]interface{}{ 345 | "majorDimension": majorDimensionRows, 346 | "range": "Sheet1!A1:A2", 347 | "values": [][]interface{}{ 348 | {"1", "2"}, 349 | {"3", "4"}, 350 | }, 351 | } 352 | 353 | gock.New("https://sheets.googleapis.com"). 354 | Put("/v4/spreadsheets/123/values/Sheet1!A1:A2"). 355 | MatchParams(expectedParams). 356 | JSON(expectedReqBody). 357 | Reply(http.StatusInternalServerError) 358 | 359 | values := [][]interface{}{{"1", "2"}, {"3", "4"}} 360 | res, err := wrapper.UpdateRows(context.Background(), "123", "Sheet1!A1:A2", values) 361 | 362 | assert.NotNil(t, err, "should have error inserting a new row") 363 | assert.Equal(t, NewA1Range(""), res.UpdatedRange) 364 | assert.Equal(t, int64(0), res.UpdatedRows) 365 | assert.Equal(t, int64(0), res.UpdatedColumns) 366 | assert.Equal(t, int64(0), res.UpdatedCells) 367 | assert.Equal(t, [][]interface{}(nil), res.UpdatedValues) 368 | }) 369 | } 370 | 371 | func TestBatchUpdateRows(t *testing.T) { 372 | path := fixtures.PathToFixture("service_account.json") 373 | 374 | auth, err := auth.NewServiceFromFile(path, []string{}, auth.ServiceConfig{}) 375 | assert.Nil(t, err, "should not have any error instantiating a new service account client") 376 | 377 | wrapper, err := NewWrapper(auth) 378 | assert.Nil(t, err, "should not have any error instantiating a new sheets wrapper") 379 | 380 | gock.InterceptClient(auth.HTTPClient()) 381 | 382 | t.Run("successful", func(t *testing.T) { 383 | expectedReqBody := map[string]interface{}{ 384 | "includeValuesInResponse": true, 385 | "responseValueRenderOption": responseValueRenderFormatted, 386 | "valueInputOption": valueInputUserEntered, 387 | "data": []map[string]interface{}{ 388 | { 389 | "majorDimension": majorDimensionRows, 390 | "range": "Sheet1!A1:A2", 391 | "values": [][]interface{}{ 392 | {"VA1"}, 393 | {"VA2"}, 394 | }, 395 | }, 396 | { 397 | "majorDimension": majorDimensionRows, 398 | "range": "Sheet1!B1:B2", 399 | "values": [][]interface{}{ 400 | {"VB1"}, 401 | {"VB2"}, 402 | }, 403 | }, 404 | }, 405 | } 406 | resp := map[string]interface{}{ 407 | "spreadsheetId": "123", 408 | "totalUpdatedRows": 4, 409 | "totalUpdatedColumns": 2, 410 | "totalUpdatedCells": 4, 411 | "totalUpdatedSheets": 1, 412 | "responses": []map[string]interface{}{ 413 | { 414 | "spreadsheetId": "123", 415 | "updatedRange": "Sheet1!A1:A2", 416 | "updatedRows": 2, 417 | "updatedColumns": 1, 418 | "updatedCells": 2, 419 | "updatedData": map[string]interface{}{ 420 | "range": "Sheet1!A1:A2", 421 | "majorDimension": majorDimensionRows, 422 | "values": [][]interface{}{ 423 | {"VA1"}, 424 | {"VA2"}, 425 | }, 426 | }, 427 | }, 428 | { 429 | "spreadsheetId": "123", 430 | "updatedRange": "Sheet1!B1:B2", 431 | "updatedRows": 2, 432 | "updatedColumns": 1, 433 | "updatedCells": 2, 434 | "updatedData": map[string]interface{}{ 435 | "range": "Sheet1!B1:B2", 436 | "majorDimension": majorDimensionRows, 437 | "values": [][]interface{}{ 438 | {"VB1"}, 439 | {"VB2"}, 440 | }, 441 | }, 442 | }, 443 | }, 444 | } 445 | 446 | gock.New("https://sheets.googleapis.com"). 447 | Post("/v4/spreadsheets/123/values:batchUpdate"). 448 | JSON(expectedReqBody). 449 | Reply(http.StatusOK). 450 | JSON(resp) 451 | 452 | requests := []BatchUpdateRowsRequest{ 453 | { 454 | A1Range: "Sheet1!A1:A2", 455 | Values: [][]interface{}{{"VA1"}, {"VA2"}}, 456 | }, 457 | { 458 | A1Range: "Sheet1!B1:B2", 459 | Values: [][]interface{}{{"VB1"}, {"VB2"}}, 460 | }, 461 | } 462 | res, err := wrapper.BatchUpdateRows(context.Background(), "123", requests) 463 | 464 | expected := BatchUpdateRowsResult{ 465 | { 466 | UpdatedRange: NewA1Range("Sheet1!A1:A2"), 467 | UpdatedRows: 2, 468 | UpdatedColumns: 1, 469 | UpdatedCells: 2, 470 | UpdatedValues: [][]interface{}{{"VA1"}, {"VA2"}}, 471 | }, 472 | { 473 | UpdatedRange: NewA1Range("Sheet1!B1:B2"), 474 | UpdatedRows: 2, 475 | UpdatedColumns: 1, 476 | UpdatedCells: 2, 477 | UpdatedValues: [][]interface{}{{"VB1"}, {"VB2"}}, 478 | }, 479 | } 480 | assert.Nil(t, err, "should not have any error inserting rows") 481 | assert.Equal(t, expected, res) 482 | }) 483 | 484 | t.Run("http500", func(t *testing.T) { 485 | expectedReqBody := map[string]interface{}{ 486 | "includeValuesInResponse": true, 487 | "responseValueRenderOption": responseValueRenderFormatted, 488 | "valueInputOption": valueInputUserEntered, 489 | "data": []map[string]interface{}{ 490 | { 491 | "majorDimension": majorDimensionRows, 492 | "range": "Sheet1!A1:A2", 493 | "values": [][]interface{}{ 494 | {"VA1"}, 495 | {"VA2"}, 496 | }, 497 | }, 498 | { 499 | "majorDimension": majorDimensionRows, 500 | "range": "Sheet1!B1:B2", 501 | "values": [][]interface{}{ 502 | {"VB1"}, 503 | {"VB2"}, 504 | }, 505 | }, 506 | }, 507 | } 508 | 509 | gock.New("https://sheets.googleapis.com"). 510 | Put("/v4/spreadsheets/123/values:batchUpdate"). 511 | JSON(expectedReqBody). 512 | Reply(http.StatusInternalServerError) 513 | 514 | requests := []BatchUpdateRowsRequest{ 515 | { 516 | A1Range: "Sheet1!A1:A2", 517 | Values: [][]interface{}{{"VA1"}, {"VA2"}}, 518 | }, 519 | { 520 | A1Range: "Sheet1!B1:B2", 521 | Values: [][]interface{}{{"VB1"}, {"VB2"}}, 522 | }, 523 | } 524 | res, err := wrapper.BatchUpdateRows(context.Background(), "123", requests) 525 | 526 | assert.NotNil(t, err, "should have error inserting a new row") 527 | assert.Equal(t, BatchUpdateRowsResult{}, res) 528 | }) 529 | } 530 | 531 | func TestClear(t *testing.T) { 532 | path := fixtures.PathToFixture("service_account.json") 533 | 534 | auth, err := auth.NewServiceFromFile(path, []string{}, auth.ServiceConfig{}) 535 | assert.Nil(t, err, "should not have any error instantiating a new service account client") 536 | 537 | wrapper, err := NewWrapper(auth) 538 | assert.Nil(t, err, "should not have any error instantiating a new sheets wrapper") 539 | 540 | gock.InterceptClient(auth.HTTPClient()) 541 | 542 | t.Run("successful", func(t *testing.T) { 543 | expectedReqBody := map[string][]string{ 544 | "ranges": {"Sheet1!A1:B3", "Sheet1!B4:C5"}, 545 | } 546 | resp := map[string]interface{}{ 547 | "spreadsheetId": "123", 548 | "clearedRanges": []string{"Sheet1!A1:B3", "Sheet1!B4:C5"}, 549 | } 550 | 551 | gock.New("https://sheets.googleapis.com"). 552 | Post("/v4/spreadsheets/123/values:batchClear"). 553 | JSON(expectedReqBody). 554 | Reply(http.StatusOK). 555 | JSON(resp) 556 | 557 | res, err := wrapper.Clear(context.Background(), "123", []string{"Sheet1!A1:B3", "Sheet1!B4:C5"}) 558 | assert.Nil(t, err, "should not have any error clearing rows") 559 | assert.Equal(t, res, []string{"Sheet1!A1:B3", "Sheet1!B4:C5"}) 560 | }) 561 | 562 | t.Run("http500", func(t *testing.T) { 563 | expectedReqBody := map[string][]string{ 564 | "ranges": {"Sheet1!A1:B3", "Sheet1!B4:C5"}, 565 | } 566 | 567 | gock.New("https://sheets.googleapis.com"). 568 | Post("/v4/spreadsheets/123/values:batchClear"). 569 | JSON(expectedReqBody). 570 | Reply(http.StatusInternalServerError) 571 | 572 | res, err := wrapper.Clear(context.Background(), "123", []string{"Sheet1!A1:B3", "Sheet1!B4:C5"}) 573 | assert.NotNil(t, err, "should have error clearing rows") 574 | assert.Equal(t, res, []string(nil)) 575 | }) 576 | } 577 | 578 | func TestQueryRows(t *testing.T) { 579 | path := fixtures.PathToFixture("service_account.json") 580 | 581 | auth, err := auth.NewServiceFromFile(path, []string{}, auth.ServiceConfig{}) 582 | assert.Nil(t, err, "should not have any error instantiating a new service account client") 583 | 584 | wrapper, err := NewWrapper(auth) 585 | assert.Nil(t, err, "should not have any error instantiating a new sheets wrapper") 586 | 587 | gock.InterceptClient(auth.HTTPClient()) 588 | 589 | t.Run("successful", func(t *testing.T) { 590 | expectedParams := map[string]string{ 591 | "sheet": "s1", 592 | "tqx": "responseHandler:freedb", 593 | "tq": "select A, B", 594 | "headers": "1", 595 | } 596 | resp := ` 597 | /*O_o*/ 598 | freedb({"version":"0.6","reqId":"0","status":"ok","sig":"141753603","table":{"cols":[{"id":"A","label":"","type":"string"},{"id":"B","label":"","type":"number","pattern":"General"}],"rows":[{"c":[{"v":"k1"},{"v":103.51,"f":"103.51"}]},{"c":[{"v":"k2"},{"v":111.0,"f":"111"}]},{"c":[{"v":"k3"},{"v":123.0,"f":"123"}]}],"parsedNumHeaders":0}}) 599 | ` 600 | 601 | gock.New("https://docs.google.com"). 602 | Get("/spreadsheets/d/spreadsheetID/gviz/tq"). 603 | MatchParams(expectedParams). 604 | Reply(http.StatusOK). 605 | BodyString(resp) 606 | 607 | res, err := wrapper.QueryRows( 608 | context.Background(), 609 | "spreadsheetID", 610 | "s1", 611 | "select A, B", 612 | true, 613 | ) 614 | assert.Nil(t, err) 615 | 616 | expected := QueryRowsResult{Rows: [][]interface{}{ 617 | {"k1", 103.51}, 618 | {"k2", float64(111)}, 619 | {"k3", float64(123)}, 620 | }} 621 | assert.Equal(t, expected, res) 622 | }) 623 | } 624 | -------------------------------------------------------------------------------- /internal/google/store/kv.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/FreeLeh/GoFreeDB/internal/codec" 8 | "github.com/FreeLeh/GoFreeDB/internal/common" 9 | "github.com/FreeLeh/GoFreeDB/internal/models" 10 | 11 | "github.com/FreeLeh/GoFreeDB/internal/google/sheets" 12 | ) 13 | 14 | // GoogleSheetKVStoreConfig defines a list of configurations that can be used to customise how the GoogleSheetKVStore works. 15 | type GoogleSheetKVStoreConfig struct { 16 | Mode models.KVMode 17 | codec Codec 18 | } 19 | 20 | // GoogleSheetKVStore encapsulates key-value store functionality on top of a Google Sheet. 21 | // 22 | // There are 2 operation modes for the key-value store: default and append only mode. 23 | // 24 | // For more details on how they differ, please read the explanations for each method or the protocol page: 25 | // https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md. 26 | type GoogleSheetKVStore struct { 27 | wrapper sheetsWrapper 28 | spreadsheetID string 29 | sheetName string 30 | scratchpadSheetName string 31 | scratchpadLocation sheets.A1Range 32 | config GoogleSheetKVStoreConfig 33 | } 34 | 35 | // Get retrieves the value associated with the given key. 36 | // If the key exists in the store, the raw bytes value and no error will be returned. 37 | // If the key does not exist in the store, a nil []byte and a wrapped ErrKeyNotFound will be returned. 38 | // 39 | // In default mode, 40 | // - There will be only one row with the given key. It will return the value for that in that row. 41 | // - There is only 1 API call behind the scene. 42 | // 43 | // In append only mode, 44 | // - As there could be multiple rows with the same key, we need to only use the latest row as it 45 | // contains the last updated value. 46 | // - Note that deletion using append only mode results in a new row with a tombstone value. 47 | // This method will also recognise and handle such cases. 48 | // - There is only 1 API call behind the scene. 49 | func (s *GoogleSheetKVStore) Get(ctx context.Context, key string) ([]byte, error) { 50 | query := fmt.Sprintf(kvGetDefaultQueryTemplate, key, common.GetA1Range(s.sheetName, defaultKVTableRange)) 51 | if s.config.Mode == models.KVModeAppendOnly { 52 | query = fmt.Sprintf(kvGetAppendQueryTemplate, key, common.GetA1Range(s.sheetName, defaultKVTableRange)) 53 | } 54 | 55 | result, err := s.wrapper.UpdateRows( 56 | ctx, 57 | s.spreadsheetID, 58 | s.scratchpadLocation.Original, 59 | [][]interface{}{{query}}, 60 | ) 61 | if err != nil { 62 | return nil, err 63 | } 64 | if len(result.UpdatedValues) == 0 || len(result.UpdatedValues[0]) == 0 { 65 | return nil, fmt.Errorf("%w: %s", models.ErrKeyNotFound, key) 66 | } 67 | 68 | value := result.UpdatedValues[0][0] 69 | if value == models.NAValue || value == "" { 70 | return nil, fmt.Errorf("%w: %s", models.ErrKeyNotFound, key) 71 | } 72 | return s.config.codec.Decode(value.(string)) 73 | } 74 | 75 | // Set inserts the key-value pair into the key-value store. 76 | // 77 | // In default mode, 78 | // - If the key is not in the store, `Set` will create a new row and store the key value pair there. 79 | // - If the key is in the store, `Set` will update the previous row with the new value and timestamp. 80 | // - There are exactly 2 API calls behind the scene: getting the row for the key and creating/updating with the given key value data. 81 | // 82 | // In append only mode, 83 | // - It always creates a new row at the bottom of the sheet with the latest value and timestamp. 84 | // - There is only 1 API call behind the scene. 85 | func (s *GoogleSheetKVStore) Set(ctx context.Context, key string, value []byte) error { 86 | encoded, err := s.config.codec.Encode(value) 87 | if err != nil { 88 | return err 89 | } 90 | if s.config.Mode == models.KVModeAppendOnly { 91 | return s.setAppendOnly(ctx, key, encoded) 92 | } 93 | return s.setDefault(ctx, key, encoded) 94 | } 95 | 96 | func (s *GoogleSheetKVStore) setAppendOnly(ctx context.Context, key string, encoded string) error { 97 | _, err := s.wrapper.InsertRows( 98 | ctx, 99 | s.spreadsheetID, 100 | common.GetA1Range(s.sheetName, defaultKVTableRange), 101 | [][]interface{}{{key, encoded, common.CurrentTimeMs()}}, 102 | ) 103 | return err 104 | } 105 | 106 | func (s *GoogleSheetKVStore) setDefault(ctx context.Context, key string, encoded string) error { 107 | a1Range, err := s.findKeyA1Range(ctx, key) 108 | if errors.Is(err, models.ErrKeyNotFound) { 109 | _, err := s.wrapper.OverwriteRows( 110 | ctx, 111 | s.spreadsheetID, 112 | common.GetA1Range(s.sheetName, defaultKVFirstRowRange), 113 | [][]interface{}{{key, encoded, common.CurrentTimeMs()}}, 114 | ) 115 | return err 116 | } 117 | 118 | if err != nil { 119 | return err 120 | } 121 | 122 | _, err = s.wrapper.UpdateRows( 123 | ctx, 124 | s.spreadsheetID, 125 | a1Range.Original, 126 | [][]interface{}{{key, encoded, common.CurrentTimeMs()}}, 127 | ) 128 | return err 129 | } 130 | 131 | func (s *GoogleSheetKVStore) findKeyA1Range(ctx context.Context, key string) (sheets.A1Range, error) { 132 | result, err := s.wrapper.UpdateRows( 133 | ctx, 134 | s.spreadsheetID, 135 | s.scratchpadLocation.Original, 136 | [][]interface{}{{fmt.Sprintf(kvFindKeyA1RangeQueryTemplate, key, common.GetA1Range(s.sheetName, defaultKVKeyColRange))}}, 137 | ) 138 | if err != nil { 139 | return sheets.A1Range{}, err 140 | } 141 | if len(result.UpdatedValues) == 0 || len(result.UpdatedValues[0]) == 0 { 142 | return sheets.A1Range{}, fmt.Errorf("%w: %s", models.ErrKeyNotFound, key) 143 | } 144 | 145 | offset := result.UpdatedValues[0][0].(string) 146 | if offset == models.NAValue || offset == "" { 147 | return sheets.A1Range{}, fmt.Errorf("%w: %s", models.ErrKeyNotFound, key) 148 | } 149 | 150 | // Note that the MATCH() query only returns the relative offset from the given range. 151 | // Here we need to return the full range where the key is found. 152 | // Hence, we need to get the row offset first, and assume that each row has only 3 rows: A B C. 153 | // Otherwise, the DELETE() function will not work properly (we need to clear the full row, not just the key cell). 154 | a1Range := common.GetA1Range(s.sheetName, fmt.Sprintf("A%s:C%s", offset, offset)) 155 | return sheets.NewA1Range(a1Range), nil 156 | } 157 | 158 | // Delete deletes the given key from the key-value store. 159 | // 160 | // In default mode, 161 | // - If the key is not in the store, it will not do anything. 162 | // - If the key is in the store, it will remove that row. 163 | // - There are up to 2 API calls behind the scene: getting the row for the key and remove the row (if the key exists). 164 | // 165 | // In append only mode, 166 | // - It creates a new row at the bottom of the sheet with a tombstone value and timestamp. 167 | // - There is only 1 API call behind the scene. 168 | func (s *GoogleSheetKVStore) Delete(ctx context.Context, key string) error { 169 | if s.config.Mode == models.KVModeAppendOnly { 170 | return s.deleteAppendOnly(ctx, key) 171 | } 172 | return s.deleteDefault(ctx, key) 173 | } 174 | 175 | func (s *GoogleSheetKVStore) deleteAppendOnly(ctx context.Context, key string) error { 176 | return s.setAppendOnly(ctx, key, "") 177 | } 178 | 179 | func (s *GoogleSheetKVStore) deleteDefault(ctx context.Context, key string) error { 180 | a1Range, err := s.findKeyA1Range(ctx, key) 181 | if errors.Is(err, models.ErrKeyNotFound) { 182 | return nil 183 | } 184 | if err != nil { 185 | return err 186 | } 187 | 188 | _, err = s.wrapper.Clear(ctx, s.spreadsheetID, []string{a1Range.Original}) 189 | return err 190 | } 191 | 192 | // Close cleans up all held resources like the scratchpad cell booked for this specific GoogleSheetKVStore instance. 193 | func (s *GoogleSheetKVStore) Close(ctx context.Context) error { 194 | _, err := s.wrapper.Clear(ctx, s.spreadsheetID, []string{s.scratchpadLocation.Original}) 195 | return err 196 | } 197 | 198 | // NewGoogleSheetKVStore creates an instance of the key-value store with the given configuration. 199 | // It will also try to create the sheet, in case it does not exist yet. 200 | func NewGoogleSheetKVStore( 201 | auth sheets.AuthClient, 202 | spreadsheetID string, 203 | sheetName string, 204 | config GoogleSheetKVStoreConfig, 205 | ) *GoogleSheetKVStore { 206 | wrapper, err := sheets.NewWrapper(auth) 207 | if err != nil { 208 | panic(fmt.Errorf("error creating sheets wrapper: %w", err)) 209 | } 210 | 211 | scratchpadSheetName := sheetName + scratchpadSheetNameSuffix 212 | config = applyGoogleSheetKVStoreConfig(config) 213 | 214 | store := &GoogleSheetKVStore{ 215 | wrapper: wrapper, 216 | spreadsheetID: spreadsheetID, 217 | sheetName: sheetName, 218 | scratchpadSheetName: scratchpadSheetName, 219 | config: config, 220 | } 221 | 222 | _ = ensureSheets(store.wrapper, store.spreadsheetID, store.sheetName) 223 | _ = ensureSheets(store.wrapper, store.spreadsheetID, store.scratchpadSheetName) 224 | 225 | scratchpadLocation, err := findScratchpadLocation(store.wrapper, store.spreadsheetID, store.scratchpadSheetName) 226 | if err != nil { 227 | panic(fmt.Errorf("error finding a scratchpad location in sheet %s: %w", store.scratchpadSheetName, err)) 228 | } 229 | store.scratchpadLocation = scratchpadLocation 230 | 231 | return store 232 | } 233 | 234 | func applyGoogleSheetKVStoreConfig(config GoogleSheetKVStoreConfig) GoogleSheetKVStoreConfig { 235 | config.codec = codec.NewBasic() 236 | return config 237 | } 238 | -------------------------------------------------------------------------------- /internal/google/store/kv_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/FreeLeh/GoFreeDB/internal/common" 7 | "github.com/FreeLeh/GoFreeDB/internal/models" 8 | "testing" 9 | "time" 10 | 11 | "github.com/FreeLeh/GoFreeDB/google/auth" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestGoogleSheetKVStore_AppendOnly_Integration(t *testing.T) { 16 | spreadsheetID, authJSON, shouldRun := getIntegrationTestInfo() 17 | if !shouldRun { 18 | t.Skip("integration test should be run only in GitHub Actions") 19 | } 20 | sheetName := fmt.Sprintf("integration_kv_append_only_%d", common.CurrentTimeMs()) 21 | 22 | googleAuth, err := auth.NewServiceFromJSON([]byte(authJSON), auth.GoogleSheetsReadWrite, auth.ServiceConfig{}) 23 | if err != nil { 24 | t.Fatalf("error when instantiating google auth: %s", err) 25 | } 26 | 27 | kv := NewGoogleSheetKVStore( 28 | googleAuth, 29 | spreadsheetID, 30 | sheetName, 31 | GoogleSheetKVStoreConfig{Mode: models.KVModeAppendOnly}, 32 | ) 33 | defer func() { 34 | time.Sleep(time.Second) 35 | deleteSheet(t, kv.wrapper, spreadsheetID, []string{kv.sheetName, kv.scratchpadSheetName}) 36 | _ = kv.Close(context.Background()) 37 | }() 38 | 39 | time.Sleep(time.Second) 40 | value, err := kv.Get(context.Background(), "k1") 41 | assert.Nil(t, value) 42 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 43 | 44 | time.Sleep(time.Second) 45 | err = kv.Set(context.Background(), "k1", []byte("test")) 46 | assert.Nil(t, err) 47 | 48 | time.Sleep(time.Second) 49 | value, err = kv.Get(context.Background(), "k1") 50 | assert.Equal(t, []byte("test"), value) 51 | assert.Nil(t, err) 52 | 53 | time.Sleep(time.Second) 54 | err = kv.Delete(context.Background(), "k1") 55 | assert.Nil(t, err) 56 | 57 | time.Sleep(time.Second) 58 | value, err = kv.Get(context.Background(), "k1") 59 | assert.Nil(t, value) 60 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 61 | } 62 | 63 | func TestNewGoogleSheetKVStore_Default_Integration(t *testing.T) { 64 | spreadsheetID, authJSON, shouldRun := getIntegrationTestInfo() 65 | if !shouldRun { 66 | t.Skip("integration test should be run only in GitHub Actions") 67 | } 68 | sheetName := fmt.Sprintf("integration_kv_default_%d", common.CurrentTimeMs()) 69 | 70 | googleAuth, err := auth.NewServiceFromJSON([]byte(authJSON), auth.GoogleSheetsReadWrite, auth.ServiceConfig{}) 71 | if err != nil { 72 | t.Fatalf("error when instantiating google auth: %s", err) 73 | } 74 | 75 | kv := NewGoogleSheetKVStore( 76 | googleAuth, 77 | spreadsheetID, 78 | sheetName, 79 | GoogleSheetKVStoreConfig{Mode: models.KVModeDefault}, 80 | ) 81 | defer func() { 82 | time.Sleep(time.Second) 83 | deleteSheet(t, kv.wrapper, spreadsheetID, []string{kv.sheetName, kv.scratchpadSheetName}) 84 | _ = kv.Close(context.Background()) 85 | }() 86 | 87 | time.Sleep(time.Second) 88 | value, err := kv.Get(context.Background(), "k1") 89 | assert.Nil(t, value) 90 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 91 | 92 | time.Sleep(time.Second) 93 | err = kv.Set(context.Background(), "k1", []byte("test")) 94 | assert.Nil(t, err) 95 | 96 | time.Sleep(time.Second) 97 | value, err = kv.Get(context.Background(), "k1") 98 | assert.Equal(t, []byte("test"), value) 99 | assert.Nil(t, err) 100 | 101 | time.Sleep(time.Second) 102 | err = kv.Set(context.Background(), "k1", []byte("test2")) 103 | assert.Nil(t, err) 104 | 105 | time.Sleep(time.Second) 106 | err = kv.Delete(context.Background(), "k1") 107 | assert.Nil(t, err) 108 | 109 | time.Sleep(time.Second) 110 | value, err = kv.Get(context.Background(), "k1") 111 | assert.Nil(t, value) 112 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 113 | } 114 | -------------------------------------------------------------------------------- /internal/google/store/kv_v2.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/FreeLeh/GoFreeDB/internal/codec" 8 | "github.com/FreeLeh/GoFreeDB/internal/google/sheets" 9 | "github.com/FreeLeh/GoFreeDB/internal/models" 10 | ) 11 | 12 | type googleSheetKVStoreV2Row struct { 13 | Key string `db:"key"` 14 | Value string `db:"value"` 15 | } 16 | 17 | // GoogleSheetKVStoreV2Config defines a list of configurations that can be used to customise 18 | // how the GoogleSheetKVStoreV2 works. 19 | type GoogleSheetKVStoreV2Config struct { 20 | Mode models.KVMode 21 | codec Codec 22 | } 23 | 24 | // GoogleSheetKVStoreV2 implements a key-value store using the row store abstraction. 25 | type GoogleSheetKVStoreV2 struct { 26 | rowStore *GoogleSheetRowStore 27 | mode models.KVMode 28 | codec Codec 29 | } 30 | 31 | // Get retrieves the value associated with the given key. 32 | func (s *GoogleSheetKVStoreV2) Get(ctx context.Context, key string) ([]byte, error) { 33 | var rows []googleSheetKVStoreV2Row 34 | var err error 35 | 36 | if s.mode == models.KVModeDefault { 37 | err = s.rowStore.Select(&rows, "value"). 38 | Where("key = ?", key). 39 | Limit(1). 40 | Exec(ctx) 41 | } else { 42 | err = s.rowStore.Select(&rows, "value"). 43 | Where("key = ?", key). 44 | OrderBy([]models.ColumnOrderBy{ 45 | {Column: "_rid", OrderBy: models.OrderByDesc}, 46 | }). 47 | Limit(1). 48 | Exec(ctx) 49 | } 50 | if err != nil { 51 | return nil, err 52 | } 53 | if len(rows) == 0 { 54 | return nil, fmt.Errorf("%w: %s", models.ErrKeyNotFound, key) 55 | } 56 | 57 | value := rows[0].Value 58 | if value == "" { 59 | return nil, fmt.Errorf("%w: %s", models.ErrKeyNotFound, key) 60 | } 61 | return s.codec.Decode(value) 62 | } 63 | 64 | // Set inserts or updates the key-value pair in the store. 65 | func (s *GoogleSheetKVStoreV2) Set(ctx context.Context, key string, value []byte) error { 66 | encoded, err := s.codec.Encode(value) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | if s.mode == models.KVModeDefault { 72 | if err := s.rowStore.Delete(). 73 | Where("key = ?", key). 74 | Exec(ctx); err != nil { 75 | return err 76 | } 77 | } 78 | 79 | row := googleSheetKVStoreV2Row{ 80 | Key: key, 81 | Value: encoded, 82 | } 83 | return s.rowStore.Insert(row).Exec(ctx) 84 | } 85 | 86 | // Delete removes the key from the store. 87 | func (s *GoogleSheetKVStoreV2) Delete(ctx context.Context, key string) error { 88 | if s.mode == models.KVModeDefault { 89 | return s.rowStore.Delete(). 90 | Where("key = ?", key). 91 | Exec(ctx) 92 | } else { 93 | return s.rowStore.Insert(googleSheetKVStoreV2Row{ 94 | Key: key, 95 | Value: "", 96 | }).Exec(ctx) 97 | } 98 | } 99 | 100 | // Close cleans up resources used by the store. 101 | func (s *GoogleSheetKVStoreV2) Close(ctx context.Context) error { 102 | return s.rowStore.Close(ctx) 103 | } 104 | 105 | // NewGoogleSheetKVStoreV2 creates a new instance of the key-value store using row store. 106 | // You cannot use this V2 store with the V1 store as the sheet format is different. 107 | func NewGoogleSheetKVStoreV2( 108 | auth sheets.AuthClient, 109 | spreadsheetID string, 110 | sheetName string, 111 | config GoogleSheetKVStoreV2Config, 112 | ) *GoogleSheetKVStoreV2 { 113 | rowStore := NewGoogleSheetRowStore( 114 | auth, 115 | spreadsheetID, 116 | sheetName, 117 | GoogleSheetRowStoreConfig{ 118 | Columns: []string{"key", "value"}, 119 | }, 120 | ) 121 | 122 | config = applyGoogleSheetKVStoreV2Config(config) 123 | return &GoogleSheetKVStoreV2{ 124 | rowStore: rowStore, 125 | mode: config.Mode, 126 | codec: config.codec, 127 | } 128 | } 129 | 130 | func applyGoogleSheetKVStoreV2Config(config GoogleSheetKVStoreV2Config) GoogleSheetKVStoreV2Config { 131 | config.codec = codec.NewBasic() 132 | return config 133 | } 134 | -------------------------------------------------------------------------------- /internal/google/store/kv_v2_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/FreeLeh/GoFreeDB/internal/common" 10 | "github.com/FreeLeh/GoFreeDB/internal/models" 11 | 12 | "github.com/FreeLeh/GoFreeDB/google/auth" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestGoogleSheetKVStoreV2_AppendOnly_Integration(t *testing.T) { 17 | spreadsheetID, authJSON, shouldRun := getIntegrationTestInfo() 18 | if !shouldRun { 19 | t.Skip("integration test should be run only in GitHub Actions") 20 | } 21 | sheetName := fmt.Sprintf("integration_kv_v2_append_only_%d", common.CurrentTimeMs()) 22 | 23 | googleAuth, err := auth.NewServiceFromJSON([]byte(authJSON), auth.GoogleSheetsReadWrite, auth.ServiceConfig{}) 24 | if err != nil { 25 | t.Fatalf("error when instantiating google auth: %s", err) 26 | } 27 | 28 | kv := NewGoogleSheetKVStoreV2( 29 | googleAuth, 30 | spreadsheetID, 31 | sheetName, 32 | GoogleSheetKVStoreV2Config{Mode: models.KVModeAppendOnly}, 33 | ) 34 | defer func() { 35 | time.Sleep(time.Second) 36 | deleteSheet(t, kv.rowStore.wrapper, spreadsheetID, []string{kv.rowStore.sheetName}) 37 | _ = kv.Close(context.Background()) 38 | }() 39 | 40 | time.Sleep(time.Second) 41 | value, err := kv.Get(context.Background(), "k1") 42 | assert.Nil(t, value) 43 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 44 | 45 | time.Sleep(time.Second) 46 | err = kv.Set(context.Background(), "k1", []byte("test")) 47 | assert.Nil(t, err) 48 | 49 | time.Sleep(time.Second) 50 | value, err = kv.Get(context.Background(), "k1") 51 | assert.Equal(t, []byte("test"), value) 52 | assert.Nil(t, err) 53 | 54 | time.Sleep(time.Second) 55 | err = kv.Delete(context.Background(), "k1") 56 | assert.Nil(t, err) 57 | 58 | time.Sleep(time.Second) 59 | value, err = kv.Get(context.Background(), "k1") 60 | assert.Nil(t, value) 61 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 62 | } 63 | 64 | func TestNewGoogleSheetKVStoreV2_Default_Integration(t *testing.T) { 65 | spreadsheetID, authJSON, shouldRun := getIntegrationTestInfo() 66 | if !shouldRun { 67 | t.Skip("integration test should be run only in GitHub Actions") 68 | } 69 | sheetName := fmt.Sprintf("integration_kv_v2_default_%d", common.CurrentTimeMs()) 70 | 71 | googleAuth, err := auth.NewServiceFromJSON([]byte(authJSON), auth.GoogleSheetsReadWrite, auth.ServiceConfig{}) 72 | if err != nil { 73 | t.Fatalf("error when instantiating google auth: %s", err) 74 | } 75 | 76 | kv := NewGoogleSheetKVStoreV2( 77 | googleAuth, 78 | spreadsheetID, 79 | sheetName, 80 | GoogleSheetKVStoreV2Config{Mode: models.KVModeDefault}, 81 | ) 82 | defer func() { 83 | time.Sleep(time.Second) 84 | deleteSheet(t, kv.rowStore.wrapper, spreadsheetID, []string{kv.rowStore.sheetName}) 85 | _ = kv.Close(context.Background()) 86 | }() 87 | 88 | time.Sleep(time.Second) 89 | value, err := kv.Get(context.Background(), "k1") 90 | assert.Nil(t, value) 91 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 92 | 93 | time.Sleep(time.Second) 94 | err = kv.Set(context.Background(), "k1", []byte("test")) 95 | assert.Nil(t, err) 96 | 97 | time.Sleep(time.Second) 98 | value, err = kv.Get(context.Background(), "k1") 99 | assert.Equal(t, []byte("test"), value) 100 | assert.Nil(t, err) 101 | 102 | time.Sleep(time.Second) 103 | err = kv.Set(context.Background(), "k1", []byte("test2")) 104 | assert.Nil(t, err) 105 | 106 | time.Sleep(time.Second) 107 | value, err = kv.Get(context.Background(), "k1") 108 | assert.Equal(t, []byte("test2"), value) 109 | assert.Nil(t, err) 110 | 111 | time.Sleep(time.Second) 112 | err = kv.Delete(context.Background(), "k1") 113 | assert.Nil(t, err) 114 | 115 | time.Sleep(time.Second) 116 | value, err = kv.Get(context.Background(), "k1") 117 | assert.Nil(t, value) 118 | assert.ErrorIs(t, err, models.ErrKeyNotFound) 119 | } 120 | -------------------------------------------------------------------------------- /internal/google/store/models.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "github.com/FreeLeh/GoFreeDB/internal/common" 6 | "regexp" 7 | 8 | "github.com/FreeLeh/GoFreeDB/internal/google/sheets" 9 | ) 10 | 11 | const ( 12 | // Currently limited to 26. 13 | // Otherwise, the sheet creation must extend the column as well to make the rowGetIndicesQueryTemplate formula works. 14 | // TODO(edocsss): add an option to extend the number of columns. 15 | maxColumn = 26 16 | 17 | scratchpadBooked = "BOOKED" 18 | scratchpadSheetNameSuffix = "_scratch" 19 | 20 | defaultKVTableRange = "A1:C5000000" 21 | defaultKVKeyColRange = "A1:A5000000" 22 | defaultKVFirstRowRange = "A1:C1" 23 | 24 | kvGetAppendQueryTemplate = "=VLOOKUP(\"%s\", SORT(%s, 3, FALSE), 2, FALSE)" 25 | kvGetDefaultQueryTemplate = "=VLOOKUP(\"%s\", %s, 2, FALSE)" 26 | kvFindKeyA1RangeQueryTemplate = "=MATCH(\"%s\", %s, 0)" 27 | 28 | rowIdxCol = "_rid" 29 | rowIdxFormula = "=ROW()" 30 | ) 31 | 32 | var ( 33 | defaultRowHeaderRange = "A1:" + common.GenerateColumnName(maxColumn-1) + "1" 34 | defaultRowFullTableRange = "A2:" + common.GenerateColumnName(maxColumn-1) 35 | rowDeleteRangeTemplate = "A%d:" + common.GenerateColumnName(maxColumn-1) + "%d" 36 | 37 | // The first condition `_rid IS NOT NULL` is necessary to ensure we are just updating rows that are non-empty. 38 | // This is required for UPDATE without WHERE clause (otherwise it will see every row as update target). 39 | rowWhereNonEmptyConditionTemplate = rowIdxCol + " is not null AND %s" 40 | rowWhereEmptyConditionTemplate = rowIdxCol + " is not null" 41 | 42 | googleSheetSelectStmtStringKeyword = regexp.MustCompile("^(date|datetime|timeofday)") 43 | ) 44 | 45 | // Codec is an interface for encoding and decoding the data provided by the client. 46 | // At the moment, only key-value store requires data encoding. 47 | type Codec interface { 48 | Encode(value []byte) (string, error) 49 | Decode(value string) ([]byte, error) 50 | } 51 | 52 | type sheetsWrapper interface { 53 | CreateSpreadsheet(ctx context.Context, title string) (string, error) 54 | GetSheetNameToID(ctx context.Context, spreadsheetID string) (map[string]int64, error) 55 | CreateSheet(ctx context.Context, spreadsheetID string, sheetName string) error 56 | DeleteSheets(ctx context.Context, spreadsheetID string, sheetIDs []int64) error 57 | InsertRows(ctx context.Context, spreadsheetID string, a1Range string, values [][]interface{}) (sheets.InsertRowsResult, error) 58 | OverwriteRows(ctx context.Context, spreadsheetID string, a1Range string, values [][]interface{}) (sheets.InsertRowsResult, error) 59 | UpdateRows(ctx context.Context, spreadsheetID string, a1Range string, values [][]interface{}) (sheets.UpdateRowsResult, error) 60 | BatchUpdateRows(ctx context.Context, spreadsheetID string, requests []sheets.BatchUpdateRowsRequest) (sheets.BatchUpdateRowsResult, error) 61 | QueryRows(ctx context.Context, spreadsheetID string, sheetName string, query string, skipHeader bool) (sheets.QueryRowsResult, error) 62 | Clear(ctx context.Context, spreadsheetID string, ranges []string) ([]string, error) 63 | } 64 | -------------------------------------------------------------------------------- /internal/google/store/models_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func getIntegrationTestInfo() (string, string, bool) { 10 | spreadsheetID := os.Getenv("INTEGRATION_TEST_SPREADSHEET_ID") 11 | authJSON := os.Getenv("INTEGRATION_TEST_AUTH_JSON") 12 | _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS") 13 | return spreadsheetID, authJSON, isGithubActions && spreadsheetID != "" && authJSON != "" 14 | } 15 | 16 | func deleteSheet(t *testing.T, wrapper sheetsWrapper, spreadsheetID string, sheetNames []string) { 17 | sheetNameToID, err := wrapper.GetSheetNameToID(context.Background(), spreadsheetID) 18 | if err != nil { 19 | t.Fatalf("failed getting mapping of sheet names to IDs: %s", err) 20 | } 21 | 22 | sheetIDs := make([]int64, 0, len(sheetNames)) 23 | for _, sheetName := range sheetNames { 24 | id, ok := sheetNameToID[sheetName] 25 | if !ok { 26 | t.Fatalf("sheet ID for the given name is not found") 27 | } 28 | sheetIDs = append(sheetIDs, id) 29 | } 30 | 31 | if err := wrapper.DeleteSheets(context.Background(), spreadsheetID, sheetIDs); err != nil { 32 | t.Logf("failed deleting sheets: %s", err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/google/store/row.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/FreeLeh/GoFreeDB/internal/common" 8 | "time" 9 | 10 | "github.com/FreeLeh/GoFreeDB/internal/google/sheets" 11 | ) 12 | 13 | // GoogleSheetRowStoreConfig defines a list of configurations that can be used to customise how the GoogleSheetRowStore works. 14 | type GoogleSheetRowStoreConfig struct { 15 | // Columns defines the list of column names. 16 | // Note that the column ordering matters. 17 | // The column ordering will be used for arranging the real columns in Google Sheet. 18 | // Changing the column ordering in this config but not in Google Sheet will result in unexpected behaviour. 19 | Columns []string 20 | 21 | // ColumnsWithFormula defines the list of column names containing a Google Sheet formula. 22 | // Note that only string fields can have a formula. 23 | ColumnsWithFormula []string 24 | } 25 | 26 | func (c GoogleSheetRowStoreConfig) validate() error { 27 | if len(c.Columns) == 0 { 28 | return errors.New("columns must have at least one column") 29 | } 30 | if len(c.Columns) > maxColumn { 31 | return fmt.Errorf("you can only have up to %d columns", maxColumn) 32 | } 33 | return nil 34 | } 35 | 36 | // GoogleSheetRowStore encapsulates row store functionality on top of a Google Sheet. 37 | type GoogleSheetRowStore struct { 38 | wrapper sheetsWrapper 39 | spreadsheetID string 40 | sheetName string 41 | colsMapping common.ColsMapping 42 | colsWithFormula *common.Set[string] 43 | config GoogleSheetRowStoreConfig 44 | } 45 | 46 | // Select specifies which columns to return from the Google Sheet when querying and the output variable 47 | // the data should be stored. 48 | // You can think of this operation like a SQL SELECT statement (with limitations). 49 | // 50 | // If "columns" is an empty slice of string, then all columns will be returned. 51 | // If a column is not found in the provided list of columns in `GoogleSheetRowStoreConfig.Columns`, that column will be ignored. 52 | // 53 | // "output" must be a pointer to a slice of a data type. 54 | // The conversion from the Google Sheet data into the slice will be done using https://github.com/mitchellh/mapstructure. 55 | // 56 | // If you are providing a slice of structs into the "output" parameter, and you want to define the mapping between the 57 | // column name with the field name, you should add a "db" struct tag. 58 | // 59 | // // Without the `db` struct tag, the column name used will be "Name" and "Age". 60 | // type Person struct { 61 | // Name string `db:"name"` 62 | // Age int `db:"age"` 63 | // } 64 | // 65 | // Please note that calling Select() does not execute the query yet. 66 | // Call GoogleSheetSelectStmt.Exec to actually execute the query. 67 | func (s *GoogleSheetRowStore) Select(output interface{}, columns ...string) *GoogleSheetSelectStmt { 68 | return newGoogleSheetSelectStmt(s, output, columns) 69 | } 70 | 71 | // Insert specifies the rows to be inserted into the Google Sheet. 72 | // 73 | // The underlying data type of each row must be a struct or a pointer to a struct. 74 | // Providing other data types will result in an error. 75 | // 76 | // By default, the column name will be following the struct field name (case-sensitive). 77 | // If you want to map the struct field name into another name, you can add a "db" struct tag 78 | // (see GoogleSheetRowStore.Select docs for more details). 79 | // 80 | // Please note that calling Insert() does not execute the insertion yet. 81 | // Call GoogleSheetInsertStmt.Exec() to actually execute the insertion. 82 | func (s *GoogleSheetRowStore) Insert(rows ...interface{}) *GoogleSheetInsertStmt { 83 | return newGoogleSheetInsertStmt(s, rows) 84 | } 85 | 86 | // Update specifies the new value for each of the targeted columns. 87 | // 88 | // The "colToValue" parameter specifies what value should be updated for which column. 89 | // Each value in the map[string]interface{} is going to be JSON marshalled. 90 | // If "colToValue" is empty, an error will be returned when GoogleSheetUpdateStmt.Exec() is called. 91 | func (s *GoogleSheetRowStore) Update(colToValue map[string]interface{}) *GoogleSheetUpdateStmt { 92 | return newGoogleSheetUpdateStmt(s, colToValue) 93 | } 94 | 95 | // Delete prepares rows deletion operation. 96 | // 97 | // Please note that calling Delete() does not execute the deletion yet. 98 | // Call GoogleSheetDeleteStmt.Exec() to actually execute the deletion. 99 | func (s *GoogleSheetRowStore) Delete() *GoogleSheetDeleteStmt { 100 | return newGoogleSheetDeleteStmt(s) 101 | } 102 | 103 | // Count prepares rows counting operation. 104 | // 105 | // Please note that calling Count() does not execute the query yet. 106 | // Call GoogleSheetCountStmt.Exec() to actually execute the query. 107 | func (s *GoogleSheetRowStore) Count() *GoogleSheetCountStmt { 108 | return newGoogleSheetCountStmt(s) 109 | } 110 | 111 | // Close cleans up all held resources if any. 112 | func (s *GoogleSheetRowStore) Close(_ context.Context) error { 113 | return nil 114 | } 115 | 116 | func (s *GoogleSheetRowStore) ensureHeaders() error { 117 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 118 | defer cancel() 119 | 120 | if _, err := s.wrapper.Clear( 121 | ctx, 122 | s.spreadsheetID, 123 | []string{common.GetA1Range(s.sheetName, defaultRowHeaderRange)}, 124 | ); err != nil { 125 | return err 126 | } 127 | 128 | cols := make([]interface{}, len(s.config.Columns)) 129 | for i := range s.config.Columns { 130 | cols[i] = s.config.Columns[i] 131 | } 132 | 133 | if _, err := s.wrapper.UpdateRows( 134 | ctx, 135 | s.spreadsheetID, 136 | common.GetA1Range(s.sheetName, defaultRowHeaderRange), 137 | [][]interface{}{cols}, 138 | ); err != nil { 139 | return err 140 | } 141 | return nil 142 | } 143 | 144 | // NewGoogleSheetRowStore creates an instance of the row based store with the given configuration. 145 | // It will also try to create the sheet, in case it does not exist yet. 146 | func NewGoogleSheetRowStore( 147 | auth sheets.AuthClient, 148 | spreadsheetID string, 149 | sheetName string, 150 | config GoogleSheetRowStoreConfig, 151 | ) *GoogleSheetRowStore { 152 | if err := config.validate(); err != nil { 153 | panic(err) 154 | } 155 | 156 | wrapper, err := sheets.NewWrapper(auth) 157 | if err != nil { 158 | panic(fmt.Errorf("error creating sheets wrapper: %w", err)) 159 | } 160 | 161 | config = injectTimestampCol(config) 162 | store := &GoogleSheetRowStore{ 163 | wrapper: wrapper, 164 | spreadsheetID: spreadsheetID, 165 | sheetName: sheetName, 166 | colsMapping: common.GenerateColumnMapping(config.Columns), 167 | colsWithFormula: common.NewSet(config.ColumnsWithFormula), 168 | config: config, 169 | } 170 | 171 | _ = ensureSheets(store.wrapper, store.spreadsheetID, store.sheetName) 172 | if err := store.ensureHeaders(); err != nil { 173 | panic(fmt.Errorf("error checking headers: %w", err)) 174 | } 175 | return store 176 | } 177 | 178 | // The additional rowIdxCol column is needed to differentiate which row is truly empty and which one is not. 179 | // Currently, we use this for detecting which rows are really empty for UPDATE without WHERE clause. 180 | // Otherwise, it will always update all rows (instead of the non-empty rows only). 181 | func injectTimestampCol(config GoogleSheetRowStoreConfig) GoogleSheetRowStoreConfig { 182 | newCols := make([]string, 0, len(config.Columns)+1) 183 | newCols = append(newCols, rowIdxCol) 184 | newCols = append(newCols, config.Columns...) 185 | config.Columns = newCols 186 | 187 | return config 188 | } 189 | -------------------------------------------------------------------------------- /internal/google/store/row_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "testing" 8 | "time" 9 | 10 | "github.com/FreeLeh/GoFreeDB/internal/common" 11 | "github.com/FreeLeh/GoFreeDB/internal/models" 12 | 13 | "github.com/FreeLeh/GoFreeDB/google/auth" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | type testPerson struct { 18 | Name string `json:"name" db:"name"` 19 | Age int64 `json:"age" db:"age"` 20 | DOB string `json:"dob" db:"dob"` 21 | } 22 | 23 | func TestGoogleSheetRowStore_Integration(t *testing.T) { 24 | spreadsheetID, authJSON, shouldRun := getIntegrationTestInfo() 25 | if !shouldRun { 26 | t.Skip("integration test should be run only in GitHub Actions") 27 | } 28 | sheetName := fmt.Sprintf("integration_row_%d", common.CurrentTimeMs()) 29 | 30 | googleAuth, err := auth.NewServiceFromJSON([]byte(authJSON), auth.GoogleSheetsReadWrite, auth.ServiceConfig{}) 31 | if err != nil { 32 | t.Fatalf("error when instantiating google auth: %s", err) 33 | } 34 | 35 | db := NewGoogleSheetRowStore( 36 | googleAuth, 37 | spreadsheetID, 38 | sheetName, 39 | GoogleSheetRowStoreConfig{Columns: []string{"name", "age", "dob"}}, 40 | ) 41 | defer func() { 42 | time.Sleep(time.Second) 43 | deleteSheet(t, db.wrapper, spreadsheetID, []string{db.sheetName}) 44 | _ = db.Close(context.Background()) 45 | }() 46 | 47 | time.Sleep(time.Second) 48 | res, err := db.Count().Exec(context.Background()) 49 | assert.Equal(t, uint64(0), res) 50 | assert.Nil(t, err) 51 | 52 | var out []testPerson 53 | 54 | time.Sleep(time.Second) 55 | err = db.Select(&out, "name", "age").Offset(10).Limit(10).Exec(context.Background()) 56 | assert.Nil(t, err) 57 | assert.Empty(t, out) 58 | 59 | time.Sleep(time.Second) 60 | err = db.Insert( 61 | testPerson{"name1", 10, "1999-01-01"}, 62 | testPerson{"name2", 11, "2000-01-01"}, 63 | ).Exec(context.Background()) 64 | assert.Nil(t, err) 65 | 66 | // Nil type 67 | time.Sleep(time.Second) 68 | err = db.Insert(nil).Exec(context.Background()) 69 | assert.NotNil(t, err) 70 | 71 | time.Sleep(time.Second) 72 | err = db.Insert(testPerson{ 73 | Name: "name3", 74 | Age: 9007199254740992, 75 | DOB: "2001-01-01", 76 | }).Exec(context.Background()) 77 | assert.Nil(t, err) 78 | 79 | time.Sleep(time.Second) 80 | err = db.Update(map[string]interface{}{"name": "name4"}). 81 | Where("age = ?", 10). 82 | Exec(context.Background()) 83 | assert.Nil(t, err) 84 | 85 | expected := []testPerson{ 86 | {"name2", 11, "2000-01-01"}, 87 | {"name3", 9007199254740992, "2001-01-01"}, 88 | } 89 | 90 | time.Sleep(time.Second) 91 | err = db.Select(&out, "name", "age", "dob"). 92 | Where("name = ? OR name = ?", "name2", "name3"). 93 | OrderBy([]models.ColumnOrderBy{{"name", models.OrderByAsc}}). 94 | Limit(2). 95 | Exec(context.Background()) 96 | assert.Nil(t, err) 97 | assert.Equal(t, expected, out) 98 | 99 | time.Sleep(time.Second) 100 | count, err := db.Count(). 101 | Where("name = ? OR name = ?", "name2", "name3"). 102 | Exec(context.Background()) 103 | assert.Nil(t, err) 104 | assert.Equal(t, uint64(2), count) 105 | 106 | time.Sleep(time.Second) 107 | err = db.Delete().Where("name = ?", "name4").Exec(context.Background()) 108 | assert.Nil(t, err) 109 | } 110 | 111 | func TestGoogleSheetRowStore_Integration_EdgeCases(t *testing.T) { 112 | spreadsheetID, authJSON, shouldRun := getIntegrationTestInfo() 113 | if !shouldRun { 114 | t.Skip("integration test should be run only in GitHub Actions") 115 | } 116 | sheetName := fmt.Sprintf("integration_row_%d", common.CurrentTimeMs()) 117 | 118 | googleAuth, err := auth.NewServiceFromJSON([]byte(authJSON), auth.GoogleSheetsReadWrite, auth.ServiceConfig{}) 119 | if err != nil { 120 | t.Fatalf("error when instantiating google auth: %s", err) 121 | } 122 | 123 | db := NewGoogleSheetRowStore( 124 | googleAuth, 125 | spreadsheetID, 126 | sheetName, 127 | GoogleSheetRowStoreConfig{Columns: []string{"name", "age", "dob"}}, 128 | ) 129 | defer func() { 130 | time.Sleep(time.Second) 131 | deleteSheet(t, db.wrapper, spreadsheetID, []string{db.sheetName}) 132 | _ = db.Close(context.Background()) 133 | }() 134 | 135 | // Non-struct types 136 | time.Sleep(time.Second) 137 | err = db.Insert([]interface{}{"name3", 12, "2001-01-01"}).Exec(context.Background()) 138 | assert.NotNil(t, err) 139 | 140 | // IEEE 754 unsafe integer 141 | time.Sleep(time.Second) 142 | err = db.Insert([]interface{}{"name3", 9007199254740993, "2001-01-01"}).Exec(context.Background()) 143 | assert.NotNil(t, err) 144 | 145 | // IEEE 754 unsafe integer 146 | time.Sleep(time.Second) 147 | err = db.Insert( 148 | testPerson{"name1", 10, "1999-01-01"}, 149 | testPerson{"name2", 11, "2000-01-01"}, 150 | ).Exec(context.Background()) 151 | assert.Nil(t, err) 152 | 153 | time.Sleep(time.Second) 154 | err = db.Update(map[string]interface{}{"name": "name4", "age": int64(9007199254740993)}). 155 | Exec(context.Background()) 156 | assert.NotNil(t, err) 157 | } 158 | 159 | type formulaWriteModel struct { 160 | Value string `json:"value" db:"value"` 161 | } 162 | 163 | type formulaReadModel struct { 164 | Value int `json:"value" db:"value"` 165 | } 166 | 167 | func TestGoogleSheetRowStore_Formula(t *testing.T) { 168 | spreadsheetID, authJSON, shouldRun := getIntegrationTestInfo() 169 | if !shouldRun { 170 | t.Skip("integration test should be run only in GitHub Actions") 171 | } 172 | sheetName := fmt.Sprintf("integration_row_%d", common.CurrentTimeMs()) 173 | 174 | googleAuth, err := auth.NewServiceFromJSON([]byte(authJSON), auth.GoogleSheetsReadWrite, auth.ServiceConfig{}) 175 | if err != nil { 176 | t.Fatalf("error when instantiating google auth: %s", err) 177 | } 178 | 179 | db := NewGoogleSheetRowStore( 180 | googleAuth, 181 | spreadsheetID, 182 | sheetName, 183 | GoogleSheetRowStoreConfig{ 184 | Columns: []string{"value"}, 185 | ColumnsWithFormula: []string{"value"}, 186 | }, 187 | ) 188 | defer func() { 189 | time.Sleep(time.Second) 190 | deleteSheet(t, db.wrapper, spreadsheetID, []string{db.sheetName}) 191 | _ = db.Close(context.Background()) 192 | }() 193 | 194 | var out []formulaReadModel 195 | 196 | time.Sleep(time.Second) 197 | err = db.Insert(formulaWriteModel{Value: "=ROW()-1"}).Exec(context.Background()) 198 | assert.Nil(t, err) 199 | 200 | time.Sleep(time.Second) 201 | err = db.Select(&out).Exec(context.Background()) 202 | assert.Nil(t, err) 203 | assert.ElementsMatch(t, []formulaReadModel{{Value: 1}}, out) 204 | 205 | time.Sleep(time.Second) 206 | err = db.Update(map[string]interface{}{"value": "=ROW()"}).Exec(context.Background()) 207 | assert.Nil(t, err) 208 | 209 | time.Sleep(time.Second) 210 | err = db.Select(&out).Exec(context.Background()) 211 | assert.Nil(t, err) 212 | assert.ElementsMatch(t, []formulaReadModel{{Value: 2}}, out) 213 | } 214 | 215 | func TestInjectTimestampCol(t *testing.T) { 216 | result := injectTimestampCol(GoogleSheetRowStoreConfig{Columns: []string{"col1", "col2"}}) 217 | assert.Equal(t, GoogleSheetRowStoreConfig{Columns: []string{rowIdxCol, "col1", "col2"}}, result) 218 | } 219 | 220 | func TestGoogleSheetRowStoreConfig_validate(t *testing.T) { 221 | t.Run("empty_columns", func(t *testing.T) { 222 | conf := GoogleSheetRowStoreConfig{Columns: []string{}} 223 | assert.NotNil(t, conf.validate()) 224 | }) 225 | 226 | t.Run("too_many_columns", func(t *testing.T) { 227 | columns := make([]string, 0) 228 | for i := 0; i < 27; i++ { 229 | columns = append(columns, strconv.FormatInt(int64(i), 10)) 230 | } 231 | 232 | conf := GoogleSheetRowStoreConfig{Columns: columns} 233 | assert.NotNil(t, conf.validate()) 234 | }) 235 | 236 | t.Run("no_error", func(t *testing.T) { 237 | columns := make([]string, 0) 238 | for i := 0; i < 10; i++ { 239 | columns = append(columns, strconv.FormatInt(int64(i), 10)) 240 | } 241 | 242 | conf := GoogleSheetRowStoreConfig{Columns: columns} 243 | assert.Nil(t, conf.validate()) 244 | }) 245 | } 246 | -------------------------------------------------------------------------------- /internal/google/store/stmt.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "reflect" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/FreeLeh/GoFreeDB/internal/common" 12 | "github.com/FreeLeh/GoFreeDB/internal/models" 13 | 14 | "github.com/FreeLeh/GoFreeDB/internal/google/sheets" 15 | ) 16 | 17 | type whereInterceptorFunc func(where string) string 18 | 19 | type queryBuilder struct { 20 | replacer *strings.Replacer 21 | columns []string 22 | where string 23 | whereArgs []interface{} 24 | whereInterceptor whereInterceptorFunc 25 | orderBy []string 26 | limit uint64 27 | offset uint64 28 | } 29 | 30 | func (q *queryBuilder) Where(condition string, args ...interface{}) *queryBuilder { 31 | q.where = condition 32 | q.whereArgs = args 33 | return q 34 | } 35 | 36 | func (q *queryBuilder) OrderBy(ordering []models.ColumnOrderBy) *queryBuilder { 37 | orderBy := make([]string, 0, len(ordering)) 38 | for _, o := range ordering { 39 | orderBy = append(orderBy, o.Column+" "+string(o.OrderBy)) 40 | } 41 | 42 | q.orderBy = orderBy 43 | return q 44 | } 45 | 46 | func (q *queryBuilder) Limit(limit uint64) *queryBuilder { 47 | q.limit = limit 48 | return q 49 | } 50 | 51 | func (q *queryBuilder) Offset(offset uint64) *queryBuilder { 52 | q.offset = offset 53 | return q 54 | } 55 | 56 | func (q *queryBuilder) Generate() (string, error) { 57 | stmt := &strings.Builder{} 58 | stmt.WriteString("select") 59 | 60 | if err := q.writeCols(stmt); err != nil { 61 | return "", err 62 | } 63 | if err := q.writeWhere(stmt); err != nil { 64 | return "", err 65 | } 66 | if err := q.writeOrderBy(stmt); err != nil { 67 | return "", err 68 | } 69 | if err := q.writeOffset(stmt); err != nil { 70 | return "", err 71 | } 72 | if err := q.writeLimit(stmt); err != nil { 73 | return "", err 74 | } 75 | 76 | return stmt.String(), nil 77 | } 78 | 79 | func (q *queryBuilder) writeCols(stmt *strings.Builder) error { 80 | stmt.WriteString(" ") 81 | 82 | translated := make([]string, 0, len(q.columns)) 83 | for _, col := range q.columns { 84 | translated = append(translated, q.replacer.Replace(col)) 85 | } 86 | 87 | stmt.WriteString(strings.Join(translated, ", ")) 88 | return nil 89 | } 90 | 91 | func (q *queryBuilder) writeWhere(stmt *strings.Builder) error { 92 | where := q.where 93 | if q.whereInterceptor != nil { 94 | where = q.whereInterceptor(q.where) 95 | } 96 | 97 | nArgs := strings.Count(where, "?") 98 | if nArgs != len(q.whereArgs) { 99 | return fmt.Errorf("number of arguments required in the 'where' clause (%d) is not the same as the number of provided arguments (%d)", nArgs, len(q.whereArgs)) 100 | } 101 | 102 | where = q.replacer.Replace(where) 103 | tokens := strings.Split(where, "?") 104 | 105 | result := make([]string, 0) 106 | result = append(result, strings.TrimSpace(tokens[0])) 107 | 108 | for i, token := range tokens[1:] { 109 | arg, err := q.convertArg(q.whereArgs[i]) 110 | if err != nil { 111 | return fmt.Errorf("failed converting 'where' arguments: %v, %w", arg, err) 112 | } 113 | result = append(result, arg, strings.TrimSpace(token)) 114 | } 115 | 116 | stmt.WriteString(" where ") 117 | stmt.WriteString(strings.Join(result, " ")) 118 | return nil 119 | } 120 | 121 | func (q *queryBuilder) convertArg(arg interface{}) (string, error) { 122 | switch converted := arg.(type) { 123 | case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: 124 | return q.convertInt(arg) 125 | case float32, float64: 126 | return q.convertFloat(arg) 127 | case string, []byte: 128 | return q.convertString(arg) 129 | case bool: 130 | return strconv.FormatBool(converted), nil 131 | default: 132 | return "", errors.New("unsupported argument type") 133 | } 134 | } 135 | 136 | func (q *queryBuilder) convertInt(arg interface{}) (string, error) { 137 | switch converted := arg.(type) { 138 | case int: 139 | return strconv.FormatInt(int64(converted), 10), nil 140 | case int8: 141 | return strconv.FormatInt(int64(converted), 10), nil 142 | case int16: 143 | return strconv.FormatInt(int64(converted), 10), nil 144 | case int32: 145 | return strconv.FormatInt(int64(converted), 10), nil 146 | case int64: 147 | return strconv.FormatInt(converted, 10), nil 148 | case uint: 149 | return strconv.FormatUint(uint64(converted), 10), nil 150 | case uint8: 151 | return strconv.FormatUint(uint64(converted), 10), nil 152 | case uint16: 153 | return strconv.FormatUint(uint64(converted), 10), nil 154 | case uint32: 155 | return strconv.FormatUint(uint64(converted), 10), nil 156 | case uint64: 157 | return strconv.FormatUint(converted, 10), nil 158 | default: 159 | return "", errors.New("unsupported argument type") 160 | } 161 | } 162 | 163 | func (q *queryBuilder) convertFloat(arg interface{}) (string, error) { 164 | switch converted := arg.(type) { 165 | case float32: 166 | return strconv.FormatFloat(float64(converted), 'f', -1, 64), nil 167 | case float64: 168 | return strconv.FormatFloat(converted, 'f', -1, 64), nil 169 | default: 170 | return "", errors.New("unsupported argument type") 171 | } 172 | } 173 | 174 | func (q *queryBuilder) convertString(arg interface{}) (string, error) { 175 | switch converted := arg.(type) { 176 | case string: 177 | cleaned := strings.ToLower(strings.TrimSpace(converted)) 178 | if googleSheetSelectStmtStringKeyword.MatchString(cleaned) { 179 | return converted, nil 180 | } 181 | return strconv.Quote(converted), nil 182 | case []byte: 183 | return strconv.Quote(string(converted)), nil 184 | default: 185 | return "", errors.New("unsupported argument type") 186 | } 187 | } 188 | 189 | func (q *queryBuilder) writeOrderBy(stmt *strings.Builder) error { 190 | if len(q.orderBy) == 0 { 191 | return nil 192 | } 193 | 194 | stmt.WriteString(" order by ") 195 | result := make([]string, 0, len(q.orderBy)) 196 | 197 | for _, o := range q.orderBy { 198 | result = append(result, q.replacer.Replace(o)) 199 | } 200 | 201 | stmt.WriteString(strings.Join(result, ", ")) 202 | return nil 203 | } 204 | 205 | func (q *queryBuilder) writeOffset(stmt *strings.Builder) error { 206 | if q.offset == 0 { 207 | return nil 208 | } 209 | 210 | stmt.WriteString(" offset ") 211 | stmt.WriteString(strconv.FormatInt(int64(q.offset), 10)) 212 | return nil 213 | } 214 | 215 | func (q *queryBuilder) writeLimit(stmt *strings.Builder) error { 216 | if q.limit == 0 { 217 | return nil 218 | } 219 | 220 | stmt.WriteString(" limit ") 221 | stmt.WriteString(strconv.FormatInt(int64(q.limit), 10)) 222 | return nil 223 | } 224 | 225 | func newQueryBuilder( 226 | colReplacements map[string]string, 227 | whereInterceptor whereInterceptorFunc, 228 | colSelected []string, 229 | ) *queryBuilder { 230 | replacements := make([]string, 0, 2*len(colReplacements)) 231 | for col, repl := range colReplacements { 232 | replacements = append(replacements, col, repl) 233 | } 234 | 235 | return &queryBuilder{ 236 | replacer: strings.NewReplacer(replacements...), 237 | columns: colSelected, 238 | whereInterceptor: whereInterceptor, 239 | } 240 | } 241 | 242 | // GoogleSheetSelectStmt encapsulates information required to query the row store. 243 | type GoogleSheetSelectStmt struct { 244 | store *GoogleSheetRowStore 245 | columns []string 246 | queryBuilder *queryBuilder 247 | output interface{} 248 | } 249 | 250 | // Where specifies the condition to meet for a row to be included. 251 | // 252 | // "condition" specifies the WHERE clause. 253 | // Values in the WHERE clause should be replaced by a placeholder "?". 254 | // The actual values used for each placeholder (ordering matters) are provided via the "args" parameter. 255 | // 256 | // "args" specifies the real value to replace each placeholder in the WHERE clause. 257 | // Note that the first "args" value will replace the first placeholder "?" in the WHERE clause. 258 | // 259 | // If you want to understand the reason behind this design, please read the protocol page: https://github.com/FreeLeh/docs/blob/main/freedb/protocols.md. 260 | // 261 | // All conditions supported by Google Sheet "QUERY" function are supported by this library. 262 | // You can read the full information in https://developers.google.com/chart/interactive/docs/querylanguage#where. 263 | func (s *GoogleSheetSelectStmt) Where(condition string, args ...interface{}) *GoogleSheetSelectStmt { 264 | s.queryBuilder.Where(condition, args...) 265 | return s 266 | } 267 | 268 | // OrderBy specifies the column ordering. 269 | // 270 | // The default value is no ordering specified. 271 | func (s *GoogleSheetSelectStmt) OrderBy(ordering []models.ColumnOrderBy) *GoogleSheetSelectStmt { 272 | s.queryBuilder.OrderBy(ordering) 273 | return s 274 | } 275 | 276 | // Limit specifies the number of rows to retrieve. 277 | // 278 | // The default value is 0. 279 | func (s *GoogleSheetSelectStmt) Limit(limit uint64) *GoogleSheetSelectStmt { 280 | s.queryBuilder.Limit(limit) 281 | return s 282 | } 283 | 284 | // Offset specifies the number of rows to skip before starting to include the rows. 285 | // 286 | // The default value is 0. 287 | func (s *GoogleSheetSelectStmt) Offset(offset uint64) *GoogleSheetSelectStmt { 288 | s.queryBuilder.Offset(offset) 289 | return s 290 | } 291 | 292 | // Exec retrieves rows matching with the given condition. 293 | // 294 | // There is only 1 API call behind the scene. 295 | func (s *GoogleSheetSelectStmt) Exec(ctx context.Context) error { 296 | if err := s.ensureOutputSlice(); err != nil { 297 | return err 298 | } 299 | 300 | stmt, err := s.queryBuilder.Generate() 301 | if err != nil { 302 | return err 303 | } 304 | 305 | result, err := s.store.wrapper.QueryRows( 306 | ctx, 307 | s.store.spreadsheetID, 308 | s.store.sheetName, 309 | stmt, 310 | true, 311 | ) 312 | if err != nil { 313 | return err 314 | } 315 | 316 | m := s.buildQueryResultMap(result) 317 | return common.MapStructureDecode(m, s.output) 318 | } 319 | 320 | func (s *GoogleSheetSelectStmt) buildQueryResultMap(original sheets.QueryRowsResult) []map[string]interface{} { 321 | result := make([]map[string]interface{}, len(original.Rows)) 322 | 323 | for rowIdx, row := range original.Rows { 324 | result[rowIdx] = make(map[string]interface{}, len(row)) 325 | 326 | for colIdx, value := range row { 327 | col := s.columns[colIdx] 328 | result[rowIdx][col] = value 329 | } 330 | } 331 | 332 | return result 333 | } 334 | 335 | func (s *GoogleSheetSelectStmt) ensureOutputSlice() error { 336 | // Passing an uninitialised slice will not compare to nil due to this: https://yourbasic.org/golang/gotcha-why-nil-error-not-equal-nil/ 337 | // Only if passing an untyped `nil` will compare to the `nil` in the line below. 338 | // Observations as below: 339 | // 340 | // var o []int 341 | // o == nil --> this is true because the compiler knows `o` is nil and of type `[]int`, so the `nil` on the right side is of the same `[]int` type. 342 | // 343 | // var x interface{} = o 344 | // x == nil --> this is false because `o` has been boxed by `x` and the `nil` on the right side is of type `nil` (i.e. nil value of nil type). 345 | // x == []int(nil) --> this is true because the `nil` has been casted explicitly to `nil` of type `[]int`. 346 | if s.output == nil { 347 | return errors.New("select statement output cannot be empty or nil") 348 | } 349 | 350 | t := reflect.TypeOf(s.output) 351 | if t.Kind() != reflect.Ptr { 352 | return errors.New("select statement output must be a pointer to a slice of something") 353 | } 354 | 355 | elem := t.Elem() 356 | if elem.Kind() != reflect.Slice { 357 | return fmt.Errorf("select statement output must be a pointer to a slice of something; current output type: %s", t.Kind().String()) 358 | } 359 | 360 | return nil 361 | } 362 | 363 | func newGoogleSheetSelectStmt(store *GoogleSheetRowStore, output interface{}, columns []string) *GoogleSheetSelectStmt { 364 | if len(columns) == 0 { 365 | columns = store.config.Columns 366 | } 367 | 368 | return &GoogleSheetSelectStmt{ 369 | store: store, 370 | columns: columns, 371 | queryBuilder: newQueryBuilder(store.colsMapping.NameMap(), ridWhereClauseInterceptor, columns), 372 | output: output, 373 | } 374 | } 375 | 376 | // GoogleSheetInsertStmt encapsulates information required to insert new rows into the Google Sheet. 377 | type GoogleSheetInsertStmt struct { 378 | store *GoogleSheetRowStore 379 | rows []interface{} 380 | } 381 | 382 | func (s *GoogleSheetInsertStmt) convertRowToSlice(row interface{}) ([]interface{}, error) { 383 | if row == nil { 384 | return nil, errors.New("row type must not be nil") 385 | } 386 | 387 | t := reflect.TypeOf(row) 388 | if t.Kind() == reflect.Ptr { 389 | t = t.Elem() 390 | } 391 | if t.Kind() != reflect.Struct { 392 | return nil, errors.New("row type must be either a struct or a slice") 393 | } 394 | 395 | var output map[string]interface{} 396 | if err := common.MapStructureDecode(row, &output); err != nil { 397 | return nil, err 398 | } 399 | 400 | result := make([]interface{}, len(s.store.colsMapping)) 401 | result[0] = rowIdxFormula 402 | 403 | for col, value := range output { 404 | if colIdx, ok := s.store.colsMapping[col]; ok { 405 | escapedValue, err := escapeValue(col, value, s.store.colsWithFormula) 406 | if err != nil { 407 | return nil, err 408 | } 409 | if err = common.CheckIEEE754SafeInteger(escapedValue); err != nil { 410 | return nil, err 411 | } 412 | result[colIdx.Idx] = escapedValue 413 | } 414 | } 415 | 416 | return result, nil 417 | } 418 | 419 | // Exec inserts the provided new rows data into Google Sheet. 420 | // This method calls the relevant Google Sheet APIs to actually insert the new rows. 421 | // 422 | // There is only 1 API call behind the scene. 423 | func (s *GoogleSheetInsertStmt) Exec(ctx context.Context) error { 424 | if len(s.rows) == 0 { 425 | return nil 426 | } 427 | 428 | convertedRows := make([][]interface{}, 0, len(s.rows)) 429 | for _, row := range s.rows { 430 | r, err := s.convertRowToSlice(row) 431 | if err != nil { 432 | return fmt.Errorf("cannot execute google sheet insert statement due to row conversion error: %w", err) 433 | } 434 | convertedRows = append(convertedRows, r) 435 | } 436 | 437 | _, err := s.store.wrapper.OverwriteRows( 438 | ctx, 439 | s.store.spreadsheetID, 440 | common.GetA1Range(s.store.sheetName, defaultRowFullTableRange), 441 | convertedRows, 442 | ) 443 | return err 444 | } 445 | 446 | func newGoogleSheetInsertStmt(store *GoogleSheetRowStore, rows []interface{}) *GoogleSheetInsertStmt { 447 | return &GoogleSheetInsertStmt{ 448 | store: store, 449 | rows: rows, 450 | } 451 | } 452 | 453 | // GoogleSheetUpdateStmt encapsulates information required to update rows. 454 | type GoogleSheetUpdateStmt struct { 455 | store *GoogleSheetRowStore 456 | colToValue map[string]interface{} 457 | queryBuilder *queryBuilder 458 | } 459 | 460 | // Where specifies the condition to choose which rows are affected. 461 | // 462 | // It works just like the GoogleSheetSelectStmt.Where() method. 463 | // Please read GoogleSheetSelectStmt.Where() for more details. 464 | func (s *GoogleSheetUpdateStmt) Where(condition string, args ...interface{}) *GoogleSheetUpdateStmt { 465 | s.queryBuilder.Where(condition, args...) 466 | return s 467 | } 468 | 469 | // Exec updates rows matching the condition with the new values for affected columns. 470 | // 471 | // There are 2 API calls behind the scene. 472 | func (s *GoogleSheetUpdateStmt) Exec(ctx context.Context) error { 473 | if len(s.colToValue) == 0 { 474 | return errors.New("empty colToValue, at least one column must be updated") 475 | } 476 | 477 | selectStmt, err := s.queryBuilder.Generate() 478 | if err != nil { 479 | return err 480 | } 481 | 482 | indices, err := getRowIndices(ctx, s.store, selectStmt) 483 | if err != nil { 484 | return err 485 | } 486 | if len(indices) == 0 { 487 | return nil 488 | } 489 | 490 | requests, err := s.generateBatchUpdateRequests(indices) 491 | if err != nil { 492 | return err 493 | } 494 | 495 | _, err = s.store.wrapper.BatchUpdateRows(ctx, s.store.spreadsheetID, requests) 496 | return err 497 | } 498 | 499 | func (s *GoogleSheetUpdateStmt) generateBatchUpdateRequests(rowIndices []int64) ([]sheets.BatchUpdateRowsRequest, error) { 500 | requests := make([]sheets.BatchUpdateRowsRequest, 0) 501 | 502 | for col, value := range s.colToValue { 503 | colIdx, ok := s.store.colsMapping[col] 504 | if !ok { 505 | return nil, fmt.Errorf("failed to update, unknown column name provided: %s", col) 506 | } 507 | 508 | escapedValue, err := escapeValue(col, value, s.store.colsWithFormula) 509 | if err != nil { 510 | return nil, err 511 | } 512 | if err = common.CheckIEEE754SafeInteger(escapedValue); err != nil { 513 | return nil, err 514 | } 515 | 516 | for _, rowIdx := range rowIndices { 517 | a1Range := colIdx.Name + strconv.FormatInt(rowIdx, 10) 518 | requests = append(requests, sheets.BatchUpdateRowsRequest{ 519 | A1Range: common.GetA1Range(s.store.sheetName, a1Range), 520 | Values: [][]interface{}{{escapedValue}}, 521 | }) 522 | } 523 | } 524 | 525 | return requests, nil 526 | } 527 | 528 | func newGoogleSheetUpdateStmt(store *GoogleSheetRowStore, colToValue map[string]interface{}) *GoogleSheetUpdateStmt { 529 | return &GoogleSheetUpdateStmt{ 530 | store: store, 531 | colToValue: colToValue, 532 | queryBuilder: newQueryBuilder(store.colsMapping.NameMap(), ridWhereClauseInterceptor, []string{rowIdxCol}), 533 | } 534 | } 535 | 536 | // GoogleSheetDeleteStmt encapsulates information required to delete rows. 537 | type GoogleSheetDeleteStmt struct { 538 | store *GoogleSheetRowStore 539 | queryBuilder *queryBuilder 540 | } 541 | 542 | // Where specifies the condition to choose which rows are affected. 543 | // 544 | // It works just like the GoogleSheetSelectStmt.Where() method. 545 | // Please read GoogleSheetSelectStmt.Where() for more details. 546 | func (s *GoogleSheetDeleteStmt) Where(condition string, args ...interface{}) *GoogleSheetDeleteStmt { 547 | s.queryBuilder.Where(condition, args...) 548 | return s 549 | } 550 | 551 | // Exec deletes rows matching the condition. 552 | // 553 | // There are 2 API calls behind the scene. 554 | func (s *GoogleSheetDeleteStmt) Exec(ctx context.Context) error { 555 | selectStmt, err := s.queryBuilder.Generate() 556 | if err != nil { 557 | return err 558 | } 559 | 560 | indices, err := getRowIndices(ctx, s.store, selectStmt) 561 | if err != nil { 562 | return err 563 | } 564 | if len(indices) == 0 { 565 | return nil 566 | } 567 | 568 | _, err = s.store.wrapper.Clear(ctx, s.store.spreadsheetID, generateRowA1Ranges(s.store.sheetName, indices)) 569 | return err 570 | } 571 | 572 | func newGoogleSheetDeleteStmt(store *GoogleSheetRowStore) *GoogleSheetDeleteStmt { 573 | return &GoogleSheetDeleteStmt{ 574 | store: store, 575 | queryBuilder: newQueryBuilder(store.colsMapping.NameMap(), ridWhereClauseInterceptor, []string{rowIdxCol}), 576 | } 577 | } 578 | 579 | // GoogleSheetCountStmt encapsulates information required to count the number of rows matching some conditions. 580 | type GoogleSheetCountStmt struct { 581 | store *GoogleSheetRowStore 582 | queryBuilder *queryBuilder 583 | } 584 | 585 | // Where specifies the condition to choose which rows are affected. 586 | // 587 | // It works just like the GoogleSheetSelectStmt.Where() method. 588 | // Please read GoogleSheetSelectStmt.Where() for more details. 589 | func (s *GoogleSheetCountStmt) Where(condition string, args ...interface{}) *GoogleSheetCountStmt { 590 | s.queryBuilder.Where(condition, args...) 591 | return s 592 | } 593 | 594 | // Exec counts the number of rows matching the provided condition. 595 | // 596 | // There is only 1 API call behind the scene. 597 | func (s *GoogleSheetCountStmt) Exec(ctx context.Context) (uint64, error) { 598 | selectStmt, err := s.queryBuilder.Generate() 599 | if err != nil { 600 | return 0, err 601 | } 602 | 603 | result, err := s.store.wrapper.QueryRows(ctx, s.store.spreadsheetID, s.store.sheetName, selectStmt, true) 604 | if err != nil { 605 | return 0, err 606 | } 607 | 608 | // When COUNT() returns 0, somehow it returns empty row slice. 609 | if len(result.Rows) < 1 || len(result.Rows[0]) < 1 { 610 | return 0, nil 611 | } 612 | if len(result.Rows) != 1 || len(result.Rows[0]) != 1 { 613 | return 0, errors.New("unexpected number of rows or columns") 614 | } 615 | 616 | count := result.Rows[0][0].(float64) 617 | return uint64(count), nil 618 | } 619 | 620 | func newGoogleSheetCountStmt(store *GoogleSheetRowStore) *GoogleSheetCountStmt { 621 | countClause := fmt.Sprintf("COUNT(%s)", rowIdxCol) 622 | return &GoogleSheetCountStmt{ 623 | store: store, 624 | queryBuilder: newQueryBuilder(store.colsMapping.NameMap(), ridWhereClauseInterceptor, []string{countClause}), 625 | } 626 | } 627 | 628 | func getRowIndices(ctx context.Context, store *GoogleSheetRowStore, selectStmt string) ([]int64, error) { 629 | result, err := store.wrapper.QueryRows(ctx, store.spreadsheetID, store.sheetName, selectStmt, true) 630 | if err != nil { 631 | return nil, err 632 | } 633 | if len(result.Rows) == 0 { 634 | return nil, nil 635 | } 636 | 637 | rowIndices := make([]int64, 0) 638 | for _, row := range result.Rows { 639 | if len(row) != 1 { 640 | return nil, fmt.Errorf("error retrieving row indices: %+v", result) 641 | } 642 | 643 | idx, ok := row[0].(float64) 644 | if !ok { 645 | return nil, fmt.Errorf("error converting row indices, value: %+v", row[0]) 646 | } 647 | 648 | rowIndices = append(rowIndices, int64(idx)) 649 | } 650 | 651 | return rowIndices, nil 652 | } 653 | 654 | func generateRowA1Ranges(sheetName string, indices []int64) []string { 655 | locations := make([]string, len(indices)) 656 | for i := range indices { 657 | locations[i] = common.GetA1Range( 658 | sheetName, 659 | fmt.Sprintf(rowDeleteRangeTemplate, indices[i], indices[i]), 660 | ) 661 | } 662 | return locations 663 | } 664 | 665 | func ridWhereClauseInterceptor(where string) string { 666 | if where == "" { 667 | return rowWhereEmptyConditionTemplate 668 | } 669 | return fmt.Sprintf(rowWhereNonEmptyConditionTemplate, where) 670 | } 671 | 672 | func escapeValue( 673 | col string, 674 | value any, 675 | colsWithFormula *common.Set[string], 676 | ) (any, error) { 677 | if !colsWithFormula.Contains(col) { 678 | return common.EscapeValue(value), nil 679 | } 680 | 681 | _, ok := value.(string) 682 | if !ok { 683 | return nil, fmt.Errorf("value of column %s is not a string, but expected to contain formula", col) 684 | } 685 | return value, nil 686 | } 687 | -------------------------------------------------------------------------------- /internal/google/store/stmt_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github.com/FreeLeh/GoFreeDB/internal/common" 7 | "github.com/FreeLeh/GoFreeDB/internal/models" 8 | "testing" 9 | 10 | "github.com/FreeLeh/GoFreeDB/internal/google/sheets" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | type person struct { 15 | Name string `db:"name,omitempty"` 16 | Age int64 `db:"age,omitempty"` 17 | DOB string `db:"dob,omitempty"` 18 | } 19 | 20 | func TestGenerateQuery(t *testing.T) { 21 | colsMapping := common.ColsMapping{ 22 | rowIdxCol: {"A", 0}, 23 | "col1": {"B", 1}, 24 | "col2": {"C", 2}, 25 | } 26 | 27 | t.Run("successful_basic", func(t *testing.T) { 28 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2"}) 29 | result, err := builder.Generate() 30 | assert.Nil(t, err) 31 | assert.Equal(t, "select B, C where A is not null", result) 32 | }) 33 | 34 | t.Run("unsuccessful_basic_wrong_column", func(t *testing.T) { 35 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2", "col3"}) 36 | result, err := builder.Generate() 37 | assert.Nil(t, err) 38 | assert.Equal(t, "select B, C, col3 where A is not null", result) 39 | }) 40 | 41 | t.Run("successful_with_where", func(t *testing.T) { 42 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2"}) 43 | builder.Where("(col1 > ? AND col2 <= ?) OR (col1 != ? AND col2 == ?)", 100, true, "value", 3.14) 44 | 45 | result, err := builder.Generate() 46 | assert.Nil(t, err) 47 | assert.Equal(t, "select B, C where A is not null AND (B > 100 AND C <= true ) OR (B != \"value\" AND C == 3.14 )", result) 48 | }) 49 | 50 | t.Run("unsuccessful_with_where_wrong_arg_count", func(t *testing.T) { 51 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2"}) 52 | builder.Where("(col1 > ? AND col2 <= ?) OR (col1 != ? AND col2 == ?)", 100, true) 53 | 54 | result, err := builder.Generate() 55 | assert.NotNil(t, err) 56 | assert.Equal(t, "", result) 57 | }) 58 | 59 | t.Run("unsuccessful_with_where_unsupported_arg_type", func(t *testing.T) { 60 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2"}) 61 | builder.Where("(col1 > ? AND col2 <= ?) OR (col1 != ? AND col2 == ?)", 100, true, nil, []string{}) 62 | 63 | result, err := builder.Generate() 64 | assert.NotNil(t, err) 65 | assert.Equal(t, "", result) 66 | }) 67 | 68 | t.Run("successful_with_limit_offset", func(t *testing.T) { 69 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2"}) 70 | builder.Limit(10).Offset(100) 71 | 72 | result, err := builder.Generate() 73 | assert.Nil(t, err) 74 | assert.Equal(t, "select B, C where A is not null offset 100 limit 10", result) 75 | }) 76 | 77 | t.Run("successful_with_order_by", func(t *testing.T) { 78 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2"}) 79 | builder.OrderBy([]models.ColumnOrderBy{{Column: "col2", OrderBy: models.OrderByDesc}, {Column: "col1", OrderBy: models.OrderByAsc}}) 80 | 81 | result, err := builder.Generate() 82 | assert.Nil(t, err) 83 | assert.Equal(t, "select B, C where A is not null order by C DESC, B ASC", result) 84 | }) 85 | 86 | t.Run("test_argument_types", func(t *testing.T) { 87 | builder := newQueryBuilder(colsMapping.NameMap(), ridWhereClauseInterceptor, []string{"col1", "col2"}) 88 | tc := []struct { 89 | input interface{} 90 | output string 91 | err error 92 | }{ 93 | { 94 | input: 1, 95 | output: "1", 96 | err: nil, 97 | }, 98 | { 99 | input: int8(1), 100 | output: "1", 101 | err: nil, 102 | }, 103 | { 104 | input: int16(1), 105 | output: "1", 106 | err: nil, 107 | }, 108 | { 109 | input: int32(1), 110 | output: "1", 111 | err: nil, 112 | }, 113 | { 114 | input: int64(1), 115 | output: "1", 116 | err: nil, 117 | }, 118 | { 119 | input: uint(1), 120 | output: "1", 121 | err: nil, 122 | }, 123 | { 124 | input: uint8(1), 125 | output: "1", 126 | err: nil, 127 | }, 128 | { 129 | input: uint16(1), 130 | output: "1", 131 | err: nil, 132 | }, 133 | { 134 | input: uint32(1), 135 | output: "1", 136 | err: nil, 137 | }, 138 | { 139 | input: uint64(1), 140 | output: "1", 141 | err: nil, 142 | }, 143 | { 144 | input: float32(1.5), 145 | output: "1.5", 146 | err: nil, 147 | }, 148 | { 149 | input: 1.5, 150 | output: "1.5", 151 | err: nil, 152 | }, 153 | { 154 | input: "something", 155 | output: "\"something\"", 156 | err: nil, 157 | }, 158 | { 159 | input: "date", 160 | output: "date", 161 | err: nil, 162 | }, 163 | { 164 | input: "datetime", 165 | output: "datetime", 166 | err: nil, 167 | }, 168 | { 169 | input: "timeofday", 170 | output: "timeofday", 171 | err: nil, 172 | }, 173 | { 174 | input: true, 175 | output: "true", 176 | err: nil, 177 | }, 178 | { 179 | input: []byte("something"), 180 | output: "\"something\"", 181 | err: nil, 182 | }, 183 | } 184 | 185 | for _, c := range tc { 186 | result, err := builder.convertArg(c.input) 187 | assert.Equal(t, c.output, result) 188 | assert.Equal(t, c.err, err) 189 | } 190 | }) 191 | } 192 | 193 | func TestSelectStmt_AllColumns(t *testing.T) { 194 | store := &GoogleSheetRowStore{ 195 | colsMapping: common.ColsMapping{ 196 | rowIdxCol: {"A", 0}, 197 | "col1": {"B", 1}, 198 | "col2": {"C", 2}, 199 | }, 200 | config: GoogleSheetRowStoreConfig{Columns: []string{"col1", "col2"}}, 201 | } 202 | stmt := newGoogleSheetSelectStmt(store, nil, []string{}) 203 | 204 | result, err := stmt.queryBuilder.Generate() 205 | assert.Nil(t, err) 206 | assert.Equal(t, "select B, C where A is not null", result) 207 | } 208 | 209 | func TestSelectStmt_Exec(t *testing.T) { 210 | t.Run("non_slice_output", func(t *testing.T) { 211 | wrapper := &sheets.MockWrapper{} 212 | store := &GoogleSheetRowStore{ 213 | wrapper: wrapper, 214 | colsMapping: map[string]common.ColIdx{ 215 | rowIdxCol: {"A", 0}, 216 | "col1": {"B", 1}, 217 | "col2": {"C", 2}, 218 | }, 219 | } 220 | o := 0 221 | stmt := newGoogleSheetSelectStmt(store, &o, []string{"col1", "col2"}) 222 | 223 | assert.NotNil(t, stmt.Exec(context.Background())) 224 | }) 225 | 226 | t.Run("non_pointer_to_slice_output", func(t *testing.T) { 227 | wrapper := &sheets.MockWrapper{} 228 | store := &GoogleSheetRowStore{ 229 | wrapper: wrapper, 230 | colsMapping: map[string]common.ColIdx{rowIdxCol: {"A", 0}, "col1": {"B", 1}, "col2": {"C", 2}}, 231 | } 232 | var o []int 233 | stmt := newGoogleSheetSelectStmt(store, o, []string{"col1", "col2"}) 234 | 235 | assert.NotNil(t, stmt.Exec(context.Background())) 236 | }) 237 | 238 | t.Run("nil_output", func(t *testing.T) { 239 | wrapper := &sheets.MockWrapper{} 240 | store := &GoogleSheetRowStore{ 241 | wrapper: wrapper, 242 | colsMapping: map[string]common.ColIdx{rowIdxCol: {"A", 0}, "col1": {"B", 1}, "col2": {"C", 2}}, 243 | } 244 | stmt := newGoogleSheetSelectStmt(store, nil, []string{"col1", "col2"}) 245 | 246 | assert.NotNil(t, stmt.Exec(context.Background())) 247 | }) 248 | 249 | t.Run("has_query_error", func(t *testing.T) { 250 | wrapper := &sheets.MockWrapper{QueryRowsError: errors.New("some error")} 251 | store := &GoogleSheetRowStore{ 252 | wrapper: wrapper, 253 | colsMapping: map[string]common.ColIdx{rowIdxCol: {"A", 0}, "col1": {"B", 1}, "col2": {"C", 2}}, 254 | } 255 | var out []int 256 | stmt := newGoogleSheetSelectStmt(store, &out, []string{"col1", "col2"}) 257 | 258 | err := stmt.Exec(context.Background()) 259 | assert.NotNil(t, err) 260 | }) 261 | 262 | t.Run("successful", func(t *testing.T) { 263 | wrapper := &sheets.MockWrapper{QueryRowsResult: sheets.QueryRowsResult{Rows: [][]interface{}{ 264 | {10, "17-01-2001"}, 265 | {11, "18-01-2000"}, 266 | }}} 267 | store := &GoogleSheetRowStore{ 268 | wrapper: wrapper, 269 | colsMapping: map[string]common.ColIdx{rowIdxCol: {"A", 0}, "name": {"B", 1}, "age": {"C", 2}, "dob": {"D", 3}}, 270 | config: GoogleSheetRowStoreConfig{ 271 | Columns: []string{"name", "age", "dob"}, 272 | }, 273 | } 274 | var out []person 275 | stmt := newGoogleSheetSelectStmt(store, &out, []string{"age", "dob"}) 276 | 277 | expected := []person{ 278 | {Age: 10, DOB: "17-01-2001"}, 279 | {Age: 11, DOB: "18-01-2000"}, 280 | } 281 | err := stmt.Exec(context.Background()) 282 | 283 | assert.Nil(t, err) 284 | assert.Equal(t, expected, out) 285 | }) 286 | 287 | t.Run("successful_select_all", func(t *testing.T) { 288 | wrapper := &sheets.MockWrapper{QueryRowsResult: sheets.QueryRowsResult{Rows: [][]interface{}{ 289 | {"name1", 10, "17-01-2001"}, 290 | {"name2", 11, "18-01-2000"}, 291 | }}} 292 | store := &GoogleSheetRowStore{ 293 | wrapper: wrapper, 294 | colsMapping: map[string]common.ColIdx{ 295 | rowIdxCol: {"A", 0}, 296 | "name": {"B", 1}, 297 | "age": {"C", 2}, 298 | "dob": {"D", 3}, 299 | }, 300 | colsWithFormula: common.NewSet([]string{"name"}), 301 | config: GoogleSheetRowStoreConfig{ 302 | Columns: []string{"name", "age", "dob"}, 303 | ColumnsWithFormula: []string{"name"}}, 304 | } 305 | var out []person 306 | stmt := newGoogleSheetSelectStmt(store, &out, []string{}) 307 | 308 | expected := []person{ 309 | {Name: "name1", Age: 10, DOB: "17-01-2001"}, 310 | {Name: "name2", Age: 11, DOB: "18-01-2000"}, 311 | } 312 | err := stmt.Exec(context.Background()) 313 | 314 | assert.Nil(t, err) 315 | assert.Equal(t, expected, out) 316 | }) 317 | } 318 | 319 | func TestGoogleSheetInsertStmt_convertRowToSlice(t *testing.T) { 320 | wrapper := &sheets.MockWrapper{} 321 | store := &GoogleSheetRowStore{ 322 | wrapper: wrapper, 323 | colsMapping: map[string]common.ColIdx{ 324 | rowIdxCol: {"A", 0}, 325 | "name": {"B", 1}, 326 | "age": {"C", 2}, 327 | "dob": {"D", 3}, 328 | }, 329 | colsWithFormula: common.NewSet([]string{"name"}), 330 | config: GoogleSheetRowStoreConfig{ 331 | Columns: []string{"name", "age", "dob"}, 332 | ColumnsWithFormula: []string{"name"}, 333 | }, 334 | } 335 | 336 | t.Run("non_struct", func(t *testing.T) { 337 | stmt := newGoogleSheetInsertStmt(store, nil) 338 | 339 | result, err := stmt.convertRowToSlice(nil) 340 | assert.Nil(t, result) 341 | assert.NotNil(t, err) 342 | 343 | result, err = stmt.convertRowToSlice(1) 344 | assert.Nil(t, result) 345 | assert.NotNil(t, err) 346 | 347 | result, err = stmt.convertRowToSlice("1") 348 | assert.Nil(t, result) 349 | assert.NotNil(t, err) 350 | 351 | result, err = stmt.convertRowToSlice([]int{1, 2, 3}) 352 | assert.Nil(t, result) 353 | assert.NotNil(t, err) 354 | }) 355 | 356 | t.Run("struct", func(t *testing.T) { 357 | stmt := newGoogleSheetInsertStmt(store, nil) 358 | 359 | result, err := stmt.convertRowToSlice(person{Name: "blah", Age: 10, DOB: "2021"}) 360 | assert.Equal(t, []interface{}{rowIdxFormula, "blah", int64(10), "'2021"}, result) 361 | assert.Nil(t, err) 362 | 363 | result, err = stmt.convertRowToSlice(&person{Name: "blah", Age: 10, DOB: "2021"}) 364 | assert.Equal(t, []interface{}{rowIdxFormula, "blah", int64(10), "'2021"}, result) 365 | assert.Nil(t, err) 366 | 367 | result, err = stmt.convertRowToSlice(person{Name: "blah", DOB: "2021"}) 368 | assert.Equal(t, []interface{}{rowIdxFormula, "blah", nil, "'2021"}, result) 369 | assert.Nil(t, err) 370 | 371 | type dummy struct { 372 | Name string `db:"name"` 373 | } 374 | 375 | result, err = stmt.convertRowToSlice(dummy{Name: "blah"}) 376 | assert.Equal(t, []interface{}{rowIdxFormula, "blah", nil, nil}, result) 377 | assert.Nil(t, err) 378 | }) 379 | 380 | t.Run("ieee754_safe_integers", func(t *testing.T) { 381 | stmt := newGoogleSheetInsertStmt(store, nil) 382 | 383 | result, err := stmt.convertRowToSlice(person{Name: "blah", Age: 9007199254740992, DOB: "2021"}) 384 | assert.Equal(t, []interface{}{rowIdxFormula, "blah", int64(9007199254740992), "'2021"}, result) 385 | assert.Nil(t, err) 386 | 387 | result, err = stmt.convertRowToSlice(person{Name: "blah", Age: 9007199254740993, DOB: "2021"}) 388 | assert.Nil(t, result) 389 | assert.NotNil(t, err) 390 | }) 391 | } 392 | 393 | func TestGoogleSheetUpdateStmt_generateBatchUpdateRequests(t *testing.T) { 394 | wrapper := &sheets.MockWrapper{} 395 | store := &GoogleSheetRowStore{ 396 | wrapper: wrapper, 397 | sheetName: "sheet1", 398 | colsMapping: map[string]common.ColIdx{ 399 | rowIdxCol: {"A", 0}, 400 | "name": {"B", 1}, 401 | "age": {"C", 2}, 402 | "dob": {"D", 3}, 403 | }, 404 | colsWithFormula: common.NewSet([]string{"name"}), 405 | config: GoogleSheetRowStoreConfig{ 406 | Columns: []string{"name", "age", "dob"}, 407 | ColumnsWithFormula: []string{"name"}, 408 | }, 409 | } 410 | 411 | t.Run("successful", func(t *testing.T) { 412 | stmt := newGoogleSheetUpdateStmt( 413 | store, 414 | map[string]interface{}{ 415 | "name": "name1", 416 | "age": int64(100), 417 | "dob": "hello", 418 | }, 419 | ) 420 | 421 | requests, err := stmt.generateBatchUpdateRequests([]int64{1, 2}) 422 | expected := []sheets.BatchUpdateRowsRequest{ 423 | { 424 | A1Range: common.GetA1Range(store.sheetName, "B1"), 425 | Values: [][]interface{}{{"name1"}}, 426 | }, 427 | { 428 | A1Range: common.GetA1Range(store.sheetName, "B2"), 429 | Values: [][]interface{}{{"name1"}}, 430 | }, 431 | { 432 | A1Range: common.GetA1Range(store.sheetName, "C1"), 433 | Values: [][]interface{}{{int64(100)}}, 434 | }, 435 | { 436 | A1Range: common.GetA1Range(store.sheetName, "C2"), 437 | Values: [][]interface{}{{int64(100)}}, 438 | }, 439 | { 440 | A1Range: common.GetA1Range(store.sheetName, "D1"), 441 | Values: [][]interface{}{{"'hello"}}, 442 | }, 443 | { 444 | A1Range: common.GetA1Range(store.sheetName, "D2"), 445 | Values: [][]interface{}{{"'hello"}}, 446 | }, 447 | } 448 | 449 | assert.ElementsMatch(t, expected, requests) 450 | assert.Nil(t, err) 451 | }) 452 | 453 | t.Run("ieee754_safe_integers_successful", func(t *testing.T) { 454 | stmt := newGoogleSheetUpdateStmt(store, map[string]interface{}{ 455 | "name": "name1", 456 | "age": int64(9007199254740992), 457 | }) 458 | 459 | requests, err := stmt.generateBatchUpdateRequests([]int64{1, 2}) 460 | expected := []sheets.BatchUpdateRowsRequest{ 461 | { 462 | A1Range: common.GetA1Range(store.sheetName, "B1"), 463 | Values: [][]interface{}{{"name1"}}, 464 | }, 465 | { 466 | A1Range: common.GetA1Range(store.sheetName, "B2"), 467 | Values: [][]interface{}{{"name1"}}, 468 | }, 469 | { 470 | A1Range: common.GetA1Range(store.sheetName, "C1"), 471 | Values: [][]interface{}{{int64(9007199254740992)}}, 472 | }, 473 | { 474 | A1Range: common.GetA1Range(store.sheetName, "C2"), 475 | Values: [][]interface{}{{int64(9007199254740992)}}, 476 | }, 477 | } 478 | 479 | assert.ElementsMatch(t, expected, requests) 480 | assert.Nil(t, err) 481 | }) 482 | 483 | t.Run("ieee754_safe_integers_unsuccessful", func(t *testing.T) { 484 | stmt := newGoogleSheetUpdateStmt( 485 | store, 486 | map[string]interface{}{ 487 | "name": "name1", 488 | "age": int64(9007199254740993), 489 | }, 490 | ) 491 | 492 | requests, err := stmt.generateBatchUpdateRequests([]int64{1, 2}) 493 | assert.Nil(t, requests) 494 | assert.NotNil(t, err) 495 | }) 496 | } 497 | 498 | func TestEscapeValue(t *testing.T) { 499 | t.Run("not in cols with formula", func(t *testing.T) { 500 | value, err := escapeValue("A", 123, common.NewSet([]string{"B"})) 501 | assert.Nil(t, err) 502 | assert.Equal(t, 123, value) 503 | 504 | value, err = escapeValue("A", "123", common.NewSet([]string{"B"})) 505 | assert.Nil(t, err) 506 | assert.Equal(t, "'123", value) 507 | }) 508 | 509 | t.Run("in cols with formula, but not string", func(t *testing.T) { 510 | value, err := escapeValue("A", 123, common.NewSet([]string{"A"})) 511 | assert.NotNil(t, err) 512 | assert.Equal(t, nil, value) 513 | }) 514 | 515 | t.Run("in cols with formula, but string", func(t *testing.T) { 516 | value, err := escapeValue("A", "123", common.NewSet([]string{"A"})) 517 | assert.Nil(t, err) 518 | assert.Equal(t, "123", value) 519 | }) 520 | } 521 | -------------------------------------------------------------------------------- /internal/google/store/utils.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/FreeLeh/GoFreeDB/internal/google/sheets" 8 | ) 9 | 10 | func ensureSheets(wrapper sheetsWrapper, spreadsheetID string, sheetName string) error { 11 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) 12 | defer cancel() 13 | 14 | return wrapper.CreateSheet(ctx, spreadsheetID, sheetName) 15 | } 16 | 17 | func findScratchpadLocation( 18 | wrapper sheetsWrapper, 19 | spreadsheetID string, 20 | scratchpadSheetName string, 21 | ) (sheets.A1Range, error) { 22 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) 23 | defer cancel() 24 | 25 | result, err := wrapper.OverwriteRows( 26 | ctx, 27 | spreadsheetID, 28 | scratchpadSheetName+"!"+defaultKVTableRange, 29 | [][]interface{}{{scratchpadBooked}}, 30 | ) 31 | if err != nil { 32 | return sheets.A1Range{}, err 33 | } 34 | return result.UpdatedRange, nil 35 | } 36 | -------------------------------------------------------------------------------- /internal/models/kv.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "errors" 4 | 5 | // KVMode defines the mode of the key value store. 6 | // For more details, please read the README file. 7 | type KVMode int 8 | 9 | const ( 10 | KVModeDefault KVMode = 0 11 | KVModeAppendOnly KVMode = 1 12 | 13 | NAValue = "#N/A" 14 | ErrorValue = "#ERROR!" 15 | ) 16 | 17 | // ErrKeyNotFound is returned only for the key-value store and when the key does not exist. 18 | var ( 19 | ErrKeyNotFound = errors.New("error key not found") 20 | ) 21 | -------------------------------------------------------------------------------- /internal/models/row.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | // OrderBy defines the type of column ordering used for GoogleSheetRowStore.Select(). 4 | type OrderBy string 5 | 6 | const ( 7 | OrderByAsc OrderBy = "ASC" 8 | OrderByDesc OrderBy = "DESC" 9 | ) 10 | 11 | // ColumnOrderBy defines what ordering is required for a particular column. 12 | // This is used for GoogleSheetRowStore.Select(). 13 | type ColumnOrderBy struct { 14 | Column string 15 | OrderBy OrderBy 16 | } 17 | -------------------------------------------------------------------------------- /kv.go: -------------------------------------------------------------------------------- 1 | package freedb 2 | 3 | import ( 4 | "github.com/FreeLeh/GoFreeDB/internal/google/store" 5 | "github.com/FreeLeh/GoFreeDB/internal/models" 6 | ) 7 | 8 | type ( 9 | GoogleSheetKVStore = store.GoogleSheetKVStore 10 | GoogleSheetKVStoreConfig = store.GoogleSheetKVStoreConfig 11 | KVMode = models.KVMode 12 | 13 | GoogleSheetKVStoreV2 = store.GoogleSheetKVStoreV2 14 | GoogleSheetKVStoreV2Config = store.GoogleSheetKVStoreV2Config 15 | ) 16 | 17 | var ( 18 | NewGoogleSheetKVStore = store.NewGoogleSheetKVStore 19 | NewGoogleSheetKVStoreV2 = store.NewGoogleSheetKVStoreV2 20 | 21 | KVModeDefault = models.KVModeDefault 22 | KVModeAppendOnly = models.KVModeAppendOnly 23 | ) 24 | -------------------------------------------------------------------------------- /kv_test.go: -------------------------------------------------------------------------------- 1 | package freedb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/FreeLeh/GoFreeDB/google/auth" 8 | ) 9 | 10 | func ExampleGoogleSheetKVStore() { 11 | googleAuth, err := auth.NewServiceFromFile( 12 | "", 13 | GoogleAuthScopes, 14 | auth.ServiceConfig{}, 15 | ) 16 | if err != nil { 17 | panic(err) 18 | } 19 | 20 | store := NewGoogleSheetKVStore( 21 | googleAuth, 22 | "spreadsheet_id", 23 | "sheet_name", 24 | GoogleSheetKVStoreConfig{ 25 | Mode: KVModeDefault, 26 | }, 27 | ) 28 | 29 | val, err := store.Get(context.Background(), "key1") 30 | if err != nil { 31 | panic(err) 32 | } 33 | fmt.Println("get key", val) 34 | 35 | err = store.Set(context.Background(), "key1", []byte("value1")) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | err = store.Delete(context.Background(), "key1") 41 | if err != nil { 42 | panic(err) 43 | } 44 | } 45 | 46 | func ExampleGoogleSheetKVStoreV2() { 47 | googleAuth, err := auth.NewServiceFromFile( 48 | "", 49 | GoogleAuthScopes, 50 | auth.ServiceConfig{}, 51 | ) 52 | if err != nil { 53 | panic(err) 54 | } 55 | 56 | storeV2 := NewGoogleSheetKVStoreV2( 57 | googleAuth, 58 | "spreadsheet_id", 59 | "sheet_name", 60 | GoogleSheetKVStoreV2Config{ 61 | Mode: KVModeDefault, 62 | }, 63 | ) 64 | 65 | val, err := storeV2.Get(context.Background(), "key1") 66 | if err != nil { 67 | panic(err) 68 | } 69 | fmt.Println("get key", val) 70 | 71 | err = storeV2.Set(context.Background(), "key1", []byte("value1")) 72 | if err != nil { 73 | panic(err) 74 | } 75 | 76 | err = storeV2.Delete(context.Background(), "key1") 77 | if err != nil { 78 | panic(err) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /row.go: -------------------------------------------------------------------------------- 1 | package freedb 2 | 3 | import ( 4 | "github.com/FreeLeh/GoFreeDB/internal/google/store" 5 | "github.com/FreeLeh/GoFreeDB/internal/models" 6 | ) 7 | 8 | type ( 9 | GoogleSheetRowStore = store.GoogleSheetRowStore 10 | GoogleSheetRowStoreConfig = store.GoogleSheetRowStoreConfig 11 | 12 | GoogleSheetSelectStmt = store.GoogleSheetSelectStmt 13 | GoogleSheetInsertStmt = store.GoogleSheetInsertStmt 14 | GoogleSheetUpdateStmt = store.GoogleSheetUpdateStmt 15 | GoogleSheetDeleteStmt = store.GoogleSheetDeleteStmt 16 | 17 | ColumnOrderBy = models.ColumnOrderBy 18 | OrderBy = models.OrderBy 19 | ) 20 | 21 | var ( 22 | NewGoogleSheetRowStore = store.NewGoogleSheetRowStore 23 | 24 | OrderByAsc = models.OrderByAsc 25 | OrderByDesc = models.OrderByDesc 26 | ) 27 | -------------------------------------------------------------------------------- /row_test.go: -------------------------------------------------------------------------------- 1 | package freedb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/FreeLeh/GoFreeDB/google/auth" 8 | ) 9 | 10 | func ExampleGoogleSheetRowStore() { 11 | // Initialize authentication 12 | googleAuth, err := auth.NewServiceFromFile( 13 | "", 14 | GoogleAuthScopes, 15 | auth.ServiceConfig{}, 16 | ) 17 | if err != nil { 18 | panic(err) 19 | } 20 | 21 | // Create row store with columns definition 22 | store := NewGoogleSheetRowStore( 23 | googleAuth, 24 | "", 25 | "", 26 | GoogleSheetRowStoreConfig{ 27 | Columns: []string{"name", "age", "email"}, 28 | }, 29 | ) 30 | 31 | // Insert some rows 32 | type Person struct { 33 | Name string `db:"name"` 34 | Age int `db:"age"` 35 | Email string `db:"email"` 36 | } 37 | 38 | err = store.Insert( 39 | Person{Name: "Alice", Age: 30, Email: "alice@example.com"}, 40 | Person{Name: "Bob", Age: 25, Email: "bob@example.com"}, 41 | ).Exec(context.Background()) 42 | if err != nil { 43 | panic(err) 44 | } 45 | 46 | // Query rows 47 | var people []Person 48 | err = store.Select(&people). 49 | Where("age > ?", 20). 50 | OrderBy([]ColumnOrderBy{{Column: "age", OrderBy: OrderByAsc}}). 51 | Limit(10). 52 | Exec(context.Background()) 53 | if err != nil { 54 | panic(err) 55 | } 56 | fmt.Println("Selected people:", people) 57 | 58 | // Update rows 59 | update := map[string]interface{}{"age": 31} 60 | err = store.Update(update).Where("name = ?", "Alice"). 61 | Exec(context.Background()) 62 | if err != nil { 63 | panic(err) 64 | } 65 | 66 | // Count rows 67 | count, err := store.Count(). 68 | Where("age > ?", 20). 69 | Exec(context.Background()) 70 | if err != nil { 71 | panic(err) 72 | } 73 | fmt.Println("Number of people over 20:", count) 74 | 75 | // Delete rows 76 | err = store.Delete(). 77 | Where("name = ?", "Bob"). 78 | Exec(context.Background()) 79 | if err != nil { 80 | panic(err) 81 | } 82 | 83 | // Clean up 84 | err = store.Close(context.Background()) 85 | if err != nil { 86 | panic(err) 87 | } 88 | } 89 | --------------------------------------------------------------------------------