├── nim.cfg ├── src ├── googleapi.nim └── googleapi │ ├── documents.nim │ ├── compute.nim │ ├── drive.nim │ ├── datastore.nim │ ├── sheets.nim │ ├── storage.nim │ ├── connection.nim │ └── bigquery.nim ├── googleapi.nimble ├── .github └── workflows │ ├── build.yml │ └── docs.yml ├── LICENSE └── README.md /nim.cfg: -------------------------------------------------------------------------------- 1 | --d:ssl 2 | --passL:"-lcrypto" -------------------------------------------------------------------------------- /src/googleapi.nim: -------------------------------------------------------------------------------- 1 | import googleapi/[bigquery, compute, connection, datastore, documents, drive, sheets, storage] 2 | export bigquery, compute, connection, datastore, documents, drive, sheets, storage 3 | -------------------------------------------------------------------------------- /googleapi.nimble: -------------------------------------------------------------------------------- 1 | # Package 2 | 3 | version = "0.2.0" 4 | author = "Andre von Houck" 5 | description = "GoogleAPI - Growing collection of google APIs for nim." 6 | license = "MIT" 7 | srcDir = "src" 8 | 9 | # Dependencies 10 | 11 | requires "nim >= 1.0.0" 12 | requires "jwt >= 0.2.0" 13 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Github Actions 2 | on: [push, pull_request] 3 | jobs: 4 | build: 5 | strategy: 6 | fail-fast: false 7 | matrix: 8 | os: [ubuntu-latest, windows-latest] 9 | 10 | runs-on: ${{ matrix.os }} 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: jiro4989/setup-nim-action@v1 15 | with: 16 | repo-token: ${{ secrets.GITHUB_TOKEN }} 17 | - run: nimble test -y 18 | - run: nimble test --gc:orc -y 19 | -------------------------------------------------------------------------------- /src/googleapi/documents.nim: -------------------------------------------------------------------------------- 1 | import asyncdispatch, connection, json, os, streams, strformat 2 | 3 | const docsRoot = "https://docs.googleapis.com/v1" 4 | 5 | proc getDocument*( 6 | conn: Connection, 7 | documentId: string 8 | ): Future[JsonNode] {.async.} = 9 | 10 | return await conn.get( 11 | &"{docsRoot}/documents/{documentId}" 12 | ) 13 | 14 | when isMainModule: 15 | 16 | proc main() {.async.} = 17 | 18 | var conn = waitFor newConnection("your_service_account.json") 19 | 20 | let documentId = "... get this value from the sheet url ..." 21 | var documentJson = await conn.getDocument(documentId) 22 | echo documentJson 23 | 24 | waitFor main() 25 | -------------------------------------------------------------------------------- /src/googleapi/compute.nim: -------------------------------------------------------------------------------- 1 | import asyncdispatch, connection, json, os, streams, strformat 2 | 3 | const cmRoot = "https://www.googleapis.com/compute/v1" 4 | 5 | proc instancesList*( 6 | conn: Connection, projectId, zone: string 7 | ): Future[JsonNode] {.async.} = 8 | return await conn.get(&"{cmRoot}/projects/{projectId}/zones/{zone}/instances") 9 | 10 | when isMainModule: 11 | 12 | proc main() {.async.} = 13 | var conn = await newConnection("your_service_account.json") 14 | var res = await conn.instancesList("your-project", "us-central1-c") 15 | echo pretty res 16 | for instnace in res["items"]: 17 | echo instnace["name"] 18 | echo instnace["networkInterfaces"][0]["networkIP"] 19 | 20 | waitFor main() 21 | -------------------------------------------------------------------------------- /src/googleapi/drive.nim: -------------------------------------------------------------------------------- 1 | 2 | import asyncdispatch, connection, json, os, streams, strformat, strutils, uri 3 | 4 | const driveRoot = "https://www.googleapis.com/drive/v3" 5 | 6 | proc list*(conn: Connection, q: string): Future[JsonNode] {.async.} = 7 | return await conn.get(&"{driveRoot}/files?q={encodeUrl(q)}") 8 | 9 | proc list*( 10 | conn: Connection, q: string, 11 | fields: seq[string] 12 | ): Future[JsonNode] {.async.} = 13 | let fieldsStr = encodeUrl(fields.join(",")) 14 | return await conn.get(&"{driveRoot}/files?q={encodeUrl(q)}&fields={fieldsStr}") 15 | 16 | when isMainModule: 17 | 18 | proc main() {.async.} = 19 | var conn = await newConnection("your_service_account.json") 20 | var res = await conn.list("") 21 | echo pretty res 22 | 23 | waitFor main() 24 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | branches: 5 | - master 6 | env: 7 | nim-version: 'stable' 8 | nim-src: src/${{ github.event.repository.name }}.nim 9 | deploy-dir: .gh-pages 10 | jobs: 11 | docs: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: jiro4989/setup-nim-action@v1 16 | with: 17 | nim-version: ${{ env.nim-version }} 18 | - run: nimble install -Y 19 | - run: nimble doc --index:on --project --git.url:https://github.com/${{ github.repository }} --git.commit:master --out:${{ env.deploy-dir }} ${{ env.nim-src }} 20 | - name: "Copy to index.html" 21 | run: cp ${{ env.deploy-dir }}/${{ github.event.repository.name }}.html ${{ env.deploy-dir }}/index.html 22 | - name: Deploy documents 23 | uses: peaceiris/actions-gh-pages@v3 24 | with: 25 | github_token: ${{ secrets.GITHUB_TOKEN }} 26 | publish_dir: ${{ env.deploy-dir }} 27 | -------------------------------------------------------------------------------- /src/googleapi/datastore.nim: -------------------------------------------------------------------------------- 1 | import asyncdispatch, connection, json, os, streams, strformat 2 | 3 | const dsRoot = "https://datastore.googleapis.com/v1" 4 | 5 | proc runQuery*( 6 | conn: Connection, projectId, queryString: string 7 | ): Future[JsonNode] {.async.} = 8 | let data = %* { 9 | "gqlQuery": { 10 | "allowLiterals": true, 11 | "queryString": queryString 12 | } 13 | } 14 | return await conn.post(&"{dsRoot}/projects/{projectId}:runQuery", data) 15 | 16 | when isMainModule: 17 | 18 | proc main() {.async.} = 19 | var conn = await newConnection("your_service_account.json") 20 | var res = await conn.runQuery( 21 | "your project", 22 | "select 1" 23 | ) 24 | var page = res["batch"]["entityResults"][0] 25 | echo page["entity"]["properties"]["name"]["stringValue"].getStr() 26 | echo page["entity"]["properties"]["security_level"]["integerValue"].getStr() 27 | echo page["entity"]["properties"]["data_json"]["blobValue"].getStr() 28 | 29 | waitFor main() 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2021 Andre von Houck 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /src/googleapi/sheets.nim: -------------------------------------------------------------------------------- 1 | import asyncdispatch, connection, json, os, streams, strformat 2 | 3 | const sheetsRoot = "https://sheets.googleapis.com/v4" 4 | 5 | proc getSpreadsheet*( 6 | conn: Connection, 7 | spreadsheetId: string 8 | ): Future[JsonNode] {.async.} = 9 | 10 | return await conn.get( 11 | &"{sheetsRoot}/spreadsheets/{spreadsheetId}" 12 | ) 13 | 14 | proc getValues*( 15 | conn: Connection, 16 | spreadsheetId, 17 | valueRange: string 18 | ): Future[JsonNode] {.async.} = 19 | 20 | return await conn.get( 21 | &"{sheetsRoot}/spreadsheets/{spreadsheetId}/values/{valueRange}" 22 | ) 23 | 24 | proc setValues*( 25 | conn: Connection, 26 | spreadsheetId, 27 | valueRange: string, 28 | data: JsonNode 29 | ): Future[JsonNode] {.async.} = 30 | 31 | return await conn.put( 32 | &"{sheetsRoot}/spreadsheets/{spreadsheetId}/values/{valueRange}?valueInputOption=USER_ENTERED", 33 | data 34 | ) 35 | 36 | when isMainModule: 37 | 38 | proc main() {.async.} = 39 | 40 | var conn = waitFor newConnection("your_service_account.json") 41 | 42 | let spreadsheetId = "... get this value from the sheet url ..." 43 | var spreadsheetJson = await conn.getSpreadsheet(spreadsheetId) 44 | echo spreadsheetJson 45 | 46 | waitFor main() 47 | -------------------------------------------------------------------------------- /src/googleapi/storage.nim: -------------------------------------------------------------------------------- 1 | import asyncdispatch, connection, httpclient, json, mimetypes, os, ospaths, 2 | streams, strformat, uri 3 | 4 | const storageRoot = "https://www.googleapis.com/storage/v1" 5 | const uploadRoot = "https://www.googleapis.com/upload/storage/v1" 6 | 7 | var m = newMimetypes() 8 | proc extractFileExt(filePath: string): string = 9 | var (_, _, ext) = splitFile(filePath) 10 | return ext 11 | 12 | proc upload*( 13 | conn: Connection, 14 | bucketId: string, 15 | objectId: string, 16 | data: string): 17 | Future[JsonNode] {.async.} = 18 | 19 | let url = &"{uploadRoot}/b/{bucketId}/o?uploadType=media&name={encodeUrl(objectId)}" 20 | 21 | var client = newAsyncHttpClient() 22 | client.headers = newHttpHeaders({ 23 | "Authorization": "Bearer " & await conn.getAuthToken(), 24 | "Content-Length": $data.len, 25 | "Content-Type": m.getMimetype(objectId.extractFileExt()) 26 | }) 27 | let resp = await client.post(url, data) 28 | let resultStr = await resp.bodyStream.readAll() 29 | result = parseJson(resultStr) 30 | client.close() 31 | 32 | proc download*( 33 | conn: Connection, 34 | bucketId: string, 35 | objectId: string): 36 | Future[string] {.async.} = 37 | 38 | let url = &"{storageRoot}/b/{bucketId}/o/{objectId}?alt=media" 39 | var client = newAsyncHttpClient() 40 | client.headers = newHttpHeaders({ 41 | "Authorization": "Bearer " & await conn.getAuthToken(), 42 | }) 43 | let resp = await client.get(url) 44 | result = await resp.bodyStream.readAll() 45 | 46 | proc getMeta*( 47 | conn: Connection, 48 | bucketId: string, 49 | objectId: string): 50 | Future[JsonNode] {.async.} = 51 | 52 | return await conn.get(&"{storageRoot}/b/{bucketId}/o/{objectId}") 53 | 54 | proc list*( 55 | conn: Connection, 56 | bucketId: string, 57 | prefix: string): 58 | Future[JsonNode] {.async.} = 59 | return await conn.get(&"{storageRoot}/b/{bucketId}/o?prefix={encodeUrl(prefix)}") 60 | 61 | when isMainModule: 62 | 63 | proc main() {.async.} = 64 | 65 | var conn = waitFor newConnection("your_service_account.json") 66 | 67 | let data = "this is contents that can be binary too!" 68 | var err = conn.upload("your_bucket", "path/key/to/object.txt", data) 69 | var data2 = await conn.download("your_bucket", "path/key/to/object.txt") 70 | var meta = await conn.getMeta("your_bucket", "path/key/to/object.txt") 71 | 72 | waitFor main() 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Google API - for Nim. 2 | 3 | `nimble install googleapi` 4 | 5 | ![Github Actions](https://github.com/treeform/googleapi/workflows/Github%20Actions/badge.svg) 6 | 7 | [API reference](https://treeform.github.io/googleapi) 8 | 9 | ## About 10 | 11 | This is a growing collection of google APIs for nim. 12 | 13 | So far it has basic support for: 14 | 15 | * BigQuery 16 | * Compute 17 | * Datastore 18 | * Spreadsheets 19 | * Documents 20 | * Drive 21 | * Storage 22 | 23 | Feel free to add new services as they are wrapped. But it might just be easier to use REST Api. 24 | 25 | ## Different jwt backends: 26 | 27 | * default: https://github.com/yglukhov/nim-jwt (takes really long to compile) 28 | * -d:googleApiUseQuickJwt `quickjwt`: https://github.com/treeform/quickjwt (segfaults with wrong libcrypto setup) 29 | * -d:googleApiUseJwtea `jwtea`: https://github.com/guzba/jwtea (pure nim but can take a second to compute) 30 | 31 | ## REST API 32 | 33 | But you can always access Google API through REST. For example find the REST documniation you want like: 34 | 35 | https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery 36 | 37 | There you see a url and the json to send: 38 | 39 | ```POST https://datastore.googleapis.com/v1/projects/{projectId}:runQuery``` 40 | 41 | Just construct your own JSON and use `con.get`, `con.post`, `con.put` or `con.delete`: 42 | 43 | ```nim 44 | const dsRoot = "https://datastore.googleapis.com/v1" 45 | let data = %* { 46 | "gqlQuery": { 47 | "allowLiterals": true, 48 | "queryString": queryString 49 | } 50 | } 51 | return await conn.post(&"{dsRoot}/projects/{projectId}:runQuery", data) 52 | ``` 53 | 54 | No now you can use any google library with this trick! 55 | 56 | ## Service accounts 57 | 58 | There are many ways to access google APIs but the best way is service accounts. In the end you'll end up using service accounts so might as well start now. 59 | 60 | You can create a service account here: https://cloud.google.com/iam/docs/creating-managing-service-accounts 61 | 62 | Then you download the JSON for your service account and load that to create a connection. 63 | 64 | ```nim 65 | var conn = await newConnection("your_service_account.json") 66 | ``` 67 | 68 | Service account is like an email + public key (with with some extra JSON) that you use to access any google service. 69 | 70 | You might run into permission issues with your service account, simply share any google project/service/dataset/page/document/sheet/table ... with the email of the service account, like as if it was a real person. 71 | -------------------------------------------------------------------------------- /src/googleapi/connection.nim: -------------------------------------------------------------------------------- 1 | import asyncdispatch, cgi, httpclient, json, os, streams, strformat, times 2 | 3 | when defined(googleApiUseQuickJwt): 4 | import quickjwt 5 | elif defined(googleApiUseJwtea): 6 | import jwtea 7 | else: 8 | import jwt 9 | 10 | const bqRoot = "https://www.googleapis.com/bigquery/v2" 11 | 12 | type 13 | Connection* = ref object 14 | authToken: string 15 | authTokenExpireTime: float64 16 | email*: string 17 | privateKey: string 18 | scope: string 19 | 20 | GoogleException* = object of Exception 21 | 22 | proc loadServiceAccount( 23 | conn: Connection, 24 | clientEmail: string, 25 | privateKey: string, 26 | ) = 27 | conn.email = clientEmail 28 | conn.privateKey = privateKey 29 | # Define needed scopes 30 | conn.scope = "https://www.googleapis.com/auth/cloud-platform " & 31 | "https://www.googleapis.com/auth/logging.write " & 32 | "https://www.googleapis.com/auth/drive " & 33 | "https://www.googleapis.com/auth/datastore" 34 | 35 | proc loadServiceAccount( 36 | conn: Connection, 37 | serviceAccountPath: string 38 | ) = 39 | let serviceAccount = parseJson(readFile(serviceAccountPath)) 40 | conn.loadServiceAccount( 41 | serviceAccount["client_email"].getStr(), 42 | serviceAccount["private_key"].getStr() 43 | ) 44 | 45 | proc newConnection*(serviceAccountPath: string): Future[Connection] {.async.} = 46 | var conn = Connection() 47 | conn.loadServiceAccount(serviceAccountPath) 48 | return conn 49 | 50 | proc newConnection*(clientEmail, privateKey: string): Future[Connection] {.async.} = 51 | var conn = Connection() 52 | conn.loadServiceAccount(clientEmail, privateKey) 53 | return conn 54 | 55 | proc getAuthToken*(conn: Connection): Future[string] {.async.} = 56 | ## add a buffer of 5 minutes for how long the token is good for 57 | if conn.authTokenExpireTime > epochTime() + (5 * 60): 58 | return conn.authToken 59 | 60 | when defined(googleApiUseQuickJwt): 61 | var token = quickjwt.sign( 62 | header = %*{ 63 | "alg": "RS256", 64 | "typ": "JWT" 65 | }, 66 | claim = %*{ 67 | "iss": conn.email, 68 | "scope": conn.scope, 69 | "aud": "https://www.googleapis.com/oauth2/v4/token", 70 | "exp": int(epochTime() + 60 * 60), 71 | "iat": int(epochTime()) 72 | }, 73 | secret = conn.privateKey 74 | ) 75 | elif defined(googleApiUseJwtea): 76 | var token = signJwt( 77 | header = %*{ 78 | "alg": "RS256", 79 | "typ": "JWT" 80 | }, 81 | claims = %*{ 82 | "iss": conn.email, 83 | "scope": conn.scope, 84 | "aud": "https://www.googleapis.com/oauth2/v4/token", 85 | "exp": int(epochTime() + 60 * 60), 86 | "iat": int(epochTime()) 87 | }, 88 | secret = conn.privateKey 89 | ) 90 | else: 91 | let header = %*{ 92 | "alg": "RS256", 93 | "typ": "JWT" 94 | } 95 | let claims = %*{ 96 | "iss": conn.email, 97 | "scope": conn.scope, 98 | "aud": "https://www.googleapis.com/oauth2/v4/token", 99 | "exp": int(epochTime() + 60 * 60), 100 | "iat": int(epochTime()) 101 | } 102 | var jwtObj = initJWT(header.toHeader, claims.toClaims) 103 | jwtObj.sign(conn.privateKey) 104 | var token = $jwtObj 105 | 106 | let postdata = "grant_type=" & encodeUrl( 107 | "urn:ietf:params:oauth:grant-type:jwt-bearer") & "&assertion=" & token 108 | 109 | proc request(url: string, body: string): string = 110 | var client = newHttpClient() 111 | client.headers = newHttpHeaders({ 112 | "Content-Length": $body.len, 113 | "Content-Type": "application/x-www-form-urlencoded" 114 | }) 115 | result = client.postContent(url, body) 116 | client.close() 117 | 118 | let dataJson = request( 119 | "https://www.googleapis.com/oauth2/v4/token", postdata).parseJson() 120 | 121 | if "access_token" notin dataJson: 122 | raise newException(GoogleException, "Could not get google AuthToken") 123 | 124 | conn.authToken = dataJson["access_token"].str 125 | conn.authTokenExpireTime = float64(epochTime() + 60 * 60) 126 | 127 | return conn.authToken 128 | 129 | proc get*(conn: Connection, url: string): 130 | Future[JsonNode] {.async.} = 131 | ## Generic get request 132 | var client = newAsyncHttpClient() 133 | client.headers = newHttpHeaders({ 134 | "Authorization": "Bearer " & await conn.getAuthToken(), 135 | "Content-Type": "application/json" 136 | }) 137 | let resp = await client.get(url) 138 | let resultStr = await resp.bodyStream.readAll() 139 | result = parseJson(resultStr) 140 | client.close() 141 | 142 | proc post*(conn: Connection, url: string, body: JsonNode): 143 | Future[JsonNode] {.async.} = 144 | ## Generic post request 145 | var client = newAsyncHttpClient() 146 | client.headers = newHttpHeaders({ 147 | "Authorization": "Bearer " & await conn.getAuthToken(), 148 | "Content-Type": "application/json" 149 | }) 150 | let resp = await client.post(url, $body) 151 | let resultStr = await resp.bodyStream.readAll() 152 | result = parseJson(resultStr) 153 | client.close() 154 | 155 | proc patch*(conn: Connection, url: string, body: JsonNode): 156 | Future[JsonNode] {.async.} = 157 | ## Generic patch request 158 | var client = newAsyncHttpClient() 159 | client.headers = newHttpHeaders({ 160 | "Authorization": "Bearer " & await conn.getAuthToken(), 161 | "Content-Type": "application/json" 162 | }) 163 | let resp = await client.request(url, httpMethod = HttpPatch, $body) 164 | let resultStr = await resp.bodyStream.readAll() 165 | result = parseJson(resultStr) 166 | client.close() 167 | 168 | proc put*(conn: Connection, url: string, body: JsonNode): 169 | Future[JsonNode] {.async.} = 170 | ## Generic patch request 171 | var client = newAsyncHttpClient() 172 | client.headers = newHttpHeaders({ 173 | "Authorization": "Bearer " & await conn.getAuthToken(), 174 | "Content-Type": "application/json" 175 | }) 176 | let resp = await client.request(url, httpMethod = HttpPut, $body) 177 | let resultStr = await resp.bodyStream.readAll() 178 | result = parseJson(resultStr) 179 | client.close() 180 | 181 | proc delete*(conn: Connection, url: string): 182 | Future[JsonNode] {.async.} = 183 | ## Generic patch request 184 | var client = newAsyncHttpClient() 185 | client.headers = newHttpHeaders({ 186 | "Authorization": "Bearer " & await conn.getAuthToken(), 187 | "Content-Type": "application/json" 188 | }) 189 | let resp = await client.request(url, httpMethod = HttpDelete) 190 | let resultStr = await resp.bodyStream.readAll() 191 | result = parseJson(resultStr) 192 | client.close() 193 | -------------------------------------------------------------------------------- /src/googleapi/bigquery.nim: -------------------------------------------------------------------------------- 1 | import asyncdispatch, connection, json, os, streams, strformat 2 | 3 | const bqRoot = "https://www.googleapis.com/bigquery/v2" 4 | 5 | type 6 | DatasetReference* = ref object 7 | datasetId*: string 8 | projectId*: string 9 | 10 | Dataset* = ref object 11 | kind*: string 12 | id*: string 13 | datasetReference*: DatasetReference 14 | location*: string 15 | 16 | TableReference* = ref object 17 | datasetId*: string 18 | projectId*: string 19 | tableId*: string 20 | 21 | Table* = ref object 22 | kind*: string 23 | id*: string 24 | tableReference*: TableReference 25 | `type`*: string 26 | creationTime*: string 27 | 28 | proc getDatasets*( 29 | conn: Connection, 30 | projectId: string 31 | ): Future[seq[Dataset]] {.async.} = 32 | let dataJson = await conn.get(&"{bqRoot}/projects/{projectId}/datasets") 33 | return to(dataJson["datasets"], seq[Dataset]) 34 | 35 | proc getTables*( 36 | conn: Connection, 37 | projectId: string, 38 | datasetId: string 39 | ): Future[seq[Table]] {.async.} = 40 | let dataJson = await conn.get(&"{bqRoot}/projects/{projectId}/datasets/{datasetId}/tables?maxResults=10000") 41 | if "tables" notin dataJson: 42 | return 43 | return to(dataJson["tables"], seq[Table]) 44 | 45 | proc getTable*( 46 | conn: Connection, projectId, datasetId, tableId: string 47 | ): Future[JsonNode] {.async.} = 48 | return await conn.get(&"{bqRoot}/projects/{projectId}/datasets/{datasetId}/tables/{tableId}/") 49 | 50 | proc insertQueryJob*( 51 | conn: Connection, 52 | projectId: string, 53 | sqlQuery: string, 54 | cache: bool = true, 55 | useLegacySql: bool = false, 56 | maxResults: int = 10_000_000 57 | ): Future[string] {.async.} = 58 | ## starts a bigquery query job 59 | let body = %*{ 60 | "configuration": { 61 | "query": { 62 | "query": sqlQuery, 63 | "useQueryCache": cache, 64 | "maxResults": maxResults, 65 | "useLegacySql": useLegacySql 66 | } 67 | } 68 | } 69 | var url = &"{bqRoot}/projects/" & projectId & "/jobs" 70 | var jsonData = await conn.post(url, body) 71 | if "error" in jsonData: 72 | raise newException(Exception, $jsonData["error"]) 73 | return jsonData["jobReference"]["jobId"].str 74 | 75 | proc insertQueryJobIntoTable*( 76 | conn: Connection, 77 | projectId: string, 78 | sqlQuery: string, 79 | toProjectId: string, 80 | toDatasetId: string, 81 | toTableId: string, 82 | overwrite = false 83 | ): 84 | Future[string] {.async.} = 85 | ## starts a bigquery query job to put results into a table 86 | let body = %*{ 87 | "configuration": { 88 | "query": { 89 | "query": sqlQuery, 90 | "createDisposition": "CREATE_IF_NEEDED", 91 | "writeDisposition": "WRITE_TRUNCATE", 92 | "destinationTable": { 93 | "projectId": toProjectId, 94 | "datasetId": toDatasetId, 95 | "tableId": toTableId 96 | } 97 | } 98 | } 99 | } 100 | var url = &"{bqRoot}/projects/" & projectId & "/jobs" 101 | var jsonData = await conn.post(url, body) 102 | if "error" in jsonData: 103 | raise newException(Exception, $jsonData["error"]) 104 | return jsonData["jobReference"]["jobId"].str 105 | 106 | proc pollQueryJob*( 107 | conn: Connection, projectId: string, 108 | jobId: string, 109 | maxResults: int = 10000000, 110 | pageToken: string = "" 111 | ): Future[JsonNode] {.async.} = 112 | ## ask google results, if they are not done you will get jobComplete = false 113 | var url = &"{bqRoot}/projects/{projectId}/queries/{jobId}?maxResults={maxResults}" 114 | if pageToken != "": 115 | url &= "&pageToken=" & pageToken 116 | return await conn.get(url) 117 | 118 | proc cancelQueryJob*(conn: Connection, projectId: string, 119 | jobId: string): Future[JsonNode] {.async.} = 120 | ## ask google results, if they are not done you will get jobComplete = false 121 | var url = &"{bqRoot}/projects/" & projectId & "/jobs/" & jobId & "/cancel" 122 | return await conn.post(url, %*{}) 123 | 124 | proc getJob*( 125 | conn: Connection, projectId, jobId: string 126 | ): Future[JsonNode] {.async.} = 127 | ## Gets the job information 128 | return await conn.get(&"{bqRoot}/projects/{projectId}/jobs/{jobId}") 129 | 130 | proc tableInsertAll*(conn: Connection, projectId, datasetId, tableId: string, 131 | rows: seq[JsonNode]): Future[JsonNode] {.async.} = 132 | ## insert data into bigquery table 133 | assert rows.len > 0 134 | var newRows: seq[JsonNode] 135 | for row in rows: 136 | newRows.add %*{ 137 | "json": row 138 | } 139 | let body = %*{ 140 | "kind": "bigquery#tableDataInsertAllRequest", 141 | "skipInvalidRows": true, 142 | "ignoreUnknownValues": false, 143 | "rows": newRows 144 | } 145 | var url = &"{bqRoot}/projects/{projectId}/datasets/{datasetId}/tables/{tableId}/insertAll" 146 | var jsonResp = await conn.post(url, body) 147 | return jsonResp 148 | 149 | proc tableInsert*( 150 | conn: Connection, 151 | projectId, datasetId: string, 152 | table: JsonNode 153 | ): Future[JsonNode] {.async.} = 154 | ## Creates a new bigquery table 155 | var url = &"{bqRoot}/projects/{projectId}/datasets/{datasetId}/tables" 156 | var jsonResp = await conn.post(url, table) 157 | return jsonResp 158 | 159 | proc tablePatch*( 160 | conn: Connection, 161 | projectId, datasetId, tableId: string, 162 | table: JsonNode 163 | ): Future[JsonNode] {.async.} = 164 | ## Alters table keepting the data, if table can't be updated safely returns error. 165 | var url = &"{bqRoot}/projects/{projectId}/datasets/{datasetId}/tables/{tableId}" 166 | var jsonResp = await conn.patch(url, table) 167 | return jsonResp 168 | 169 | proc tableDelete*( 170 | conn: Connection, 171 | projectId, datasetId, tableId: string 172 | ): Future[JsonNode] {.async.} = 173 | ## Overwrites a table potentially deleting all data. 174 | var url = &"{bqRoot}/projects/{projectId}/datasets/{datasetId}/tables/{tableId}" 175 | var jsonResp = await conn.delete(url) 176 | return jsonResp 177 | 178 | proc tableUpdate*( 179 | conn: Connection, 180 | projectId, datasetId, tableId: string, 181 | table: JsonNode 182 | ): Future[JsonNode] {.async.} = 183 | ## Overwrites a table potentially deleting all data. 184 | var url = &"{bqRoot}/projects/{projectId}/datasets/{datasetId}/tables/{tableId}" 185 | var jsonResp = await conn.put(url, table) 186 | return jsonResp 187 | 188 | when isMainModule: 189 | import print 190 | proc main() {.async.} = 191 | var conn = await newConnection("your_service_account.json") 192 | 193 | block: 194 | for ds in await conn.getDatasets("your-project"): 195 | print ds.datasetReference.datasetId, ds.location 196 | for tb in await conn.getTables("your-project", ds.datasetReference.datasetId): 197 | print " ", tb.tableReference.tableId 198 | 199 | block: 200 | let jobId = await conn.insertQueryJob("your-project", "select 1") 201 | print jobId 202 | while true: 203 | let resultsJson = await conn.pollQueryJob("your-project", jobId) 204 | echo pretty resultsJson 205 | if resultsJson["jobComplete"].getBool == true: 206 | break 207 | 208 | block: 209 | let jobId = await conn.insertQueryJob("your-project", "select 1") 210 | print jobId 211 | let resultsJson = await conn.cancelQueryJob("your-project", jobId) 212 | echo pretty resultsJson 213 | 214 | waitFor main() 215 | --------------------------------------------------------------------------------