├── .github └── FUNDING.yml ├── LICENSE ├── README.md └── cloud-datastore └── CloudDatastore.gs /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: ["syslogic"] 2 | custom: ["https://www.paypal.me/syslogic"] 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Martin Zeitler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CloudDatastore.gs 2 | CloudDatastore.gs is a client for Google Cloud Datastore, which runs as a Service Account. 3 | 4 | It is written in Google Apps Script - and so far, it can select, insert, update and delete entities. 5 | 6 | ## Setup 7 | 8 | a) Add the script to https://script.google.com 9 | 10 | b) Add the `OAuth2 for Apps Script` library: `1B7FSrk5Zi6L1rSxxTDgDEUsPzlukDsi4KGuTMorsTQHhGBzBkMun4iDF`. 11 | 12 | c) Upload the service account configuration file to your Google Drive (to be downloaded from https://console.cloud.google.com/project/_/iam-admin and then uploaded to https://drive.google.com). 13 | 14 | The service account needs to have the "Cloud Datastore User" role asigned. 15 | 16 | d) Adjust the script according to the filename of the configuration file. 17 | 18 | ## Reference 19 | 20 | @see https://github.com/googlesamples/apps-script-oauth2 21 | 22 | @see https://stackoverflow.com/questions/49112189/49113976#49113976 23 | 24 | @see https://cloud.google.com/datastore/docs/reference/data/rest/ 25 | -------------------------------------------------------------------------------- /cloud-datastore/CloudDatastore.gs: -------------------------------------------------------------------------------- 1 | /* 2 | Apps Script: Accessing Google Cloud Datastore under a Service Account. 3 | @author Martin Zeitler 4 | */ 5 | 6 | /* 7 | a Service Account configuration file on Google Drive 8 | (the "Cloud Datastore User" role must be assigned). 9 | */ 10 | var CONFIG = "serviceaccount.json"; 11 | 12 | /* Kind, ID and Name of an Entity, which is used by below functional tests */ 13 | var test = {KIND: "strings", ID: "5558520099373056", NAME: "2ja7h"}; 14 | 15 | /* API wrapper */ 16 | var datastore = { 17 | 18 | /* verbose logging */ 19 | debug: true, 20 | 21 | /* api related */ 22 | scopes: "https://www.googleapis.com/auth/datastore https://www.googleapis.com/auth/drive", 23 | baseUrl: "https://datastore.googleapis.com/v1", 24 | httpMethod: "POST", 25 | currentUrl: false, 26 | 27 | /* authentication */ 28 | oauth: false, 29 | projectId: false, 30 | clientId: false, 31 | clientEmail: false, 32 | privateKey: false, 33 | 34 | /* transactions */ 35 | transactionId: false, 36 | 37 | /* pagination */ 38 | startCursor: false, 39 | queryInProgress: false, 40 | currentPage: 1, 41 | totalPages: 1, 42 | perPage: 5, 43 | 44 | /* returns an instance */ 45 | getInstance: function() { 46 | 47 | /* configure the client on demand */ 48 | if(! this.config) {this.getConfig(CONFIG);} 49 | 50 | /* authenticate the client on demand */ 51 | if(! this.oauth) {this.createService();} 52 | 53 | return this; 54 | }, 55 | 56 | /* loads the configuration file from Google Drive */ 57 | getConfig: function(filename) { 58 | var it = DriveApp.getFilesByName(filename); 59 | while (it.hasNext()) { 60 | var file = it.next(); 61 | var data = JSON.parse(file.getAs("application/json").getDataAsString()); 62 | this.projectId = data.project_id; 63 | this.privateKey = data.private_key; 64 | this.clientEmail = data.client_email; 65 | this.clientId = data.client_id; 66 | continue; 67 | } 68 | }, 69 | 70 | /* creates the oAuth2 service */ 71 | createService: function() { 72 | this.oauth = OAuth2.createService("Datastore") 73 | .setTokenUrl("https://www.googleapis.com/oauth2/v4/token") 74 | .setPropertyStore(PropertiesService.getScriptProperties()) 75 | // .setSubject(Session.getActiveUser().getEmail()) 76 | .setPrivateKey(this.privateKey) 77 | .setIssuer(this.clientEmail) 78 | .setScope(this.scopes); 79 | }, 80 | 81 | /* sets the request url per method */ 82 | setCurrentUrl: function(method){ 83 | this.currentUrl = this.baseUrl + "/projects/" + this.projectId + ":" + method; 84 | }, 85 | 86 | /* gets the request options */ 87 | getOptions: function(payload) { 88 | if (this.oauth.hasAccess()) { 89 | return { 90 | headers: {Authorization: 'Bearer ' + this.oauth.getAccessToken()}, 91 | payload: JSON.stringify(payload), 92 | contentType: "application/json", 93 | muteHttpExceptions: true, 94 | method: this.httpMethod 95 | }; 96 | } 97 | }, 98 | 99 | /** 100 | * Queries for entities. 101 | * @param payload ~ partitionId, readOptions, query, gqlQuery 102 | **/ 103 | runQuery: function(payload) {return this.request("runQuery", payload);}, 104 | 105 | /** 106 | * Begins a new transaction. 107 | * @param payload ~ transactionOptions 108 | **/ 109 | beginTransaction: function(payload) {return this.request("beginTransaction", payload);}, 110 | 111 | /** 112 | * Commits a transaction, optionally creating, deleting or modifying some entities. 113 | * @param payload ~ mode, mutations, transaction 114 | **/ 115 | commit: function(payload) {return this.request("commit", payload);}, 116 | 117 | /** 118 | * Rolls back a transaction. 119 | * @param payload ~ transaction 120 | **/ 121 | rollback: function(payload) {return this.request("rollback", payload);}, 122 | 123 | /** 124 | * Allocates IDs for the given keys, which is useful for referencing an entity before it is inserted. 125 | * @param payload ~ keys 126 | **/ 127 | allocateIds: function(payload) {return this.request("allocateIds", payload);}, 128 | 129 | /** 130 | * Prevents the supplied keys' IDs from being auto-allocated by Cloud Datastore. 131 | * @param payload ~ databaseId, keys 132 | **/ 133 | reserveIds: function(payload) {return this.request("reserveIds", payload);}, 134 | 135 | /** 136 | * Looks up entities by key. 137 | * @param payload ~ readOptions, keys 138 | **/ 139 | lookup: function(payload) {return this.request("lookup", payload);}, 140 | 141 | /* API wrapper */ 142 | request: function(method, payload) { 143 | 144 | /* wait for the previous request to complete, check every 200 ms */ 145 | while(this.queryInProgress) {Utilities.sleep(200);} 146 | 147 | if (this.oauth.hasAccess()) { 148 | 149 | /* set queryInProgress flag to true */ 150 | this.queryInProgress = true; 151 | 152 | /* configure the request */ 153 | var options = this.getOptions(payload); 154 | this.log(method + " > " + options.payload); 155 | this.setCurrentUrl(method); 156 | 157 | /* the individual api methods can be handled here */ 158 | switch(method) { 159 | 160 | /* projects.runQuery */ 161 | case "runQuery": 162 | break; 163 | 164 | /* projects.beginTransaction */ 165 | case "beginTransaction": 166 | break; 167 | 168 | /* projects.commit */ 169 | case "commit": 170 | if(! this.transactionId){ 171 | this.log("cannot commit() while there is no ongoing transaction."); 172 | return false; 173 | } else { 174 | payload.transaction = this.transactionId; 175 | } 176 | break; 177 | 178 | /* projects.rollback */ 179 | case "rollback": 180 | if(! this.transactionId){ 181 | this.log("cannot rollback() while there is no ongoing transaction."); 182 | return false; 183 | } else { 184 | payload.transaction = this.transactionId; 185 | } 186 | break; 187 | 188 | /* projects.allocateIds */ 189 | case "allocateIds": 190 | break; 191 | 192 | /* projects.reserveIds */ 193 | case "reserveIds": 194 | break; 195 | 196 | /* projects.lookup */ 197 | case "lookup": 198 | break; 199 | 200 | default: 201 | this.log("invalid api method: "+ method); 202 | return false; 203 | } 204 | 205 | /* execute the request */ 206 | var response = UrlFetchApp.fetch(this.currentUrl, options); 207 | var result = JSON.parse(response.getContentText()); 208 | this.handleResult(method, result); 209 | 210 | /* it returns the actual result of the request */ 211 | return result; 212 | 213 | } else { 214 | this.log(this.oauth.getLastError()); 215 | return false; 216 | } 217 | }, 218 | 219 | /* handles the result */ 220 | handleResult: function(method, result) { 221 | 222 | /* the individual api responses can be handled here */ 223 | switch(method) { 224 | 225 | /* projects.runQuery */ 226 | case "runQuery": 227 | 228 | /* result.query */ 229 | if(typeof(result.query) !== "undefined") { 230 | for(i=0; i < result.query.length; i++) { 231 | this.log(JSON.stringify(result.query[i])); 232 | } 233 | } 234 | 235 | /* result.batch */ 236 | if(typeof(result.batch) !== "undefined") { 237 | 238 | /* log the entityResults */ 239 | for(i=0; i < result.batch["entityResults"].length; i++) { 240 | this.log(JSON.stringify(result.batch['entityResults'][i])); 241 | } 242 | 243 | /* set the endCursor as the next one startCursor */ 244 | if(typeof(result.batch["moreResults"]) !== "undefined") { 245 | 246 | switch(result.batch["moreResults"]) { 247 | 248 | /* There may be additional batches to fetch from this query. */ 249 | case "NOT_FINISHED": 250 | 251 | /* The query is finished, but there may be more results after the limit. */ 252 | case "MORE_RESULTS_AFTER_LIMIT": 253 | 254 | /* The query is finished, but there may be more results after the end cursor. */ 255 | case "MORE_RESULTS_AFTER_CURSOR": 256 | this.startCursor = result.batch["endCursor"]; 257 | break; 258 | 259 | /* The query is finished, and there are no more results. */ 260 | case "NO_MORE_RESULTS": 261 | this.startCursor = false; 262 | break; 263 | } 264 | 265 | } else { 266 | this.startCursor = false; 267 | } 268 | } 269 | break; 270 | 271 | /* projects.beginTransaction */ 272 | case "beginTransaction": 273 | if(typeof(result.transaction) !== "undefined" && result.transaction != "") { 274 | this.log(method + " > " + result.transaction); 275 | this.transactionId = result.transaction; 276 | } 277 | break; 278 | 279 | /* projects.commit */ 280 | case "commit": 281 | if(typeof(result.error) !== "undefined") { 282 | 283 | /* resetting the transaction in progress */ 284 | this.transactionId = false; 285 | 286 | } else { 287 | 288 | /* log the mutationResults */ 289 | if(typeof(result.mutationResults) !== "undefined") { 290 | for(i=0; i < result.mutationResults.length; i++) { 291 | this.log(JSON.stringify(result.mutationResults[i])); 292 | } 293 | } 294 | } 295 | break; 296 | 297 | /* projects.rollback */ 298 | case "rollback": 299 | /* resetting the transaction in progress */ 300 | this.transactionId = false; 301 | break; 302 | 303 | /* projects.allocateIds */ 304 | case "allocateIds": 305 | 306 | /* log allocated keys */ 307 | if(typeof(result.keys) !== "undefined") { 308 | for(i=0; i < result.keys.length; i++) { 309 | this.log(JSON.stringify(result.keys[i])); 310 | } 311 | } 312 | break; 313 | 314 | /* projects.reserveIds (the response is empty by default) */ 315 | case "reserveIds": 316 | break; 317 | 318 | /* projects.lookup */ 319 | case "lookup": 320 | 321 | /* log found entities */ 322 | if(typeof(result.found) !== "undefined") { 323 | for(i=0; i < result.found.length; i++) { 324 | this.log(JSON.stringify(result.found[i])); 325 | } 326 | } 327 | 328 | /* log missing entities */ 329 | if(typeof(result.missing) !== "undefined") { 330 | for(i=0; i < result.missing.length; i++) { 331 | this.log(JSON.stringify(result.missing[i])); 332 | } 333 | } 334 | 335 | /* log deferred entities */ 336 | if(typeof(result.deferred) !== "undefined") { 337 | for(i=0; i < result.deferred.length; i++) { 338 | this.log(JSON.stringify(result.deferred[i])); 339 | } 340 | } 341 | break; 342 | } 343 | 344 | /* set queryInProgress flag to false */ 345 | this.queryInProgress = false; 346 | 347 | /* always log remote errors */ 348 | if(typeof(result.error) !== "undefined") { 349 | Logger.log(method + " > ERROR " + result.error.code + ": " + result.error.message); 350 | return false; 351 | } 352 | }, 353 | 354 | randomString: function() { 355 | var str = ""; 356 | while (str === "") { 357 | str = Math.random().toString(36).substr(2, 5); 358 | } 359 | return str; 360 | }, 361 | 362 | /* logs while this.debug is true */ 363 | log: function(message){ 364 | if(this.debug) {Logger.log(message);} 365 | }, 366 | 367 | /* resets the authorization state */ 368 | resetAuth: function() { 369 | this.oauth.reset(); 370 | }, 371 | 372 | /** 373 | * runs a simple GQL query 374 | * @see https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#GqlQuery 375 | **/ 376 | runGql: function(query_string) { 377 | if(! this.startCursor) { 378 | var options = { 379 | gqlQuery: { 380 | query_string: query_string, 381 | allowLiterals: true 382 | } 383 | }; 384 | } else { 385 | var options = { 386 | gqlQuery: { 387 | query_string: query_string, 388 | allowLiterals: true, 389 | namedBindings: { 390 | startCursor: {cursor: this.startCursor} 391 | } 392 | } 393 | }; 394 | } 395 | this.log(JSON.stringify(options)); 396 | return this.runQuery(options); 397 | }, 398 | 399 | /* queries for entities by the name of their kind */ 400 | queryByKind: function(value) { 401 | return this.runQuery({query: {kind:[{name: value}]}}); 402 | }, 403 | 404 | lookupById: function(value) { 405 | return this.lookup({ 406 | "keys": [{ 407 | "partitionId": {"projectId": this.projectId}, 408 | "path": [{"kind": "strings", "id": value}] 409 | }] 410 | }); 411 | }, 412 | 413 | lookupByName: function(value) { 414 | return this.lookup({ 415 | "keys": [{ 416 | "partitionId": {"projectId": this.projectId}, 417 | "path": [{"kind": "strings", "name": value}] 418 | }] 419 | }); 420 | }, 421 | 422 | /* deletes an entity by the name of it's kind and it's id */ 423 | deleteByKindAndId: function(value, id) { 424 | this.beginTransaction({}); 425 | return this.commit({ 426 | "transaction": this.transactionId, 427 | "mutations": { 428 | "delete": { 429 | "partitionId": {"projectId": this.projectId}, 430 | "path": [{"kind": value, "id": id}] 431 | } 432 | } 433 | }); 434 | } 435 | }; 436 | 437 | 438 | /* Test: looks up entities of kind `strings` with id TEST_ID */ 439 | function lookupById() { 440 | var ds = datastore.getInstance(); 441 | var result = ds.lookupById(test.ID); 442 | } 443 | 444 | /* Test: looks up entities of kind `strings` with name TEST_NAME */ 445 | function lookupByName() { 446 | var ds = datastore.getInstance(); 447 | var result = ds.lookupByName(test.NAME); 448 | } 449 | 450 | /* Test: queries for entities of kind `strings` */ 451 | function queryByKind() { 452 | var ds = datastore.getInstance(); 453 | var result = ds.queryByKind(test.KIND); 454 | } 455 | 456 | /* Test: run a GQL query */ 457 | function queryByKindPaged() { 458 | var ds = datastore.getInstance(); 459 | var i=0; 460 | 461 | /* run the first query, yielding a startCursor */ 462 | var result = ds.runGql("SELECT * FROM " + test.KIND + " ORDER BY name ASC LIMIT " + ds.perPage + " OFFSET 0"); 463 | 464 | /* when the startCursor is false,the last page had been reached */ 465 | while(ds.startCursor && i < 3){ 466 | var result = ds.runGql("SELECT * FROM " + test.KIND + " ORDER BY name ASC LIMIT " + ds.perPage + " OFFSET @startCursor"); 467 | i++; 468 | } 469 | } 470 | 471 | /* Test: deletes an entity of kind `strings` with id */ 472 | function deleteByKindAndId() { 473 | var ds = datastore.getInstance(); 474 | ds.deleteByKindAndId("strings", test.ID); 475 | } 476 | 477 | /* Test: inserts an entity */ 478 | function insertEntity() { 479 | 480 | /* it inserts an entity of kind `strings` with a random string as property `name` */ 481 | var ds = datastore.getInstance(); 482 | ds.beginTransaction({}); 483 | ds.commit({ 484 | "transaction": ds.transactionId, 485 | "mutations": { 486 | "insert": { 487 | "key": { 488 | "partitionId": {"projectId": ds.projectId}, 489 | "path": [{"kind": test.KIND}] 490 | }, 491 | "properties":{ 492 | "name": {"stringValue": ds.randomString()} 493 | } 494 | } 495 | } 496 | }); 497 | } 498 | 499 | /* Test: updates an entity */ 500 | function updateEntity() { 501 | 502 | /* it selects of an entity of kind `strings` by it's id and updates it's property `name` with a random string */ 503 | var ds = datastore.getInstance(); 504 | ds.beginTransaction({}); 505 | ds.commit({ 506 | "transaction": ds.transactionId, 507 | "mutations": { 508 | "update": { 509 | "key": { 510 | "partitionId": {"projectId": ds.projectId}, 511 | "path": [{"kind": test.KIND, "id": test.ID}] 512 | }, 513 | "properties":{ 514 | "name": {"stringValue": ds.randomString()} 515 | 516 | } 517 | } 518 | } 519 | }); 520 | } 521 | 522 | /* Test: upserts an entity */ 523 | function upsertEntity() { 524 | 525 | /* it selects of an entity of kind `strings` by it's id and updates it's property `name` with a random string */ 526 | var ds = datastore.getInstance(); 527 | ds.beginTransaction({}); 528 | ds.commit({ 529 | "transaction": ds.transactionId, 530 | "mutations": { 531 | "upsert": { 532 | "key": { 533 | "partitionId": {"projectId": ds.projectId}, 534 | "path": [{"kind": test.KIND, "id": test.ID}] 535 | }, 536 | "properties":{ 537 | "name": {"stringValue": ds.randomString()} 538 | 539 | } 540 | } 541 | } 542 | }); 543 | } 544 | 545 | /* Test: allocates ids for entities of kind `strings` */ 546 | function allocateIds() { 547 | var ds = datastore.getInstance(); 548 | var result = ds.allocateIds({ 549 | "keys": [{ 550 | "partitionId": {"projectId": ds.projectId}, 551 | "path": [{"kind": test.KIND}] 552 | }] 553 | }); 554 | } 555 | 556 | /* Test: reserves ids for entities of kind `strings` */ 557 | function reserveIds() { 558 | var ds = datastore.getInstance(); 559 | ds.reserveIds({ 560 | "keys": [{ 561 | "partitionId": {"projectId": ds.projectId}, 562 | "path": [{"kind": test.KIND, "id": "6750768661004291"}] 563 | }, { 564 | "partitionId": {"projectId": ds.projectId}, 565 | "path": [{"kind": test.KIND, "id": "6750768661004292"}] 566 | }] 567 | }); 568 | } 569 | --------------------------------------------------------------------------------