├── .dialyzer.ignore-warnings ├── .gitignore ├── .iex.exs ├── .tool-versions ├── .travis.yml ├── CHANGELOG.md ├── README.md ├── config ├── config.exs ├── dev.exs ├── test-credentials.json └── test.exs ├── lib ├── datastore_v1.proto ├── datastore_v1beta3.proto ├── diplomat.ex └── diplomat │ ├── client.ex │ ├── cursor.ex │ ├── entity.ex │ ├── key.ex │ ├── key_utils.ex │ ├── query.ex │ ├── query_result_batch.ex │ ├── transaction.ex │ └── value.ex ├── mix.exs ├── mix.lock └── test ├── diplomat ├── client_test.exs ├── cursor_test.exs ├── entity │ ├── insert_test.exs │ └── upsert_test.exs ├── entity_test.exs ├── key │ └── allocate_ids_test.exs ├── key_test.exs ├── key_utils_test.exs ├── query_result_batch_test.exs ├── query_test.exs ├── transaction_test.exs └── value_test.exs ├── diplomat_test.exs ├── support └── test_struct.ex └── test_helper.exs /.dialyzer.ignore-warnings: -------------------------------------------------------------------------------- 1 | lib/exprotobuf 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /cover 3 | /deps 4 | erl_crash.dump 5 | *.ez 6 | config/*credentials.json 7 | !config/test-credentials.json 8 | /log 9 | .#* 10 | /doc/ 11 | .elixir_ls/ 12 | -------------------------------------------------------------------------------- /.iex.exs: -------------------------------------------------------------------------------- 1 | try do 2 | import_file "~/.iex.exs" 3 | rescue 4 | _ -> :ok 5 | end 6 | 7 | alias Diplomat.{Key, Entity, Property, Value, PropertyList, Client, Query} 8 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | elixir 1.10.3 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | elixir: 3 | - 1.5 4 | - 1.6 5 | - 1.7 6 | - 1.8 7 | otp_release: 8 | - 17.5 9 | - 18.2 10 | - 19.0 11 | - 20.0 12 | - 21.0 13 | matrix: 14 | exclude: 15 | - elixir: 1.5 16 | otp_release: 21.0 17 | - elixir: 1.6 18 | otp_release: 17.5 19 | - elixir: 1.6 20 | otp_release: 18.2 21 | - elixir: 1.7 22 | otp_release: 17.5 23 | - elixir: 1.7 24 | otp_release: 18.2 25 | - elixir: 1.8 26 | otp_release: 17.5 27 | - elixir: 1.8 28 | otp_release: 18.2 29 | - elixir: 1.8 30 | otp_release: 19.0 31 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### v0.1.0 2 | * **switch to the v1beta3 API** This includes a brand new protocol buffer file and an update to the underlying protocol buffer syntax version 3 | * support lat/long geo point values 4 | * refactor property and value implementations 5 | * update to use the new DateTime structs in Elixir 1.3 6 | 7 | #### Backwards Incompatible Changes 8 | * DateTime property values must now be `%DateTime{}` structs. Previously, timestamps were passed in via a three-element tuple, which will now cause Diplomat to throw an exception when it tries to encode the value. 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/peburrows/diplomat.svg?branch=master)](https://travis-ci.org/peburrows/diplomat) 2 | 3 | # Diplomat 4 | 5 | Diplomat is an Elixir library for interacting with Google's Cloud Datastore. 6 | 7 | ## Installation 8 | 9 | 1. Add datastore to your list of dependencies in `mix.exs`: 10 | 11 | ```elixir 12 | def deps do 13 | [{:diplomat, "~> 0.2"}] 14 | end 15 | ``` 16 | 17 | 2. Make sure you've configured [Goth](https://github.com/peburrows/goth) with your credentials: 18 | 19 | ```elixir 20 | config :goth, 21 | json: {:system, "GCP_CREDENTIALS_JSON"} 22 | ``` 23 | 24 | ## Usage 25 | 26 | #### Insert an Entity: 27 | 28 | ```elixir 29 | Diplomat.Entity.new( 30 | %{"name" => "My awesome book", "author" => "Phil Burrows"}, 31 | "Book", 32 | "my-unique-book-id" 33 | ) |> Diplomat.Entity.insert 34 | ``` 35 | 36 | #### Find an Entity via a GQL Query: 37 | 38 | ```elixir 39 | Diplomat.Query.new( 40 | "select * from `Book` where name = @name", 41 | %{name: "20,000 Leagues Under The Sea"} 42 | ) |> Diplomat.Query.execute 43 | ``` 44 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # 3rd-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure for your application as: 12 | # 13 | # config :diplomat, key: :value 14 | # 15 | # And access this configuration in your application as: 16 | # 17 | # Application.get_env(:diplomat, :key) 18 | # 19 | # Or configure a 3rd-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | 31 | import_config "#{Mix.env}.exs" 32 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | try do 4 | config :goth, 5 | json: "config/credentials.json" |> Path.expand |> File.read! 6 | rescue 7 | _ -> 8 | config :goth, json: "{}" 9 | end 10 | -------------------------------------------------------------------------------- /config/test-credentials.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "tokyo-amphora-437", 4 | "private_key_id": "854414a51270519ed74ec9112389e495eec1ccd1", 5 | "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCN1IUO2QhgdI+Q\nUTIWV9X7Gm9Cw6kPgHeZ+f+RXXknSL6/CAwVz1EG2VIgCBJv45mF0Vsw2vM8/0Sx\nSDoEb7skelCahYCQUOIp72egAk3InOINGo/n+A1ai7fmR0EzQ6WFr3pZmzcX/7ZB\n0TjDkXX81NJIreJSfqSCvMg7uZfzihv7RbmljyofxoXwP8FoMOS5BcHo8ZkBZyJx\ny0uLKrFvCkTRS/OhuGRVcJC6VrZUA2MhQPkqjHNcttEXIajL+jl4jmVQ8irwR4LO\nXnFaBoEXexciTAteO4vjrrV2iIh0x24vgD2SemhUW/pOTZ/AMNUjwjnvmWFdvvyf\nFYDTtHL1AgMBAAECggEBAILyYBchUpabh6EbFj+CwVGhSnA97e0eE07afpdb0evv\nQg1mBKJuUsUcCLMCQOOFI81lSeiFfmYm2OlFYiuObR50v86qy9RymR1WqDoXZnF+\nR0cJ6yuk3c9niFbYGt6V6lDPfwsUP32s3j1OSjZmKqVQaQYpZPf9bS431jcuV5jF\n0tJEFZTY+FS3BW3JefpDCBW1SmyXtA4BiZdP37I9hKOohC7iQuOna5g9iaCbFKC4\nw80FZngDB4MTpSypjYBOR4SROOcIMd3cXyDJEuYoJqKpc3Ke9QZrHPSZPKREug7u\nG7v5TwFXwn2lLtlV7KXAknl2CUNGHEzDOyMRP0PVZtECgYEA9ywGoPFP2ejMVJ+e\nvsSo5x5mXL0hczYyUT1ryohi1+4Rq4S4StfvLKZ9hKp8xzOOwChV0QNT6ZdLTOQo\nSQWQ0tqZfzIVOBFxeqRPokaojQ0MEvXcDTnUzCSh7q+GvNFkN5kMcOCaPYGsCWzU\np/BYjyijX/SB3Y/vWCIlRWQpQx8CgYEAkuVRtn5nOwlWcykRE/PYZo9HNPcjdW2Q\nAIki1ntfHZTikLRf3cRpWYgWqbYJMiTq4Mkwhye0jgKRVs8urHJwVxYTXyfPl6UM\n17DaCwnX2VuMDEM9cbBxF4MdbIBuQ7YJUmajrh63E/hx2NJso6/nvYk5V4v5v5lZ\nwTKB+7+a+2sCgYEAoWeSfI6YAkhPBgOl+hUZ5rKnTXAD4+REP2DIft1JDpBb4ZEt\nd1JC0Pl3haZ/DOXSFhFA2Ng/d45gkbl7xRNpWwd8rN7blF1vqRKbHfDeKB2ZANij\n9c8J8rUJOYBNkAd8VgIPabaBgiCnYxA6XeBJNFLpPMPB+hj/xqGljQa3GykCgYAo\nLyNTUPDcbYmAp1NMqgAgzkEkdBb3IKmr+9fT5Jv4c6om+7Dd8cUAAQJyGqIZXZAD\nPgZQcsQptPodTT/vXL7uk9NozHM1gKkqt+5t5pttkmWVVS+R0jqdu/honhmL3Fhg\nekN8dlqO1AAQ2D9v58b1SnytPlVr3H95Il/8hkXXUQKBgDvylw5n2rW4ER2j91Lg\nW74L2D9VXIFp8Trrb+QE5G87GQDXq+WaixEScC0tdOV1MnOHQFRLbMzXQcuf34uu\nLu1yTECyOrRwI2tDcCCnNXQx+e10lGhf8sbWTR9jNjWX5QIBiGdOIq7CV8174IuH\nI7pFKB+yxZJd4tT/F4IbrUBU\n-----END PRIVATE KEY-----\n", 6 | "client_email": "testing-private-key-encryption@tokyo-amphora-437.iam.gserviceaccount.com", 7 | "client_id": "102915290076242385238", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://accounts.google.com/o/oauth2/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/testing-private-key-encryption%40tokyo-amphora-437.iam.gserviceaccount.com" 12 | } 13 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :goth, 4 | json: "config/test-credentials.json" |> Path.expand |> File.read! 5 | 6 | config :diplomat, 7 | token_module: Diplomat.TestToken 8 | -------------------------------------------------------------------------------- /lib/datastore_v1.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Google Inc. All Rights Reserved. 2 | // 3 | // The datastore v1 service proto definitions 4 | 5 | // Had to remove the following line to get exprotobuf to parse this file 6 | // syntax = "proto2"; 7 | 8 | package pb; 9 | option java_package = "com.google.api.services.datastore"; 10 | 11 | 12 | // An identifier for a particular subset of entities. 13 | // 14 | // Entities are partitioned into various subsets, each used by different 15 | // datasets and different namespaces within a dataset and so forth. 16 | // 17 | // All input partition IDs are normalized before use. 18 | // A partition ID is normalized as follows: 19 | // If the partition ID is unset or is set to an empty partition ID, replace it 20 | // with the context partition ID. 21 | // Otherwise, if the partition ID has no dataset ID, assign it the context 22 | // partition ID's dataset ID. 23 | // Unless otherwise documented, the context partition ID has the dataset ID set 24 | // to the context dataset ID and no other partition dimension set. 25 | // 26 | // A partition ID is empty if all of its fields are unset. 27 | // 28 | // Partition dimension: 29 | // A dimension may be unset. 30 | // A dimension's value must never be "". 31 | // A dimension's value must match [A-Za-z\d\.\-_]{1,100} 32 | // If the value of any dimension matches regex "__.*__", 33 | // the partition is reserved/read-only. 34 | // A reserved/read-only partition ID is forbidden in certain documented contexts. 35 | // 36 | // Dataset ID: 37 | // A dataset id's value must never be "". 38 | // A dataset id's value must match 39 | // ([a-z\d\-]{1,100}~)?([a-z\d][a-z\d\-\.]{0,99}:)?([a-z\d][a-z\d\-]{0,99} 40 | message PartitionId { 41 | // The dataset ID. 42 | optional string dataset_id = 3; 43 | // The namespace. 44 | optional string namespace = 4; 45 | } 46 | 47 | // A unique identifier for an entity. 48 | // If a key's partition id or any of its path kinds or names are 49 | // reserved/read-only, the key is reserved/read-only. 50 | // A reserved/read-only key is forbidden in certain documented contexts. 51 | message Key { 52 | // Entities are partitioned into subsets, currently identified by a dataset 53 | // (usually implicitly specified by the project) and namespace ID. 54 | // Queries are scoped to a single partition. 55 | optional PartitionId partition_id = 1; 56 | 57 | // A (kind, ID/name) pair used to construct a key path. 58 | // 59 | // At most one of name or ID may be set. 60 | // If either is set, the element is complete. 61 | // If neither is set, the element is incomplete. 62 | message PathElement { 63 | // The kind of the entity. 64 | // A kind matching regex "__.*__" is reserved/read-only. 65 | // A kind must not contain more than 500 characters. 66 | // Cannot be "". 67 | required string kind = 1; 68 | // The ID of the entity. 69 | // Never equal to zero. Values less than zero are discouraged and will not 70 | // be supported in the future. 71 | optional int64 id = 2; 72 | // The name of the entity. 73 | // A name matching regex "__.*__" is reserved/read-only. 74 | // A name must not be more than 500 characters. 75 | // Cannot be "". 76 | optional string name = 3; 77 | } 78 | 79 | // The entity path. 80 | // An entity path consists of one or more elements composed of a kind and a 81 | // string or numerical identifier, which identify entities. The first 82 | // element identifies a root entity, the second element identifies 83 | // a child of the root entity, the third element a child of the 84 | // second entity, and so forth. The entities identified by all prefixes of 85 | // the path are called the element's ancestors. 86 | // An entity path is always fully complete: ALL of the entity's ancestors 87 | // are required to be in the path along with the entity identifier itself. 88 | // The only exception is that in some documented cases, the identifier in the 89 | // last path element (for the entity) itself may be omitted. A path can never 90 | // be empty. 91 | repeated PathElement path_element = 2; 92 | } 93 | 94 | // A message that can hold any of the supported value types and associated 95 | // metadata. 96 | // 97 | // At most one of the Value fields may be set. 98 | // If none are set the value is "null". 99 | // 100 | message Value { 101 | // A boolean value. 102 | optional bool boolean_value = 1; 103 | // An integer value. 104 | optional int64 integer_value = 2; 105 | // A double value. 106 | optional double double_value = 3; 107 | // A timestamp value. 108 | optional int64 timestamp_microseconds_value = 4; 109 | // A key value. 110 | optional Key key_value = 5; 111 | // A blob key value. 112 | optional string blob_key_value = 16; 113 | // A UTF-8 encoded string value. 114 | optional string string_value = 17; 115 | // A blob value. 116 | optional bytes blob_value = 18; 117 | // An entity value. 118 | // May have no key. 119 | // May have a key with an incomplete key path. 120 | // May have a reserved/read-only key. 121 | optional Entity entity_value = 6; 122 | // A list value. 123 | // Cannot contain another list value. 124 | // Cannot also have a meaning and indexing set. 125 | repeated Value list_value = 7; 126 | 127 | // The meaning field is reserved and should not be used. 128 | optional int32 meaning = 14; 129 | 130 | // If the value should be indexed. 131 | // 132 | // The indexed property may be set for a 133 | // null value. 134 | // When indexed is true, stringValue 135 | // is limited to 500 characters and the blob value is limited to 500 bytes. 136 | // Exception: If meaning is set to 2, string_value is limited to 2038 137 | // characters regardless of indexed. 138 | // When indexed is true, meaning 15 and 22 are not allowed, and meaning 16 139 | // will be ignored on input (and will never be set on output). 140 | // Input values by default have indexed set to 141 | // true; however, you can explicitly set indexed to 142 | // true if you want. (An output value never has 143 | // indexed explicitly set to true.) If a value is 144 | // itself an entity, it cannot have indexed set to 145 | // true. 146 | // Exception: An entity value with meaning 9, 20 or 21 may be indexed. 147 | optional bool indexed = 15 [default = true]; 148 | } 149 | 150 | // An entity property. 151 | message Property { 152 | // The name of the property. 153 | // A property name matching regex "__.*__" is reserved. 154 | // A reserved property name is forbidden in certain documented contexts. 155 | // The name must not contain more than 500 characters. 156 | // Cannot be "". 157 | required string name = 1; 158 | 159 | // The value(s) of the property. 160 | // Each value can have only one value property populated. For example, 161 | // you cannot have a values list of { value: { integerValue: 22, 162 | // stringValue: "a" } }, but you can have { value: { listValue: 163 | // [ { integerValue: 22 }, { stringValue: "a" } ] }. 164 | required Value value = 4; 165 | } 166 | 167 | // An entity. 168 | // 169 | // An entity is limited to 1 megabyte when stored. That roughly 170 | // corresponds to a limit of 1 megabyte for the serialized form of this 171 | // message. 172 | message Entity { 173 | // The entity's key. 174 | // 175 | // An entity must have a key, unless otherwise documented (for example, 176 | // an entity in Value.entityValue may have no key). 177 | // An entity's kind is its key's path's last element's kind, 178 | // or null if it has no key. 179 | optional Key key = 1; 180 | // The entity's properties. 181 | // Each property's name must be unique for its entity. 182 | repeated Property property = 2; 183 | } 184 | 185 | // The result of fetching an entity from the datastore. 186 | message EntityResult { 187 | // Specifies what data the 'entity' field contains. 188 | // A ResultType is either implied (for example, in LookupResponse.found it 189 | // is always FULL) or specified by context (for example, in message 190 | // QueryResultBatch, field 'entity_result_type' specifies a ResultType 191 | // for all the values in field 'entity_result'). 192 | enum ResultType { 193 | FULL = 1; // The entire entity. 194 | PROJECTION = 2; // A projected subset of properties. 195 | // The entity may have no key. 196 | // A property value may have meaning 18. 197 | KEY_ONLY = 3; // Only the key. 198 | } 199 | 200 | // The resulting entity. 201 | required Entity entity = 1; 202 | } 203 | 204 | // A query. 205 | message Query { 206 | // The projection to return. If not set the entire entity is returned. 207 | repeated PropertyExpression projection = 2; 208 | 209 | // The kinds to query (if empty, returns entities from all kinds). 210 | repeated KindExpression kind = 3; 211 | 212 | // The filter to apply (optional). 213 | optional Filter filter = 4; 214 | 215 | // The order to apply to the query results (if empty, order is unspecified). 216 | repeated PropertyOrder order = 5; 217 | 218 | // The properties to group by (if empty, no grouping is applied to the 219 | // result set). 220 | repeated PropertyReference group_by = 6; 221 | 222 | // A starting point for the query results. Optional. Query cursors are 223 | // returned in query result batches. 224 | optional bytes /* serialized QueryCursor */ start_cursor = 7; 225 | 226 | // An ending point for the query results. Optional. Query cursors are 227 | // returned in query result batches. 228 | optional bytes /* serialized QueryCursor */ end_cursor = 8; 229 | 230 | // The number of results to skip. Applies before limit, but after all other 231 | // constraints (optional, defaults to 0). 232 | optional int32 offset = 10 [default=0]; 233 | 234 | // The maximum number of results to return. Applies after all other 235 | // constraints. Optional. 236 | optional int32 limit = 11; 237 | } 238 | 239 | // A representation of a kind. 240 | message KindExpression { 241 | // The name of the kind. 242 | required string name = 1; 243 | } 244 | 245 | // A reference to a property relative to the kind expressions. 246 | // exactly. 247 | message PropertyReference { 248 | // The name of the property. 249 | required string name = 2; 250 | } 251 | 252 | // A representation of a property in a projection. 253 | message PropertyExpression { 254 | enum AggregationFunction { 255 | FIRST = 1; 256 | } 257 | // The property to project. 258 | required PropertyReference property = 1; 259 | // The aggregation function to apply to the property. Optional. 260 | // Can only be used when grouping by at least one property. Must 261 | // then be set on all properties in the projection that are not 262 | // being grouped by. 263 | optional AggregationFunction aggregation_function = 2; 264 | } 265 | 266 | // The desired order for a specific property. 267 | message PropertyOrder { 268 | enum Direction { 269 | ASCENDING = 1; 270 | DESCENDING = 2; 271 | } 272 | // The property to order by. 273 | required PropertyReference property = 1; 274 | // The direction to order by. 275 | optional Direction direction = 2 [default=ASCENDING]; 276 | } 277 | 278 | // A holder for any type of filter. Exactly one field should be specified. 279 | message Filter { 280 | // A composite filter. 281 | optional CompositeFilter composite_filter = 1; 282 | // A filter on a property. 283 | optional PropertyFilter property_filter = 2; 284 | } 285 | 286 | // A filter that merges the multiple other filters using the given operation. 287 | message CompositeFilter { 288 | enum Operator { 289 | AND = 1; 290 | } 291 | 292 | // The operator for combining multiple filters. 293 | required Operator operator = 1; 294 | // The list of filters to combine. 295 | // Must contain at least one filter. 296 | repeated Filter filter = 2; 297 | } 298 | 299 | // A filter on a specific property. 300 | message PropertyFilter { 301 | enum Operator { 302 | LESS_THAN = 1; 303 | LESS_THAN_OR_EQUAL = 2; 304 | GREATER_THAN = 3; 305 | GREATER_THAN_OR_EQUAL = 4; 306 | EQUAL = 5; 307 | 308 | HAS_ANCESTOR = 11; 309 | } 310 | 311 | // The property to filter by. 312 | required PropertyReference property = 1; 313 | // The operator to filter by. 314 | required Operator operator = 2; 315 | // The value to compare the property to. 316 | required Value value = 3; 317 | } 318 | 319 | // A GQL query. 320 | message GqlQuery { 321 | required string query_string = 1; 322 | // When false, the query string must not contain a literal. 323 | optional bool allow_literal = 2 [default = false]; 324 | // A named argument must set field GqlQueryArg.name. 325 | // No two named arguments may have the same name. 326 | // For each non-reserved named binding site in the query string, 327 | // there must be a named argument with that name, 328 | // but not necessarily the inverse. 329 | repeated GqlQueryArg name_arg = 3; 330 | // Numbered binding site @1 references the first numbered argument, 331 | // effectively using 1-based indexing, rather than the usual 0. 332 | // A numbered argument must NOT set field GqlQueryArg.name. 333 | // For each binding site numbered i in query_string, 334 | // there must be an ith numbered argument. 335 | // The inverse must also be true. 336 | repeated GqlQueryArg number_arg = 4; 337 | } 338 | 339 | // A binding argument for a GQL query. 340 | // Exactly one of fields value and cursor must be set. 341 | message GqlQueryArg { 342 | // Must match regex "[A-Za-z_$][A-Za-z_$0-9]*". 343 | // Must not match regex "__.*__". 344 | // Must not be "". 345 | optional string name = 1; 346 | optional Value value = 2; 347 | optional bytes cursor = 3; 348 | } 349 | 350 | // A batch of results produced by a query. 351 | message QueryResultBatch { 352 | // The possible values for the 'more_results' field. 353 | enum MoreResultsType { 354 | NOT_FINISHED = 1; // There are additional batches to fetch from this query. 355 | MORE_RESULTS_AFTER_LIMIT = 2; // The query is finished, but there are more 356 | // results after the limit. 357 | NO_MORE_RESULTS = 3; // The query has been exhausted. 358 | } 359 | 360 | // The result type for every entity in entityResults. 361 | required EntityResult.ResultType entity_result_type = 1; 362 | // The results for this batch. 363 | repeated EntityResult entity_result = 2; 364 | 365 | // A cursor that points to the position after the last result in the batch. 366 | // May be absent. 367 | optional bytes /* serialized QueryCursor */ end_cursor = 4; 368 | 369 | // The state of the query after the current batch. 370 | required MoreResultsType more_results = 5; 371 | 372 | // The number of results skipped because of Query.offset. 373 | optional int32 skipped_results = 6; 374 | } 375 | 376 | // A set of changes to apply. 377 | // 378 | // No entity in this message may have a reserved property name, 379 | // not even a property in an entity in a value. 380 | // No value in this message may have meaning 18, 381 | // not even a value in an entity in another value. 382 | // 383 | // If entities with duplicate keys are present, an arbitrary choice will 384 | // be made as to which is written. 385 | message Mutation { 386 | // Entities to upsert. 387 | // Each upserted entity's key must have a complete path and 388 | // must not be reserved/read-only. 389 | repeated Entity upsert = 1; 390 | // Entities to update. 391 | // Each updated entity's key must have a complete path and 392 | // must not be reserved/read-only. 393 | repeated Entity update = 2; 394 | // Entities to insert. 395 | // Each inserted entity's key must have a complete path and 396 | // must not be reserved/read-only. 397 | repeated Entity insert = 3; 398 | // Insert entities with a newly allocated ID. 399 | // Each inserted entity's key must omit the final identifier in its path and 400 | // must not be reserved/read-only. 401 | repeated Entity insert_auto_id = 4; 402 | // Keys of entities to delete. 403 | // Each key must have a complete key path and must not be reserved/read-only. 404 | repeated Key delete = 5; 405 | // Ignore a user specified read-only period. Optional. 406 | optional bool force = 6; 407 | } 408 | 409 | // The result of applying a mutation. 410 | message MutationResult { 411 | // Number of index writes. 412 | required int32 index_updates = 1; 413 | // Keys for insertAutoId entities. One per entity from the 414 | // request, in the same order. 415 | repeated Key insert_auto_id_key = 2; 416 | } 417 | 418 | // Options shared by read requests. 419 | message ReadOptions { 420 | enum ReadConsistency { 421 | DEFAULT = 0; 422 | STRONG = 1; 423 | EVENTUAL = 2; 424 | } 425 | 426 | // The read consistency to use. 427 | // Cannot be set when transaction is set. 428 | // Lookup and ancestor queries default to STRONG, global queries default to 429 | // EVENTUAL and cannot be set to STRONG. 430 | optional ReadConsistency read_consistency = 1 [default=DEFAULT]; 431 | 432 | // The transaction to use. Optional. 433 | optional bytes /* serialized Transaction */ transaction = 2; 434 | } 435 | 436 | // The request for Lookup. 437 | message LookupRequest { 438 | 439 | // Options for this lookup request. Optional. 440 | optional ReadOptions read_options = 1; 441 | // Keys of entities to look up from the datastore. 442 | repeated Key key = 3; 443 | } 444 | 445 | // The response for Lookup. 446 | message LookupResponse { 447 | 448 | // The order of results in these fields is undefined and has no relation to 449 | // the order of the keys in the input. 450 | 451 | // Entities found as ResultType.FULL entities. 452 | repeated EntityResult found = 1; 453 | 454 | // Entities not found as ResultType.KEY_ONLY entities. 455 | repeated EntityResult missing = 2; 456 | 457 | // A list of keys that were not looked up due to resource constraints. 458 | repeated Key deferred = 3; 459 | } 460 | 461 | 462 | // The request for RunQuery. 463 | message RunQueryRequest { 464 | 465 | // The options for this query. 466 | optional ReadOptions read_options = 1; 467 | 468 | // Entities are partitioned into subsets, identified by a dataset (usually 469 | // implicitly specified by the project) and namespace ID. Queries are scoped 470 | // to a single partition. 471 | // This partition ID is normalized with the standard default context 472 | // partition ID, but all other partition IDs in RunQueryRequest are 473 | // normalized with this partition ID as the context partition ID. 474 | optional PartitionId partition_id = 2; 475 | 476 | // The query to run. 477 | // Either this field or field gql_query must be set, but not both. 478 | optional Query query = 3; 479 | // The GQL query to run. 480 | // Either this field or field query must be set, but not both. 481 | optional GqlQuery gql_query = 7; 482 | } 483 | 484 | // The response for RunQuery. 485 | message RunQueryResponse { 486 | 487 | // A batch of query results (always present). 488 | optional QueryResultBatch batch = 1; 489 | 490 | } 491 | 492 | // The request for BeginTransaction. 493 | message BeginTransactionRequest { 494 | 495 | enum IsolationLevel { 496 | SNAPSHOT = 0; // Read from a consistent snapshot. Concurrent transactions 497 | // conflict if their mutations conflict. For example: 498 | // Read(A),Write(B) may not conflict with Read(B),Write(A), 499 | // but Read(B),Write(B) does conflict with Read(B),Write(B). 500 | SERIALIZABLE = 1; // Read from a consistent snapshot. Concurrent 501 | // transactions conflict if they cannot be serialized. 502 | // For example Read(A),Write(B) does conflict with 503 | // Read(B),Write(A) but Read(A) may not conflict with 504 | // Write(A). 505 | } 506 | 507 | // The transaction isolation level. 508 | optional IsolationLevel isolation_level = 1 [default=SNAPSHOT]; 509 | } 510 | 511 | // The response for BeginTransaction. 512 | message BeginTransactionResponse { 513 | 514 | // The transaction identifier (always present). 515 | optional bytes /* serialized Transaction */ transaction = 1; 516 | } 517 | 518 | // The request for Rollback. 519 | message RollbackRequest { 520 | 521 | // The transaction identifier, returned by a call to 522 | // beginTransaction. 523 | required bytes /* serialized Transaction */ transaction = 1; 524 | } 525 | 526 | // The response for Rollback. 527 | message RollbackResponse { 528 | // Empty 529 | } 530 | 531 | // The request for Commit. 532 | message CommitRequest { 533 | 534 | enum Mode { 535 | TRANSACTIONAL = 1; 536 | NON_TRANSACTIONAL = 2; 537 | } 538 | 539 | // The transaction identifier, returned by a call to 540 | // beginTransaction. Must be set when mode is TRANSACTIONAL. 541 | optional bytes /* serialized Transaction */ transaction = 1; 542 | // The mutation to perform. Optional. 543 | optional Mutation mutation = 2; 544 | // The type of commit to perform. Either TRANSACTIONAL or NON_TRANSACTIONAL. 545 | optional Mode mode = 5 [default=TRANSACTIONAL]; 546 | } 547 | 548 | // The response for Commit. 549 | message CommitResponse { 550 | 551 | // The result of performing the mutation (if any). 552 | optional MutationResult mutation_result = 1; 553 | } 554 | 555 | // The request for AllocateIds. 556 | message AllocateIdsRequest { 557 | 558 | // A list of keys with incomplete key paths to allocate IDs for. 559 | // No key may be reserved/read-only. 560 | repeated Key key = 1; 561 | } 562 | 563 | // The response for AllocateIds. 564 | message AllocateIdsResponse { 565 | 566 | // The keys specified in the request (in the same order), each with 567 | // its key path completed with a newly allocated ID. 568 | repeated Key key = 1; 569 | } 570 | 571 | // Each rpc normalizes the partition IDs of the keys in its input entities, 572 | // and always returns entities with keys with normalized partition IDs. 573 | // (Note that applies to all entities, including entities in values.) 574 | service DatastoreService { 575 | // Look up some entities by key. 576 | rpc Lookup(LookupRequest) returns (LookupResponse) { 577 | // POST /DatastoreService/Lookup ? 578 | }; 579 | // Query for entities. 580 | rpc RunQuery(RunQueryRequest) returns (RunQueryResponse) { 581 | // POST /DatastoreService/RunQ ? 582 | }; 583 | // Begin a new transaction. 584 | rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse) { 585 | }; 586 | // Commit a transaction, optionally creating, deleting or modifying some 587 | // entities. 588 | rpc Commit(CommitRequest) returns (CommitResponse) { 589 | }; 590 | // Roll back a transaction. 591 | rpc Rollback(RollbackRequest) returns (RollbackResponse) { 592 | }; 593 | // Allocate IDs for incomplete keys (useful for referencing an entity before 594 | // it is inserted). 595 | rpc AllocateIds(AllocateIdsRequest) returns (AllocateIdsResponse) { 596 | }; 597 | } 598 | -------------------------------------------------------------------------------- /lib/datastore_v1beta3.proto: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // syntax = "proto3"; 16 | 17 | package google.datastore.v1beta3; 18 | 19 | // import "google/protobuf/wrappers.proto"; 20 | message Int32Value { 21 | // The int32 value. 22 | optional int32 value = 1; 23 | } 24 | 25 | // import "google/protobuf/timestamp.proto"; 26 | message Timestamp { 27 | 28 | // Represents seconds of UTC time since Unix epoch 29 | // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to 30 | // 9999-12-31T23:59:59Z inclusive. 31 | optional int64 seconds = 1; 32 | 33 | // Non-negative fractions of a second at nanosecond resolution. Negative 34 | // second values with fractions must still have non-negative nanos values 35 | // that count forward in time. Must be from 0 to 999,999,999 36 | // inclusive. 37 | optional int32 nanos = 2; 38 | } 39 | 40 | // import "google/protobuf/struct.proto"; 41 | enum NullValue { 42 | NULL_VALUE = 0; 43 | } 44 | 45 | // import "google/type/latlng.proto"; 46 | // An object representing a latitude/longitude pair. This is expressed as a pair 47 | // of doubles representing degrees latitude and degrees longitude. Unless 48 | // specified otherwise, this must conform to the 49 | // WGS84 50 | // standard. Values must be within normalized ranges. 51 | // 52 | // Example of normalization code in Python: 53 | // 54 | // def NormalizeLongitude(longitude): 55 | // """Wraps decimal degrees longitude to [-180.0, 180.0].""" 56 | // q, r = divmod(longitude, 360.0) 57 | // if r > 180.0 or (r == 180.0 and q <= -1.0): 58 | // return r - 360.0 59 | // return r 60 | // 61 | // def NormalizeLatLng(latitude, longitude): 62 | // """Wraps decimal degrees latitude and longitude to 63 | // [-180.0, 180.0] and [-90.0, 90.0], respectively.""" 64 | // r = latitude % 360.0 65 | // if r <= 90.0: 66 | // return r, NormalizeLongitude(longitude) 67 | // elif r >= 270.0: 68 | // return r - 360, NormalizeLongitude(longitude) 69 | // else: 70 | // return 180 - r, NormalizeLongitude(longitude + 180.0) 71 | // 72 | // assert 180.0 == NormalizeLongitude(180.0) 73 | // assert -180.0 == NormalizeLongitude(-180.0) 74 | // assert -179.0 == NormalizeLongitude(181.0) 75 | // assert (0.0, 0.0) == NormalizeLatLng(360.0, 0.0) 76 | // assert (0.0, 0.0) == NormalizeLatLng(-360.0, 0.0) 77 | // assert (85.0, 180.0) == NormalizeLatLng(95.0, 0.0) 78 | // assert (-85.0, -170.0) == NormalizeLatLng(-95.0, 10.0) 79 | // assert (90.0, 10.0) == NormalizeLatLng(90.0, 10.0) 80 | // assert (-90.0, -10.0) == NormalizeLatLng(-90.0, -10.0) 81 | // assert (0.0, -170.0) == NormalizeLatLng(-180.0, 10.0) 82 | // assert (0.0, -170.0) == NormalizeLatLng(180.0, 10.0) 83 | // assert (-90.0, 10.0) == NormalizeLatLng(270.0, 10.0) 84 | // assert (90.0, 10.0) == NormalizeLatLng(-270.0, 10.0) 85 | message LatLng { 86 | // The latitude in degrees. It must be in the range [-90.0, +90.0]. 87 | optional double latitude = 1; 88 | 89 | // The longitude in degrees. It must be in the range [-180.0, +180.0]. 90 | optional double longitude = 2; 91 | } 92 | 93 | // import "google/protobuf/any.proto"; 94 | message Any { 95 | // A URL/resource name whose content describes the type of the 96 | // serialized protocol buffer message. 97 | // 98 | // For URLs which use the scheme `http`, `https`, or no scheme, the 99 | // following restrictions and interpretations apply: 100 | // 101 | // * If no scheme is provided, `https` is assumed. 102 | // * The last segment of the URL's path must represent the fully 103 | // qualified name of the type (as in `path/google.protobuf.Duration`). 104 | // The name should be in a canonical form (e.g., leading "." is 105 | // not accepted). 106 | // * An HTTP GET on the URL must yield a [google.protobuf.Type][] 107 | // value in binary format, or produce an error. 108 | // * Applications are allowed to cache lookup results based on the 109 | // URL, or have them precompiled into a binary to avoid any 110 | // lookup. Therefore, binary compatibility needs to be preserved 111 | // on changes to types. (Use versioned type names to manage 112 | // breaking changes.) 113 | // 114 | // Schemes other than `http`, `https` (or the empty scheme) might be 115 | // used with implementation specific semantics. 116 | // 117 | string type_url = 1; 118 | 119 | // Must be a valid serialized protocol buffer of the above specified type. 120 | bytes value = 2; 121 | } 122 | 123 | // import google/rpc/status.proto 124 | message Status { 125 | // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. 126 | int32 code = 1; 127 | 128 | // A developer-facing error message, which should be in English. Any 129 | // user-facing error message should be localized and sent in the 130 | // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. 131 | string message = 2; 132 | 133 | // A list of messages that carry the error details. There will be a 134 | // common set of message types for APIs to use. 135 | repeated Any details = 3; 136 | } 137 | 138 | 139 | // A partition ID identifies a grouping of entities. The grouping is always 140 | // by project and namespace, however the namespace ID may be empty. 141 | // 142 | // A partition ID contains several dimensions: 143 | // project ID and namespace ID. 144 | // 145 | // Partition dimensions: 146 | // 147 | // - May be `""`. 148 | // - Must be valid UTF-8 bytes. 149 | // - Must have values that match regex `[A-Za-z\d\.\-_]{1,100}` 150 | // If the value of any dimension matches regex `__.*__`, the partition is 151 | // reserved/read-only. 152 | // A reserved/read-only partition ID is forbidden in certain documented 153 | // contexts. 154 | // 155 | // Foreign partition IDs (in which the project ID does 156 | // not match the context project ID ) are discouraged. 157 | // Reads and writes of foreign partition IDs may fail if the project is not in an active state. 158 | message PartitionId { 159 | // The ID of the project to which the entities belong. 160 | optional string project_id = 2; 161 | 162 | // If not empty, the ID of the namespace to which the entities belong. 163 | optional string namespace_id = 4; 164 | } 165 | 166 | // A unique identifier for an entity. 167 | // If a key's partition ID or any of its path kinds or names are 168 | // reserved/read-only, the key is reserved/read-only. 169 | // A reserved/read-only key is forbidden in certain documented contexts. 170 | message Key { 171 | // A (kind, ID/name) pair used to construct a key path. 172 | // 173 | // If either name or ID is set, the element is complete. 174 | // If neither is set, the element is incomplete. 175 | message PathElement { 176 | // The kind of the entity. 177 | // A kind matching regex `__.*__` is reserved/read-only. 178 | // A kind must not contain more than 1500 bytes when UTF-8 encoded. 179 | // Cannot be `""`. 180 | optional string kind = 1; 181 | 182 | // The type of ID. 183 | oneof id_type { 184 | // The auto-allocated ID of the entity. 185 | // Never equal to zero. Values less than zero are discouraged and may not 186 | // be supported in the future. 187 | int64 id = 2; 188 | 189 | // The name of the entity. 190 | // A name matching regex `__.*__` is reserved/read-only. 191 | // A name must not be more than 1500 bytes when UTF-8 encoded. 192 | // Cannot be `""`. 193 | string name = 3; 194 | } 195 | } 196 | 197 | // Entities are partitioned into subsets, currently identified by a project 198 | // ID and namespace ID. 199 | // Queries are scoped to a single partition. 200 | optional PartitionId partition_id = 1; 201 | 202 | // The entity path. 203 | // An entity path consists of one or more elements composed of a kind and a 204 | // string or numerical identifier, which identify entities. The first 205 | // element identifies a _root entity_, the second element identifies 206 | // a _child_ of the root entity, the third element identifies a child of the 207 | // second entity, and so forth. The entities identified by all prefixes of 208 | // the path are called the element's _ancestors_. 209 | // 210 | // An entity path is always fully complete: *all* of the entity's ancestors 211 | // are required to be in the path along with the entity identifier itself. 212 | // The only exception is that in some documented cases, the identifier in the 213 | // last path element (for the entity) itself may be omitted. For example, 214 | // the last path element of the key of `Mutation.insert` may have no 215 | // identifier. 216 | // 217 | // A path can never be empty, and a path can have at most 100 elements. 218 | repeated PathElement path = 2; 219 | } 220 | 221 | // An array value. 222 | message ArrayValue { 223 | // Values in the array. 224 | // The order of this array may not be preserved if it contains a mix of 225 | // indexed and unindexed values. 226 | repeated Value values = 1; 227 | } 228 | 229 | // A message that can hold any of the supported value types and associated 230 | // metadata. 231 | message Value { 232 | // Must have a value set. 233 | oneof value_type { 234 | // A null value. 235 | NullValue null_value = 11; 236 | 237 | // A boolean value. 238 | bool boolean_value = 1; 239 | 240 | // An integer value. 241 | int64 integer_value = 2; 242 | 243 | // A double value. 244 | double double_value = 3; 245 | 246 | // A timestamp value. 247 | // When stored in the Datastore, precise only to microseconds; 248 | // any additional precision is rounded down. 249 | Timestamp timestamp_value = 10; 250 | 251 | // A key value. 252 | Key key_value = 5; 253 | 254 | // A UTF-8 encoded string value. 255 | // When `exclude_from_indexes` is false (it is indexed) , may have at most 1500 bytes. 256 | // Otherwise, may be set to at least 1,000,000 bytes. 257 | string string_value = 17; 258 | 259 | // A blob value. 260 | // May have at most 1,000,000 bytes. 261 | // When `exclude_from_indexes` is false, may have at most 1500 bytes. 262 | // In JSON requests, must be base64-encoded. 263 | bytes blob_value = 18; 264 | 265 | // A geo point value representing a point on the surface of Earth. 266 | LatLng geo_point_value = 8; 267 | 268 | // An entity value. 269 | // 270 | // - May have no key. 271 | // - May have a key with an incomplete key path. 272 | // - May have a reserved/read-only key. 273 | Entity entity_value = 6; 274 | 275 | // An array value. 276 | // Cannot contain another array value. 277 | // A `Value` instance that sets field `array_value` must not set fields 278 | // `meaning` or `exclude_from_indexes`. 279 | ArrayValue array_value = 9; 280 | } 281 | 282 | // The `meaning` field should only be populated for backwards compatibility. 283 | optional int32 meaning = 14; 284 | 285 | // If the value should be excluded from all indexes including those defined 286 | // explicitly. 287 | optional bool exclude_from_indexes = 19; 288 | } 289 | 290 | // A Datastore data object. 291 | // 292 | // An entity is limited to 1 megabyte when stored. That _roughly_ 293 | // corresponds to a limit of 1 megabyte for the serialized form of this 294 | // message. 295 | message Entity { 296 | // The entity's key. 297 | // 298 | // An entity must have a key, unless otherwise documented (for example, 299 | // an entity in `Value.entity_value` may have no key). 300 | // An entity's kind is its key path's last element's kind, 301 | // or null if it has no key. 302 | optional Key key = 1; 303 | 304 | // The entity's properties. 305 | // The map's keys are property names. 306 | // A property name matching regex `__.*__` is reserved. 307 | // A reserved property name is forbidden in certain documented contexts. 308 | // The name must not contain more than 500 characters. 309 | // The name cannot be `""`. 310 | 311 | map properties = 3; 312 | } 313 | 314 | // The result of fetching an entity from Datastore. 315 | message EntityResult { 316 | // Specifies what data the 'entity' field contains. 317 | // A `ResultType` is either implied (for example, in `LookupResponse.missing` 318 | // from `datastore.proto`, it is always `KEY_ONLY`) or specified by context 319 | // (for example, in message `QueryResultBatch`, field `entity_result_type` 320 | // specifies a `ResultType` for all the values in field `entity_results`). 321 | enum ResultType { 322 | // Unspecified. This value is never used. 323 | RESULT_TYPE_UNSPECIFIED = 0; 324 | 325 | // The key and properties. 326 | FULL = 1; 327 | 328 | // A projected subset of properties. The entity may have no key. 329 | PROJECTION = 2; 330 | 331 | // Only the key. 332 | KEY_ONLY = 3; 333 | } 334 | 335 | // The resulting entity. 336 | optional Entity entity = 1; 337 | 338 | // A cursor that points to the position after the result entity. 339 | // Set only when the `EntityResult` is part of a `QueryResultBatch` message. 340 | optional bytes cursor = 3; 341 | } 342 | 343 | // A query for entities. 344 | message Query { 345 | // The projection to return. Defaults to returning all properties. 346 | repeated Projection projection = 2; 347 | 348 | // The kinds to query (if empty, returns entities of all kinds). 349 | // Currently at most 1 kind may be specified. 350 | repeated KindExpression kind = 3; 351 | 352 | // The filter to apply. 353 | optional Filter filter = 4; 354 | 355 | // The order to apply to the query results (if empty, order is unspecified). 356 | repeated PropertyOrder order = 5; 357 | 358 | // The properties to make distinct. The query results will contain the first 359 | // result for each distinct combination of values for the given properties 360 | // (if empty, all results are returned). 361 | repeated PropertyReference distinct_on = 6; 362 | 363 | // A starting point for the query results. Query cursors are 364 | // returned in query result batches. 365 | optional bytes start_cursor = 7; 366 | 367 | // An ending point for the query results. Query cursors are 368 | // returned in query result batches. 369 | optional bytes end_cursor = 8; 370 | 371 | // The number of results to skip. Applies before limit, but after all other 372 | // constraints. Optional. Must be >= 0 if specified. 373 | optional int32 offset = 10; 374 | 375 | // The maximum number of results to return. Applies after all other 376 | // constraints. Optional. 377 | // Unspecified is interpreted as no limit. 378 | // Must be >= 0 if specified. 379 | optional Int32Value limit = 12; 380 | } 381 | 382 | // A representation of a kind. 383 | message KindExpression { 384 | // The name of the kind. 385 | optional string name = 1; 386 | } 387 | 388 | // A reference to a property relative to the kind expressions. 389 | message PropertyReference { 390 | // The name of the property. 391 | // If name includes "."s, it may be interpreted as a property name path. 392 | optional string name = 2; 393 | } 394 | 395 | // A representation of a property in a projection. 396 | message Projection { 397 | // The property to project. 398 | optional PropertyReference property = 1; 399 | } 400 | 401 | // The desired order for a specific property. 402 | message PropertyOrder { 403 | // The sort direction. 404 | enum Direction { 405 | // Unspecified. This value must not be used. 406 | DIRECTION_UNSPECIFIED = 0; 407 | 408 | // Ascending. 409 | ASCENDING = 1; 410 | 411 | // Descending. 412 | DESCENDING = 2; 413 | } 414 | 415 | // The property to order by. 416 | optional PropertyReference property = 1; 417 | 418 | // The direction to order by. Defaults to `ASCENDING`. 419 | optional Direction direction = 2; 420 | } 421 | 422 | // A holder for any type of filter. 423 | message Filter { 424 | // The type of filter. 425 | oneof filter_type { 426 | // A composite filter. 427 | CompositeFilter composite_filter = 1; 428 | 429 | // A filter on a property. 430 | PropertyFilter property_filter = 2; 431 | } 432 | } 433 | 434 | // A filter that merges multiple other filters using the given operator. 435 | message CompositeFilter { 436 | // A composite filter operator. 437 | enum Operator { 438 | // Unspecified. This value must not be used. 439 | OPERATOR_UNSPECIFIED = 0; 440 | 441 | // The results are required to satisfy each of the combined filters. 442 | AND = 1; 443 | } 444 | 445 | // The operator for combining multiple filters. 446 | optional Operator op = 1; 447 | 448 | // The list of filters to combine. 449 | // Must contain at least one filter. 450 | repeated Filter filters = 2; 451 | } 452 | 453 | // A filter on a specific property. 454 | message PropertyFilter { 455 | // A property filter operator. 456 | enum Operator { 457 | // Unspecified. This value must not be used. 458 | OPERATOR_UNSPECIFIED = 0; 459 | 460 | // Less than. 461 | LESS_THAN = 1; 462 | 463 | // Less than or equal. 464 | LESS_THAN_OR_EQUAL = 2; 465 | 466 | // Greater than. 467 | GREATER_THAN = 3; 468 | 469 | // Greater than or equal. 470 | GREATER_THAN_OR_EQUAL = 4; 471 | 472 | // Equal. 473 | EQUAL = 5; 474 | 475 | // Has ancestor. 476 | HAS_ANCESTOR = 11; 477 | } 478 | 479 | // The property to filter by. 480 | optional PropertyReference property = 1; 481 | 482 | // The operator to filter by. 483 | optional Operator op = 2; 484 | 485 | // The value to compare the property to. 486 | optional Value value = 3; 487 | } 488 | 489 | // A [GQL query](https://cloud.google.com/datastore/docs/apis/gql/gql_reference). 490 | message GqlQuery { 491 | // A string of the format described 492 | // [here](https://cloud.google.com/datastore/docs/apis/gql/gql_reference). 493 | optional string query_string = 1; 494 | 495 | // When false, the query string must not contain any literals and instead 496 | // must bind all values. For example, 497 | // `SELECT * FROM Kind WHERE a = 'string literal'` is not allowed, while 498 | // `SELECT * FROM Kind WHERE a = @value` is. 499 | optional bool allow_literals = 2; 500 | 501 | // For each non-reserved named binding site in the query string, 502 | // there must be a named parameter with that name, 503 | // but not necessarily the inverse. 504 | // Key must match regex `[A-Za-z_$][A-Za-z_$0-9]*`, must not match regex 505 | // `__.*__`, and must not be `""`. 506 | map named_bindings = 5; 507 | 508 | // Numbered binding site @1 references the first numbered parameter, 509 | // effectively using 1-based indexing, rather than the usual 0. 510 | // For each binding site numbered i in `query_string`, 511 | // there must be an i-th numbered parameter. 512 | // The inverse must also be true. 513 | repeated GqlQueryParameter positional_bindings = 4; 514 | } 515 | 516 | // A binding parameter for a GQL query. 517 | message GqlQueryParameter { 518 | // The type of parameter. 519 | oneof parameter_type { 520 | // A value parameter. 521 | Value value = 2; 522 | 523 | // A query cursor. Query cursors are returned in query 524 | // result batches. 525 | bytes cursor = 3; 526 | } 527 | } 528 | 529 | // A batch of results produced by a query. 530 | message QueryResultBatch { 531 | // The possible values for the `more_results` field. 532 | enum MoreResultsType { 533 | // Unspecified. This value is never used. 534 | MORE_RESULTS_TYPE_UNSPECIFIED = 0; 535 | 536 | // There may be additional batches to fetch from this query. 537 | NOT_FINISHED = 1; 538 | 539 | // The query is finished, but there may be more results after the limit. 540 | MORE_RESULTS_AFTER_LIMIT = 2; 541 | 542 | // The query is finished, but there may be more results after the end cursor. 543 | MORE_RESULTS_AFTER_CURSOR = 4; 544 | 545 | // The query has been exhausted. 546 | NO_MORE_RESULTS = 3; 547 | } 548 | 549 | // The number of results skipped, typically because of an offset. 550 | optional int32 skipped_results = 6; 551 | 552 | // A cursor that points to the position after the last skipped result. 553 | // Will be set when `skipped_results` != 0. 554 | optional bytes skipped_cursor = 3; 555 | 556 | // The result type for every entity in `entity_results`. 557 | optional EntityResult.ResultType entity_result_type = 1; 558 | 559 | // The results for this batch. 560 | repeated EntityResult entity_results = 2; 561 | 562 | // A cursor that points to the position after the last result in the batch. 563 | optional bytes end_cursor = 4; 564 | 565 | // The state of the query after the current batch. 566 | optional MoreResultsType more_results = 5; 567 | } 568 | 569 | // The request for [google.datastore.v1beta3.Datastore.Lookup][google.datastore.v1beta3.Datastore.Lookup]. 570 | message LookupRequest { 571 | // The ID of the project against which to make the request. 572 | optional string project_id = 8; 573 | 574 | // The options for this lookup request. 575 | optional ReadOptions read_options = 1; 576 | 577 | // Keys of entities to look up. 578 | repeated Key keys = 3; 579 | } 580 | 581 | // The response for [google.datastore.v1beta3.Datastore.Lookup][google.datastore.v1beta3.Datastore.Lookup]. 582 | message LookupResponse { 583 | // Entities found as `ResultType.FULL` entities. The order of results in this 584 | // field is undefined and has no relation to the order of the keys in the 585 | // input. 586 | repeated EntityResult found = 1; 587 | 588 | // Entities not found as `ResultType.KEY_ONLY` entities. The order of results 589 | // in this field is undefined and has no relation to the order of the keys 590 | // in the input. 591 | repeated EntityResult missing = 2; 592 | 593 | // A list of keys that were not looked up due to resource constraints. The 594 | // order of results in this field is undefined and has no relation to the 595 | // order of the keys in the input. 596 | repeated Key deferred = 3; 597 | } 598 | 599 | // The request for [google.datastore.v1beta3.Datastore.RunQuery][google.datastore.v1beta3.Datastore.RunQuery]. 600 | message RunQueryRequest { 601 | // The ID of the project against which to make the request. 602 | optional string project_id = 8; 603 | 604 | // Entities are partitioned into subsets, identified by a partition ID. 605 | // Queries are scoped to a single partition. 606 | // This partition ID is normalized with the standard default context 607 | // partition ID. 608 | optional PartitionId partition_id = 2; 609 | 610 | // The options for this query. 611 | optional ReadOptions read_options = 1; 612 | 613 | // The type of query. 614 | oneof query_type { 615 | // The query to run. 616 | Query query = 3; 617 | 618 | // The GQL query to run. 619 | GqlQuery gql_query = 7; 620 | } 621 | } 622 | 623 | // The response for [google.datastore.v1beta3.Datastore.RunQuery][google.datastore.v1beta3.Datastore.RunQuery]. 624 | message RunQueryResponse { 625 | // A batch of query results (always present). 626 | optional QueryResultBatch batch = 1; 627 | 628 | // The parsed form of the `GqlQuery` from the request, if it was set. 629 | optional Query query = 2; 630 | } 631 | 632 | // The request for [google.datastore.v1beta3.Datastore.BeginTransaction][google.datastore.v1beta3.Datastore.BeginTransaction]. 633 | message BeginTransactionRequest { 634 | // The ID of the project against which to make the request. 635 | optional string project_id = 8; 636 | } 637 | 638 | // The response for [google.datastore.v1beta3.Datastore.BeginTransaction][google.datastore.v1beta3.Datastore.BeginTransaction]. 639 | message BeginTransactionResponse { 640 | // The transaction identifier (always present). 641 | optional bytes transaction = 1; 642 | } 643 | 644 | // The request for [google.datastore.v1beta3.Datastore.Rollback][google.datastore.v1beta3.Datastore.Rollback]. 645 | message RollbackRequest { 646 | // The ID of the project against which to make the request. 647 | optional string project_id = 8; 648 | 649 | // The transaction identifier, returned by a call to 650 | // [google.datastore.v1beta3.Datastore.BeginTransaction][google.datastore.v1beta3.Datastore.BeginTransaction]. 651 | optional bytes transaction = 1; 652 | } 653 | 654 | // The response for [google.datastore.v1beta3.Datastore.Rollback][google.datastore.v1beta3.Datastore.Rollback] 655 | // (an empty message). 656 | message RollbackResponse { 657 | 658 | } 659 | 660 | // The request for [google.datastore.v1beta3.Datastore.Commit][google.datastore.v1beta3.Datastore.Commit]. 661 | message CommitRequest { 662 | // The modes available for commits. 663 | enum Mode { 664 | // Unspecified. This value must not be used. 665 | MODE_UNSPECIFIED = 0; 666 | 667 | // Transactional: The mutations are either all applied, or none are applied. 668 | // Learn about transactions [here](https://cloud.google.com/datastore/docs/concepts/transactions). 669 | TRANSACTIONAL = 1; 670 | 671 | // Non-transactional: The mutations may not apply as all or none. 672 | NON_TRANSACTIONAL = 2; 673 | } 674 | 675 | // The ID of the project against which to make the request. 676 | optional string project_id = 8; 677 | 678 | // The type of commit to perform. Defaults to `TRANSACTIONAL`. 679 | optional Mode mode = 5; 680 | 681 | // Must be set when mode is `TRANSACTIONAL`. 682 | oneof transaction_selector { 683 | // The identifier of the transaction associated with the commit. A 684 | // transaction identifier is returned by a call to 685 | // [BeginTransaction][google.datastore.v1beta3.Datastore.BeginTransaction]. 686 | bytes transaction = 1; 687 | } 688 | 689 | // The mutations to perform. 690 | // 691 | // When mode is `TRANSACTIONAL`, mutations affecting a single entity are 692 | // applied in order. The following sequences of mutations affecting a single 693 | // entity are not permitted in a single `Commit` request: 694 | // 695 | // - `insert` followed by `insert` 696 | // - `update` followed by `insert` 697 | // - `upsert` followed by `insert` 698 | // - `delete` followed by `update` 699 | // 700 | // When mode is `NON_TRANSACTIONAL`, no two mutations may affect a single 701 | // entity. 702 | repeated Mutation mutations = 6; 703 | } 704 | 705 | // The response for [google.datastore.v1beta3.Datastore.Commit][google.datastore.v1beta3.Datastore.Commit]. 706 | message CommitResponse { 707 | // The result of performing the mutations. 708 | // The i-th mutation result corresponds to the i-th mutation in the request. 709 | repeated MutationResult mutation_results = 3; 710 | 711 | // The number of index entries updated during the commit, or zero if none were 712 | // updated. 713 | optional int32 index_updates = 4; 714 | } 715 | 716 | // The request for [google.datastore.v1beta3.Datastore.AllocateIds][google.datastore.v1beta3.Datastore.AllocateIds]. 717 | message AllocateIdsRequest { 718 | // The ID of the project against which to make the request. 719 | optional string project_id = 8; 720 | 721 | // A list of keys with incomplete key paths for which to allocate IDs. 722 | // No key may be reserved/read-only. 723 | repeated Key keys = 1; 724 | } 725 | 726 | // The response for [google.datastore.v1beta3.Datastore.AllocateIds][google.datastore.v1beta3.Datastore.AllocateIds]. 727 | message AllocateIdsResponse { 728 | // The keys specified in the request (in the same order), each with 729 | // its key path completed with a newly allocated ID. 730 | repeated Key keys = 1; 731 | } 732 | 733 | // A mutation to apply to an entity. 734 | message Mutation { 735 | // The mutation operation. 736 | // 737 | // For `insert`, `update`, and `upsert`: 738 | // - The entity's key must not be reserved/read-only. 739 | // - No property in the entity may have a reserved name, 740 | // not even a property in an entity in a value. 741 | // - No value in the entity may have meaning 18, 742 | // not even a value in an entity in another value. 743 | oneof operation { 744 | // The entity to insert. The entity must not already exist. 745 | // The entity key's final path element may be incomplete. 746 | Entity insert = 4; 747 | 748 | // The entity to update. The entity must already exist. 749 | // Must have a complete key path. 750 | Entity update = 5; 751 | 752 | // The entity to upsert. The entity may or may not already exist. 753 | // The entity key's final path element may be incomplete. 754 | Entity upsert = 6; 755 | 756 | // The key of the entity to delete. The entity may or may not already exist. 757 | // Must have a complete key path and must not be reserved/read-only. 758 | Key delete = 7; 759 | } 760 | } 761 | 762 | // The result of applying a mutation. 763 | message MutationResult { 764 | // The automatically allocated key. 765 | // Set only when the mutation allocated a key. 766 | optional Key key = 3; 767 | } 768 | 769 | // The options shared by read requests. 770 | message ReadOptions { 771 | // The possible values for read consistencies. 772 | enum ReadConsistency { 773 | // Unspecified. This value must not be used. 774 | READ_CONSISTENCY_UNSPECIFIED = 0; 775 | 776 | // Strong consistency. 777 | STRONG = 1; 778 | 779 | // Eventual consistency. 780 | EVENTUAL = 2; 781 | } 782 | 783 | // If not specified, lookups and ancestor queries default to 784 | // `read_consistency`=`STRONG`, global queries default to 785 | // `read_consistency`=`EVENTUAL`. 786 | oneof consistency_type { 787 | // The non-transactional read consistency to use. 788 | // Cannot be set to `STRONG` for global queries. 789 | ReadConsistency read_consistency = 1; 790 | 791 | // The transaction in which to read. 792 | bytes transaction = 2; 793 | } 794 | } 795 | -------------------------------------------------------------------------------- /lib/diplomat.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat do 2 | @moduledoc """ 3 | `Diplomat` is a library for interacting with Google's Cloud Datastore APIs. 4 | 5 | It provides simple interfaces for creating, updating, and deleting Entities, 6 | and also has support for querying via Datastore's GQL language (which is 7 | similar, but not exactly like, SQL). 8 | """ 9 | defmodule Proto do 10 | use Protobuf, from: Path.expand("datastore_v1beta3.proto", __DIR__), doc: false 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/diplomat/client.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Client do 2 | alias Diplomat.{Entity, QueryResultBatch, Key} 3 | 4 | alias Diplomat.Proto.{ 5 | AllocateIdsRequest, 6 | AllocateIdsResponse, 7 | CommitRequest, 8 | CommitResponse, 9 | BeginTransactionRequest, 10 | BeginTransactionResponse, 11 | RollbackRequest, 12 | RollbackResponse, 13 | RunQueryRequest, 14 | RunQueryResponse, 15 | LookupRequest, 16 | LookupResponse, 17 | Status 18 | } 19 | 20 | @moduledoc """ 21 | Low level Google DataStore RPC client functions. 22 | """ 23 | 24 | @api_version "v1" 25 | 26 | @type error :: {:error, Status.t()} 27 | @typep method :: :allocateIds | :beginTransaction | :commit | :lookup | :rollback | :runQuery 28 | 29 | @spec allocate_ids(AllocateIdsRequest.t()) :: list(Key.t()) | error 30 | @doc "Allocate ids for a list of keys with incomplete key paths" 31 | def allocate_ids(req) do 32 | req 33 | |> AllocateIdsRequest.encode() 34 | |> call(:allocateIds) 35 | |> case do 36 | {:ok, body} -> 37 | body 38 | |> AllocateIdsResponse.decode() 39 | |> Key.from_allocate_ids_proto() 40 | 41 | any -> 42 | any 43 | end 44 | end 45 | 46 | @spec commit(CommitRequest.t()) :: {:ok, CommitResponse.t()} | error 47 | @doc "Commit a transaction optionally performing any number of mutations" 48 | def commit(req) do 49 | req 50 | |> CommitRequest.encode() 51 | |> call(:commit) 52 | |> case do 53 | {:ok, body} -> {:ok, CommitResponse.decode(body)} 54 | any -> any 55 | end 56 | end 57 | 58 | @spec begin_transaction(BeginTransactionRequest.t()) :: 59 | {:ok, BeginTransactionResponse.t()} | error 60 | @doc "Begin a new transaction" 61 | def begin_transaction(req) do 62 | req 63 | |> BeginTransactionRequest.encode() 64 | |> call(:beginTransaction) 65 | |> case do 66 | {:ok, body} -> {:ok, BeginTransactionResponse.decode(body)} 67 | any -> any 68 | end 69 | end 70 | 71 | @spec rollback(RollbackRequest.t()) :: {:ok, RollbackResponse.t()} | error 72 | @doc "Roll back a transaction specified by a transaction id" 73 | def rollback(req) do 74 | req 75 | |> RollbackRequest.encode() 76 | |> call(:rollback) 77 | |> case do 78 | {:ok, body} -> {:ok, RollbackResponse.decode(body)} 79 | any -> any 80 | end 81 | end 82 | 83 | @spec run_query(RunQueryRequest.t()) :: list(Entity.t()) | error 84 | @doc "Query for entities" 85 | def run_query(req) do 86 | req 87 | |> RunQueryRequest.encode() 88 | |> call(:runQuery) 89 | |> case do 90 | {:ok, body} -> 91 | result = body |> RunQueryResponse.decode() 92 | 93 | Enum.map(result.batch.entity_results, fn e -> 94 | Entity.from_proto(e.entity) 95 | end) 96 | 97 | any -> 98 | any 99 | end 100 | end 101 | 102 | @spec run_query_with_pagination(RunQueryRequest.t()) :: QueryResultBatch.t() | error 103 | @doc "Query for entities with pagination metadata" 104 | def run_query_with_pagination(req) do 105 | req 106 | |> RunQueryRequest.encode() 107 | |> call(:runQuery) 108 | |> case do 109 | {:ok, body} -> 110 | body 111 | |> RunQueryResponse.decode() 112 | |> Map.get(:batch) 113 | |> QueryResultBatch.from_proto() 114 | 115 | any -> 116 | any 117 | end 118 | end 119 | 120 | @spec lookup(LookupRequest.t()) :: list(Entity.t()) | error 121 | @doc "Lookup entities by key" 122 | def lookup(req) do 123 | req 124 | |> LookupRequest.encode() 125 | |> call(:lookup) 126 | |> case do 127 | {:ok, body} -> 128 | result = body |> LookupResponse.decode() 129 | 130 | Enum.map(result.found, fn e -> 131 | Entity.from_proto(e.entity) 132 | end) 133 | 134 | any -> 135 | any 136 | end 137 | end 138 | 139 | @spec call(String.t(), method()) :: {:ok, String.t()} | error | {:error, any} 140 | defp call(data, method) do 141 | url(method) 142 | |> HTTPoison.post(data, [auth_header(), proto_header()]) 143 | |> case do 144 | {:ok, %{body: body, status_code: code}} when code in 200..299 -> 145 | {:ok, body} 146 | 147 | {:ok, %{body: body}} -> 148 | {:error, Status.decode(body)} 149 | 150 | {:error, %HTTPoison.Error{reason: reason}} -> 151 | {:error, reason} 152 | end 153 | end 154 | 155 | defp url(method), do: url(@api_version, method) 156 | 157 | defp url("v1", method) do 158 | Path.join([endpoint(), @api_version, "projects", "#{project()}:#{method}"]) 159 | end 160 | 161 | defp endpoint, do: Application.get_env(:diplomat, :endpoint, default_endpoint(@api_version)) 162 | 163 | defp default_endpoint("v1"), do: "https://datastore.googleapis.com" 164 | 165 | defp token_module, do: Application.get_env(:diplomat, :token_module, Goth.Token) 166 | 167 | defp project do 168 | {:ok, project_id} = Goth.Config.get(:project_id) 169 | project_id 170 | end 171 | 172 | defp api_scope, do: api_scope(@api_version) 173 | defp api_scope("v1"), do: "https://www.googleapis.com/auth/datastore" 174 | 175 | defp auth_header do 176 | {:ok, token} = token_module().for_scope(api_scope()) 177 | {"Authorization", "#{token.type} #{token.token}"} 178 | end 179 | 180 | defp proto_header do 181 | {"Content-Type", "application/x-protobuf"} 182 | end 183 | end 184 | -------------------------------------------------------------------------------- /lib/diplomat/cursor.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Cursor do 2 | @type t :: %__MODULE__{value: String.t()} 3 | 4 | defstruct [:value] 5 | 6 | @spec new(String.t(), Keyword.t()) :: t 7 | def new(cursor_value, opts \\ []) 8 | def new(cursor_value, encode: true), do: %__MODULE__{value: encode(cursor_value)} 9 | def new(cursor_value, _opts), do: %__MODULE__{value: cursor_value} 10 | 11 | @spec encode(String.t()) :: String.t() 12 | def encode(cursor_value), do: Base.url_encode64(cursor_value) 13 | 14 | @spec decode(t | String.t()) :: String.t() 15 | def decode(%__MODULE__{} = cursor), do: decode(cursor.value) 16 | def decode(value) when is_bitstring(value), do: Base.url_decode64!(value) 17 | end 18 | -------------------------------------------------------------------------------- /lib/diplomat/entity.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Entity do 2 | alias Diplomat.Proto.Entity, as: PbEntity 3 | alias Diplomat.Proto.{CommitRequest, CommitResponse, Mutation, Mode} 4 | alias Diplomat.{Key, Value, Entity, Client} 5 | 6 | @type mutation :: {operation(), t} 7 | @type operation :: :insert | :upsert | :update | :delete 8 | 9 | @type t :: %__MODULE__{ 10 | kind: String.t() | nil, 11 | key: Diplomat.Key.t() | nil, 12 | properties: %{optional(String.t()) => Diplomat.Value.t()} 13 | } 14 | 15 | defstruct kind: nil, key: nil, properties: %{} 16 | 17 | @spec new( 18 | props :: struct() | map(), 19 | kind_or_key_or_opts :: Key.t() | String.t() | Keyword.t(), 20 | id_or_name_or_opts :: String.t() | integer | Keyword.t(), 21 | opts :: Keyword.t() 22 | ) :: t 23 | @doc """ 24 | Creates a new `Diplomat.Entity` with the given properties. 25 | 26 | Instead of building a `Diplomat.Entity` struct manually, `new` is the way you 27 | should create your entities. `new` wraps and nests properties correctly, and 28 | ensures that your entities have a valid `Key` (among other things). 29 | 30 | ## Options 31 | 32 | * `:exclude_from_indexes` - An atom, list of atoms, or Keyword list of 33 | properties that will not be indexed. 34 | * `:truncate` - Boolean, whether or not to truncate string values that are 35 | over 1500 bytes (the max length of an indexed string in Datastore). 36 | Defaults to `false`. 37 | * `:sanitize_keys` - Boolean or String. If `true`, dots (`.`) in property 38 | keys will be replaced with an underscore (`_`). If a string, dots will 39 | be replaced with the string passed. Defaults to `false`. 40 | 41 | ## Examples 42 | 43 | ### Without a key 44 | 45 | Entity.new(%{"foo" => "bar"}) 46 | 47 | ### With a kind but without a name or id 48 | 49 | Entity.new(%{"foo" => "bar"}, "ExampleKind") 50 | 51 | ### With a kind and name or id 52 | 53 | Entity.new(%{"foo" => "bar"}, "ExampleKind", "1") 54 | 55 | ### With a key 56 | 57 | Entity.new(%{"foo" => "bar"}, Diplomat.Key.new("ExampleKind", "1")) 58 | 59 | ### With excluded fields 60 | 61 | Entity.new(%{"foo" => %{"bar" => "baz"}, "qux" => true}, 62 | exclude_from_indexes: [:qux, [foo: :bar]]) 63 | 64 | The above will exclude the `:qux` field from the top level entity and the `:bar` 65 | field from the entity nested at `:foo`. 66 | """ 67 | def new(props, kind_or_key_or_opts \\ [], id_or_opts \\ [], opts \\ []) 68 | 69 | def new(props = %{__struct__: _}, kind_or_key_or_opts, id_or_opts, opts), 70 | do: props |> Map.from_struct() |> new(kind_or_key_or_opts, id_or_opts, opts) 71 | 72 | def new(props, opts, [], []) when is_list(opts), 73 | do: %Entity{properties: value_properties(props, opts)} 74 | 75 | def new(props, kind, opts, []) when is_binary(kind) and is_list(opts), 76 | do: new(props, Key.new(kind), opts) 77 | 78 | def new(props, key = %Key{kind: kind}, opts, []) when is_list(opts), 79 | do: %Entity{kind: kind, key: key, properties: value_properties(props, opts)} 80 | 81 | def new(props, kind, id, opts) when is_binary(kind) and is_list(opts), 82 | do: new(props, Key.new(kind, id), opts) 83 | 84 | @spec proto(map() | t) :: Diplomat.Proto.Entity.t() 85 | @doc """ 86 | Generate a `Diplomat.Proto.Entity` from a given `Diplomat.Entity`. This can 87 | then be used to generate the binary protocol buffer representation of the 88 | `Diplomat.Entity` 89 | """ 90 | def proto(%Entity{key: key, properties: properties}) do 91 | pb_properties = 92 | properties 93 | |> Map.to_list() 94 | |> Enum.map(fn {name, value} -> 95 | {to_string(name), Value.proto(value)} 96 | end) 97 | 98 | %PbEntity{ 99 | key: key |> Key.proto(), 100 | properties: pb_properties 101 | } 102 | end 103 | 104 | def proto(properties) when is_map(properties) do 105 | properties 106 | |> new() 107 | |> proto() 108 | end 109 | 110 | @spec from_proto(PbEntity.t()) :: t 111 | @doc "Create a `Diplomat.Entity` from a `Diplomat.Proto.Entity`" 112 | def from_proto(%PbEntity{key: nil, properties: pb_properties}), 113 | do: %Entity{properties: values_from_proto(pb_properties)} 114 | 115 | def from_proto(%PbEntity{key: pb_key, properties: pb_properties}) do 116 | key = Key.from_proto(pb_key) 117 | 118 | %Entity{ 119 | kind: key.kind, 120 | key: key, 121 | properties: values_from_proto(pb_properties) 122 | } 123 | end 124 | 125 | @spec properties(t) :: map() 126 | @doc """ 127 | Extract a `Diplomat.Entity`'s properties as a map. 128 | 129 | The properties are stored on the struct as a map with string keys and 130 | `Diplomat.Value` values. This function will allow you to extract the properties 131 | as a map with string keys and Elixir built-in values. 132 | 133 | For example, creating an `Entity` looks like the following: 134 | ``` 135 | iex> entity = Entity.new(%{"hello" => "world"}) 136 | # => %Diplomat.Entity{key: nil, kind: nil, 137 | # properties: %{"hello" => %Diplomat.Value{value: "world"}}} 138 | ``` 139 | 140 | `Diplomat.Entity.properties/1` allows you to extract those properties to get 141 | the following: `%{"hello" => "world"}` 142 | """ 143 | def properties(%Entity{properties: properties}) do 144 | properties 145 | |> Enum.map(fn {key, value} -> 146 | {key, value |> recurse_properties} 147 | end) 148 | |> Enum.into(%{}) 149 | end 150 | 151 | defp recurse_properties(value) do 152 | case value do 153 | %Entity{} -> value |> properties 154 | %Value{value: value} -> value |> recurse_properties 155 | value when is_list(value) -> value |> Enum.map(&recurse_properties/1) 156 | _ -> value 157 | end 158 | end 159 | 160 | @spec insert([t] | t) :: [Key.t()] | Client.error() 161 | def insert(%Entity{} = entity), do: insert([entity]) 162 | 163 | def insert(entities) when is_list(entities) do 164 | entities 165 | |> Enum.map(fn e -> {:insert, e} end) 166 | |> commit_request 167 | |> Diplomat.Client.commit() 168 | |> case do 169 | {:ok, resp} -> Key.from_commit_proto(resp) 170 | any -> any 171 | end 172 | end 173 | 174 | # at some point we should validate the entity keys 175 | @spec upsert([t] | t) :: {:ok, CommitResponse.t()} | Client.error() 176 | def upsert(%Entity{} = entity), do: upsert([entity]) 177 | 178 | def upsert(entities) when is_list(entities) do 179 | entities 180 | |> Enum.map(fn e -> {:upsert, e} end) 181 | |> commit_request 182 | |> Diplomat.Client.commit() 183 | |> case do 184 | {:ok, resp} -> resp 185 | any -> any 186 | end 187 | end 188 | 189 | @spec commit_request([mutation()]) :: CommitResponse.t() 190 | @doc false 191 | def commit_request(opts), do: commit_request(opts, :NON_TRANSACTIONAL) 192 | 193 | @spec commit_request([mutation()], Mode.t()) :: CommitResponse.t() 194 | @doc false 195 | def commit_request(opts, mode) do 196 | CommitRequest.new( 197 | mode: mode, 198 | mutations: extract_mutations(opts, []) 199 | ) 200 | end 201 | 202 | @spec commit_request([mutation()], Mode.t(), Transaction.t()) :: CommitResponse.t() 203 | @doc false 204 | def commit_request(opts, mode, trans) do 205 | CommitRequest.new( 206 | mode: mode, 207 | transaction_selector: {:transaction, trans.id}, 208 | mutations: extract_mutations(opts, []) 209 | ) 210 | end 211 | 212 | @spec extract_mutations([mutation()], [Mutation.t()]) :: [Mutation.t()] 213 | def extract_mutations([], acc), do: Enum.reverse(acc) 214 | 215 | def extract_mutations([{:insert, ent} | tail], acc) do 216 | extract_mutations(tail, [Mutation.new(operation: {:insert, proto(ent)}) | acc]) 217 | end 218 | 219 | def extract_mutations([{:upsert, ent} | tail], acc) do 220 | extract_mutations(tail, [Mutation.new(operation: {:upsert, proto(ent)}) | acc]) 221 | end 222 | 223 | def extract_mutations([{:update, ent} | tail], acc) do 224 | extract_mutations(tail, [Mutation.new(operation: {:update, proto(ent)}) | acc]) 225 | end 226 | 227 | def extract_mutations([{:delete, key} | tail], acc) do 228 | extract_mutations(tail, [Mutation.new(operation: {:delete, Key.proto(key)}) | acc]) 229 | end 230 | 231 | defp value_properties(props = %{__struct__: _struct}, opts) do 232 | props 233 | |> Map.from_struct() 234 | |> value_properties(opts) 235 | end 236 | 237 | defp value_properties(props, opts) when is_map(props) do 238 | exclude = opts |> Keyword.get(:exclude_from_indexes, []) |> get_excluded() 239 | 240 | props 241 | |> Map.to_list() 242 | |> Enum.map(fn {name, value} -> 243 | { 244 | sanitize_key(name, Keyword.get(opts, :sanitize_keys)), 245 | Value.new( 246 | value, 247 | sanitize_keys: Keyword.get(opts, :sanitize_keys), 248 | truncate: Keyword.get(opts, :truncate), 249 | exclude_from_indexes: get_exclude_value(exclude, name) 250 | ) 251 | } 252 | end) 253 | |> Enum.into(%{}) 254 | end 255 | 256 | defp sanitize_key(key, false), do: key |> to_string 257 | defp sanitize_key(key, nil), do: key |> to_string 258 | defp sanitize_key(key, true), do: key |> to_string |> String.replace(".", "_") 259 | defp sanitize_key(key, r), do: key |> to_string |> String.replace(".", r) 260 | 261 | defp get_excluded(fields) when is_list(fields), do: fields 262 | defp get_excluded(field), do: [field] 263 | 264 | defp get_exclude_value(excluded, name) when is_atom(name) do 265 | (name in excluded) || Keyword.get(excluded, name, false) 266 | end 267 | defp get_exclude_value(excluded, name) when is_binary(name) do 268 | Enum.find_value(excluded, false, fn 269 | {field, value} -> if Atom.to_string(field) == name, do: value, else: false 270 | field -> Atom.to_string(field) == name 271 | end) 272 | end 273 | 274 | defp values_from_proto(pb_properties) do 275 | pb_properties 276 | |> Enum.map(fn {name, pb_value} -> {name, Value.from_proto(pb_value)} end) 277 | |> Enum.into(%{}) 278 | end 279 | end 280 | -------------------------------------------------------------------------------- /lib/diplomat/key.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Key do 2 | alias Diplomat.Key 3 | alias Diplomat.Proto.Key, as: PbKey 4 | alias Diplomat.Proto.Key.PathElement, as: PbPathElement 5 | alias Diplomat.Proto.PartitionId, as: PbPartition 6 | 7 | alias Diplomat.Proto.{ 8 | CommitResponse, 9 | LookupRequest, 10 | AllocateIdsRequest, 11 | AllocateIdsResponse 12 | } 13 | 14 | @typedoc """ 15 | A Key for uniquely identifying a `Diplomat.Entity`. 16 | 17 | A Key must have a unique identifier, which is either a `name` or an `id`. 18 | Most often, if setting a unique identifier manually, you will use the name 19 | field. However, if neither a name nor an id is defined, `Diplomat` will 20 | auto-assign an id by calling the API to allocate an id for the `Key`. 21 | """ 22 | @type t :: %__MODULE__{ 23 | id: integer | nil, 24 | name: String.t() | nil, 25 | kind: String.t(), 26 | parent: Diplomat.Key.t() | nil, 27 | project_id: String.t() | nil, 28 | namespace: String.t() | nil 29 | } 30 | 31 | @type key_pair :: [String.t() | integer | nil] 32 | 33 | defstruct [:id, :name, :kind, :parent, :project_id, :namespace] 34 | 35 | @spec new(String.t()) :: t 36 | @doc "Creates a new `Diplomat.Key` from a kind" 37 | def new(kind), 38 | do: %__MODULE__{kind: kind} 39 | 40 | @spec new(String.t(), String.t() | integer) :: t 41 | @doc "Creates a new `Diplomat.Key` from a kind and id" 42 | def new(kind, id) when is_integer(id), 43 | do: %__MODULE__{kind: kind, id: id} 44 | 45 | @doc "Creates a new `Diplomat.Key` form a kind and a name" 46 | def new(kind, name), 47 | do: %__MODULE__{kind: kind, name: name} 48 | 49 | @spec new(String.t(), String.t() | integer, t) :: t 50 | @doc "Creates a new `Diplomat.Key` from a kind, an id or name, and a parent Entity" 51 | def new(kind, id_or_name, %__MODULE__{} = parent), 52 | do: %{new(kind, id_or_name) | parent: parent} 53 | 54 | @spec from_path(key_pair | [key_pair]) :: t 55 | @doc """ 56 | Creates a new `Diplomat.Key` from a path. The path should be either a list 57 | with two elements or a list of lists of two elements. Each two element list 58 | represents a segment of the key path. 59 | """ 60 | def from_path([[kind, id] | tail]), 61 | do: from_path(tail, new(kind, id)) 62 | 63 | def from_path([_, _] = path), 64 | do: from_path([path]) 65 | 66 | defp from_path([], parent), 67 | do: parent 68 | 69 | defp from_path([[kind, id] | tail], parent), 70 | do: from_path(tail, new(kind, id, parent)) 71 | 72 | @spec proto(nil | t) :: nil | PbKey.t() 73 | @doc """ 74 | Convert a `Diplomat.Key` to its protobuf struct. 75 | 76 | It should be noted that this function does not convert the struct to its 77 | binary representation, but instead returns a `Diplomat.Proto.Key` struct 78 | (which is later converted to the binary format). 79 | """ 80 | def proto(nil), do: nil 81 | 82 | def proto(%__MODULE__{} = key) do 83 | path_els = 84 | key 85 | |> path 86 | |> path_to_proto([]) 87 | |> Enum.reverse() 88 | 89 | partition = 90 | case key.project_id || key.namespace do 91 | nil -> 92 | nil 93 | 94 | _ -> 95 | {:ok, global_project_id} = Goth.Config.get(:project_id) 96 | 97 | PbPartition.new( 98 | project_id: key.project_id || global_project_id, 99 | namespace_id: key.namespace 100 | ) 101 | end 102 | 103 | PbKey.new(partition_id: partition, path: path_els) 104 | end 105 | 106 | @spec from_proto(nil | PbKey.t()) :: t 107 | @doc """ 108 | Creates a new `Diplomat.Key` from a `Diplomat.Proto.Key` struct 109 | """ 110 | def from_proto(nil), do: nil 111 | 112 | def from_proto(%PbKey{partition_id: nil, path: path_el}), 113 | do: from_path_proto(path_el, []) 114 | 115 | def from_proto(%PbKey{ 116 | partition_id: %PbPartition{project_id: pid, namespace_id: ns}, 117 | path: path_el 118 | }), 119 | do: %{from_path_proto(path_el, []) | project_id: pid, namespace: ns} 120 | 121 | @spec path(t) :: [key_pair] 122 | @doc """ 123 | Generates a path list ffor a given key. The path identifies the Enity's 124 | location nested within another `Diplomat.Entity`. 125 | """ 126 | def path(key) do 127 | key 128 | |> ancestors_and_self([]) 129 | |> generate_path([]) 130 | end 131 | 132 | @spec from_commit_proto(CommitResponse.t()) :: [t] 133 | def from_commit_proto(%CommitResponse{mutation_results: results}) do 134 | results 135 | |> Enum.map(&Key.from_proto(&1.key)) 136 | end 137 | 138 | @spec incomplete?(t) :: boolean 139 | @doc """ 140 | Determines whether or not a `Key` is incomplete. This is most useful when 141 | figuring out whether or not we must first call the API to allocate an ID for 142 | the associated entity. 143 | """ 144 | def incomplete?(%__MODULE__{id: nil, name: nil}), do: true 145 | def incomplete?(%__MODULE__{}), do: false 146 | 147 | @spec complete?(t) :: boolean 148 | def complete?(%__MODULE__{} = k), do: !incomplete?(k) 149 | 150 | @spec from_allocate_ids_proto(AllocateIdsResponse.t()) :: [t] 151 | def from_allocate_ids_proto(%AllocateIdsResponse{keys: keys}) do 152 | keys |> Enum.map(&from_proto(&1)) 153 | end 154 | 155 | @spec allocate_ids(String.t(), pos_integer) :: [t] | Client.error() 156 | def allocate_ids(type, count \\ 1) do 157 | keys = 158 | Enum.map(1..count, fn _ -> 159 | type |> new() |> proto() 160 | end) 161 | 162 | AllocateIdsRequest.new(key: keys) 163 | |> Diplomat.Client.allocate_ids() 164 | end 165 | 166 | @spec get([t] | t) :: [Entity.t()] | Client.error() 167 | def get(keys) when is_list(keys) do 168 | %LookupRequest{ 169 | keys: Enum.map(keys, &proto(&1)) 170 | } 171 | |> Diplomat.Client.lookup() 172 | end 173 | 174 | def get(%__MODULE__{} = key) do 175 | get([key]) 176 | end 177 | 178 | @spec path_to_proto([key_pair], [PbPathElement.t()]) :: [PbPathElement.t()] 179 | defp path_to_proto([], acc), do: acc 180 | 181 | defp path_to_proto([[kind, id] | tail], acc) when is_nil(id) do 182 | path_to_proto(tail, [PbPathElement.new(kind: kind, id_type: nil) | acc]) 183 | end 184 | 185 | defp path_to_proto([[kind, id] | tail], acc) when is_integer(id) do 186 | path_to_proto(tail, [PbPathElement.new(kind: kind, id_type: {:id, id}) | acc]) 187 | end 188 | 189 | defp path_to_proto([[kind, name] | tail], acc) do 190 | path_to_proto(tail, [PbPathElement.new(kind: kind, id_type: {:name, name}) | acc]) 191 | end 192 | 193 | defp from_path_proto([], acc), 194 | do: acc |> Enum.reverse() |> from_path 195 | 196 | defp from_path_proto([head | tail], acc) do 197 | case head.id_type do 198 | {:id, id} -> from_path_proto(tail, [[head.kind, id] | acc]) 199 | # in case value return as char list 200 | {:name, name} -> from_path_proto(tail, [[head.kind, to_string(name)] | acc]) 201 | nil -> from_path_proto(tail, [[head.kind, nil] | acc]) 202 | end 203 | end 204 | 205 | defp ancestors_and_self(nil, acc), do: Enum.reverse(acc) 206 | 207 | defp ancestors_and_self(key, acc) do 208 | ancestors_and_self(key.parent, [key | acc]) 209 | end 210 | 211 | defp generate_path([], acc), do: acc 212 | 213 | defp generate_path([key | tail], acc) do 214 | generate_path(tail, [[key.kind, key.id || key.name] | acc]) 215 | end 216 | end 217 | 218 | defimpl Jason.Encoder, for: Diplomat.Key do 219 | def encode(key, options) do 220 | Jason.Encode.list(Diplomat.Key.path(key), options) 221 | end 222 | end 223 | -------------------------------------------------------------------------------- /lib/diplomat/key_utils.ex: -------------------------------------------------------------------------------- 1 | defmodule KeyUtils do 2 | use Bitwise 3 | alias Diplomat.Key 4 | 5 | @spec urlsafe(Key.t()) :: String.t() 6 | def urlsafe(%Key{} = key) do 7 | key 8 | |> encode 9 | |> Base.url_encode64(padding: false) 10 | end 11 | 12 | @spec from_urlsafe(bitstring) :: {:ok, Key.t()} | {:error, String.t()} 13 | def from_urlsafe(value) when is_bitstring(value) do 14 | key = 15 | value 16 | |> Base.url_decode64!(padding: false) 17 | |> decode 18 | 19 | {:ok, key} 20 | rescue 21 | _ -> {:error, "Invalid data"} 22 | end 23 | 24 | ############################################################################# 25 | # PRIVATE FUNCTIONS 26 | ############################################################################# 27 | 28 | # encode function 29 | 30 | defp encode(%Key{} = key) do 31 | result = 32 | put_int(106) <> 33 | put_prefix_string(key.project_id) <> 34 | put_int(114) <> put_int(path_byte_size(key)) <> encode(Key.path(key)) 35 | 36 | if key.namespace do 37 | result <> put_int(162) <> put_prefix_string(key.namespace) 38 | else 39 | result 40 | end 41 | end 42 | 43 | defp encode([_head | _tail] = list) do 44 | list 45 | |> Enum.map(&encode_path_item(&1)) 46 | |> Enum.reduce(<<>>, &(&2 <> &1)) 47 | end 48 | 49 | defp encode_path_item([kind, id]) when is_integer(id) do 50 | put_int(11) <> 51 | put_int(18) <> put_prefix_string(kind) <> put_int(24) <> put_int_64(id) <> put_int(12) 52 | end 53 | 54 | defp encode_path_item([kind, name]) when is_bitstring(name) do 55 | put_int(11) <> 56 | put_int(18) <> 57 | put_prefix_string(kind) <> put_int(34) <> put_prefix_string(name) <> put_int(12) 58 | end 59 | 60 | defp put_int(v) when is_integer(v) do 61 | if (v &&& 127) == v do 62 | <> 63 | else 64 | do_put_int(v, <<>>) 65 | end 66 | end 67 | 68 | defp put_int_64(v) when is_integer(v) do 69 | do_put_int(v, <<>>) 70 | end 71 | 72 | defp do_put_int(0, acc) do 73 | acc 74 | end 75 | 76 | defp do_put_int(v, acc) when v < 0 do 77 | do_put_int((v + 1) <<< 64, acc) 78 | end 79 | 80 | defp do_put_int(v, acc) do 81 | next_v = v >>> 7 82 | next_bit = v &&& 127 83 | 84 | if next_v > 0 do 85 | do_put_int(next_v, acc <> <>) 86 | else 87 | acc <> <> 88 | end 89 | end 90 | 91 | defp put_prefix_string(v) when is_bitstring(v) do 92 | put_int(String.length(v)) <> v 93 | end 94 | 95 | # decode functions 96 | 97 | defp decode(data) do 98 | {106, data} = get_int(data) 99 | {project_id, data} = get_prefix_string(data) 100 | {114, data} = get_int(data) 101 | {path_size, data} = get_int(data) 102 | <> = data 103 | key = path_data |> decode_path([]) |> Key.from_path() 104 | 105 | if left == "" do 106 | %{key | project_id: project_id} 107 | else 108 | {162, data} = get_int(left) 109 | {namespace, _data} = get_prefix_string(data) 110 | %{key | project_id: project_id, namespace: namespace} 111 | end 112 | end 113 | 114 | defp decode_path(<<>>, path) do 115 | path 116 | end 117 | 118 | defp decode_path(data, path) do 119 | {11, data} = get_int(data) 120 | {18, data} = get_int(data) 121 | {kind, data} = get_prefix_string(data) 122 | {value, data} = get_int(data) 123 | 124 | case value do 125 | 24 -> 126 | {id, data} = get_int_64(data) 127 | {12, data} = get_int(data) 128 | decode_path(data, path ++ [[kind, id]]) 129 | 130 | 34 -> 131 | {name, data} = get_prefix_string(data) 132 | {12, data} = get_int(data) 133 | decode_path(data, path ++ [[kind, name]]) 134 | end 135 | end 136 | 137 | defp get_int(<>) do 138 | if (b &&& 128) == 0 do 139 | {b, data} 140 | else 141 | {result, next_data} = do_get_int(data, b, 0, 0) 142 | 143 | result = 144 | if result > 1 <<< 63 do 145 | result - (1 <<< 64) 146 | else 147 | result 148 | end 149 | 150 | {result, next_data} 151 | end 152 | end 153 | 154 | defp do_get_int(data, b, shift, result) do 155 | result = result ||| (b &&& 127) <<< shift 156 | shift = shift + 7 157 | 158 | if (b &&& 128) == 0 do 159 | {result, data} 160 | else 161 | <> = data 162 | do_get_int(next_data, next_b, shift, result) 163 | end 164 | end 165 | 166 | defp get_int_64(data) do 167 | {result, next_data} = do_get_int_64(data, 0, 0) 168 | 169 | result = 170 | if result > 1 <<< 63 do 171 | result - (1 <<< 64) 172 | else 173 | result 174 | end 175 | 176 | {result, next_data} 177 | end 178 | 179 | defp do_get_int_64(<>, result, shift) do 180 | result = result ||| (b &&& 127) <<< shift 181 | 182 | if (b &&& 128) == 0 do 183 | {result, next_data} 184 | else 185 | do_get_int_64(next_data, result, shift + 7) 186 | end 187 | end 188 | 189 | defp get_prefix_string(data) do 190 | {size, left_data} = get_int(data) 191 | <> = left_data 192 | {result, bin} 193 | end 194 | 195 | # get size functions 196 | 197 | defp path_byte_size(%Key{} = key) do 198 | key 199 | |> Key.path() 200 | |> Enum.map(&path_item_byte_size(&1)) 201 | |> Enum.reduce(0, &(&1 + &2)) 202 | end 203 | 204 | defp path_item_byte_size([kind, id]) when is_integer(id) do 205 | length_string(String.length(kind)) + length_var_int_64(id) + 4 206 | end 207 | 208 | defp path_item_byte_size([kind, name]) when is_bitstring(name) do 209 | length_string(String.length(kind)) + length_string(String.length(name)) + 4 210 | end 211 | 212 | defp length_string(v), do: length_var_int_64(v) + v 213 | 214 | defp length_var_int_64(v) when v < 0, do: 10 215 | defp length_var_int_64(v), do: do_length_var_int_64(v, 0) 216 | 217 | defp do_length_var_int_64(0, acc), do: acc 218 | defp do_length_var_int_64(v, acc), do: do_length_var_int_64(v >>> 7, acc + 1) 219 | end 220 | -------------------------------------------------------------------------------- /lib/diplomat/query.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Query do 2 | alias Diplomat.{Cursor, Query, Value} 3 | alias Diplomat.Proto.{GqlQuery, GqlQueryParameter, RunQueryRequest, PartitionId} 4 | 5 | defstruct query: nil, numbered_args: [], named_args: %{} 6 | 7 | @type t :: %__MODULE__{ 8 | query: String.t() | nil, 9 | numbered_args: args_list, 10 | named_args: args_map 11 | } 12 | 13 | @type args_list :: [any] 14 | @type args_map :: %{optional(atom) => any} 15 | 16 | @spec new(String.t()) :: t 17 | def new(query), do: new(query, []) 18 | 19 | @spec new(String.t(), args_map | args_list) :: t 20 | def new(query, args) when is_list(args) and is_binary(query) do 21 | %Query{ 22 | query: query, 23 | numbered_args: args 24 | } 25 | end 26 | 27 | def new(query, args) when is_map(args) and is_binary(query) do 28 | %Query{ 29 | query: query, 30 | named_args: args 31 | } 32 | end 33 | 34 | @spec proto(t) :: GqlQuery.t() 35 | def proto(%Query{query: q, numbered_args: num, named_args: named}) do 36 | GqlQuery.new( 37 | query_string: q, 38 | allow_literals: allow_literals(num, named), 39 | positional_bindings: positional_bindings(num), 40 | named_bindings: named_bindings(named) 41 | ) 42 | end 43 | 44 | @spec execute(t, String.t() | nil) :: [Entity.t()] | Client.error() 45 | def execute(%__MODULE__{} = q, namespace \\ nil) do 46 | {:ok, project} = Goth.Config.get(:project_id) 47 | 48 | RunQueryRequest.new( 49 | query_type: {:gql_query, q |> Query.proto()}, 50 | partition_id: PartitionId.new(namespace_id: namespace, project_id: project) 51 | ) 52 | |> Diplomat.Client.run_query() 53 | end 54 | 55 | @spec execute_with_pagination(t, String.t() | nil) :: [QueryResultBatch.t()] | Client.error() 56 | def execute_with_pagination(%__MODULE__{} = q, namespace \\ nil) do 57 | {:ok, project} = Goth.Config.get(:project_id) 58 | 59 | RunQueryRequest.new( 60 | query_type: {:gql_query, q |> Query.proto()}, 61 | partition_id: PartitionId.new(namespace_id: namespace, project_id: project) 62 | ) 63 | |> Diplomat.Client.run_query_with_pagination() 64 | end 65 | 66 | @spec positional_bindings(args_list) :: [GqlQueryParameter.t()] 67 | defp positional_bindings(args) do 68 | Enum.map(args, fn 69 | %Cursor{} = cursor -> 70 | GqlQueryParameter.new(parameter_type: {:cursor, Cursor.decode(cursor)}) 71 | 72 | i -> 73 | val = i |> Value.new() |> Value.proto() 74 | GqlQueryParameter.new(parameter_type: {:value, val}) 75 | end) 76 | end 77 | 78 | @spec named_bindings(args_map) :: [{String.t(), GqlQueryParameter.t()}] 79 | defp named_bindings(args) do 80 | args 81 | |> Enum.map(fn {k, v} -> 82 | val = v |> Value.new() |> Value.proto() 83 | {to_string(k), GqlQueryParameter.new(parameter_type: {:value, val})} 84 | end) 85 | end 86 | 87 | defp allow_literals([], {}), do: false 88 | defp allow_literals(_, _), do: true 89 | end 90 | -------------------------------------------------------------------------------- /lib/diplomat/query_result_batch.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.QueryResultBatch do 2 | alias Diplomat.{Cursor, Entity} 3 | alias Diplomat.Proto.QueryResultBatch, as: PBQueryResultBatch 4 | 5 | @type t :: %__MODULE__{entity_results: [Entity.t()], end_cursor: Cursor.t() | nil} 6 | 7 | defstruct entity_results: [], end_cursor: nil 8 | 9 | @spec from_proto(PBQueryResultBatch.t()) :: t 10 | def from_proto(%PBQueryResultBatch{entity_results: entity_results, end_cursor: end_cursor}) do 11 | %__MODULE__{ 12 | entity_results: entities_from_proto(entity_results), 13 | end_cursor: maybe_create_cursor(end_cursor) 14 | } 15 | end 16 | 17 | defp entities_from_proto(entity_results) do 18 | Enum.map(entity_results, &Entity.from_proto(&1.entity)) 19 | end 20 | 21 | defp maybe_create_cursor(nil), do: nil 22 | defp maybe_create_cursor(end_cursor), do: Cursor.new(end_cursor, encode: true) 23 | end 24 | -------------------------------------------------------------------------------- /lib/diplomat/transaction.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Transaction do 2 | @moduledoc """ 3 | ``` 4 | Transaction.begin 5 | |> Transaction.save(entity) 6 | |> Transaction.save(entity2) 7 | |> Transaction.commit 8 | ``` 9 | 10 | OR 11 | 12 | ``` 13 | Transaction.begin fn t -> 14 | # auto-begin 15 | t 16 | |> Transaction.save(entity) 17 | |> Transaction.save(entity2) 18 | # auto-commits on exit 19 | end 20 | ``` 21 | """ 22 | 23 | alias Diplomat.Proto.BeginTransactionResponse, as: TransResponse 24 | alias Diplomat.Proto.BeginTransactionRequest, as: TransRequest 25 | 26 | alias Diplomat.Proto.{ 27 | CommitRequest, 28 | CommitResponse, 29 | RollbackRequest, 30 | RollbackResponse, 31 | LookupRequest, 32 | ReadOptions 33 | } 34 | 35 | alias Diplomat.{Transaction, Entity, Key, Client} 36 | 37 | @type t :: %__MODULE__{ 38 | id: integer, 39 | state: :init | :begun, 40 | mutations: [Entity.mutation()] 41 | } 42 | 43 | defstruct id: nil, state: :init, mutations: [] 44 | 45 | @spec from_begin_response(TransResponse.t()) :: t 46 | def from_begin_response(%TransResponse{transaction: id}) do 47 | %Transaction{id: id, state: :begun} 48 | end 49 | 50 | @spec begin() :: t | Client.error() 51 | def begin do 52 | TransRequest.new() 53 | |> Diplomat.Client.begin_transaction() 54 | |> case do 55 | {:ok, resp} -> 56 | resp |> Transaction.from_begin_response() 57 | 58 | other -> 59 | other 60 | end 61 | end 62 | 63 | @spec begin((() -> t)) :: {:ok, CommitResponse.t()} | Client.error() | {:error, any} 64 | def begin(block) when is_function(block) do 65 | # the try block defines a new scope that isn't accessible in the rescue block 66 | # so we need to begin the transaction here so both have access to the var 67 | transaction = begin() 68 | 69 | try do 70 | transaction 71 | |> block.() 72 | |> commit 73 | rescue 74 | e -> 75 | rollback(transaction) 76 | {:error, e} 77 | end 78 | end 79 | 80 | @spec commit(t) :: {:ok, CommitResponse.t()} | Client.error() 81 | def commit(%Transaction{} = transaction) do 82 | transaction 83 | |> to_commit_proto 84 | |> Diplomat.Client.commit() 85 | end 86 | 87 | # require the transaction to have an ID 88 | @spec rollback(t) :: {:ok, RollbackResponse.t()} | Client.error() 89 | def rollback(%Transaction{id: id}) when not is_nil(id) do 90 | RollbackRequest.new(transaction: id) 91 | |> Diplomat.Client.rollback() 92 | end 93 | 94 | # handle the case where we attempt to rollback something that isn't a transaction 95 | # this should only happen when beginning a transaction fails 96 | def rollback(t), do: t 97 | 98 | @spec find(Transaction.t(), Key.t() | [Key.t()]) :: list(Entity.t()) | Client.error() 99 | def find(%Transaction{id: id}, keys) when is_list(keys) do 100 | %LookupRequest{ 101 | keys: Enum.map(keys, &Key.proto(&1)), 102 | read_options: %ReadOptions{consistency_type: {:transaction, id}} 103 | } 104 | |> Diplomat.Client.lookup() 105 | end 106 | 107 | def find(transaction, key) do 108 | find(transaction, [key]) 109 | end 110 | 111 | # we could clean this up with some macros 112 | @spec insert(t, Entity.t() | [Entity.t()]) :: t 113 | def insert(%Transaction{} = t, %Entity{} = e), do: insert(t, [e]) 114 | def insert(%Transaction{} = t, []), do: t 115 | 116 | def insert(%Transaction{} = t, [%Entity{} = e | tail]) do 117 | insert(%{t | mutations: [{:insert, e} | t.mutations]}, tail) 118 | end 119 | 120 | @spec upsert(t, Entity.t() | [Entity.t()]) :: t 121 | def upsert(%Transaction{} = t, %Entity{} = e), do: upsert(t, [e]) 122 | def upsert(%Transaction{} = t, []), do: t 123 | 124 | def upsert(%Transaction{} = t, [%Entity{} = e | tail]) do 125 | upsert(%{t | mutations: [{:upsert, e} | t.mutations]}, tail) 126 | end 127 | 128 | @spec update(t, Entity.t() | [Entity.t()]) :: t 129 | def update(%Transaction{} = t, %Entity{} = e), do: update(t, [e]) 130 | def update(%Transaction{} = t, []), do: t 131 | 132 | def update(%Transaction{} = t, [%Entity{} = e | tail]) do 133 | update(%{t | mutations: [{:update, e} | t.mutations]}, tail) 134 | end 135 | 136 | @spec delete(t, Key.t() | [Key.t()]) :: t 137 | def delete(%Transaction{} = t, %Key{} = k), do: delete(t, [k]) 138 | def delete(%Transaction{} = t, []), do: t 139 | 140 | def delete(%Transaction{} = t, [%Key{} = k | tail]) do 141 | delete(%{t | mutations: [{:delete, k} | t.mutations]}, tail) 142 | end 143 | 144 | @spec to_commit_proto(t) :: CommitRequest.t() 145 | def to_commit_proto(%Transaction{} = transaction) do 146 | CommitRequest.new( 147 | mode: :TRANSACTIONAL, 148 | transaction_selector: {:transaction, transaction.id}, 149 | mutations: Entity.extract_mutations(transaction.mutations, []) 150 | ) 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /lib/diplomat/value.ex: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Value do 2 | alias Diplomat.Proto.Value, as: PbVal 3 | alias Diplomat.Proto.Key, as: PbKey 4 | alias Diplomat.Proto.Entity, as: PbEntity 5 | alias Diplomat.Proto.ArrayValue, as: PbArray 6 | alias Diplomat.Proto.Timestamp, as: PbTimestamp 7 | alias Diplomat.Proto.LatLng, as: PbLatLng 8 | alias Diplomat.{Entity, Key} 9 | 10 | @type t :: %__MODULE__{ 11 | value: any, 12 | exclude_from_indexes: boolean 13 | } 14 | 15 | defstruct value: nil, exclude_from_indexes: false 16 | 17 | @spec new(any, Keyword.t()) :: t 18 | def new(val, opts \\ []) 19 | 20 | def new(val = %{__struct__: struct}, opts) 21 | when struct in [Diplomat.Entity, Diplomat.Key, Diplomat.Value, DateTime, NaiveDateTime], 22 | do: %__MODULE__{ 23 | value: val, 24 | exclude_from_indexes: Keyword.get(opts, :exclude_from_indexes) == true 25 | } 26 | 27 | def new(val = %{__struct__: _struct}, opts), 28 | do: new(Map.from_struct(val), opts) 29 | 30 | def new(val, opts) when is_map(val), 31 | do: %__MODULE__{ 32 | value: Entity.new(val, opts), 33 | exclude_from_indexes: Keyword.get(opts, :exclude_from_indexes) == true 34 | } 35 | 36 | def new(val, opts) when is_list(val), 37 | do: %__MODULE__{ 38 | value: Enum.map(val, &new(&1, opts)), 39 | exclude_from_indexes: Keyword.get(opts, :exclude_from_indexes) == true 40 | } 41 | 42 | # match a string that is > 1500 bytes 43 | def new(<> = full, opts) do 44 | _ = 45 | opts 46 | |> Keyword.get(:truncate) 47 | |> case do 48 | true -> 49 | new(first, opts) 50 | 51 | _ -> 52 | %__MODULE__{ 53 | value: full, 54 | exclude_from_indexes: Keyword.get(opts, :exclude_from_indexes) == true 55 | } 56 | end 57 | end 58 | 59 | def new(val, opts), 60 | do: %__MODULE__{ 61 | value: val, 62 | exclude_from_indexes: Keyword.get(opts, :exclude_from_indexes) == true 63 | } 64 | 65 | @spec from_proto(PbVal.t()) :: t 66 | def from_proto(%PbVal{value_type: {:boolean_value, val}}) when is_boolean(val), 67 | do: new(val) 68 | 69 | def from_proto(%PbVal{value_type: {:integer_value, val}}) when is_integer(val), 70 | do: new(val) 71 | 72 | def from_proto(%PbVal{value_type: {:double_value, val}}) when is_float(val), 73 | do: new(val) 74 | 75 | def from_proto(%PbVal{value_type: {:string_value, val}}), 76 | do: new(to_string(val)) 77 | 78 | def from_proto(%PbVal{value_type: {:blob_value, val}}) when is_bitstring(val), 79 | do: new(val) 80 | 81 | def from_proto(%PbVal{value_type: {:key_value, %PbKey{} = val}}), 82 | do: val |> Diplomat.Key.from_proto() |> new 83 | 84 | def from_proto(%PbVal{value_type: {:entity_value, %PbEntity{} = val}}), 85 | do: val |> Diplomat.Entity.from_proto() |> new 86 | 87 | def from_proto(%PbVal{value_type: {:array_value, %PbArray{} = val}}) do 88 | %__MODULE__{value: Enum.map(val.values, &from_proto/1)} 89 | end 90 | 91 | def from_proto(%PbVal{value_type: {:timestamp_value, %PbTimestamp{} = val}}) do 92 | (val.seconds * 1_000_000_000 + (val.nanos || 0)) 93 | |> DateTime.from_unix!(:nanosecond) 94 | |> new 95 | end 96 | 97 | def from_proto(%PbVal{value_type: {:geo_point_value, %PbLatLng{} = val}}), 98 | do: new({val.latitude, val.longitude}) 99 | 100 | def from_proto(_), 101 | do: new(nil) 102 | 103 | # convert to protocol buffer struct 104 | @spec proto(any, Keyword.t()) :: PbVal.t() 105 | def proto(val, opts \\ []) 106 | 107 | def proto(%PbVal{} = val, opts) do 108 | %PbVal{val | exclude_from_indexes: Keyword.get(opts, :exclude_from_indexes, false)} 109 | end 110 | 111 | def proto(nil, opts), 112 | do: proto(PbVal.new(value_type: {:null_value, :NULL_VALUE}), opts) 113 | 114 | def proto(%__MODULE__{value: val, exclude_from_indexes: exclude}, opts) do 115 | opts = 116 | Keyword.merge(opts, [exclude_from_indexes: exclude], fn 117 | _, v1, v2 when is_list(v1) and is_list(v2) -> 118 | (v1 ++ v2) |> Enum.uniq() 119 | 120 | _, v1, _ -> 121 | v1 122 | end) 123 | 124 | proto(val, opts) 125 | end 126 | 127 | def proto(val, opts) when is_boolean(val), 128 | do: proto(PbVal.new(value_type: {:boolean_value, val}), opts) 129 | 130 | def proto(val, opts) when is_integer(val), 131 | do: proto(PbVal.new(value_type: {:integer_value, val}), opts) 132 | 133 | def proto(val, opts) when is_float(val), 134 | do: proto(PbVal.new(value_type: {:double_value, val}), opts) 135 | 136 | def proto(val, opts) when is_atom(val), 137 | do: val |> to_string() |> proto(opts) 138 | 139 | def proto(val, opts) when is_binary(val) do 140 | case String.valid?(val) do 141 | true -> 142 | proto(PbVal.new(value_type: {:string_value, val}), opts) 143 | 144 | false -> 145 | proto(PbVal.new(value_type: {:blob_value, val}), opts) 146 | end 147 | end 148 | 149 | def proto(val, opts) when is_bitstring(val), 150 | do: proto(PbVal.new(value_type: {:blob_value, val}), opts) 151 | 152 | def proto(val, opts) when is_list(val), 153 | do: proto_list(val, [], opts) 154 | 155 | def proto(%DateTime{} = val, opts) do 156 | timestamp = DateTime.to_unix(val, :nanosecond) 157 | 158 | PbVal.new( 159 | value_type: { 160 | :timestamp_value, 161 | %PbTimestamp{seconds: div(timestamp, 1_000_000_000), nanos: rem(timestamp, 1_000_000_000)} 162 | } 163 | ) 164 | |> proto(opts) 165 | end 166 | 167 | def proto(%Key{} = val, opts), 168 | do: proto(PbVal.new(value_type: {:key_value, Key.proto(val)}), opts) 169 | 170 | def proto(%{} = val, opts), 171 | do: proto(PbVal.new(value_type: {:entity_value, Diplomat.Entity.proto(val)}), opts) 172 | 173 | # might need to be more explicit about this... 174 | def proto({latitude, longitude}, opts) when is_float(latitude) and is_float(longitude), 175 | do: 176 | proto( 177 | PbVal.new( 178 | value_type: {:geo_point_value, %PbLatLng{latitude: latitude, longitude: longitude}} 179 | ), 180 | opts 181 | ) 182 | 183 | defp proto_list([], acc, opts) do 184 | PbVal.new( 185 | value_type: { 186 | :array_value, 187 | %PbArray{values: acc} 188 | } 189 | ) 190 | |> proto(opts) 191 | end 192 | 193 | defp proto_list([head | tail], acc, opts) do 194 | proto_list(tail, acc ++ [proto(head, opts)], opts) 195 | end 196 | end 197 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :diplomat, 7 | version: "0.12.1", 8 | elixir: "~> 1.5", 9 | description: "A library for interacting with Google's Cloud Datastore", 10 | package: package(), 11 | deps: deps(), 12 | elixirc_paths: elixirc_paths(Mix.env()), 13 | dialyzer: [ignore_warnings: ".dialyzer.ignore-warnings"] 14 | ] 15 | end 16 | 17 | def application do 18 | [] 19 | end 20 | 21 | # Specifies which paths to compile per environment. 22 | defp elixirc_paths(:test), do: ["lib", "test/support"] 23 | defp elixirc_paths(_), do: ["lib"] 24 | 25 | defp deps do 26 | [ 27 | {:goth, "~> 1.0"}, 28 | {:exprotobuf, "~> 1.2"}, 29 | {:httpoison, "~> 1.0"}, 30 | {:jason, "~> 1.1"}, 31 | {:credo, "~> 1.0", only: [:dev, :test]}, 32 | {:bypass, "~> 0.8", only: :test}, 33 | {:plug_cowboy, "~> 1.0", only: :test}, 34 | {:mix_test_watch, "~> 0.4", only: :dev}, 35 | {:ex_doc, "~> 0.19", only: :dev}, 36 | {:uuid, "~> 1.1", only: :test}, 37 | {:dialyxir, "~> 0.5", only: [:dev, :test], runtime: false} 38 | ] 39 | end 40 | 41 | defp package do 42 | [ 43 | maintainers: ["Phil Burrows"], 44 | licenses: ["MIT"], 45 | links: %{"GitHub" => "https://github.com/peburrows/diplomat"} 46 | ] 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm", "fab09b20e3f5db886725544cbcf875b8e73ec93363954eb8a1a9ed834aa8c1f9"}, 3 | "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, 4 | "bypass": {:hex, :bypass, "0.8.1", "16d409e05530ece4a72fabcf021a3e5c7e15dcc77f911423196a0c551f2a15ca", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "61fc89e67e448785905d35b2b06a3a36ae0cf0857c343fd65c753af42406f31a"}, 5 | "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "01d479edba0569a7b7a2c8bf923feeb6dc6a358edc2965ef69aea9ba288bb243"}, 6 | "cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "f4763bbe08233eceed6f24bc4fcc8d71c17cfeafa6439157c57349aa1bb4f17c"}, 7 | "cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], [], "hexpm", "db622da03aa039e6366ab953e31186cc8190d32905e33788a1acb22744e6abd2"}, 8 | "credo": {:hex, :credo, "1.0.1", "5a5bc382cf0a12cc7db64cc018526aee05db572c60e867f6bc4b647d7ef9fc61", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "7006af85d751454a6ac01af37ea098304c398ae53ad51976640e9ea4bd8cc699"}, 9 | "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm", "6c32a70ed5d452c6650916555b1f96c79af5fc4bf286997f8b15f213de786f73"}, 10 | "earmark": {:hex, :earmark, "1.3.1", "73812f447f7a42358d3ba79283cfa3075a7580a3a2ed457616d6517ac3738cb9", [:mix], [], "hexpm", "000aaeff08919e95e7aea13e4af7b2b9734577b3e6a7c50ee31ee88cab6ec4fb"}, 11 | "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0e11d67e662142fc3945b0ee410c73c8c956717fbeae4ad954b418747c734973"}, 12 | "exprotobuf": {:hex, :exprotobuf, "1.2.9", "f3cac1b0d0444da3c72cdfe80e394d721275dc80b1d7703ead9dad9267e93822", [:mix], [{:gpb, "~> 3.24", [hex: :gpb, repo: "hexpm", optional: false]}], "hexpm", "9789b99ee31e1d54107491ef14a417924e4b7e44d18c8370a2c197bedba9c839"}, 13 | "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm", "0d50da6b04c58e101a3793b1600f9a03b86e3a8057b192ac1766013d35706fa6"}, 14 | "goth": {:hex, :goth, "1.0.1", "191773b527db4ae6695d9bab62602bda6645a367a95e2acaca0d51f61fe20e4d", [:mix], [{:httpoison, "~> 0.11 or ~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.0", [hex: :joken, repo: "hexpm", optional: false]}], "hexpm", "79df601b116c3dcd5c979db46640bde0a062816665761ab023dfe35b68b2197c"}, 15 | "gpb": {:hex, :gpb, "3.28.1", "6849b2f0004dc4e7644f4f67e7cdd18e893f0ab87eb7ad82b9cb1483ce60eed0", [:make, :rebar], [], "hexpm", "1e645d0cbe23c6c7574bdf8e38483156e02ba898e0eab28fb736d2f71ad78cc9"}, 16 | "hackney": {:hex, :hackney, "1.15.0", "287a5d2304d516f63e56c469511c42b016423bcb167e61b611f6bad47e3ca60e", [:rebar3], [{:certifi, "2.4.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "b69d97134f1876ba8e4e2f405e9da8cba7cf4f2da0b7cc24a5ccef8dcf1b46b2"}, 17 | "httpoison": {:hex, :httpoison, "1.5.0", "71ae9f304bdf7f00e9cd1823f275c955bdfc68282bc5eb5c85c3a9ade865d68e", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "e9d994aea63fab9e29307920492ab95f87339b56fbc5c8c4b1f65ea20d3ba9a4"}, 18 | "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, 19 | "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, 20 | "joken": {:hex, :joken, "2.0.0", "ff10fca10ec539d7a73874da303f4a7a975fea53fcd59b1b89dda2a71ecb4c6b", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "03950bcdfa594e649193a976cc30cda7afa9e5439dce9d699ea6d4895593cb13"}, 21 | "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm", "6429c4fee52b2dda7861ee19a4f09c8c1ffa213bee3a1ec187828fde95d447ed"}, 22 | "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5fbc8e549aa9afeea2847c0769e3970537ed302f93a23ac612602e805d9d1e7f"}, 23 | "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "adf0218695e22caeda2820eaba703fa46c91820d53813a2223413da3ef4ba515"}, 24 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, 25 | "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"}, 26 | "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm", "7a4c8e1115a2732a67d7624e28cf6c9f30c66711a9e92928e745c255887ba465"}, 27 | "mix_test_watch": {:hex, :mix_test_watch, "0.6.0", "5e206ed04860555a455de2983937efd3ce79f42bd8536fc6b900cc286f5bb830", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "ea6f2a3766f18c2f53ca5b2d40b623ce2831c1646f36ff2b608607e20fc6c63c"}, 28 | "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm", "5c040b8469c1ff1b10093d3186e2e10dbe483cd73d79ec017993fb3985b8a9b3"}, 29 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, 30 | "plug": {:hex, :plug, "1.7.1", "8516d565fb84a6a8b2ca722e74e2cd25ca0fc9d64f364ec9dbec09d33eb78ccd", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}], "hexpm", "daa5fee4209c12c3c48b05a96cf88c320b627c9575f987554dcdc1fdcdf2c15e"}, 31 | "plug_cowboy": {:hex, :plug_cowboy, "1.0.0", "2e2a7d3409746d335f451218b8bb0858301c3de6d668c3052716c909936eb57a", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "01d201427a8a1f4483be2465a98b45f5e82263327507fe93404a61c51eb9e9a8"}, 32 | "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm", "73c1682f0e414cfb5d9b95c8e8cd6ffcfdae699e3b05e1db744e58b7be857759"}, 33 | "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], [], "hexpm", "6e56493a862433fccc3aca3025c946d6720d8eedf6e3e6fb911952a7071c357f"}, 34 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm", "603561dc0fd62f4f2ea9b890f4e20e1a0d388746d6e20557cafb1b16950de88c"}, 35 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"}, 36 | "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm", "c790593b4c3b601f5dc2378baae7efaf5b3d73c4c6456ba85759905be792f2ac"}, 37 | } 38 | -------------------------------------------------------------------------------- /test/diplomat/client_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.ClientTest do 2 | use ExUnit.Case 3 | end 4 | -------------------------------------------------------------------------------- /test/diplomat/cursor_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.CursorTest do 2 | use ExUnit.Case 3 | 4 | alias Diplomat.Cursor 5 | 6 | @decoded_cursor_value <<255, 127, 254, 252>> 7 | 8 | describe "Cursor.new/1" do 9 | test "it create a new cursor" do 10 | assert %Cursor{value: "_3_-_A=="} = Cursor.new("_3_-_A==") 11 | end 12 | 13 | test "it returns a new Cursor with an encoded value with 'encode' flag" do 14 | assert %Cursor{value: "_3_-_A=="} = Cursor.new(@decoded_cursor_value, encode: true) 15 | end 16 | end 17 | 18 | describe "Cursor.encode/1" do 19 | test "encodes a cursor value into a base64 string" do 20 | assert "_3_-_A==" == Cursor.encode(@decoded_cursor_value) 21 | end 22 | end 23 | 24 | describe "Cursor.decode/1" do 25 | test "decodes a Cursor struct" do 26 | cursor = Cursor.new(@decoded_cursor_value, encode: true) 27 | 28 | assert @decoded_cursor_value == Cursor.decode(cursor) 29 | end 30 | 31 | test "decodes a base64 encoded cursor value" do 32 | assert @decoded_cursor_value == Cursor.decode("_3_-_A==") 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/diplomat/entity/insert_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Entity.InsertTest do 2 | use ExUnit.Case 3 | alias Diplomat.Proto.CommitResponse 4 | alias Diplomat.Proto.CommitRequest 5 | alias Diplomat.Proto.MutationResult 6 | alias Diplomat.Proto.Mutation 7 | 8 | alias Diplomat.{Key, Entity} 9 | 10 | setup do 11 | bypass = Bypass.open() 12 | Application.put_env(:diplomat, :endpoint, "http://localhost:#{bypass.port}") 13 | {:ok, bypass: bypass} 14 | end 15 | 16 | test "extracting keys from CommitResponse" do 17 | response = 18 | CommitResponse.new( 19 | mutation_results: [ 20 | MutationResult.new(key: Key.new("Thing", 1) |> Key.proto()), 21 | MutationResult.new(key: Key.new("Thing", 2) |> Key.proto()) 22 | ], 23 | index_updates: 2 24 | ) 25 | 26 | keys = response |> Key.from_commit_proto() 27 | assert Enum.count(keys) == 2 28 | 29 | Enum.each(keys, fn k -> 30 | assert %Key{} = k 31 | end) 32 | end 33 | 34 | test "building a CommitRequest from a single Entity mutation" do 35 | entity = Entity.new(%{"name" => "phil"}, "Person", "phil-burrows") 36 | ent_proto = Entity.proto(entity) 37 | 38 | assert %CommitRequest{ 39 | mutations: [ 40 | %Mutation{operation: {:insert, ^ent_proto}} 41 | ], 42 | mode: :NON_TRANSACTIONAL 43 | } = Entity.commit_request([{:insert, entity}]) 44 | end 45 | 46 | test "inserting a single entity", %{bypass: bypass} do 47 | {:ok, project} = Goth.Config.get(:project_id) 48 | {kind, name} = {"TestBook", "my-book-unique-id"} 49 | 50 | entity = 51 | Entity.new( 52 | %{"name" => "My awesome book", "author" => "Phil Burrows"}, 53 | kind, 54 | name 55 | ) 56 | 57 | Bypass.expect(bypass, fn conn -> 58 | {:ok, body, conn} = Plug.Conn.read_body(conn) 59 | # ensure we're passing in the correct data 60 | assert %CommitRequest{ 61 | mutations: [ 62 | %Mutation{operation: {:insert, _ent}} 63 | ] 64 | } = CommitRequest.decode(body) 65 | 66 | assert Regex.match?(~r{/v1/projects/#{project}:commit}, conn.request_path) 67 | 68 | resp = 69 | CommitResponse.new( 70 | mutation_results: [ 71 | MutationResult.new(key: Key.new(kind, name) |> Key.proto()) 72 | ], 73 | index_updates: 1 74 | ) 75 | |> CommitResponse.encode() 76 | 77 | Plug.Conn.resp(conn, 201, resp) 78 | end) 79 | 80 | keys = Entity.insert(entity) 81 | assert Enum.count(keys) == 1 82 | retkey = hd(keys) 83 | 84 | assert retkey.kind == kind 85 | assert retkey.name == name 86 | end 87 | 88 | test "a failed insert", %{bypass: bypass} do 89 | Bypass.expect(bypass, fn conn -> 90 | resp = 91 | <<8, 3, 18, 22, 69, 110, 116, 105, 116, 121, 32, 105, 115, 32, 109, 105, 115, 115, 105, 92 | 110, 103, 32, 107, 101, 121, 46>> 93 | 94 | Plug.Conn.resp(conn, 400, resp) 95 | end) 96 | 97 | assert {:error, status} = Entity.new(%{"a" => 1}) |> Entity.insert() 98 | 99 | assert %Diplomat.Proto.Status{ 100 | code: 3, 101 | details: [], 102 | message: "Entity is missing key." 103 | } = status 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /test/diplomat/entity/upsert_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Entity.UpsertTest do 2 | use ExUnit.Case 3 | 4 | alias Diplomat.Entity 5 | alias Diplomat.Proto.{CommitResponse, CommitRequest, Mutation, MutationResult} 6 | 7 | setup do 8 | bypass = Bypass.open() 9 | Application.put_env(:diplomat, :endpoint, "http://localhost:#{bypass.port}") 10 | {:ok, bypass: bypass} 11 | end 12 | 13 | test "upserting a single Entity", %{bypass: bypass} do 14 | # entity = Entity.new(%{title: "20k Leagues", author: "Jules Verne"}, "Book", "20k-key") 15 | 16 | {:ok, project} = Goth.Config.get(:project_id) 17 | {kind, name} = {"TestBook", "my-book-unique-id"} 18 | 19 | entity = 20 | Entity.new( 21 | %{"name" => "My awesome book", "author" => "Phil Burrows"}, 22 | kind, 23 | name 24 | ) 25 | 26 | Bypass.expect(bypass, fn conn -> 27 | {:ok, body, conn} = Plug.Conn.read_body(conn) 28 | 29 | assert %CommitRequest{ 30 | mutations: [ 31 | %Mutation{operation: {:upsert, _ent}} 32 | ] 33 | } = CommitRequest.decode(body) 34 | 35 | assert Regex.match?(~r{/v1/projects/#{project}:commit}, conn.request_path) 36 | 37 | resp = 38 | CommitResponse.new( 39 | mutation_result: 40 | MutationResult.new( 41 | index_updates: 1 42 | # insert_auto_id_key: [ (Key.new(kind, name) |> Key.proto) ] 43 | ) 44 | ) 45 | |> CommitResponse.encode() 46 | 47 | Plug.Conn.resp(conn, 201, resp) 48 | end) 49 | 50 | resp = Entity.upsert(entity) 51 | assert %CommitResponse{} = resp 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/diplomat/entity_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.EntityTest do 2 | use ExUnit.Case 3 | alias Diplomat.{Entity, Value, Key} 4 | alias Diplomat.Proto.Value, as: PbValue 5 | alias Diplomat.Proto.Entity, as: PbEntity 6 | 7 | describe "Entity.new/1" do 8 | test "given a props struct" do 9 | props = %TestStruct{foo: "bar"} 10 | 11 | assert Entity.new(props) == 12 | %Entity{properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: false}}} 13 | end 14 | 15 | test "given a props map" do 16 | props = %{"foo" => "bar"} 17 | 18 | assert Entity.new(props) == 19 | %Entity{properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: false}}} 20 | end 21 | end 22 | 23 | describe "Entity.new/2" do 24 | test "given a props and opts" do 25 | props = %{"foo" => "bar"} 26 | opts = [exclude_from_indexes: :foo] 27 | 28 | assert Entity.new(props, opts) == 29 | %Entity{properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: true}}} 30 | end 31 | 32 | test "given props and a kind" do 33 | props = %{"foo" => "bar"} 34 | kind = "TestKind" 35 | 36 | assert Entity.new(props, kind) == 37 | %Entity{ 38 | kind: kind, 39 | key: %Key{kind: kind, id: nil}, 40 | properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: false}} 41 | } 42 | end 43 | 44 | test "given props and a key" do 45 | props = %{"foo" => "bar"} 46 | key = Key.new(kind: "TestKind", id: "1") 47 | 48 | assert Entity.new(props, key) == 49 | %Entity{ 50 | kind: key.kind, 51 | key: key, 52 | properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: false}} 53 | } 54 | end 55 | 56 | test "given props and opts with a nested entity" do 57 | props = %{"foo" => %{"bar" => "baz"}} 58 | opts = [exclude_from_indexes: [foo: :bar]] 59 | 60 | nested_entity = %Entity{ 61 | properties: %{"bar" => %Value{value: "baz", exclude_from_indexes: true}} 62 | } 63 | 64 | assert Entity.new(props, opts) == 65 | %Entity{ 66 | properties: %{"foo" => %Value{value: nested_entity, exclude_from_indexes: false}} 67 | } 68 | end 69 | end 70 | 71 | describe "Entity.new/3" do 72 | test "given props, a kind, and opts" do 73 | props = %{"foo" => "bar"} 74 | kind = "TestKind" 75 | opts = [exclude_from_indexes: :foo] 76 | 77 | assert Entity.new(props, kind, opts) == 78 | %Entity{ 79 | kind: kind, 80 | key: %Key{kind: kind, id: nil}, 81 | properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: true}} 82 | } 83 | end 84 | 85 | test "given props, a key, and opts" do 86 | props = %{"foo" => "bar"} 87 | key = Key.new(kind: "TestKind", id: "1") 88 | opts = [exclude_from_indexes: :foo] 89 | 90 | assert Entity.new(props, key, opts) == 91 | %Entity{ 92 | kind: key.kind, 93 | key: key, 94 | properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: true}} 95 | } 96 | end 97 | 98 | test "given props, a kind, and a string id" do 99 | props = %{"foo" => "bar"} 100 | kind = "TestKind" 101 | id = "1" 102 | key = %Key{kind: kind, name: id} 103 | 104 | assert Entity.new(props, kind, id) == 105 | %Entity{ 106 | kind: key.kind, 107 | key: key, 108 | properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: false}} 109 | } 110 | end 111 | end 112 | 113 | describe "Entity.new/4" do 114 | test "given props, a kind, a string id, and opts" do 115 | props = %{"foo" => "bar"} 116 | kind = "TestKind" 117 | id = "1" 118 | opts = [exclude_from_indexes: :foo] 119 | key = %Key{kind: kind, name: id} 120 | 121 | assert Entity.new(props, kind, id, opts) == 122 | %Entity{ 123 | kind: key.kind, 124 | key: key, 125 | properties: %{"foo" => %Value{value: "bar", exclude_from_indexes: true}} 126 | } 127 | end 128 | end 129 | 130 | test "some JSON w/o null values" do 131 | ent = 132 | ~s<{"id":1089,"log_type":"view","access_token":"778efaf8333b2ac840f097448154bb6b","ip_address":"127.0.0.1","created_at":"2016-01-28T23:03:27.000Z","updated_at":"2016-01-28T23:03:27.000Z","log_guid":"2016-1-0b68c093a68b4bb5b16b","user_guid":"58GQA26TZ567K3C65VVN","vbid":"12345","brand":"vst","user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"}> 133 | |> Jason.decode!() 134 | |> Diplomat.Entity.proto() 135 | 136 | assert <<_::binary>> = Diplomat.Proto.Entity.encode(ent) 137 | end 138 | 139 | test "decode proto" do 140 | ent = %PbEntity{ 141 | key: Key.new("User", 1) |> Key.proto(), 142 | properties: [ 143 | {"name", %PbValue{value_type: {:string_value, "elixir"}}} 144 | ] 145 | } 146 | 147 | ent |> PbEntity.encode() |> PbEntity.decode() 148 | end 149 | 150 | test "some JSON with null values" do 151 | ent = 152 | ~s<{"geo_lat":null,"geo_long":null,"id":1089,"log_type":"view","access_token":"778efaf8333b2ac840f097448154bb6b","ip_address":"127.0.0.1","created_at":"2016-01-28T23:03:27.000Z","updated_at":"2016-01-28T23:03:27.000Z","log_guid":"2016-1-0b68c093a68b4bb5b16b","user_guid":"58GQA26TZ567K3C65VVN","vbid":"12345","brand":"vst","user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"}> 153 | |> Jason.decode!() 154 | |> Diplomat.Entity.proto() 155 | 156 | # ensure we can encode this crazy thing 157 | assert <<_::binary>> = Diplomat.Proto.Entity.encode(ent) 158 | end 159 | 160 | test "converting to proto from Entity" do 161 | proto = %Entity{properties: %{"hello" => "world"}} |> Entity.proto() 162 | 163 | assert %Diplomat.Proto.Entity{ 164 | properties: [ 165 | {"hello", %Diplomat.Proto.Value{value_type: {:string_value, "world"}}} 166 | ] 167 | } = proto 168 | end 169 | 170 | @entity %Diplomat.Proto.Entity{ 171 | key: %Diplomat.Proto.Key{ 172 | path: [%Diplomat.Proto.Key.PathElement{kind: "Random", id_type: {:id, 1_234_567_890}}] 173 | }, 174 | properties: %{ 175 | "hello" => %Diplomat.Proto.Value{value_type: {:string_value, "world"}}, 176 | "math" => %Diplomat.Proto.Value{value_type: {:entity_value, %Diplomat.Proto.Entity{}}} 177 | } 178 | } 179 | 180 | test "converting from a protobuf struct" do 181 | assert %Entity{ 182 | key: %Key{kind: "Random", id: 1_234_567_890}, 183 | properties: %{ 184 | "math" => %Value{value: %Entity{}}, 185 | "hello" => %Value{value: "world"} 186 | } 187 | } = Entity.from_proto(@entity) 188 | end 189 | 190 | test "generating an Entity from a flat map" do 191 | map = %{ 192 | "access_token" => "778efaf8333b2ac840f097448154bb6b", 193 | "brand" => "vst", 194 | "geo_lat" => nil, 195 | "geo_long" => nil, 196 | "id" => 1089, 197 | "ip_address" => "127.0.0.1", 198 | "log_guid" => "2016-1-0b68c093a68b4bb5b16b", 199 | "log_type" => "view", 200 | "updated_at" => "2016-01-28T23:03:27.000Z", 201 | "user_agent" => 202 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36", 203 | "user_guid" => "58GQA26TZ567K3C65VVN", 204 | "vbid" => "12345" 205 | } 206 | 207 | ent = Entity.new(map, "Log") 208 | assert map |> Map.keys() |> length == ent.properties |> Map.keys() |> length 209 | assert ent.kind == "Log" 210 | end 211 | 212 | test "generating an Entity from a nested map" do 213 | ent = %{"person" => %{"firstName" => "Phil", "lastName" => "Burrows"}} |> Entity.new("Person") 214 | 215 | assert ent.kind == "Person" 216 | assert ent.properties |> Map.to_list() |> length == 1 217 | 218 | first_property = ent.properties |> Map.to_list() |> List.first() 219 | {"person", person_val} = first_property 220 | 221 | assert %Diplomat.Value{ 222 | value: %Diplomat.Entity{ 223 | properties: %{ 224 | "firstName" => %Value{value: "Phil"}, 225 | "lastName" => %Value{value: "Burrows"} 226 | } 227 | } 228 | } = person_val 229 | end 230 | 231 | defmodule Person do 232 | defstruct [:firstName, :lastName, :address, :dogs] 233 | end 234 | 235 | defmodule Dog do 236 | defstruct [:name] 237 | end 238 | 239 | defmodule Address do 240 | defstruct [:city, :state] 241 | end 242 | 243 | test "generating an Entity from a nested struct" do 244 | person = %Person{ 245 | firstName: "Phil", 246 | lastName: "Burrows", 247 | address: %Address{city: "Seattle", state: "WA"}, 248 | dogs: [%Dog{name: "fido"}] 249 | } 250 | 251 | ent = %{"person" => person} |> Entity.new("Person") 252 | 253 | assert ent.kind == "Person" 254 | assert ent.properties |> Map.to_list() |> length == 1 255 | 256 | first_property = ent.properties |> Map.to_list() |> List.first() 257 | {"person", _person_val} = first_property 258 | 259 | expected_properties = %{ 260 | "person" => %{ 261 | "firstName" => "Phil", 262 | "lastName" => "Burrows", 263 | "address" => %{"city" => "Seattle", "state" => "WA"}, 264 | "dogs" => [%{"name" => "fido"}] 265 | } 266 | } 267 | 268 | assert expected_properties == Entity.properties(ent) 269 | 270 | assert %Diplomat.Entity{ 271 | key: %Diplomat.Key{ 272 | id: nil, 273 | kind: "Person", 274 | name: nil, 275 | namespace: nil, 276 | parent: nil, 277 | project_id: nil 278 | }, 279 | kind: "Person", 280 | properties: %{ 281 | "person" => %Diplomat.Value{ 282 | value: %Diplomat.Entity{ 283 | key: nil, 284 | kind: nil, 285 | properties: %{ 286 | "address" => %Diplomat.Value{ 287 | value: %Diplomat.Entity{ 288 | key: nil, 289 | kind: nil, 290 | properties: %{ 291 | "city" => %Diplomat.Value{value: "Seattle"}, 292 | "state" => %Diplomat.Value{value: "WA"} 293 | } 294 | } 295 | }, 296 | "dogs" => %Diplomat.Value{ 297 | value: [ 298 | %Diplomat.Value{ 299 | value: %Diplomat.Entity{ 300 | key: nil, 301 | kind: nil, 302 | properties: %{"name" => %Diplomat.Value{value: "fido"}} 303 | } 304 | } 305 | ] 306 | }, 307 | "firstName" => %Diplomat.Value{value: "Phil"}, 308 | "lastName" => %Diplomat.Value{value: "Burrows"} 309 | } 310 | } 311 | } 312 | } 313 | } = ent 314 | end 315 | 316 | test "encoding an entity that has a nested entity" do 317 | ent = %{"person" => %{"firstName" => "Phil"}} |> Entity.new("Person") 318 | assert <<_::binary>> = ent |> Entity.proto() |> Diplomat.Proto.Entity.encode() 319 | end 320 | 321 | test "pulling properties properties" do 322 | ent = %{"person" => %{"firstName" => "Phil"}} |> Entity.new("Person") 323 | assert %{"person" => %{"firstName" => "Phil"}} == ent |> Entity.properties() 324 | end 325 | 326 | test "pulling properties of arrays of properties" do 327 | properties = %{ 328 | "person" => %{"firstName" => "Phil", "dogs" => [%{"name" => "Fido"}, %{"name" => "Woofer"}]} 329 | } 330 | 331 | # cast to proto 332 | ent = properties |> Entity.new("Person") |> Entity.proto() |> Entity.from_proto() 333 | assert properties == ent |> Entity.properties() 334 | end 335 | 336 | test "property names are converted to strings" do 337 | entity = Entity.new(%{:hello => "world"}, "CodeSnippet") 338 | assert %{"hello" => "world"} == Entity.properties(entity) 339 | end 340 | 341 | describe "property key sanitization" do 342 | test "property names that have dots are sanitized if option calls for it" do 343 | entity = Entity.new(%{"dear.phil" => "hello"}, "Greeting", sanitize_keys: true) 344 | assert %{"dear_phil" => "hello"} == Entity.properties(entity) 345 | end 346 | 347 | test "property names that have dots are sanitized with a replace char" do 348 | entity = Entity.new(%{"dear.phil" => "hello"}, "Greeting", sanitize_keys: "XX") 349 | assert %{"dearXXphil" => "hello"} == Entity.properties(entity) 350 | end 351 | 352 | test "nested property names are sanitized when called for" do 353 | entity = 354 | Entity.new(%{"dear.phil" => %{"nice.to" => "see you"}}, "Letter", sanitize_keys: true) 355 | 356 | assert %{"dear_phil" => %{"nice_to" => "see you"}} == Entity.properties(entity) 357 | end 358 | end 359 | 360 | test "building an entity with a custom key" do 361 | entity = Entity.new(%{"hi" => "there"}, %Key{kind: "Message", namespace: "custom"}) 362 | 363 | assert %Entity{ 364 | properties: %{}, 365 | key: %Key{ 366 | kind: "Message", 367 | namespace: "custom" 368 | } 369 | } = entity 370 | end 371 | end 372 | -------------------------------------------------------------------------------- /test/diplomat/key/allocate_ids_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.Entity.AllocateIdsTest do 2 | use ExUnit.Case 3 | alias Diplomat.Key 4 | alias Diplomat.Proto.AllocateIdsResponse, as: PbAllocateResp 5 | 6 | setup do 7 | bypass = Bypass.open() 8 | Application.put_env(:diplomat, :endpoint, "http://localhost:#{bypass.port}") 9 | {:ok, bypass: bypass} 10 | end 11 | 12 | test "generating keys from an AllocateIdsResponse" do 13 | keys = 14 | PbAllocateResp.new( 15 | keys: [ 16 | Key.new("Log", 1) |> Key.proto(), 17 | Key.new("Log", 2) |> Key.proto(), 18 | Key.new("Log", 3) |> Key.proto() 19 | ] 20 | ) 21 | |> Key.from_allocate_ids_proto() 22 | 23 | assert Enum.count(keys) == 3 24 | 25 | Enum.each(keys, fn k -> 26 | assert k.name == nil 27 | assert k.kind == "Log" 28 | refute k.id == nil 29 | end) 30 | end 31 | 32 | test "allocating ids", %{bypass: bypass} do 33 | count = 20 34 | kind = "Log" 35 | {:ok, project} = Goth.Config.get(:project_id) 36 | 37 | Bypass.expect(bypass, fn conn -> 38 | assert Regex.match?(~r{/v1/projects/#{project}:allocateIds}, conn.request_path) 39 | 40 | keys = 41 | Enum.map(1..count, fn i -> 42 | Key.new(kind, i) |> Key.proto() 43 | end) 44 | 45 | resp = PbAllocateResp.new(keys: keys) |> PbAllocateResp.encode() 46 | Plug.Conn.resp(conn, 201, resp) 47 | end) 48 | 49 | # we should get back a bunch of keys, I believe 50 | keys = Key.allocate_ids(kind, count) 51 | assert Enum.count(keys) == count 52 | 53 | Enum.each(keys, fn k -> 54 | refute k.id == nil 55 | assert k.kind == kind 56 | assert k.name == nil 57 | end) 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /test/diplomat/key_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.KeyTest do 2 | use ExUnit.Case 3 | alias Diplomat.{Key} 4 | alias Diplomat.Proto.Key, as: PbKey 5 | 6 | test "creating a new key with a name" do 7 | assert %Key{ 8 | kind: "Log", 9 | name: "testing", 10 | id: nil 11 | } = Key.new("Log", "testing") 12 | end 13 | 14 | test "creating a new key with an id" do 15 | assert %Key{ 16 | kind: "Log", 17 | name: nil, 18 | id: 123 19 | } = Key.new("Log", 123) 20 | end 21 | 22 | test "creating a key with an id and parent" do 23 | parent = Key.new("Asset") 24 | 25 | assert %Key{ 26 | parent: ^parent, 27 | kind: "Author", 28 | id: 123, 29 | name: nil 30 | } = Key.new("Author", 123, parent) 31 | end 32 | 33 | test "creating a key with a name and parent" do 34 | parent = Key.new("Asset") 35 | 36 | assert %Key{ 37 | parent: ^parent, 38 | kind: "Author", 39 | id: nil, 40 | name: "20k-author" 41 | } = Key.new("Author", "20k-author", parent) 42 | end 43 | 44 | test "creating a kew via an array" do 45 | assert %Key{ 46 | kind: "Book", 47 | id: 1 48 | } = ["Book", 1] |> Key.from_path() 49 | end 50 | 51 | test "creating a key with a parent via an array" do 52 | assert %Key{ 53 | kind: "Author", 54 | id: 2, 55 | parent: %Key{ 56 | kind: "Book", 57 | id: 1 58 | } 59 | } = [["Book", 1], ["Author", 2]] |> Key.from_path() 60 | end 61 | 62 | test "creating a key with multiple ancestors via an array" do 63 | assert %Key{ 64 | kind: "Name", 65 | id: 3, 66 | parent: %Key{ 67 | kind: "Author", 68 | id: 2, 69 | parent: %Key{ 70 | kind: "Book", 71 | id: 1 72 | } 73 | } 74 | } = [["Book", 1], ["Author", 2], ["Name", 3]] |> Key.from_path() 75 | end 76 | 77 | test "generating the path without ancestors" do 78 | assert [["Author", 123]] == Key.new("Author", 123) |> Key.path() 79 | assert [["Author", "hello"]] == Key.new("Author", "hello") |> Key.path() 80 | end 81 | 82 | test "generating a path with a single parent" do 83 | parent = Key.new("Asset", 123) 84 | assert [["Asset", 123], ["Author", 123]] == Key.new("Author", 123, parent) |> Key.path() 85 | end 86 | 87 | test "generating a path with muliple ancestors" do 88 | grandparent = Key.new("Collection", "Shakespeare") 89 | parent = Key.new("Play", "Romeo+Juliet", grandparent) 90 | child = Key.new("Act", "ActIII", parent) 91 | 92 | assert [ 93 | ["Collection", "Shakespeare"], 94 | ["Play", "Romeo+Juliet"], 95 | ["Act", "ActIII"] 96 | ] == Key.path(child) 97 | end 98 | 99 | test "converting a single key to a protobuf" do 100 | pb = Key.new("Book", "Romeo+Juliet") |> Key.proto() 101 | 102 | assert %PbKey{ 103 | path: [ 104 | %PbKey.PathElement{kind: "Book", id_type: {:name, "Romeo+Juliet"}} 105 | ] 106 | } = pb 107 | 108 | assert <<_::binary>> = pb |> PbKey.encode() 109 | end 110 | 111 | test "converting a key with ancestors to a protobuf" do 112 | pb = Key.new("Book", "Romeo+Juliet", Key.new("Collection", "Shakespeare")) |> Key.proto() 113 | 114 | assert %PbKey{ 115 | path: [ 116 | %PbKey.PathElement{kind: "Collection", id_type: {:name, "Shakespeare"}}, 117 | %PbKey.PathElement{kind: "Book", id_type: {:name, "Romeo+Juliet"}} 118 | ] 119 | } = pb 120 | 121 | assert <<_::binary>> = pb |> PbKey.encode() 122 | end 123 | 124 | test "converting an incomplete key to a protobuf" do 125 | pb = Key.new("Book") |> Key.proto() 126 | 127 | assert %PbKey{ 128 | path: [ 129 | %PbKey.PathElement{kind: "Book", id_type: nil} 130 | ] 131 | } = pb 132 | 133 | assert <<_::binary>> = pb |> PbKey.encode() 134 | end 135 | 136 | test "creating a key from a protobuf struct" do 137 | key = %Key{ 138 | kind: "User", 139 | name: "dev@philburrows.com", 140 | project_id: "diplo", 141 | namespace: "test" 142 | } 143 | 144 | pb_key = %PbKey{ 145 | partition_id: Diplomat.Proto.PartitionId.new(project_id: "diplo", namespace_id: "test"), 146 | path: [ 147 | PbKey.PathElement.new(kind: "User", id_type: {:name, "dev@philburrows.com"}) 148 | ] 149 | } 150 | 151 | assert ^key = pb_key |> Key.from_proto() 152 | end 153 | 154 | test "creating a key from a nested key protobuf struct" do 155 | assert %Key{ 156 | project_id: "diplo", 157 | namespace: "test", 158 | kind: "Name", 159 | parent: %Key{ 160 | kind: "UserDetails", 161 | id: 1, 162 | parent: %Key{ 163 | kind: "User", 164 | name: "dev@philburrows.com" 165 | } 166 | } 167 | } = 168 | %PbKey{ 169 | partition_id: 170 | Diplomat.Proto.PartitionId.new(project_id: "diplo", namespace_id: "test"), 171 | path: [ 172 | PbKey.PathElement.new(kind: "User", id_type: {:name, "dev@philburrows.com"}), 173 | PbKey.PathElement.new(kind: "UserDetails", id_type: {:id, 1}), 174 | PbKey.PathElement.new(kind: "Name", id_type: {:name, "phil-name"}) 175 | ] 176 | } 177 | |> Key.from_proto() 178 | end 179 | 180 | test "converting a key to a proto struct includes the namespace if defined" do 181 | assert %PbKey{ 182 | partition_id: %Diplomat.Proto.PartitionId{ 183 | namespace_id: "custom", 184 | project_id: "random" 185 | } 186 | } = %Key{namespace: "custom", project_id: "random"} |> Key.proto() 187 | end 188 | 189 | test "reads proto with an incomplete key" do 190 | key = %Key{kind: "Test"} 191 | assert key == key |> Key.proto() |> Key.from_proto() 192 | end 193 | 194 | test "Key.incomplete?" do 195 | assert %Key{kind: "Asset"} |> Key.incomplete?() 196 | refute %Key{id: 1} |> Key.incomplete?() 197 | refute %Key{name: "test"} |> Key.incomplete?() 198 | refute [["Book", 1], ["Author", 2]] |> Key.from_path() |> Key.incomplete?() 199 | end 200 | 201 | test "Key.complete?" do 202 | refute %Key{kind: "Asset"} |> Key.complete?() 203 | assert %Key{id: 1} |> Key.complete?() 204 | assert %Key{name: "test"} |> Key.complete?() 205 | assert [["Asset", 1], ["Random", 2]] |> Key.from_path() |> Key.complete?() 206 | end 207 | end 208 | -------------------------------------------------------------------------------- /test/diplomat/key_utils_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.KeyUtilsTest do 2 | use ExUnit.Case 3 | alias Diplomat.{Key} 4 | 5 | test "KeyUtils.urlsafe" do 6 | app = "testbed-test" 7 | key_with_id1 = %Key{kind: "User", id: 1, project_id: app} 8 | key_with_id2 = %Key{kind: "User", id: 2, project_id: app} 9 | key_with_name = %Key{kind: "User", name: "hello", project_id: app} 10 | key_with_parent = %Key{kind: "User", id: 3, parent: key_with_id2, project_id: app} 11 | assert KeyUtils.urlsafe(key_with_id1) == "agx0ZXN0YmVkLXRlc3RyCgsSBFVzZXIYAQw" 12 | assert KeyUtils.urlsafe(key_with_id2) == "agx0ZXN0YmVkLXRlc3RyCgsSBFVzZXIYAgw" 13 | assert KeyUtils.urlsafe(key_with_name) == "agx0ZXN0YmVkLXRlc3RyDwsSBFVzZXIiBWhlbGxvDA" 14 | assert KeyUtils.urlsafe(key_with_parent) == "agx0ZXN0YmVkLXRlc3RyFAsSBFVzZXIYAgwLEgRVc2VyGAMM" 15 | end 16 | 17 | test "KeyUtils.from_urlsafe" do 18 | {:ok, key} = KeyUtils.from_urlsafe("agx0ZXN0YmVkLXRlc3RyGQsSBFVzZXIiBWhlbGxvDAsSBFVzZXIYAww") 19 | assert key.kind == "User" 20 | assert key.project_id == "testbed-test" 21 | assert key.id == 3 22 | assert key.parent.kind == "User" 23 | assert key.parent.name == "hello" 24 | end 25 | 26 | test "KeyUtils.urlsafe fail" do 27 | {result, _} = KeyUtils.from_urlsafe("aaaa") 28 | assert result == :error 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /test/diplomat/query_result_batch_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.QueryResultBatchTest do 2 | use ExUnit.Case 3 | alias Diplomat.Proto.Entity, as: PbEntity 4 | alias Diplomat.Proto.QueryResultBatch, as: PbQueryResultBatch 5 | alias Diplomat.Proto.{EntityResult, Entity, PartitionId, Key, Value} 6 | alias Diplomat.{Cursor, Entity, QueryResultBatch} 7 | 8 | @query_result_batch %PbQueryResultBatch{ 9 | end_cursor: 10 | <<10, 104, 10, 34, 10, 2, 105, 100, 18, 28, 26, 26, 111, 102, 102, 95, 48, 48, 48, 48, 57, 11 | 113, 54, 49, 98, 114, 49, 77, 66, 71, 56, 49, 54, 70, 65, 109, 107, 48, 18, 62, 106, 16, 12 | 100, 117, 102, 102, 101, 108, 45, 116, 109, 112, 45, 49, 49, 51, 51, 52, 114, 42, 11, 18, 13 | 10, 97, 105, 114, 95, 111, 102, 102, 101, 114, 115, 34, 26, 111, 102, 102, 95, 48, 48, 48, 14 | 48, 57, 113, 54, 49, 98, 114, 49, 77, 66, 71, 56, 49, 54, 70, 65, 109, 107, 48, 12, 24, 0, 15 | 32, 0>>, 16 | entity_result_type: :PROJECTION, 17 | entity_results: [ 18 | %EntityResult{ 19 | cursor: 20 | <<10, 104, 10, 34, 10, 2, 105, 100, 18, 28, 26, 26, 111, 102, 102, 95, 48, 48, 48, 48, 21 | 57, 113, 54, 49, 98, 114, 49, 77, 66, 71, 56, 49, 54, 70, 65, 109, 107, 48, 18, 62, 22 | 106, 16, 100, 117, 102, 102, 101, 108, 45, 116, 109, 112, 45, 49, 49, 51, 51, 52, 114, 23 | 42, 11, 18, 10, 97, 105, 114, 95, 111, 102, 102, 101, 114, 115, 34, 26, 111, 102, 102, 24 | 95, 48, 48, 48, 48, 57, 113, 54, 49, 98, 114, 49, 77, 66, 71, 56, 49, 54, 70, 65, 109, 25 | 107, 48, 12, 24, 0, 32, 0>>, 26 | entity: %PbEntity{ 27 | key: %Key{ 28 | partition_id: %PartitionId{ 29 | namespace_id: nil, 30 | project_id: "diplomat" 31 | }, 32 | path: [ 33 | %Key.PathElement{ 34 | id_type: {:name, "123_foo"}, 35 | kind: "foo_table" 36 | } 37 | ] 38 | }, 39 | properties: [ 40 | {"id", 41 | %Value{ 42 | exclude_from_indexes: nil, 43 | meaning: 18, 44 | value_type: {:string_value, "123_foo"} 45 | }} 46 | ] 47 | } 48 | } 49 | ], 50 | more_results: :MORE_RESULTS_AFTER_LIMIT, 51 | skipped_cursor: nil, 52 | skipped_results: nil 53 | } 54 | 55 | describe "QueryResultBatch.from_proto/1" do 56 | test "given a list of entity results and an end_cursor" do 57 | pb_query_result_batch_end_cursor = @query_result_batch.end_cursor 58 | 59 | assert %QueryResultBatch{ 60 | entity_results: [%Entity{}], 61 | end_cursor: %Cursor{value: value} 62 | } = QueryResultBatch.from_proto(@query_result_batch) 63 | 64 | assert pb_query_result_batch_end_cursor == Cursor.decode(value) 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /test/diplomat/query_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.QueryTest do 2 | use ExUnit.Case 3 | alias Diplomat.{Query, Value} 4 | alias Diplomat.Proto.{GqlQuery, GqlQueryParameter} 5 | 6 | test "we can construct a query" do 7 | q = "select * from Assets where title = @1" 8 | title = "20,000 Leagues Under The Sea" 9 | query = Query.new(q, [title]) 10 | assert %Query{query: ^q, numbered_args: [^title]} = query 11 | end 12 | 13 | test "we can convert a Query to a Proto.GqlQuery" do 14 | query = "select * from whatever where yes = @1" 15 | arg = "sure" 16 | arg_val = Value.new(arg) |> Value.proto() 17 | 18 | assert %GqlQuery{ 19 | query_string: ^query, 20 | allow_literals: true, 21 | positional_bindings: [%GqlQueryParameter{parameter_type: {:value, ^arg_val}}], 22 | named_bindings: [] 23 | } = Query.new(query, [arg]) |> Query.proto() 24 | 25 | assert <<_::binary>> = Query.new(query, [arg]) |> Query.proto() |> GqlQuery.encode() 26 | end 27 | 28 | test "we can construct a query with named args" do 29 | {q, args} = {"select * from Log where user = @user", %{user: "phil"}} 30 | query = Query.new(q, args) 31 | assert %Query{query: ^q, named_args: ^args} = query 32 | end 33 | 34 | test "that atom keys in named arg maps are converted to strings" do 35 | {q, args} = {"select @what", %{what: "sure"}} 36 | query = Query.new(q, args) 37 | val = "sure" |> Value.proto() 38 | 39 | assert %GqlQuery{ 40 | named_bindings: [ 41 | {"what", %GqlQueryParameter{parameter_type: {:value, ^val}}} 42 | ] 43 | } = query |> Query.proto() 44 | end 45 | 46 | test "we can convert a Query with named args to a Proto.GqlQuery" do 47 | {q, args} = {"select * from whatever where thing = @thing", %{thing: "me"}} 48 | query = Query.new(q, args) 49 | assert <<_::binary>> = query |> Query.proto() |> GqlQuery.encode() 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/diplomat/transaction_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.TransactionTest do 2 | use ExUnit.Case 3 | 4 | alias Diplomat.{Transaction, Entity, Key} 5 | alias Diplomat.Proto.BeginTransactionResponse, as: TransResponse 6 | alias Diplomat.Proto.BeginTransactionRequest, as: TransRequest 7 | alias Diplomat.Proto.{CommitRequest, CommitResponse, MutationResult, Mutation, RollbackResponse} 8 | 9 | setup do 10 | bypass = Bypass.open() 11 | Application.put_env(:diplomat, :endpoint, "http://localhost:#{bypass.port}") 12 | {:ok, project} = Goth.Config.get(:project_id) 13 | {:ok, bypass: bypass, project: project} 14 | end 15 | 16 | test "creating a transaction from a transaction response returns a transaction" do 17 | assert %Transaction{id: <<1, 2, 4>>, state: :begun} = 18 | Transaction.from_begin_response(TransResponse.new(transaction: <<1, 2, 4>>)) 19 | end 20 | 21 | test "beginning a transaction calls the server with a BeginTransactionRequest and returns a Transaction struct", 22 | %{bypass: bypass, project: project} do 23 | Bypass.expect(bypass, fn conn -> 24 | {:ok, body, conn} = Plug.Conn.read_body(conn) 25 | 26 | assert %TransRequest{project_id: nil} = TransRequest.decode(body) 27 | 28 | assert Regex.match?(~r{/v1/projects/#{project}:beginTransaction}, conn.request_path) 29 | resp = TransResponse.new(transaction: <<40, 30, 20>>) |> TransResponse.encode() 30 | Plug.Conn.resp(conn, 201, resp) 31 | end) 32 | 33 | trans = Transaction.begin() 34 | assert %Transaction{state: :begun, id: <<40, 30, 20>>} = trans 35 | end 36 | 37 | test "converting a transaction to a CommitRequest" do 38 | t = %Transaction{ 39 | id: <<1, 2, 3>>, 40 | state: :begun, 41 | mutations: [ 42 | {:update, Entity.new(%{phil: "burrows"}, "Person", "phil-burrows")}, 43 | {:insert, Entity.new(%{jimmy: "allen"}, "Person", 12_234_324)}, 44 | {:delete, Key.new("Person", "that-one-guy")} 45 | ] 46 | } 47 | 48 | commit = 49 | CommitRequest.new( 50 | mode: :TRANSACTIONAL, 51 | transaction_selector: {:transaction, <<1, 2, 3>>}, 52 | mutations: [ 53 | Mutation.new( 54 | operation: 55 | {:update, 56 | Entity.new(%{phil: "burrows"}, "Person", "phil-burrows") |> Entity.proto()} 57 | ), 58 | Mutation.new( 59 | operation: 60 | {:insert, Entity.new(%{jimmy: "allen"}, "Person", 12_234_324) |> Entity.proto()} 61 | ), 62 | Mutation.new(operation: {:delete, Key.new("Person", "that-one-guy") |> Key.proto()}) 63 | ] 64 | ) 65 | 66 | assert ^commit = Transaction.to_commit_proto(t) 67 | end 68 | 69 | test "rolling back a transaction calls the server with the RollbackRequest", %{ 70 | bypass: bypass, 71 | project: project 72 | } do 73 | Bypass.expect(bypass, fn conn -> 74 | assert Regex.match?(~r{/v1/projects/#{project}:rollback}, conn.request_path) 75 | # the rsponse is empty 76 | Plug.Conn.resp(conn, 200, <<>>) 77 | end) 78 | 79 | {:ok, resp} = %Transaction{id: <<1>>} |> Transaction.rollback() 80 | assert %RollbackResponse{} = resp 81 | end 82 | 83 | test "committing a transaction calls the server with the right data and returns a successful response (whatever that is)", 84 | %{bypass: bypass, project: project} do 85 | commit = 86 | CommitResponse.new( 87 | index_updates: 0, 88 | mutation_results: [MutationResult.new()] 89 | ) 90 | 91 | Bypass.expect(bypass, fn conn -> 92 | assert Regex.match?(~r{/v1/projects/#{project}:commit}, conn.request_path) 93 | response = commit |> CommitResponse.encode() 94 | Plug.Conn.resp(conn, 201, response) 95 | end) 96 | 97 | assert {:ok, ^commit} = %Transaction{id: <<1>>} |> Transaction.commit() 98 | end 99 | 100 | test "a transaction block begins and commits the transaction automatically", opts do 101 | assert_begin_and_commit!(opts) 102 | Transaction.begin(fn t -> t end) 103 | end 104 | 105 | test "we can add inserts to a transaction" do 106 | e = Entity.new(%{abraham: "lincoln"}, "Body", 123) 107 | t = %Transaction{id: 123} |> Transaction.insert(e) 108 | assert Enum.count(t.mutations) == 1 109 | assert Enum.at(t.mutations, 0) == {:insert, e} 110 | end 111 | 112 | # test "we can add insert_auto_ids to a transaction" do 113 | # e = Entity.new(%{abraham: "lincoln"}, "Body") 114 | # t = %Transaction{id: 123} |> Transaction.insert(e) 115 | # assert Enum.count(t.insert_auto_ids) == 1 116 | # assert Enum.at(t.insert_auto_ids, 0) == e 117 | # end 118 | 119 | test "find an entity within the context of a transaction", %{bypass: bypass, project: project} do 120 | Bypass.expect(bypass, fn conn -> 121 | path = "/v1/projects/#{project}" 122 | 123 | cond do 124 | Regex.match?(~r{#{path}:beginTransaction}, conn.request_path) -> 125 | response = 126 | <<10, 29, 9, 166, 1, 0, 0, 0, 0, 0, 0, 18, 18, 108, 111, 121, 97, 108, 45, 103, 108, 127 | 97, 115, 115, 45, 49, 54, 51, 48, 48, 50>> 128 | 129 | Plug.Conn.resp(conn, 200, response) 130 | 131 | Regex.match?(~r{#{path}:lookup}, conn.request_path) -> 132 | response = 133 | <<10, 53, 10, 48, 10, 34, 10, 20, 18, 18, 108, 111, 121, 97, 108, 45, 103, 108, 97, 134 | 115, 115, 45, 49, 54, 51, 48, 48, 50, 18, 10, 10, 5, 84, 104, 105, 110, 103, 26, 1, 135 | 49, 26, 10, 10, 4, 116, 101, 115, 116, 18, 2, 8, 1, 32, 235, 1>> 136 | 137 | Plug.Conn.resp(conn, 200, response) 138 | 139 | true -> 140 | raise "Unknown request" 141 | end 142 | end) 143 | 144 | tx = Transaction.begin() 145 | result = Transaction.find(tx, %Key{id: 1}) 146 | 147 | assert [ 148 | %Diplomat.Entity{ 149 | key: %Diplomat.Key{ 150 | id: nil, 151 | kind: "Thing", 152 | name: "1", 153 | namespace: nil, 154 | parent: nil, 155 | project_id: _ 156 | }, 157 | kind: "Thing", 158 | properties: %{"test" => %Diplomat.Value{value: true}} 159 | } 160 | ] = result 161 | end 162 | 163 | test "we can add upserts to a transaction" do 164 | e = Entity.new(%{whatever: "yes"}, "Thing", 123) 165 | t = %Transaction{id: 123} |> Transaction.upsert(e) 166 | assert Enum.count(t.mutations) == 1 167 | assert Enum.at(t.mutations, 0) == {:upsert, e} 168 | end 169 | 170 | test "we can add updates to a transaction" do 171 | e = Entity.new(%{whatever: "yes"}, "Thing", 123) 172 | t = %Transaction{id: 123} |> Transaction.update(e) 173 | assert Enum.count(t.mutations) == 1 174 | assert Enum.at(t.mutations, 0) == {:update, e} 175 | end 176 | 177 | test "we can add deletes to a transaction" do 178 | k = Key.new("Person", 123) 179 | t = %Transaction{id: 123} |> Transaction.delete(k) 180 | assert Enum.count(t.mutations) == 1 181 | assert Enum.at(t.mutations, 0) == {:delete, k} 182 | end 183 | 184 | def assert_begin_and_commit!(%{bypass: bypass, project: project}) do 185 | Bypass.expect(bypass, fn conn -> 186 | if Regex.match?(~r{beginTransaction}, conn.request_path) do 187 | assert Regex.match?(~r{/v1/projects/#{project}:beginTransaction}, conn.request_path) 188 | resp = TransResponse.new(transaction: <<40, 30, 20>>) |> TransResponse.encode() 189 | Plug.Conn.resp(conn, 201, resp) 190 | else 191 | assert Regex.match?(~r{/v1/projects/#{project}:commit}, conn.request_path) 192 | 193 | resp = 194 | CommitResponse.new( 195 | mutation_result: 196 | MutationResult.new( 197 | index_updates: 0, 198 | insert_auto_id_key: [] 199 | ) 200 | ) 201 | |> CommitResponse.encode() 202 | 203 | Plug.Conn.resp(conn, 201, resp) 204 | end 205 | end) 206 | end 207 | end 208 | -------------------------------------------------------------------------------- /test/diplomat/value_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Diplomat.ValueTest do 2 | use ExUnit.Case 3 | alias Diplomat.{Value, Entity, Key} 4 | alias Diplomat.Proto.Value, as: PbVal 5 | alias Diplomat.Proto.Key, as: PbKey 6 | alias Diplomat.Proto.ArrayValue, as: PbArray 7 | alias Diplomat.Proto.Timestamp, as: PbTimestamp 8 | alias Diplomat.Proto.LatLng, as: PbLatLng 9 | 10 | describe "Value.new/1" do 11 | test "given an Entity" do 12 | entity = Entity.new(%{"foo" => "bar"}) 13 | assert Value.new(entity) == %Value{value: entity, exclude_from_indexes: false} 14 | end 15 | 16 | test "given a Key" do 17 | key = Key.new("TestKind", "1") 18 | assert Value.new(key) == %Value{value: key, exclude_from_indexes: false} 19 | end 20 | 21 | test "given a Value" do 22 | value = Value.new(1) 23 | assert Value.new(value) == %Value{value: value, exclude_from_indexes: false} 24 | end 25 | 26 | test "given a struct" do 27 | struct = %TestStruct{foo: "bar"} 28 | entity = Entity.new(%{"foo" => "bar"}) 29 | assert Value.new(struct) == %Value{value: entity, exclude_from_indexes: false} 30 | end 31 | 32 | test "given a map" do 33 | map = %{"foo" => "bar"} 34 | entity = Entity.new(map) 35 | assert Value.new(map) == %Value{value: entity, exclude_from_indexes: false} 36 | end 37 | 38 | test "given a list" do 39 | int = 1 40 | map = %{"foo" => "bar"} 41 | entity = Entity.new(map) 42 | list = [int, map] 43 | int_value = %Value{value: int, exclude_from_indexes: false} 44 | entity_value = %Value{value: entity, exclude_from_indexes: false} 45 | 46 | assert Value.new(list) == 47 | %Value{value: [int_value, entity_value], exclude_from_indexes: false} 48 | end 49 | 50 | test "given a string" do 51 | string = "test" 52 | assert Value.new(string) == %Value{value: string, exclude_from_indexes: false} 53 | end 54 | 55 | test "given an integer" do 56 | int = 1 57 | assert Value.new(int) == %Value{value: int, exclude_from_indexes: false} 58 | end 59 | 60 | test "given a DateTime" do 61 | time = DateTime.utc_now() 62 | assert Value.new(time) == %Value{value: time, exclude_from_indexes: false} 63 | end 64 | 65 | test "given a NaiveDateTime" do 66 | time = NaiveDateTime.utc_now() 67 | assert Value.new(time) == %Value{value: time, exclude_from_indexes: false} 68 | end 69 | end 70 | 71 | describe "Value.new/2" do 72 | test "given an Entity and exclude_from_indexes is true" do 73 | entity = Entity.new(%{"foo" => "bar"}) 74 | 75 | assert Value.new(entity, exclude_from_indexes: true) == 76 | %Value{value: entity, exclude_from_indexes: true} 77 | end 78 | 79 | test "given a nested map and exclude_from_indexes contains an path" do 80 | map = %{"foo" => %{"bar" => "baz"}} 81 | 82 | nested_entity = %Entity{ 83 | properties: %{"bar" => %Value{value: "baz", exclude_from_indexes: true}} 84 | } 85 | 86 | entity = %Entity{ 87 | properties: %{"foo" => %Value{value: nested_entity, exclude_from_indexes: false}} 88 | } 89 | 90 | assert Value.new(map, exclude_from_indexes: [foo: :bar]) == 91 | %Value{value: entity, exclude_from_indexes: false} 92 | end 93 | 94 | test "given a deeply nested map and exclude_from_indexes contains a path" do 95 | map = %{"foo" => %{"bar" => %{"baz" => "qux"}}, "foo2" => 1} 96 | 97 | deeply_nested_entity = %Entity{ 98 | properties: %{"baz" => %Value{value: "qux", exclude_from_indexes: true}} 99 | } 100 | 101 | nested_entity = %Entity{ 102 | properties: %{"bar" => %Value{value: deeply_nested_entity, exclude_from_indexes: false}} 103 | } 104 | 105 | entity = %Entity{ 106 | properties: %{ 107 | "foo" => %Value{value: nested_entity, exclude_from_indexes: false}, 108 | "foo2" => %Value{value: 1, exclude_from_indexes: true} 109 | } 110 | } 111 | 112 | assert Value.new(map, exclude_from_indexes: [:foo2, foo: [bar: :baz]]) == 113 | %Value{value: entity, exclude_from_indexes: false} 114 | end 115 | 116 | test "a string longer than 1500 bytes, with trucate: true passed" do 117 | string = 2_000 |> :crypto.strong_rand_bytes() |> Base.url_encode64() 118 | <> = string 119 | 120 | assert Value.new(string, truncate: true) == 121 | %Value{value: first, exclude_from_indexes: false} 122 | end 123 | 124 | test "a strong longer than 1500 bytes w/o truncate option passed" do 125 | string = 2_000 |> :crypto.strong_rand_bytes() |> Base.url_encode64() 126 | assert Value.new(string) == %Value{value: string, exclude_from_indexes: false} 127 | end 128 | 129 | test "a map with a string value longer than 1500 bytes w/truncate: true" do 130 | string = 2_000 |> :crypto.strong_rand_bytes() |> Base.url_encode64() 131 | <> = string 132 | 133 | assert Value.new(%{"key" => string}, truncate: true) == 134 | %Value{ 135 | value: %Entity{ 136 | properties: %{ 137 | "key" => %Value{value: truncated, exclude_from_indexes: false} 138 | } 139 | }, 140 | exclude_from_indexes: false 141 | } 142 | end 143 | end 144 | 145 | describe "Value.proto/1" do 146 | test "given a protocol buffer value" do 147 | pb_val = %PbVal{} 148 | assert Value.proto(pb_val) == %{pb_val | exclude_from_indexes: false} 149 | end 150 | 151 | test "given a nil value" do 152 | assert Value.proto(nil) == 153 | PbVal.new(value_type: {:null_value, :NULL_VALUE}, exclude_from_indexes: false) 154 | end 155 | 156 | test "given a Diplomat.Value struct" do 157 | value = Value.new(nil, exclude_from_indexes: true) 158 | 159 | assert Value.proto(value) == 160 | PbVal.new(value_type: {:null_value, :NULL_VALUE}, exclude_from_indexes: true) 161 | end 162 | 163 | test "given a boolean value" do 164 | assert Value.proto(true) == 165 | PbVal.new(value_type: {:boolean_value, true}, exclude_from_indexes: false) 166 | end 167 | 168 | test "given an integer value" do 169 | assert Value.proto(1) == 170 | PbVal.new(value_type: {:integer_value, 1}, exclude_from_indexes: false) 171 | end 172 | 173 | test "given a double value" do 174 | assert Value.proto(1.1) == 175 | PbVal.new(value_type: {:double_value, 1.1}, exclude_from_indexes: false) 176 | end 177 | 178 | test "given an atom value" do 179 | assert Value.proto(:foo) == 180 | PbVal.new(value_type: {:string_value, "foo"}, exclude_from_indexes: false) 181 | end 182 | 183 | test "given a string value" do 184 | assert Value.proto("foo") == 185 | PbVal.new(value_type: {:string_value, "foo"}, exclude_from_indexes: false) 186 | end 187 | 188 | test "given an invalid string value" do 189 | blob = <<0xFFFF::16>> 190 | 191 | assert Value.proto(blob) == 192 | PbVal.new(value_type: {:blob_value, blob}, exclude_from_indexes: false) 193 | end 194 | 195 | test "given a bitstring value" do 196 | blob = <<1::size(1)>> 197 | 198 | assert Value.proto(blob) == 199 | PbVal.new(value_type: {:blob_value, blob}, exclude_from_indexes: false) 200 | end 201 | 202 | test "given a list value" do 203 | map = %{"foo" => "bar"} 204 | array = [1, nil, map, "asdf"] 205 | pb_entity = map |> Entity.new() |> Entity.proto() 206 | 207 | pb_array = %PbArray{ 208 | values: [ 209 | %PbVal{value_type: {:integer_value, 1}, exclude_from_indexes: false}, 210 | %PbVal{value_type: {:null_value, :NULL_VALUE}, exclude_from_indexes: false}, 211 | %PbVal{value_type: {:entity_value, pb_entity}, exclude_from_indexes: false}, 212 | %PbVal{value_type: {:string_value, "asdf"}, exclude_from_indexes: false} 213 | ] 214 | } 215 | 216 | assert Value.proto(array) == 217 | PbVal.new(value_type: {:array_value, pb_array}, exclude_from_indexes: false) 218 | end 219 | 220 | test "given an empty list value" do 221 | pb_array = %PbArray{values: []} 222 | 223 | assert Value.proto([]) == 224 | PbVal.new(value_type: {:array_value, pb_array}, exclude_from_indexes: false) 225 | end 226 | 227 | test "given a DateTime value" do 228 | datetime = DateTime.utc_now() 229 | timestamp = DateTime.to_unix(datetime, :nanosecond) 230 | 231 | pb_timestamp = %PbTimestamp{ 232 | seconds: div(timestamp, 1_000_000_000), 233 | nanos: rem(timestamp, 1_000_000_000) 234 | } 235 | 236 | assert Value.proto(datetime) == 237 | PbVal.new( 238 | value_type: {:timestamp_value, pb_timestamp}, 239 | exclude_from_indexes: false 240 | ) 241 | end 242 | 243 | test "given a Diplomat.Key value" do 244 | key = Key.new("TestKind", "1") 245 | pb_key = key |> Key.proto() 246 | 247 | assert Value.proto(key) == 248 | PbVal.new(value_type: {:key_value, pb_key}, exclude_from_indexes: false) 249 | end 250 | 251 | test "given a map value" do 252 | map = %{"foo" => "bar"} 253 | pb_entity = map |> Entity.new() |> Entity.proto() 254 | 255 | assert Value.proto(map) == 256 | PbVal.new(value_type: {:entity_value, pb_entity}, exclude_from_indexes: false) 257 | end 258 | 259 | test "given a geo value" do 260 | geo = {1.0, 2.0} 261 | pb_latlng = %PbLatLng{latitude: 1.0, longitude: 2.0} 262 | 263 | assert Value.proto(geo) == 264 | PbVal.new(value_type: {:geo_point_value, pb_latlng}, exclude_from_indexes: false) 265 | end 266 | end 267 | 268 | describe "Value.proto/2" do 269 | test "given a nil value and exclude_from_indexes is true" do 270 | assert Value.proto(nil, exclude_from_indexes: true) == 271 | PbVal.new(value_type: {:null_value, :NULL_VALUE}, exclude_from_indexes: true) 272 | end 273 | end 274 | 275 | # ==== Value.from_proto ====== 276 | test "creating from protobuf struct" do 277 | [true, 35, 3.1415, "hello", nil] 278 | |> Enum.each(fn i -> 279 | proto = Value.proto(i) 280 | val = Value.new(i) 281 | assert val == Value.from_proto(proto) 282 | end) 283 | end 284 | 285 | test "create key from protobuf key" do 286 | proto = %PbVal{ 287 | value_type: { 288 | :key_value, 289 | %PbKey{ 290 | path: [ 291 | PbKey.PathElement.new(kind: "User", id_type: {:id, 1}) 292 | ] 293 | } 294 | } 295 | } 296 | 297 | key = Value.from_proto(proto).value 298 | assert key.kind == "User" 299 | assert key.id == 1 300 | end 301 | 302 | test "creating from a protobuf struct with a list value" do 303 | proto = [1, 2, 3] |> Value.proto() 304 | 305 | assert %Value{ 306 | value: [ 307 | %Value{value: 1}, 308 | %Value{value: 2}, 309 | %Value{value: 3} 310 | ] 311 | } = Value.from_proto(proto) 312 | end 313 | end 314 | -------------------------------------------------------------------------------- /test/diplomat_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DiplomatTest do 2 | use ExUnit.Case 3 | doctest Diplomat 4 | end 5 | -------------------------------------------------------------------------------- /test/support/test_struct.ex: -------------------------------------------------------------------------------- 1 | defmodule TestStruct do 2 | defstruct [:foo] 3 | end 4 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | Application.ensure_all_started(:bypass) 3 | 4 | defmodule Diplomat.TestToken do 5 | def for_scope(scope) do 6 | {:ok, 7 | %Goth.Token{ 8 | scope: scope, 9 | expires: :os.system_time(:seconds) + 3600, 10 | type: "Bearer", 11 | token: UUID.uuid1() 12 | }} 13 | end 14 | end 15 | --------------------------------------------------------------------------------