├── .gcloudignore ├── .github └── workflows │ └── build.yaml ├── .gitignore ├── README.md ├── call-local.sh ├── example.png ├── examples ├── EXAMPLES.md ├── all-emojis.csv ├── load-csv.js ├── package-lock.json └── package.json ├── gil ├── driver │ ├── driverConfiguration.js │ ├── driverConfiguration.test.js │ └── index.js ├── index.js └── sink │ ├── CUDBatch.js │ ├── CUDBatch.test.js │ ├── CUDCommand.js │ ├── CUDCommand.test.js │ ├── CypherSink.js │ ├── CypherSink.test.js │ ├── DataSink.js │ ├── Strategy.js │ └── generateCUD.js ├── index.js ├── package-lock.json ├── package.json ├── services ├── common.js ├── http │ ├── cud.js │ ├── cud.test.js │ ├── customCypher.js │ ├── cypher.js │ ├── cypher.test.js │ ├── edge.js │ ├── edge.test.js │ ├── node.js │ └── node.test.js └── pubsub │ ├── cudPubsub.js │ ├── customCypherPubsub.js │ └── cypherPubsub.js ├── test ├── cud-messages.json ├── cypher-payload.json └── index.js └── yarn.lock /.gcloudignore: -------------------------------------------------------------------------------- 1 | # Credentials are NOT ignored, must be uploaded to google to work. 2 | # creds.json 3 | 4 | # Serverless 5 | .serverless 6 | .env 7 | tmp 8 | .coveralls.yml 9 | 10 | # Google 11 | keyfile.json 12 | 13 | # Logs 14 | *.log 15 | 16 | # Directory for instrumented libs generated by jscoverage/JSCover 17 | lib-cov 18 | 19 | # Coverage directory used by tools like istanbul 20 | coverage 21 | 22 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 23 | .grunt 24 | 25 | # node-waf configuration 26 | .lock-wscript 27 | 28 | # Compiled binary addons (http://nodejs.org/api/addons.html) 29 | build/Release 30 | 31 | # Dependency directory 32 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 33 | node_modules 34 | 35 | # IDE 36 | **/.idea 37 | 38 | # OS 39 | .DS_Store 40 | .tmp 41 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: push 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v2 12 | - uses: actions/cache@v2 13 | with: 14 | path: '**/node_modules' 15 | key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }} 16 | 17 | - name: Use Node.js 18 | uses: actions/setup-node@v1 19 | with: 20 | node-version: '12.x' 21 | - run: echo $service_key > /tmp/service_key.json 22 | env: 23 | service_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} 24 | - run: yarn install 25 | - run: yarn test --runInBand --coverage --watchAll=false 26 | env: 27 | CI: true 28 | GOOGLE_APPLICATION_CREDENTIALS: /tmp/service_key.json 29 | 30 | deploy: 31 | needs: [build] 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v2 35 | - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master 36 | with: 37 | version: latest 38 | project_id: ${{ secrets.GCP_PROJECT_ID }} 39 | service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} 40 | export_default_credentials: true 41 | - run: gcloud info 42 | # - run: | 43 | # # Deploy cypher pubsub 44 | # gcloud functions deploy cypherPubsub \ 45 | # --ingress-settings=all --runtime=nodejs12 \ 46 | # --allow-unauthenticated --timeout=300 \ 47 | # --service-account=$(gcloud config list account --format "value(core.account)") \ 48 | # --set-env-vars GCP_PROJECT=${{ secrets.GCP_PROJECT_ID }} \ 49 | # --set-env-vars URI_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_URI/versions/latest \ 50 | # --set-env-vars USER_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_USER/versions/latest \ 51 | # --set-env-vars PASSWORD_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_PASSWORD/versions/latest \ 52 | # --trigger-topic cypher 53 | - run: | 54 | # Deploy custom cypher pubsub for image annotations 55 | echo "GCP_PROJECT: ${{ secrets.GCP_PROJECT_ID }}" >> /tmp/env.yaml 56 | echo "URI_SECRET: projects/graphs-are-everywhere/secrets/NEO4J_URI/versions/latest" >> /tmp/env.yaml 57 | echo "USER_SECRET: projects/graphs-are-everywhere/secrets/NEO4J_USER/versions/latest" >> /tmp/env.yaml 58 | echo "PASSWORD_SECRET: projects/graphs-are-everywhere/secrets/NEO4J_PASSWORD/versions/latest" >> /tmp/env.yaml 59 | echo 'CYPHER: "MERGE (i:Image {uri:event.uri}) MERGE (l:Label { mid: event.mid, description: event.description }) MERGE (i)-[:LABELED {score:event.score, confidence:event.confidence, topicality:event.topicality}]->(l)"' >> /tmp/env.yaml 60 | 61 | cat /tmp/env.yaml 62 | 63 | gcloud functions deploy imageAnnotationsToNeo4j \ 64 | --entry-point=customCypherPubsub \ 65 | --ingress-settings=all --runtime=nodejs12 \ 66 | --allow-unauthenticated --timeout=300 \ 67 | --service-account=$(gcloud config list account --format "value(core.account)") \ 68 | --env-vars-file /tmp/env.yaml \ 69 | --trigger-topic imageAnnotation 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Testing data 2 | cud.json 3 | # Credentials 4 | creds.json 5 | 6 | # Serverless 7 | .serverless 8 | .env 9 | tmp 10 | .coveralls.yml 11 | 12 | # Google 13 | keyfile.json 14 | 15 | # Logs 16 | *.log 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | 24 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 25 | .grunt 26 | 27 | # node-waf configuration 28 | .lock-wscript 29 | 30 | # Compiled binary addons (http://nodejs.org/api/addons.html) 31 | build/Release 32 | 33 | # Dependency directory 34 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 35 | node_modules 36 | 37 | # IDE 38 | **/.idea 39 | 40 | # OS 41 | .DS_Store 42 | .tmp 43 | 44 | coverage 45 | .neo4j 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Neo4j Cloud Functions 2 | 3 | [![Actions Status](https://github.com/moxious/neo4j-serverless-functions/workflows/CI/badge.svg)](https://github.com/moxious/neo4j-serverless-functions/actions) 4 | 5 | Cloud functions for working with Neo4j. Deploy these to Google Cloud, and you can pipe 6 | data into Neo4j from any system that can make an HTTP request, or can send a message to 7 | a PubSub topic, like Cloud Dataflow, PubSub itself, and many others. 8 | 9 | ## Features 10 | 11 | * Endpoint for running batched Cypher operations 12 | * Endpoint for piping in data via the CUD format 13 | * PubSub and HTTP availability 14 | 15 | ## Pre-Requisites 16 | 17 | - Have a Google Cloud project 18 | - Have `gcloud` CLI installed 19 | - Enable the Cloud Functions API on that project. 20 | - Enable the Secrets Manager API 21 | - Create a service account with access to APIs above 22 | 23 | ## Setup 24 | 25 | ``` 26 | yarn install 27 | ``` 28 | 29 | ## Configure 30 | 31 | On this branch you may use Google Secrets manager with the following keys, and ensure 32 | that your service account has access to the following secrets. Only the latest versions 33 | will be used. 34 | 35 | - `NEO4J_USER` 36 | - `NEO4J_PASSWORD` 37 | - `NEO4J_URI` 38 | 39 | *If you do not enable secrets manager, the secrets will be taken from environment variables of the same name* 40 | 41 | The functions will not execute correctly if these details are not provided one way or the other. 42 | 43 | As an env var, you may set `GOOGLE_PROJECT` to point to the project where the secrets 44 | should be taken from. 45 | 46 | ## Deploy 47 | 48 | *Make sure to tweak the settings in this deploy*. This deploys unsecured functions 49 | that unauthenticated users can connect to. Tailor the settings to your needs. 50 | 51 | The `.github/workflows/build.yaml` file gives a GitHub Actions pipeline example of how 52 | to build and deploy this module. The build requires a service key JSON secret `GOOGLE_APPLICATION_CREDENTIALS` and it requires a `GCP_PROJECT_ID` secret indicating the 53 | project to deploy to. 54 | 55 | Below commands are for manual deploys only, and are examples. 56 | 57 | ### PubSub Triggered Functions 58 | 59 | Make sure to customize the trigger topic and environment variables! 60 | 61 | ``` 62 | # Ensure Google Secret Manager secrets are set 63 | 64 | gcloud functions deploy cudPubsub \ 65 | --ingress-settings=all --runtime=nodejs12 --allow-unauthenticated \ 66 | --timeout=300 \ 67 | --set-env-vars GCP_PROJECT=${{ secrets.GCP_PROJECT_ID }} \ 68 | --set-env-vars URI_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_URI/versions/latest \ 69 | --set-env-vars USER_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_USER/versions/latest \ 70 | --set-env-vars PASSWORD_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_PASSWORD/versions/latest \ 71 | --trigger-topic neo4j-cud 72 | 73 | gcloud functions deploy cypherPubsub \ 74 | --ingress-settings=all --runtime=nodejs12 --allow-unauthenticated \ 75 | --timeout=300 \ 76 | --set-env-vars GCP_PROJECT=${{ secrets.GCP_PROJECT_ID }} \ 77 | --set-env-vars URI_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_URI/versions/latest \ 78 | --set-env-vars USER_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_USER/versions/latest \ 79 | --set-env-vars PASSWORD_SECRET=projects/graphs-are-everywhere/secrets/NEO4J_PASSWORD/versions/latest \ 80 | --trigger-topic cypher 81 | ``` 82 | 83 | ### HTTP Functions 84 | 85 | (On deploy carefully note the secret env vars if you want to use GSM) 86 | 87 | ``` 88 | export NEO4J_USER=neo4j 89 | export NEO4J_PASSWORD=secret 90 | export NEO4J_URI=neo4j+s://my-host:7687/ 91 | 92 | gcloud functions deploy cud \ 93 | --ingress-settings=all --runtime=nodejs12 --allow-unauthenticated \ 94 | --timeout=300 \ 95 | --set-env-vars NEO4J_USER=$NEO4J_USER,NEO4J_PASSWORD=$NEO4J_PASSWORD,NEO4J_URI=$NEO4J_URI \ 96 | --trigger-http 97 | 98 | gcloud functions deploy cypher \ 99 | --ingress-settings=all --runtime=nodejs12 --allow-unauthenticated \ 100 | --timeout=300 \ 101 | --set-env-vars NEO4J_USER=$NEO4J_USER,NEO4J_PASSWORD=$NEO4J_PASSWORD,NEO4J_URI=$NEO4J_URI \ 102 | --trigger-http 103 | 104 | gcloud functions deploy node \ 105 | --ingress-settings=all --runtime=nodejs12 --allow-unauthenticated \ 106 | --timeout=300 \ 107 | --set-env-vars NEO4J_USER=$NEO4J_USER,NEO4J_PASSWORD=$NEO4J_PASSWORD,NEO4J_URI=$NEO4J_URI \ 108 | --trigger-http 109 | 110 | gcloud functions deploy edge \ 111 | --ingress-settings=all --runtime=nodejs12 --allow-unauthenticated \ 112 | --timeout=300 \ 113 | --set-env-vars NEO4J_USER=$NEO4J_USER,NEO4J_PASSWORD=$NEO4J_PASSWORD,NEO4J_URI=$NEO4J_URI \ 114 | --trigger-http 115 | ``` 116 | 117 | [See related documentation](https://cloud.google.com/functions/docs/env-var) 118 | 119 | ## Quick Example of functions and their results 120 | 121 | ``` 122 | # Given this local deploy URL prefix (provided by local testing above) 123 | LOCALDEPLOY=http://localhost:8080/ 124 | 125 | # CUD 126 | curl --data @test/cud-messages.json \ 127 | -H "Content-Type: application/json" -X POST \ 128 | $LOCALDEPLOY 129 | 130 | # Cypher 131 | curl --data @test/cypher-payload.json \ 132 | -H "Content-Type: application/json" -X POST \ 133 | $LOCALDEPLOY 134 | 135 | # Node 136 | curl -H "Content-Type: application/json" -X POST \ 137 | -d '{"username":"xyz","name":"David"}' \ 138 | $LOCALDEPLOY/node?label=User 139 | 140 | # Node 141 | curl -H "Content-Type: application/json" -X POST \ 142 | -d '{"username":"foo","name":"Mark"}' \ 143 | $LOCALDEPLOY/node?label=User 144 | 145 | # Edge 146 | curl -H "Content-Type: application/json" -X POST \ 147 | -d '{"since":"yesterday","metadata":"whatever"}' \ 148 | $LOCALDEPLOY'/edge?fromLabel=User&fromProp=username&fromVal=xyz&toLabel=User&toProp=username&toVal=foo&relType=knows' 149 | ``` 150 | 151 | ![Example Result Graph](example.png) 152 | 153 | ## Node Function 154 | 155 | This function takes JSON body data reported into the endpoint, and creates a node with a specified label having those properties. Example: 156 | 157 | ``` 158 | curl -XPOST -d '{"name":"Bob"}' http://cloud-endpoint/node?label=Person 159 | ``` 160 | 161 | Will result in a node with the label Foo, having property names like `name:"Bob"`. 162 | 163 | If deeply nested JSON is posted to the endpoint, the dictionary will be flattened, so that: 164 | ``` 165 | { 166 | "model": { 167 | "name": "something" 168 | } 169 | } 170 | ``` 171 | 172 | Will turn into a property 173 | 174 | ``` 175 | `model.name`: "something" 176 | ``` 177 | 178 | in neo4j. 179 | 180 | By customizing the URL you use for the webhook, you can track source of data. For example, 181 | providing to the slack external webhook a URL of: `http://cloud-endpoint/node?label=SlackMessage`. 182 | 183 | ## Edge Function 184 | 185 | This function matches two nodes, and creates a relationship between them with a given set of properties. 186 | 187 | Example: 188 | 189 | ``` 190 | curl -XPOST -d '{"x":1,"y":2}' 'http://localhost:8010/test-drive-development/us-central1/edge?fromLabel=Foo&toLabel=Foo&fromProp=x&toProp=x&fro=6&toVal=5&relType=blark' 191 | ``` 192 | 193 | This is equivalent to doing this in Cypher: 194 | 195 | ``` 196 | MATCH (a:Foo { x: "5" }), (b:Foo { x: "6" }) 197 | CREATE (a)-[:blark { x: 1, y: 2 }]->(b); 198 | ``` 199 | 200 | Any POST'd JSON data will be stored as properties on the relationship. 201 | 202 | ## CUD Function 203 | 204 | The CUD function takes an array of CUD command objects. 205 | 206 | The CUD format is a tiny JSON format that allows you to specify a graph "Create, Update, 207 | or Delete" (CUD) operation on a graph. For example, a JSON message may indicate that you 208 | want to create a node with certain labels and properties. 209 | 210 | [See here for documentation on the CUD format](https://neo4j.com/docs/labs/neo4j-streams/current/consumer/#_cud_file_format) 211 | 212 | Example: 213 | 214 | ``` 215 | curl --data @test/cud-messages.json \ 216 | -H "Content-Type: application/json" -X POST \ 217 | $LOCALDEPLOY 218 | ``` 219 | 220 | ## Cypher Function 221 | 222 | It takes two simple arguments: a cypher string, and an array of batch inputs. An example 223 | input would look like this: 224 | 225 | ``` 226 | { 227 | "cypher": "CREATE (p:Person) SET p += event", 228 | "batch": [ 229 | { "name": "Sarah", "age": 22 }, 230 | { "name": "Bob", "age": 25 } 231 | ] 232 | } 233 | ``` 234 | 235 | Your query will always be prepended with the clause `UNWIND batch AS event` so that 236 | the "event" variable reference will always be defined in your query to reference an individual 237 | row of data. 238 | 239 | Here's another example which would create a set of relationships; make a simple social network 240 | in one JSON post body. 241 | 242 | ``` 243 | { 244 | "cypher": "MERGE (p1:Person{name: event.originator}) MERGE (p2:Person{name: event.accepter}) MERGE (p1)-[:FRIENDED { date: event.date }]->(p2)", 245 | "batch": [ 246 | { "originator": "John", "accepter": "Sarah", "date": "2020-01-01" }, 247 | { "originator": "Anita", "accepter": "Joe", "date": "2020-01-02" }, 248 | { "originator": "Baz", "accepter": "John", "date": "2020-01-03" }, 249 | { "originator": "Evander", "accepter": "Sarah", "date": "2020-01-04" }, 250 | { "originator": "Idris", "accepter": "Evander", "date": "2020-01-05" }, 251 | { "originator": "Sarah", "accepter": "Baz", "date": "2020-01-06" }, 252 | { "originator": "Nia", "accepter": "Joe", "date": "2020-01-01" }, 253 | { "originator": "Joe", "accepter": "Baz", "date": "2020-01-03" }, 254 | { "originator": "Bob", "accepter": "Idris", "date": "2020-01-03" }, 255 | { "originator": "Joe", "accepter": "Evander", "date": "2020-01-03" } 256 | ] 257 | } 258 | ``` 259 | 260 | ## Custom Cypher Function 261 | 262 | In the Cypher function above, note that the message to the function requires sending a Cypher query. Sometimes 263 | the publisher of the message or sender of the JSON payload won't know the Cypher queries. In this case, we want 264 | to bake in the cypher query and ensure that the function in question can only ever use 1 query. That's what the 265 | custom cypher function is for. 266 | 267 | The input format accepted is then only a batch array. Because the Cypher sink query is "baked into the function", 268 | the function you deploy can only ever run that query. 269 | 270 | Here's a deployment example of how you can use hard-wired custom cypher functions: 271 | 272 | ``` 273 | echo "GCP_PROJECT: ${{ secrets.GCP_PROJECT_ID }}" >> /tmp/env.yaml 274 | echo "URI_SECRET: projects/graphs-are-everywhere/secrets/NEO4J_URI/versions/latest" >> /tmp/env.yaml 275 | echo "USER_SECRET: projects/graphs-are-everywhere/secrets/NEO4J_USER/versions/latest" >> /tmp/env.yaml 276 | echo "PASSWORD_SECRET: projects/graphs-are-everywhere/secrets/NEO4J_PASSWORD/versions/latest" >> /tmp/env.yaml 277 | echo 'CYPHER: "MERGE (p:Person) SET p += event"' >> /tmp/env.yaml 278 | 279 | 280 | gcloud functions deploy myCustomFunction \ 281 | --entry-point customCypherPubsub \ 282 | --ingress-settings=all --runtime=nodejs12 \ 283 | --allow-unauthenticated --timeout=300 \ 284 | --service-account=(address of service account)) \ 285 | --env-vars-file /tmp/env.yaml \ 286 | --trigger-topic personFeed 287 | ``` 288 | 289 | What this does: 290 | 291 | * The topic `personFeed` is assumed to publish arrays of JSON only (just the batch data) 292 | * It takes the `customCypherPubsub` function, and wires it to a Cloud Function called `myCustomFunction` 293 | * This function will only ever be able to execute the cypher query `MERGE (p:Person) SET p += event` 294 | 295 | ## Security 296 | 297 | *It is very important you secure access to the functions in a way that is appropriate 298 | to your database*. These functions fundamentally allow users to run cypher and modify 299 | data, so take care to use the existing Google Cloud Functions utilities to secure 300 | the endpoints. 301 | 302 | The development documentation in this repo assume an insecure "anyone can call this" 303 | endpoint. I recommend using [Google Cloud identity or network-based access control](https://cloud.google.com/functions/docs/securing) 304 | 305 | ## Unit Testing 306 | 307 | ``` 308 | yarn test 309 | ``` 310 | 311 | ## Local Testing 312 | 313 | ``` 314 | ./node_modules/.bin/functions-framework --target=cud 315 | ./node_modules/.bin/functions-framework --target=cypher 316 | ./node_modules/.bin/functions-framework --target=node 317 | ./node_modules/.bin/functions-framework --target=edge 318 | ``` 319 | -------------------------------------------------------------------------------- /call-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LOCALDEPLOY=http://localhost:8010/test-drive-development/us-central1 4 | 5 | curl -H "Content-Type: application/json" -X POST -d '{"username":"xyz","name":"David"}' $LOCALDEPLOY/node?label=User 6 | curl -H "Content-Type: application/json" -X POST -d '{"username":"foo","name":"Mark"}' $LOCALDEPLOY/node?label=User 7 | curl -H "Content-Type: application/json" -X POST -d '{"since":"yesterday","metadata":"whatever"}' \ 8 | $LOCALDEPLOY'/edge?fromLabel=User&fromProp=username&fromVal=xyz&toLabel=User&toProp=username&toVal=foo&relType=knows' 9 | -------------------------------------------------------------------------------- /example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moxious/neo4j-serverless-functions/9410787f068662e021a2da66a22fe3f2229ed751/example.png -------------------------------------------------------------------------------- /examples/EXAMPLES.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | This is a separate JS project in the same repo, so make sure to run: 4 | 5 | ``` 6 | npm install 7 | ``` 8 | 9 | ## Load some CSV with Custom Cypher 10 | 11 | Note the URL, this is assuming you're running the function locally. 12 | 13 | This will load a set of Emoji along with their categories into the database that the service 14 | is pointing at. 15 | 16 | ``` 17 | CYPHER="MERGE (c:Category { name: event.category }) CREATE (e:Emoji) SET e += event CREATE (e)-[:IN]->(c)" 18 | node load-csv.js -u http://localhost:8080/ -f all-emojis.csv \ 19 | -c "$CYPHER" 20 | ``` -------------------------------------------------------------------------------- /examples/load-csv.js: -------------------------------------------------------------------------------- 1 | const csv = require('csv-load-sync'); 2 | const request = require('request-promise'); 3 | const args = require('yargs') 4 | .example('$0 -f people.csv -l Person -u http://localhost:8080/WhereCypherRuns') 5 | .describe('u', 'URL where the Cypher endpoint runs') 6 | .describe('f', 'CSV file to load') 7 | .describe('l', 'Label to apply to all nodes in the CSV') 8 | .describe('c', 'Cypher to execute') 9 | .demandOption(['u', 'f']) 10 | .argv; 11 | 12 | if ((!args.l && !args.c) || (args.l && args.c)) { 13 | throw new Error('You must specify exactly one of either -l label or -c Cypher'); 14 | } 15 | 16 | const data = csv(args.f); 17 | 18 | const cypher = args.l ? `CREATE (n:\`${args.l}\`) SET n += event` : args.c; 19 | 20 | request({ 21 | uri: args.u, 22 | method: 'POST', 23 | json: true, 24 | body: { 25 | cypher, 26 | batch: data, 27 | }, 28 | }) 29 | .then(results => { 30 | console.log(JSON.stringify(results, null, 2)); 31 | }) 32 | .catch(err => console.error(err)); 33 | -------------------------------------------------------------------------------- /examples/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "examples", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@types/color-name": { 8 | "version": "1.1.1", 9 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/@types/color-name/-/color-name-1.1.1.tgz", 10 | "integrity": "sha1-HBJhu+qhCoBVu8XYq4S3sq/IRqA=" 11 | }, 12 | "ajv": { 13 | "version": "6.12.2", 14 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/ajv/-/ajv-6.12.2.tgz", 15 | "integrity": "sha1-xinF7O0XuvMUQ3kY0tqIyZ1ZWM0=", 16 | "requires": { 17 | "fast-deep-equal": "^3.1.1", 18 | "fast-json-stable-stringify": "^2.0.0", 19 | "json-schema-traverse": "^0.4.1", 20 | "uri-js": "^4.2.2" 21 | } 22 | }, 23 | "ansi-regex": { 24 | "version": "5.0.0", 25 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/ansi-regex/-/ansi-regex-5.0.0.tgz", 26 | "integrity": "sha1-OIU59VF5vzkznIGvMKZU1p+Hy3U=" 27 | }, 28 | "ansi-styles": { 29 | "version": "4.2.1", 30 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/ansi-styles/-/ansi-styles-4.2.1.tgz", 31 | "integrity": "sha1-kK51xCTQCNJiTFvynq0xd+v881k=", 32 | "requires": { 33 | "@types/color-name": "^1.1.1", 34 | "color-convert": "^2.0.1" 35 | } 36 | }, 37 | "asn1": { 38 | "version": "0.2.4", 39 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/asn1/-/asn1-0.2.4.tgz", 40 | "integrity": "sha1-jSR136tVO7M+d7VOWeiAu4ziMTY=", 41 | "requires": { 42 | "safer-buffer": "~2.1.0" 43 | } 44 | }, 45 | "assert-plus": { 46 | "version": "1.0.0", 47 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/assert-plus/-/assert-plus-1.0.0.tgz", 48 | "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" 49 | }, 50 | "asynckit": { 51 | "version": "0.4.0", 52 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/asynckit/-/asynckit-0.4.0.tgz", 53 | "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" 54 | }, 55 | "aws-sign2": { 56 | "version": "0.7.0", 57 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/aws-sign2/-/aws-sign2-0.7.0.tgz", 58 | "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" 59 | }, 60 | "aws4": { 61 | "version": "1.9.1", 62 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/aws4/-/aws4-1.9.1.tgz", 63 | "integrity": "sha1-fjPY99RJs/ZzzXLeuavcVS2+Uo4=" 64 | }, 65 | "bcrypt-pbkdf": { 66 | "version": "1.0.2", 67 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", 68 | "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", 69 | "requires": { 70 | "tweetnacl": "^0.14.3" 71 | } 72 | }, 73 | "bluebird": { 74 | "version": "3.7.2", 75 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/bluebird/-/bluebird-3.7.2.tgz", 76 | "integrity": "sha1-nyKcFb4nJFT/qXOs4NvueaGww28=" 77 | }, 78 | "camelcase": { 79 | "version": "5.3.1", 80 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/camelcase/-/camelcase-5.3.1.tgz", 81 | "integrity": "sha1-48mzFWnhBoEd8kL3FXJaH0xJQyA=" 82 | }, 83 | "caseless": { 84 | "version": "0.12.0", 85 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/caseless/-/caseless-0.12.0.tgz", 86 | "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" 87 | }, 88 | "check-more-types": { 89 | "version": "2.24.0", 90 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/check-more-types/-/check-more-types-2.24.0.tgz", 91 | "integrity": "sha1-FCD/sQ/URNz8ebQ4kbv//TKoRgA=" 92 | }, 93 | "cliui": { 94 | "version": "6.0.0", 95 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/cliui/-/cliui-6.0.0.tgz", 96 | "integrity": "sha1-UR1wLAxOQcoVbX0OlgIfI+EyJbE=", 97 | "requires": { 98 | "string-width": "^4.2.0", 99 | "strip-ansi": "^6.0.0", 100 | "wrap-ansi": "^6.2.0" 101 | } 102 | }, 103 | "color-convert": { 104 | "version": "2.0.1", 105 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/color-convert/-/color-convert-2.0.1.tgz", 106 | "integrity": "sha1-ctOmjVmMm9s68q0ehPIdiWq9TeM=", 107 | "requires": { 108 | "color-name": "~1.1.4" 109 | } 110 | }, 111 | "color-name": { 112 | "version": "1.1.4", 113 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/color-name/-/color-name-1.1.4.tgz", 114 | "integrity": "sha1-wqCah6y95pVD3m9j+jmVyCbFNqI=" 115 | }, 116 | "combined-stream": { 117 | "version": "1.0.8", 118 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/combined-stream/-/combined-stream-1.0.8.tgz", 119 | "integrity": "sha1-w9RaizT9cwYxoRCoolIGgrMdWn8=", 120 | "requires": { 121 | "delayed-stream": "~1.0.0" 122 | } 123 | }, 124 | "core-util-is": { 125 | "version": "1.0.2", 126 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/core-util-is/-/core-util-is-1.0.2.tgz", 127 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" 128 | }, 129 | "csv-load-sync": { 130 | "version": "1.0.0", 131 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/csv-load-sync/-/csv-load-sync-1.0.0.tgz", 132 | "integrity": "sha1-UoXKmTrgcngI9xfab0NisbYhQgU=", 133 | "requires": { 134 | "check-more-types": "^2.10.0" 135 | } 136 | }, 137 | "dashdash": { 138 | "version": "1.14.1", 139 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/dashdash/-/dashdash-1.14.1.tgz", 140 | "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", 141 | "requires": { 142 | "assert-plus": "^1.0.0" 143 | } 144 | }, 145 | "decamelize": { 146 | "version": "1.2.0", 147 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/decamelize/-/decamelize-1.2.0.tgz", 148 | "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" 149 | }, 150 | "delayed-stream": { 151 | "version": "1.0.0", 152 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/delayed-stream/-/delayed-stream-1.0.0.tgz", 153 | "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" 154 | }, 155 | "ecc-jsbn": { 156 | "version": "0.1.2", 157 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", 158 | "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", 159 | "requires": { 160 | "jsbn": "~0.1.0", 161 | "safer-buffer": "^2.1.0" 162 | } 163 | }, 164 | "emoji-regex": { 165 | "version": "8.0.0", 166 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/emoji-regex/-/emoji-regex-8.0.0.tgz", 167 | "integrity": "sha1-6Bj9ac5cz8tARZT4QpY79TFkzDc=" 168 | }, 169 | "extend": { 170 | "version": "3.0.2", 171 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/extend/-/extend-3.0.2.tgz", 172 | "integrity": "sha1-+LETa0Bx+9jrFAr/hYsQGewpFfo=" 173 | }, 174 | "extsprintf": { 175 | "version": "1.3.0", 176 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/extsprintf/-/extsprintf-1.3.0.tgz", 177 | "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" 178 | }, 179 | "fast-deep-equal": { 180 | "version": "3.1.1", 181 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", 182 | "integrity": "sha1-VFFFB3xQFJHjOxXsQIwpQ3bpSuQ=" 183 | }, 184 | "fast-json-stable-stringify": { 185 | "version": "2.1.0", 186 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", 187 | "integrity": "sha1-h0v2nG9ATCtdmcSBNBOZ/VWJJjM=" 188 | }, 189 | "find-up": { 190 | "version": "4.1.0", 191 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/find-up/-/find-up-4.1.0.tgz", 192 | "integrity": "sha1-l6/n1s3AvFkoWEt8jXsW6KmqXRk=", 193 | "requires": { 194 | "locate-path": "^5.0.0", 195 | "path-exists": "^4.0.0" 196 | } 197 | }, 198 | "forever-agent": { 199 | "version": "0.6.1", 200 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/forever-agent/-/forever-agent-0.6.1.tgz", 201 | "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" 202 | }, 203 | "form-data": { 204 | "version": "2.3.3", 205 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/form-data/-/form-data-2.3.3.tgz", 206 | "integrity": "sha1-3M5SwF9kTymManq5Nr1yTO/786Y=", 207 | "requires": { 208 | "asynckit": "^0.4.0", 209 | "combined-stream": "^1.0.6", 210 | "mime-types": "^2.1.12" 211 | } 212 | }, 213 | "get-caller-file": { 214 | "version": "2.0.5", 215 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/get-caller-file/-/get-caller-file-2.0.5.tgz", 216 | "integrity": "sha1-T5RBKoLbMvNuOwuXQfipf+sDH34=" 217 | }, 218 | "getpass": { 219 | "version": "0.1.7", 220 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/getpass/-/getpass-0.1.7.tgz", 221 | "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", 222 | "requires": { 223 | "assert-plus": "^1.0.0" 224 | } 225 | }, 226 | "har-schema": { 227 | "version": "2.0.0", 228 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/har-schema/-/har-schema-2.0.0.tgz", 229 | "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" 230 | }, 231 | "har-validator": { 232 | "version": "5.1.3", 233 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/har-validator/-/har-validator-5.1.3.tgz", 234 | "integrity": "sha1-HvievT5JllV2de7ZiTEQ3DUPoIA=", 235 | "requires": { 236 | "ajv": "^6.5.5", 237 | "har-schema": "^2.0.0" 238 | } 239 | }, 240 | "http-signature": { 241 | "version": "1.2.0", 242 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/http-signature/-/http-signature-1.2.0.tgz", 243 | "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", 244 | "requires": { 245 | "assert-plus": "^1.0.0", 246 | "jsprim": "^1.2.2", 247 | "sshpk": "^1.7.0" 248 | } 249 | }, 250 | "is-fullwidth-code-point": { 251 | "version": "3.0.0", 252 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", 253 | "integrity": "sha1-8Rb4Bk/pCz94RKOJl8C3UFEmnx0=" 254 | }, 255 | "is-typedarray": { 256 | "version": "1.0.0", 257 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/is-typedarray/-/is-typedarray-1.0.0.tgz", 258 | "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" 259 | }, 260 | "isstream": { 261 | "version": "0.1.2", 262 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/isstream/-/isstream-0.1.2.tgz", 263 | "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" 264 | }, 265 | "jsbn": { 266 | "version": "0.1.1", 267 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/jsbn/-/jsbn-0.1.1.tgz", 268 | "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" 269 | }, 270 | "json-schema": { 271 | "version": "0.2.3", 272 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/json-schema/-/json-schema-0.2.3.tgz", 273 | "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" 274 | }, 275 | "json-schema-traverse": { 276 | "version": "0.4.1", 277 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", 278 | "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=" 279 | }, 280 | "json-stringify-safe": { 281 | "version": "5.0.1", 282 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", 283 | "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" 284 | }, 285 | "jsprim": { 286 | "version": "1.4.1", 287 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/jsprim/-/jsprim-1.4.1.tgz", 288 | "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", 289 | "requires": { 290 | "assert-plus": "1.0.0", 291 | "extsprintf": "1.3.0", 292 | "json-schema": "0.2.3", 293 | "verror": "1.10.0" 294 | } 295 | }, 296 | "locate-path": { 297 | "version": "5.0.0", 298 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/locate-path/-/locate-path-5.0.0.tgz", 299 | "integrity": "sha1-Gvujlq/WdqbUJQTQpno6frn2KqA=", 300 | "requires": { 301 | "p-locate": "^4.1.0" 302 | } 303 | }, 304 | "lodash": { 305 | "version": "4.17.15", 306 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/lodash/-/lodash-4.17.15.tgz", 307 | "integrity": "sha1-tEf2ZwoEVbv+7dETku/zMOoJdUg=" 308 | }, 309 | "mime-db": { 310 | "version": "1.44.0", 311 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/mime-db/-/mime-db-1.44.0.tgz", 312 | "integrity": "sha1-+hHF6wrKEzS0Izy01S8QxaYnL5I=" 313 | }, 314 | "mime-types": { 315 | "version": "2.1.27", 316 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/mime-types/-/mime-types-2.1.27.tgz", 317 | "integrity": "sha1-R5SfmOJ56lMRn1ci4PNOUpvsAJ8=", 318 | "requires": { 319 | "mime-db": "1.44.0" 320 | } 321 | }, 322 | "oauth-sign": { 323 | "version": "0.9.0", 324 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/oauth-sign/-/oauth-sign-0.9.0.tgz", 325 | "integrity": "sha1-R6ewFrqmi1+g7PPe4IqFxnmsZFU=" 326 | }, 327 | "p-limit": { 328 | "version": "2.3.0", 329 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/p-limit/-/p-limit-2.3.0.tgz", 330 | "integrity": "sha1-PdM8ZHohT9//2DWTPrCG2g3CHbE=", 331 | "requires": { 332 | "p-try": "^2.0.0" 333 | } 334 | }, 335 | "p-locate": { 336 | "version": "4.1.0", 337 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/p-locate/-/p-locate-4.1.0.tgz", 338 | "integrity": "sha1-o0KLtwiLOmApL2aRkni3wpetTwc=", 339 | "requires": { 340 | "p-limit": "^2.2.0" 341 | } 342 | }, 343 | "p-try": { 344 | "version": "2.2.0", 345 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/p-try/-/p-try-2.2.0.tgz", 346 | "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=" 347 | }, 348 | "path-exists": { 349 | "version": "4.0.0", 350 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/path-exists/-/path-exists-4.0.0.tgz", 351 | "integrity": "sha1-UTvb4tO5XXdi6METfvoZXGxhtbM=" 352 | }, 353 | "performance-now": { 354 | "version": "2.1.0", 355 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/performance-now/-/performance-now-2.1.0.tgz", 356 | "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" 357 | }, 358 | "psl": { 359 | "version": "1.8.0", 360 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/psl/-/psl-1.8.0.tgz", 361 | "integrity": "sha1-kyb4vPsBOtzABf3/BWrM4CDlHCQ=" 362 | }, 363 | "punycode": { 364 | "version": "2.1.1", 365 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/punycode/-/punycode-2.1.1.tgz", 366 | "integrity": "sha1-tYsBCsQMIsVldhbI0sLALHv0eew=" 367 | }, 368 | "qs": { 369 | "version": "6.5.2", 370 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/qs/-/qs-6.5.2.tgz", 371 | "integrity": "sha1-yzroBuh0BERYTvFUzo7pjUA/PjY=" 372 | }, 373 | "request": { 374 | "version": "2.88.2", 375 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/request/-/request-2.88.2.tgz", 376 | "integrity": "sha1-1zyRhzHLWofaBH4gcjQUb2ZNErM=", 377 | "requires": { 378 | "aws-sign2": "~0.7.0", 379 | "aws4": "^1.8.0", 380 | "caseless": "~0.12.0", 381 | "combined-stream": "~1.0.6", 382 | "extend": "~3.0.2", 383 | "forever-agent": "~0.6.1", 384 | "form-data": "~2.3.2", 385 | "har-validator": "~5.1.3", 386 | "http-signature": "~1.2.0", 387 | "is-typedarray": "~1.0.0", 388 | "isstream": "~0.1.2", 389 | "json-stringify-safe": "~5.0.1", 390 | "mime-types": "~2.1.19", 391 | "oauth-sign": "~0.9.0", 392 | "performance-now": "^2.1.0", 393 | "qs": "~6.5.2", 394 | "safe-buffer": "^5.1.2", 395 | "tough-cookie": "~2.5.0", 396 | "tunnel-agent": "^0.6.0", 397 | "uuid": "^3.3.2" 398 | } 399 | }, 400 | "request-promise": { 401 | "version": "4.2.5", 402 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/request-promise/-/request-promise-4.2.5.tgz", 403 | "integrity": "sha1-GGIixZrlEvNJff5NdanIRhvQBTw=", 404 | "requires": { 405 | "bluebird": "^3.5.0", 406 | "request-promise-core": "1.1.3", 407 | "stealthy-require": "^1.1.1", 408 | "tough-cookie": "^2.3.3" 409 | } 410 | }, 411 | "request-promise-core": { 412 | "version": "1.1.3", 413 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/request-promise-core/-/request-promise-core-1.1.3.tgz", 414 | "integrity": "sha1-6aPAgbUTgN/qZ3M2Bh/qh5qCnuk=", 415 | "requires": { 416 | "lodash": "^4.17.15" 417 | } 418 | }, 419 | "require-directory": { 420 | "version": "2.1.1", 421 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/require-directory/-/require-directory-2.1.1.tgz", 422 | "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" 423 | }, 424 | "require-main-filename": { 425 | "version": "2.0.0", 426 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/require-main-filename/-/require-main-filename-2.0.0.tgz", 427 | "integrity": "sha1-0LMp7MfMD2Fkn2IhW+aa9UqomJs=" 428 | }, 429 | "safe-buffer": { 430 | "version": "5.2.0", 431 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/safe-buffer/-/safe-buffer-5.2.0.tgz", 432 | "integrity": "sha1-t02uxJsRSPiMZLaNSbHoFcHy9Rk=" 433 | }, 434 | "safer-buffer": { 435 | "version": "2.1.2", 436 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/safer-buffer/-/safer-buffer-2.1.2.tgz", 437 | "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" 438 | }, 439 | "set-blocking": { 440 | "version": "2.0.0", 441 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/set-blocking/-/set-blocking-2.0.0.tgz", 442 | "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" 443 | }, 444 | "sshpk": { 445 | "version": "1.16.1", 446 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/sshpk/-/sshpk-1.16.1.tgz", 447 | "integrity": "sha1-+2YcC+8ps520B2nuOfpwCT1vaHc=", 448 | "requires": { 449 | "asn1": "~0.2.3", 450 | "assert-plus": "^1.0.0", 451 | "bcrypt-pbkdf": "^1.0.0", 452 | "dashdash": "^1.12.0", 453 | "ecc-jsbn": "~0.1.1", 454 | "getpass": "^0.1.1", 455 | "jsbn": "~0.1.0", 456 | "safer-buffer": "^2.0.2", 457 | "tweetnacl": "~0.14.0" 458 | } 459 | }, 460 | "stealthy-require": { 461 | "version": "1.1.1", 462 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/stealthy-require/-/stealthy-require-1.1.1.tgz", 463 | "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" 464 | }, 465 | "string-width": { 466 | "version": "4.2.0", 467 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/string-width/-/string-width-4.2.0.tgz", 468 | "integrity": "sha1-lSGCxGzHssMT0VluYjmSvRY7crU=", 469 | "requires": { 470 | "emoji-regex": "^8.0.0", 471 | "is-fullwidth-code-point": "^3.0.0", 472 | "strip-ansi": "^6.0.0" 473 | } 474 | }, 475 | "strip-ansi": { 476 | "version": "6.0.0", 477 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/strip-ansi/-/strip-ansi-6.0.0.tgz", 478 | "integrity": "sha1-CxVx3XZpzNTz4G4U7x7tJiJa5TI=", 479 | "requires": { 480 | "ansi-regex": "^5.0.0" 481 | } 482 | }, 483 | "tough-cookie": { 484 | "version": "2.5.0", 485 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/tough-cookie/-/tough-cookie-2.5.0.tgz", 486 | "integrity": "sha1-zZ+yoKodWhK0c72fuW+j3P9lreI=", 487 | "requires": { 488 | "psl": "^1.1.28", 489 | "punycode": "^2.1.1" 490 | } 491 | }, 492 | "tunnel-agent": { 493 | "version": "0.6.0", 494 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/tunnel-agent/-/tunnel-agent-0.6.0.tgz", 495 | "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", 496 | "requires": { 497 | "safe-buffer": "^5.0.1" 498 | } 499 | }, 500 | "tweetnacl": { 501 | "version": "0.14.5", 502 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/tweetnacl/-/tweetnacl-0.14.5.tgz", 503 | "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" 504 | }, 505 | "uri-js": { 506 | "version": "4.2.2", 507 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/uri-js/-/uri-js-4.2.2.tgz", 508 | "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", 509 | "requires": { 510 | "punycode": "^2.1.0" 511 | } 512 | }, 513 | "uuid": { 514 | "version": "3.4.0", 515 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/uuid/-/uuid-3.4.0.tgz", 516 | "integrity": "sha1-sj5DWK+oogL+ehAK8fX4g/AgB+4=" 517 | }, 518 | "verror": { 519 | "version": "1.10.0", 520 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/verror/-/verror-1.10.0.tgz", 521 | "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", 522 | "requires": { 523 | "assert-plus": "^1.0.0", 524 | "core-util-is": "1.0.2", 525 | "extsprintf": "^1.2.0" 526 | } 527 | }, 528 | "which-module": { 529 | "version": "2.0.0", 530 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/which-module/-/which-module-2.0.0.tgz", 531 | "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" 532 | }, 533 | "wrap-ansi": { 534 | "version": "6.2.0", 535 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/wrap-ansi/-/wrap-ansi-6.2.0.tgz", 536 | "integrity": "sha1-6Tk7oHEC5skaOyIUePAlfNKFblM=", 537 | "requires": { 538 | "ansi-styles": "^4.0.0", 539 | "string-width": "^4.1.0", 540 | "strip-ansi": "^6.0.0" 541 | } 542 | }, 543 | "y18n": { 544 | "version": "4.0.0", 545 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/y18n/-/y18n-4.0.0.tgz", 546 | "integrity": "sha1-le+U+F7MgdAHwmThkKEg8KPIVms=" 547 | }, 548 | "yargs": { 549 | "version": "15.3.1", 550 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/yargs/-/yargs-15.3.1.tgz", 551 | "integrity": "sha1-lQW0cnY5Y+VK/mAUitJ6MwgY6Ys=", 552 | "requires": { 553 | "cliui": "^6.0.0", 554 | "decamelize": "^1.2.0", 555 | "find-up": "^4.1.0", 556 | "get-caller-file": "^2.0.1", 557 | "require-directory": "^2.1.1", 558 | "require-main-filename": "^2.0.0", 559 | "set-blocking": "^2.0.0", 560 | "string-width": "^4.2.0", 561 | "which-module": "^2.0.0", 562 | "y18n": "^4.0.0", 563 | "yargs-parser": "^18.1.1" 564 | } 565 | }, 566 | "yargs-parser": { 567 | "version": "18.1.3", 568 | "resolved": "https://neo.jfrog.io/neo/api/npm/npm/yargs-parser/-/yargs-parser-18.1.3.tgz", 569 | "integrity": "sha1-vmjEl1xrKr9GkjawyHA2L6sJp7A=", 570 | "requires": { 571 | "camelcase": "^5.0.0", 572 | "decamelize": "^1.2.0" 573 | } 574 | } 575 | } 576 | } 577 | -------------------------------------------------------------------------------- /examples/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "examples", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "csv-load-sync": "^1.0.0", 13 | "request": "^2.88.2", 14 | "request-promise": "^4.2.5", 15 | "yargs": "^15.3.1" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /gil/driver/driverConfiguration.js: -------------------------------------------------------------------------------- 1 | const { SecretManagerServiceClient } = require('@google-cloud/secret-manager'); 2 | const neo4j = require('neo4j-driver'); 3 | 4 | const project = process.env.GCP_PROJECT || 'graphs-are-everywhere'; 5 | 6 | // For proper auth, set GOOGLE_APPLICATION_CREDENTIALS to the key 7 | // that contains project information & service account. 8 | const getDriverOptions = async (config=process.env) => { 9 | const DRIVER_OPTIONS = { 10 | maxConnectionLifetime: 8 * 1000 * 60, // 8 minutes 11 | connectionLivenessCheckTimeout: 2 * 1000 * 60, 12 | }; 13 | 14 | let uri, user, password; 15 | 16 | if (config.URI_SECRET && config.USER_SECRET && config.PASSWORD_SECRET) { 17 | const client = new SecretManagerServiceClient(); 18 | const val = await client.accessSecretVersion({ name: config.URI_SECRET }); 19 | console.log('SECRET ',val); 20 | 21 | const [uriResponse] = await client.accessSecretVersion({ name: config.URI_SECRET }); 22 | const [userResponse] = await client.accessSecretVersion({ name: config.USER_SECRET }); 23 | const [passwordResponse] = await client.accessSecretVersion({ name: config.PASSWORD_SECRET }); 24 | 25 | uri = uriResponse.payload.data.toString('utf8'); 26 | user = userResponse.payload.data.toString('utf8'); 27 | password = passwordResponse.payload.data.toString('utf8'); 28 | } else { 29 | console.error('Attempting to configure driver from the environment, as Google Secret Manager keys are not present'); 30 | 31 | uri = config.NEO4J_URI; 32 | user = config.NEO4J_USER; 33 | password = config.NEO4J_PASSWORD; 34 | } 35 | 36 | if (!uri || !user || !password) { 37 | throw new Error('You must either configure Google Secrets Manager for connection details, or you must specify environment variables'); 38 | } 39 | 40 | const auth = neo4j.auth.basic(user, password); 41 | 42 | return [uri, auth, DRIVER_OPTIONS]; 43 | } 44 | 45 | module.exports = getDriverOptions; 46 | -------------------------------------------------------------------------------- /gil/driver/driverConfiguration.test.js: -------------------------------------------------------------------------------- 1 | const { SecretManagerServiceClient } = require('@google-cloud/secret-manager'); 2 | const getDriverOptions = require('./driverConfiguration'); 3 | const neo4j = require('./index'); 4 | const sinon = require('sinon'); 5 | const { iteratee } = require('lodash'); 6 | const { expectation } = require('sinon'); 7 | 8 | describe('Driver Configuration', () => { 9 | let driver, session, tx, run, rec; 10 | let data; 11 | 12 | const URI = 'neo4j://fake'; 13 | const USER = 'MySpecialUser'; 14 | const PASS = 'MySecretPassword'; 15 | 16 | const fakeSecrets = { 17 | uriSecret: 'neo4j://secreturi', 18 | userSecret: 'MySecretUser', 19 | passwordSecret: 'xxx', 20 | }; 21 | 22 | let stub; 23 | 24 | beforeEach(() => { 25 | stub = sinon.stub(SecretManagerServiceClient.prototype, 'accessSecretVersion') 26 | .callsFake(args => { 27 | const response = { 28 | payload: { 29 | data: fakeSecrets[args.name], 30 | }, 31 | }; 32 | 33 | return [response]; 34 | }); 35 | }); 36 | 37 | afterEach(() => stub.restore()); 38 | 39 | const goodReturnValue = val => { 40 | expect(val).toBeInstanceOf(Array); 41 | expect(val.length).toEqual(3); 42 | expect(val[2]).toBeInstanceOf(Object); 43 | expect(val[1]).toBeInstanceOf(Object); 44 | 45 | // ['scheme', 'principal', 'credentials'].forEach(key => { 46 | // expect(val[1][key]).toBeOk(); 47 | // }); 48 | } 49 | 50 | it('throws an error when given no configuration information', async () => { 51 | const config = { 52 | NEO4J_URI: '', 53 | NEO4J_USER: '', 54 | NEO4J_PASSWORD: '', 55 | URI_SECRET: '', 56 | USER_SECRET: '', 57 | PASSWORD_SECRET: '', 58 | }; 59 | 60 | expect(getDriverOptions(config)).rejects; 61 | }); 62 | 63 | it('uses environment variables when they are specified', async () => { 64 | const config = { 65 | NEO4J_URI: URI, 66 | NEO4J_USER: USER, 67 | NEO4J_PASSWORD: PASS, 68 | URI_SECRET: '', 69 | USER_SECRET: '', 70 | PASSWORD_SECRET: '', 71 | }; 72 | 73 | const res = await getDriverOptions(config); 74 | 75 | goodReturnValue(res); 76 | expect(res[0]).toEqual(URI); 77 | console.log(res[1]); 78 | }); 79 | 80 | it('uses Google Secret Manager configuration when it is specified', async() => { 81 | const config = { 82 | NEO4J_URI: '', 83 | NEO4J_USER: '', 84 | NEO4J_PASSWORD: '', 85 | URI_SECRET: 'uriSecret', 86 | USER_SECRET: 'userSecret', 87 | PASSWORD_SECRET: 'passwordSecret', 88 | }; 89 | 90 | const res = await getDriverOptions(config); 91 | goodReturnValue(res); 92 | expect(res[0]).toEqual(fakeSecrets.uriSecret); 93 | 94 | }); 95 | }); -------------------------------------------------------------------------------- /gil/driver/index.js: -------------------------------------------------------------------------------- 1 | const neo4j = require('neo4j-driver'); 2 | const Promise = require('bluebird'); 3 | const flat = require('flat'); 4 | const _ = require('lodash'); 5 | const me = require('../../package.json'); 6 | const Neode = require('neode'); 7 | const driverConfiguration = require('./driverConfiguration'); 8 | 9 | const driverSetup = async () => { 10 | const config = await driverConfiguration(); 11 | const driver = neo4j.driver(...config); 12 | 13 | driver._userAgent = `neo4j-serverless-functions/v${me.version}`; 14 | return driver; 15 | }; 16 | 17 | let persistentDriver = null; 18 | 19 | const setupInitial = async () => { 20 | const driver = await driverSetup(); 21 | persistentDriver = driver; 22 | }; 23 | 24 | /** 25 | * Get a driver instance. Creates them lazy according to google best practices. 26 | */ 27 | const getDriver = async () => { 28 | if (!persistentDriver) { 29 | persistentDriver = await driverSetup(); 30 | } 31 | 32 | return persistentDriver; 33 | }; 34 | 35 | const createNeo4jPropertiesFromObject = obj => { 36 | const flattened = flat.flatten(obj); 37 | // Eliminate empty maps. 38 | _.forEach(flattened, (val, key) => { 39 | // Cypher can't store empty maps, so set them to null. 40 | if (_.isObject(val) && _.isEmpty(val)) { 41 | _.set(flattened, key, null); 42 | } 43 | }); 44 | 45 | return flattened; 46 | }; 47 | 48 | const getSessionConfig = () => { 49 | const database = process.env.NEO4J_DATABASE || false 50 | return { ...(database ? { database } : {}) } 51 | } 52 | 53 | module.exports = { 54 | getDriver, 55 | getSessionConfig, 56 | createNeo4jPropertiesFromObject, 57 | }; 58 | -------------------------------------------------------------------------------- /gil/index.js: -------------------------------------------------------------------------------- 1 | const CUDBatch = require('./sink/CUDBatch'); 2 | const CUDCommand = require('./sink/CUDCommand'); 3 | const CypherSink = require('./sink/CypherSink'); 4 | const DataSink = require('./sink/DataSink'); 5 | const neo4j = require('./driver'); 6 | 7 | module.exports = { 8 | CUDBatch, CUDCommand, CypherSink, DataSink, neo4j, 9 | }; -------------------------------------------------------------------------------- /gil/sink/CUDBatch.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const neo4j = require('../driver'); 3 | const Promise = require('bluebird'); 4 | const Integer = require('neo4j-driver/lib/integer.js'); 5 | const Strategy = require('./Strategy'); 6 | 7 | const keyFor = cmd => { 8 | let { op, type, ids, properties, labels, from, to } = cmd.data; 9 | 10 | return `${op}.${type}` 11 | }; 12 | 13 | let seq = 0; 14 | 15 | const MAX_BATCH_SIZE = 2000; 16 | 17 | class CUDBatch extends Strategy { 18 | constructor(sequence = seq++) { 19 | super(); 20 | this.batch = []; 21 | this.key = null; 22 | this.seq = sequence; 23 | } 24 | 25 | commands() { return this.batch; } 26 | getEvents() { return this.commands(); } 27 | getCypher() { return this.commands()[0].generate(); } 28 | isEmpty() { return this.batch.length === 0; } 29 | canHold(cmd) { return this.key === null || this.key === cmd.key(); } 30 | 31 | getKey() { return this.key; } 32 | 33 | add(command) { 34 | const key = command.key(); 35 | 36 | if (this.key === null) { 37 | this.key = key; 38 | this.batch.push(command); 39 | return command; 40 | } else if (key !== this.key) { 41 | throw new Error('Cannot add command of different type to batch with key ' + key); 42 | } 43 | 44 | this.batch.push(command); 45 | return command; 46 | } 47 | 48 | /** 49 | * Creates an array of batches for a sequence of commands. 50 | * @param {Array{CUDCommand}} commands 51 | * @returns {Array[CUDBatch]} 52 | */ 53 | static batchCommands(commands, maxBatchSize=MAX_BATCH_SIZE) { 54 | const results = []; 55 | 56 | let activeBatch = new CUDBatch(); 57 | let batches = 0; 58 | 59 | commands.forEach(command => { 60 | // Don't let batches get too big. 61 | if (activeBatch.commands().length >= maxBatchSize) { 62 | // console.log('Pushing full batch and making ', ++batches); 63 | results.push(activeBatch); 64 | activeBatch = new CUDBatch(); 65 | } 66 | 67 | if (activeBatch.canHold(command)) { 68 | // console.log('adding to batch ', batches); 69 | return activeBatch.add(command); 70 | } 71 | 72 | // console.log('created batch ', ++batches); 73 | results.push(activeBatch); 74 | activeBatch = new CUDBatch(); 75 | return activeBatch.add(command); 76 | }); 77 | 78 | if (!activeBatch.isEmpty()) { 79 | results.push(activeBatch); 80 | } 81 | 82 | return results; 83 | } 84 | 85 | /** 86 | * Batch a set of commands for optimal execution, and run each batch. 87 | * @param {Array{CUDCommand}} commands 88 | * @returns {Promise} that resolves to an array of batch results. 89 | */ 90 | static async runAll(commands) { 91 | const batches = CUDBatch.batchCommands(commands); 92 | 93 | const driver = await neo4j.getDriver(); 94 | const session = driver.session(neo4j.getSessionConfig()); 95 | 96 | return session.writeTransaction(tx => 97 | Promise.mapSeries(batches, batch => batch.run(tx))) 98 | .finally(session.close); 99 | } 100 | } 101 | 102 | module.exports = CUDBatch; 103 | -------------------------------------------------------------------------------- /gil/sink/CUDBatch.test.js: -------------------------------------------------------------------------------- 1 | const CUDBatch = require('./CUDBatch'); 2 | const CUDCommand = require('./CUDCommand'); 3 | const test = require('../../test'); 4 | 5 | const fakeCommands = () => { 6 | const messages = []; 7 | for (let i=0; i<10; i++) { 8 | messages.push({ 9 | op: 'create', 10 | type: 'node', 11 | ids: { uuid: 'A' }, 12 | labels: ['Person', 'Friend'], 13 | properties: { 14 | x: 1, 15 | }, 16 | }); 17 | } 18 | 19 | const commands = messages.map(msg => new CUDCommand(msg)); 20 | return commands; 21 | }; 22 | 23 | describe('CUD Batch', () => { 24 | it('can batch messages', () => { 25 | const commands = fakeCommands(); 26 | const batches = CUDBatch.batchCommands(commands); 27 | 28 | expect(batches.length).toBe(1); 29 | expect(batches[0].commands().length).toBe(10); 30 | 31 | batches[0].commands().forEach(cmd => { 32 | expect(cmd.key()).toEqual('create.node.Person,Friend.uuid'); 33 | }); 34 | }); 35 | 36 | it('can batch messages up to a max batch size', () => { 37 | const commands = fakeCommands(); 38 | const batches = CUDBatch.batchCommands(commands, 4); // Batches no bigger than 4. 39 | expect(batches.length).toBe(3); 40 | 41 | expect(batches[0].commands().length).toBe(4); 42 | expect(batches[1].commands().length).toBe(4); 43 | expect(batches[2].commands().length).toBe(2); 44 | }); 45 | }); -------------------------------------------------------------------------------- /gil/sink/CUDCommand.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const operations = ['create', 'merge', 'delete']; 3 | const types = ['node', 'relationship']; 4 | const Integer = require('neo4j-driver/lib/integer.js'); 5 | 6 | const validNonEmptyObject = o => { 7 | return _.isObject(o) && _.values(o).length > 0; 8 | }; 9 | 10 | const validate = data => { 11 | if (operations.indexOf(data.op) === -1 || types.indexOf(data.type) === -1) { 12 | throw new Error('CUD command missing valid operation or type'); 13 | } 14 | 15 | if (data.type === 'relationship') { 16 | if (!validNonEmptyObject(data.from)) { throw new Error('Missing from information for relationship'); } 17 | if (!validNonEmptyObject(data.to)) { throw new Error('Missing to information for relationship'); } 18 | if (!data.rel_type) { throw new Error('Missing rel_type'); } 19 | } 20 | 21 | if (data.type === 'node') { 22 | if (_.isNil(data.labels) || _.isEmpty(data.labels) || !_.isArray(data.labels)) { 23 | throw new Error('Nodes must have an array of labels specified; unlabeled nodes are not supported.'); 24 | } 25 | } 26 | 27 | if (data.type === 'node' && data.op === 'merge') { 28 | if (!validNonEmptyObject(data.ids)) { 29 | throw new Error('Missing ids field for merge operation') 30 | } 31 | } 32 | } 33 | 34 | const escape = str => '`' + str.replace(/\`/g, '') + '`'; 35 | 36 | const labels2Cypher = labels => labels.map(escape).join(':'); 37 | 38 | const matchProperties = (criteria, paramField='ids') => { 39 | const clauses = Object.keys(criteria).map(propertyName => 40 | `${escape(propertyName)}: event.${paramField}.${escape(propertyName)}`) 41 | .join(', '); 42 | 43 | return '{ ' + clauses + ' }'; 44 | }; 45 | 46 | const matchOn = (alias, criteria, paramField='ids') => { 47 | const a = escape(alias); 48 | 49 | return Object.keys(criteria).map(propertyName => 50 | `${a}.${escape(propertyName)} = event.${paramField}.${escape(propertyName)}`) 51 | .join(' AND '); 52 | }; 53 | 54 | /** 55 | * Returns a single match clause for a single node by a lookup property set. 56 | * @param {String} alias name of variable to alias 57 | * @param {Object} data key/value pairs of properties to match on. 58 | */ 59 | const nodePattern = (alias, data, paramFields='ids') => 60 | `(${alias}:${labels2Cypher(data.labels)} ${matchProperties(data.ids || {}, paramFields)})`; 61 | 62 | class CUDCommand { 63 | constructor(data={}) { 64 | validate(data); 65 | data.properties = data.properties || {}; 66 | this.data = data; 67 | } 68 | 69 | _deleteNode() { 70 | const { op, properties, ids, labels, detach } = this.data; 71 | return (` 72 | MATCH ${nodePattern('n', this.data)} 73 | ${ detach ? 'DETACH' : ''} DELETE n 74 | RETURN event.op as op, event.type as type, id(n) as id 75 | `); 76 | } 77 | 78 | _generateNode() { 79 | const { op, properties, ids, labels } = this.data; 80 | 81 | let whereClause = ''; 82 | 83 | if (op === 'merge') { 84 | 85 | } 86 | 87 | return (` 88 | ${op.toUpperCase()} ${nodePattern('n', this.data)} 89 | SET n += event.properties 90 | RETURN event.op as op, event.type as type, id(n) as id 91 | `); 92 | } 93 | 94 | _deleteRelationship() { 95 | const { op, from, rel_type, to, properties, ids, labels } = this.data; 96 | 97 | let extraMatch = ''; 98 | 99 | if (properties && !_.isEmpty(properties)) { 100 | extraMatch = 'WHERE ' + matchOn('r', properties, 'properties'); 101 | } 102 | 103 | return (` 104 | MATCH ${nodePattern('a', from, 'from.ids')}-[r:${escape(rel_type)}]->${nodePattern('b', to, 'to.ids')} 105 | ${extraMatch} 106 | DELETE r 107 | RETURN event.op as op, event.type as type, id(r) as id 108 | `); 109 | } 110 | 111 | _generateRelationship() { 112 | const { op, from, rel_type, to, properties, ids, labels } = this.data; 113 | return (` 114 | MATCH ${nodePattern('a', from, 'from.ids')} 115 | WITH a, event 116 | MATCH ${nodePattern('b', to, 'to.ids')} 117 | ${op.toUpperCase()} (a)-[r:${escape(rel_type)}]->(b) 118 | SET r += event.properties 119 | RETURN event.op as op, event.type as type, id(r) as id 120 | `); 121 | } 122 | 123 | generate() { 124 | if (this.data.type === 'node') { 125 | if (this.data.op === 'delete') { 126 | return this._deleteNode(); 127 | } 128 | return this._generateNode(); 129 | } else { 130 | if (this.data.op === 'delete') { 131 | return this._deleteRelationship(); 132 | } 133 | return this._generateRelationship(); 134 | } 135 | } 136 | 137 | /** 138 | * The key for a command ties to what kind of batch the command can be 139 | * executed with. In order to do the UNWIND strategy and batch things 140 | * together, the query form must be the same. We can't for example batch 141 | * a relationship create together with a node delete. This function gives 142 | * a "partition key" of what kind of command this is. Differing commands 143 | * with the same keys can be batched together 144 | */ 145 | key() { 146 | let { op, type, ids, rel_type, labels, from, to } = this.data; 147 | 148 | const keyparts = [op, type]; 149 | 150 | if (type === 'node') { 151 | keyparts.push((labels || []).join(',')); 152 | keyparts.push(Object.keys(ids || {}).join(',')); 153 | } else { 154 | keyparts.push(rel_type); 155 | keyparts.push('from=' + from.labels.join(',')); 156 | keyparts.push('ids=', Object.keys(from.ids).join(',')); 157 | keyparts.push('to=' + to.labels.join(',')); 158 | keyparts.push('ids=', Object.keys(to.ids).join(',')); 159 | } 160 | 161 | // console.log('key',keyparts.join('.')); 162 | return keyparts.join('.'); 163 | } 164 | 165 | /** 166 | * Run the command 167 | * @param {Transaction} tx a driver Transaction object 168 | * @returns {Promise} containing op, type, and id fields on success. 169 | */ 170 | run(tx) { 171 | // Bind the $event parameter to the event var reference, which the 172 | // query form expects. 173 | const cypher = 'WITH $input as event ' + this.generate(); 174 | const params = { input: this.data }; 175 | 176 | console.log('RUNNING ', cypher, params); 177 | return tx.run(cypher, params) 178 | .then(results => { 179 | const rec = results.records[0]; 180 | 181 | if (!rec) { 182 | console.error('Cypher produced no results', cypher, JSON.stringify(params)); 183 | return null; 184 | } 185 | 186 | const id = rec.get('id'); 187 | // console.log('PAYLOAD',rec); 188 | return { 189 | op: rec.get('op'), 190 | type: rec.get('type'), 191 | id: Integer.inSafeRange(id) ? Integer.toNumber(id) : Integer.toString(id), 192 | }; 193 | }); 194 | } 195 | }; 196 | 197 | module.exports = CUDCommand; -------------------------------------------------------------------------------- /gil/sink/CUDCommand.test.js: -------------------------------------------------------------------------------- 1 | const CUDCommand = require('./CUDCommand'); 2 | 3 | const cleanupString = str => str.trim().replace(/\s+/g, ' '); 4 | 5 | describe('CUD Command', function () { 6 | it('should create a node', () => { 7 | const data = { 8 | op: 'create', 9 | type: 'node', 10 | labels: ['Person'], 11 | properties: { 12 | a: 'b', 13 | }, 14 | }; 15 | 16 | const expected = "CREATE (n:`Person` { }) SET n += event.properties RETURN event.op as op, event.type as type, id(n) as id"; 17 | 18 | const c = new CUDCommand(data); 19 | expect(cleanupString(c.generate())).toEqual(expected); 20 | }); 21 | 22 | 23 | it('should merge a node', () => { 24 | const data = { 25 | op: 'merge', 26 | type: 'node', 27 | labels: ['Person'], 28 | ids: { 29 | foo: 'bar', 30 | baz: 'quux', 31 | }, 32 | properties: { 33 | a: 'b', 34 | }, 35 | }; 36 | 37 | const expected = "MERGE (n:`Person` { `foo`: event.ids.`foo`, `baz`: event.ids.`baz` }) SET n += event.properties RETURN event.op as op, event.type as type, id(n) as id"; 38 | 39 | const c = new CUDCommand(data); 40 | expect(cleanupString(c.generate())).toEqual(expected); 41 | }); 42 | 43 | it('should delete a node', () => { 44 | const data = { 45 | op: 'delete', 46 | type: 'node', 47 | labels: ['A', 'B', 'C'], 48 | ids: { 49 | key: 1, 50 | otherKey: 2, 51 | }, 52 | }; 53 | 54 | const expected = "MATCH (n:`A`:`B`:`C` { `key`: event.ids.`key`, `otherKey`: event.ids.`otherKey` }) DELETE n RETURN event.op as op, event.type as type, id(n) as id"; 55 | const c = new CUDCommand(data); 56 | expect(cleanupString(c.generate())).toEqual(expected); 57 | }); 58 | 59 | it('knows how to detach a node', () => { 60 | const data = { 61 | op: 'delete', 62 | type: 'node', 63 | labels: ['A'], 64 | ids: { key: 1 }, 65 | detach: true, 66 | }; 67 | 68 | const expected = "MATCH (n:`A` { `key`: event.ids.`key` }) DETACH DELETE n RETURN event.op as op, event.type as type, id(n) as id"; 69 | const c = new CUDCommand(data); 70 | expect(cleanupString(c.generate())).toEqual(expected); 71 | }); 72 | 73 | it('should generate a relationship', () => { 74 | const data = { 75 | op: 'create', 76 | type: 'relationship', 77 | from: { 78 | ids: { x: 1, y: 2 }, 79 | labels: ['Person'], 80 | }, 81 | to: { 82 | ids: { z: 3 }, 83 | labels: ['Company'], 84 | }, 85 | rel_type: 'WORKS_FOR', 86 | }; 87 | 88 | const expected = "MATCH (a:`Person` { `x`: event.from.ids.`x`, `y`: event.from.ids.`y` }) WITH a, event MATCH (b:`Company` { `z`: event.to.ids.`z` }) CREATE (a)-[r:`WORKS_FOR`]->(b) SET r += event.properties RETURN event.op as op, event.type as type, id(r) as id"; 89 | const c = new CUDCommand(data); 90 | expect(cleanupString(c.generate())).toEqual(expected); 91 | }); 92 | 93 | it('should delete a relationship', () => { 94 | const data = { 95 | op: 'delete', 96 | type: 'relationship', 97 | from: { 98 | labels: ['Foo'], 99 | ids: { uuid: '123' }, 100 | }, 101 | to: { 102 | labels: ['Bar'], 103 | ids: { uuid: 'a' }, 104 | }, 105 | rel_type: 'blorko', 106 | }; 107 | 108 | const expected = "MATCH (a:`Foo` { `uuid`: event.from.ids.`uuid` })-[r:`blorko`]->(b:`Bar` { `uuid`: event.to.ids.`uuid` }) DELETE r RETURN event.op as op, event.type as type, id(r) as id"; 109 | const c = new CUDCommand(data); 110 | expect(cleanupString(c.generate())).toEqual(expected); 111 | }); 112 | 113 | describe('Validation', () => { 114 | it('should fail a bogus message', () => 115 | expect(() => new CUDCommand()).toThrow(Error)); 116 | 117 | it('requires IDs for node merge', () => 118 | expect(() => new CUDCommand({ 119 | type: 'node', 120 | op: 'merge', 121 | labels: ['Foo'], 122 | properties: { x: 1 }, 123 | })).toThrow(Error)); 124 | 125 | it('requires labesl for node merge', () => 126 | expect(() => new CUDCommand({ 127 | type: 'node', 128 | op: 'merge', 129 | ids: { x: 1 }, 130 | properties: { x: 1 }, 131 | })).toThrow(Error)); 132 | 133 | it('requires from on relationships', () => 134 | expect(() => new CUDCommand({ 135 | type: 'relationship', 136 | op: 'merge', 137 | ids: { x: 1 }, 138 | rel_type: 'foo', 139 | to: { labels: ['X'], ids: { x: 1 } }, 140 | properties: { x: 1 }, 141 | })).toThrow(Error)); 142 | 143 | it('requires to on relationships', () => 144 | expect(() => new CUDCommand({ 145 | type: 'relationship', 146 | op: 'merge', 147 | ids: { x: 1 }, 148 | rel_type: 'foo', 149 | from: { labels: ['X'], ids: { x: 1 } }, 150 | properties: { x: 1 }, 151 | })).toThrow(Error)); 152 | 153 | it('requires rel_type on relationships', () => 154 | expect(() => new CUDCommand({ 155 | type: 'relationship', 156 | op: 'merge', 157 | ids: { x: 1 }, 158 | from: { labels: ['X'], ids: { x: 1 } }, 159 | to: { labels: ['X'], ids: { x: 1 } }, 160 | properties: { x: 1 }, 161 | })).toThrow(Error)); 162 | 163 | it('requires from/to labels on relationships', () => 164 | expect(() => new CUDCommand({ 165 | type: 'relationship', 166 | op: 'merge', 167 | ids: { x: 1 }, 168 | from: { labels: [], ids: { x: 1 } }, 169 | to: { labels: [], ids: { x: 1 } }, 170 | properties: { x: 1 }, 171 | })).toThrow(Error)); 172 | }); 173 | }); -------------------------------------------------------------------------------- /gil/sink/CypherSink.js: -------------------------------------------------------------------------------- 1 | const Strategy = require('./Strategy'); 2 | const _ = require('lodash'); 3 | 4 | /** 5 | * Example message input to constructor 6 | { 7 | "cypher": "CREATE (b:Blork) SET b += event", 8 | "batch": [ 9 | {"x": 1, "y": 2}, 10 | {"x": 3, "y": 4} 11 | ] 12 | } 13 | */ 14 | class CypherSink extends Strategy { 15 | constructor(data) { 16 | super(); 17 | 18 | if (!data || !data.cypher || !data.batch || !_.isArray(data.batch) || _.isEmpty(data.batch)) { 19 | console.error(JSON.stringify(data)); 20 | throw new Error(`Input must contain a valid cypher field and a valid batch, which must be an array. cypher=${_.get(data,'cypher')} batch=${_.get(data, 'batch')}`); 21 | } 22 | 23 | this.data = data; 24 | } 25 | 26 | getCypher() { return this.data.cypher; }; 27 | getEvents() { return this.data.batch; }; 28 | } 29 | 30 | module.exports = CypherSink; -------------------------------------------------------------------------------- /gil/sink/CypherSink.test.js: -------------------------------------------------------------------------------- 1 | const CypherSink = require('./CypherSink'); 2 | const neo4j = require('../driver'); 3 | const sinon = require('sinon'); 4 | 5 | describe('CypherSink', () => { 6 | let driver, session, tx, run, rec; 7 | let data; 8 | 9 | beforeEach(() => { 10 | rec = { 11 | get: field => data[field], 12 | }; 13 | 14 | run = sinon.stub(); 15 | run.returns(Promise.resolve({ 16 | records: [rec], 17 | })); 18 | 19 | tx = { run }; 20 | 21 | session = sinon.stub().returns({ writeTransaction: f => f(tx) }); 22 | 23 | driver = sinon.stub(neo4j, 'getDriver'); 24 | driver.returns({ session }); 25 | }); 26 | 27 | afterEach(() => { 28 | neo4j.getDriver.restore(); 29 | }); 30 | 31 | it('should validate inputs', () => { 32 | expect(() => new CypherSink({})).toThrow(Error); 33 | expect(() => new CypherSink({ cypher: 'match n return n' })).toThrow(Error); 34 | expect(() => new CypherSink({ 35 | cypher: 'CREATE (n:Foo) SET n += event', 36 | batch: [], 37 | })).toThrow(Error); 38 | }); 39 | 40 | it('should run a batch', () => { 41 | const data = { 42 | cypher: 'CREATE (n:Foo) SET n += event', 43 | batch: [ 44 | { x: 1, y: 2 }, 45 | { x: 3, y: 4 }, 46 | ], 47 | }; 48 | 49 | const fakeSession = { writeTransaction: f => f(tx) }; 50 | 51 | return new CypherSink(data).run(fakeSession) 52 | .then(result => { 53 | expect(result.batch).toBe(true); 54 | expect(result.elements).toBe(2); 55 | expect(result.started).toBeTruthy(); 56 | expect(result.finished).toBeTruthy(); 57 | }); 58 | }); 59 | }); -------------------------------------------------------------------------------- /gil/sink/DataSink.js: -------------------------------------------------------------------------------- 1 | const neo4j = require('../driver'); 2 | const Promise = require('bluebird'); 3 | 4 | class DataSink { 5 | constructor(strategies) { 6 | this.strategies = strategies; 7 | } 8 | 9 | async run() { 10 | const driver = await neo4j.getDriver(); 11 | const session = driver.session(neo4j.getSessionConfig()); 12 | 13 | return Promise.mapSeries(this.strategies, strategy => strategy.run(session)) 14 | .finally(session.close); 15 | } 16 | } 17 | 18 | module.exports = DataSink; 19 | -------------------------------------------------------------------------------- /gil/sink/Strategy.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment'); 2 | 3 | class Strategy { 4 | constructor() { 5 | } 6 | 7 | getCypher() { 8 | throw new Error('Override me (getCypher) in a subclass!'); 9 | } 10 | 11 | getEvents() { 12 | throw new Error('Override me (getEvents) in a subclass!'); 13 | } 14 | 15 | run(session) { 16 | const cypher = `UNWIND $events as event ${this.getCypher()}` 17 | const events = this.getEvents(); 18 | const elements = events.length; 19 | const params = { events }; 20 | const started = moment.utc().toISOString(); 21 | 22 | return session.writeTransaction(tx => tx.run(cypher, params)) 23 | .then(() => ({ 24 | batch: true, 25 | elements, 26 | started, 27 | finished: moment.utc().toISOString(), 28 | })); 29 | } 30 | } 31 | 32 | module.exports = Strategy; -------------------------------------------------------------------------------- /gil/sink/generateCUD.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid'); 2 | const test = require('../../test/index'); 3 | 4 | const buf = []; 5 | for (let i=0; i<100; i++) { 6 | const msg = { 7 | op: 'create', 8 | type: 'node', 9 | labels: ['Test'], 10 | properties: { 11 | index: i, 12 | x: Math.random(), 13 | uuid: uuid.v4(), 14 | }, 15 | }; 16 | 17 | buf.push(test.generateCUDMessage()); 18 | } 19 | 20 | console.log(JSON.stringify(buf)); 21 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const nodeSvc = require('./services/http/node'); 2 | const edgeSvc = require('./services/http/edge'); 3 | const cudSvc = require('./services/http/cud'); 4 | const cypherSvc = require('./services/http/cypher'); 5 | const customCypherSvc = require('./services/http/customCypher'); 6 | 7 | const cudPubsub = require('./services/pubsub/cudPubsub'); 8 | const cypherPubsub = require('./services/pubsub/cypherPubsub'); 9 | const customCypherPubsub = require('./services/pubsub/customCypherPubsub'); 10 | 11 | const moment = require('moment'); 12 | 13 | /** 14 | * Executes a function, and if it throws an error, guarantees error response. 15 | * @param {*} aFunction function to execute 16 | * @param {*} res response object 17 | * @returns an export function 18 | */ 19 | const guaranteeResponseHTTP = (aFunction, failMsg = 'Error processing response') => { 20 | return (req, res) => { 21 | try { 22 | return aFunction(req, res); 23 | } catch (err) { 24 | console.error(err); 25 | return Promise.resolve(res.status(500).json({ 26 | date: moment.utc().format(), 27 | message: failMsg, 28 | error: `${err}`, 29 | })); 30 | } 31 | }; 32 | }; 33 | 34 | const guaranteeCallbackPubsub = (aFunction) => { 35 | return (pubSubEvent, context, callback) => { 36 | try { 37 | return aFunction(pubSubEvent, context, callback); 38 | } catch(err) { 39 | return Promise.resolve(callback(err)); 40 | } 41 | }; 42 | }; 43 | 44 | module.exports = { 45 | node: guaranteeResponseHTTP(nodeSvc), 46 | edge: guaranteeResponseHTTP(edgeSvc), 47 | cud: guaranteeResponseHTTP(cudSvc), 48 | cypher: guaranteeResponseHTTP(cypherSvc), 49 | customCypher: guaranteeResponseHTTP(customCypherSvc), 50 | 51 | cudPubsub: guaranteeCallbackPubsub(cudPubsub), 52 | cypherPubsub: guaranteeCallbackPubsub(cypherPubsub), 53 | customCypherPubsub: guaranteeCallbackPubsub(customCypherPubsub), 54 | }; 55 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "neo4j-serverless-functions", 3 | "version": "0.0.1", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "scripts": { 7 | "test": "jest --coverage --watchAll=false" 8 | }, 9 | "dependencies": { 10 | "@google-cloud/functions-framework": "^1.7.1", 11 | "@google-cloud/secret-manager": "^3.6.1", 12 | "bluebird": "^3.7.2", 13 | "flat": "^5.0.1", 14 | "lodash": "^4.17.20", 15 | "moment": "^2.25.3", 16 | "neo4j-driver": "^4.2.1", 17 | "neode": "^0.4.3", 18 | "uuid": "^8.0.0" 19 | }, 20 | "devDependencies": { 21 | "jest": "^26.0.1", 22 | "sinon": "^9.2.4" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /services/common.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid'); 2 | const neo4j = require('../gil/driver'); 3 | const _ = require('lodash'); 4 | const moment = require('moment'); 5 | 6 | // What do we want to add to the request? 7 | const markers = () => { 8 | const extraStuff = {}; 9 | extraStuff._date = moment.utc().format(); 10 | extraStuff._uuid = uuid.v4(); 11 | 12 | return extraStuff; 13 | }; 14 | 15 | const getRequestProps = (req) => { 16 | const requestProps = neo4j.createNeo4jPropertiesFromObject( 17 | _.merge(markers(), req.headers, { ip: req.ip })); 18 | 19 | return requestProps; 20 | }; 21 | 22 | module.exports = { 23 | markers, 24 | getRequestProps, 25 | }; -------------------------------------------------------------------------------- /services/http/cud.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const moment = require('moment'); 3 | const Promise = require('bluebird'); 4 | const gil = require('../../gil'); 5 | 6 | const cud = (req, res) => { 7 | const records = req.body; 8 | 9 | if (_.isNil(records) || !_.isArray(records) || _.isEmpty(records)) { 10 | return Promise.resolve(res.status(400).json({ 11 | date: moment.utc().format(), 12 | error: `Required: a non-empty JSON array of CUD messages`, 13 | })); 14 | } 15 | 16 | let commands; 17 | 18 | try { 19 | commands = records.map(rec => new gil.CUDCommand(rec)); 20 | } catch(e) { 21 | // This is going to be a CUD message formatting error 22 | return Promise.resolve(res.status(400).json({ 23 | date: moment.utc().format(), 24 | error: `${e}`, 25 | })); 26 | } 27 | 28 | const batches = gil.CUDBatch.batchCommands(commands); 29 | const sink = new gil.DataSink(batches); 30 | 31 | return sink.run() 32 | .then(results => res.status(200).json(results)) 33 | .catch(err => { 34 | return res.status(500).json({ 35 | date: moment.utc().format(), 36 | error: `${err}`, 37 | stack: err.stack, 38 | }); 39 | }); 40 | }; 41 | 42 | module.exports = cud; -------------------------------------------------------------------------------- /services/http/cud.test.js: -------------------------------------------------------------------------------- 1 | const handlers = require('../../index'); 2 | const cud = handlers.cud; 3 | const cudPubsub = handlers.cudPubsub; 4 | 5 | const test = require('../../test'); 6 | const cudMessages = require('../../test/cud-messages.json'); 7 | const gil = require('../../gil'); 8 | const sinon = require("sinon"); 9 | 10 | describe('CUD Function', () => { 11 | let driver, session, tx, run, rec; 12 | let data; 13 | beforeEach(() => { 14 | rec = { 15 | get: field => data[field], 16 | }; 17 | 18 | run = sinon.stub(); 19 | run.returns(Promise.resolve({ 20 | records: [rec], 21 | })); 22 | 23 | tx = { run }; 24 | 25 | session = sinon.stub(); 26 | session.returns({ writeTransaction: f => f(tx) }); 27 | 28 | const getDriverStub = sinon.stub(gil.neo4j, 'getDriver'); 29 | getDriverStub.returns(Promise.resolve({ session })); 30 | }); 31 | 32 | afterEach(() => { 33 | gil.neo4j.getDriver.restore(); 34 | }); 35 | 36 | describe('Pubsub', () => { 37 | let callback, context; 38 | 39 | beforeEach(() => { 40 | callback = sinon.stub(); 41 | context = sinon.stub(); 42 | }); 43 | 44 | const encode = obj => Buffer.from(JSON.stringify(obj)).toString('base64'); 45 | 46 | it('should require a non-empty array', () => { 47 | const event = { data: encode([]) }; 48 | 49 | return cudPubsub(event, context, callback) 50 | .then(() => { 51 | expect(callback.calledOnce).toBe(true); 52 | expect(callback.firstCall.args[0]).toBeInstanceOf(Error); 53 | }); 54 | }); 55 | 56 | it('requires well formed JSON', () => { 57 | const event = { data: 'alskdjf;alksdjf;asdjkfa;dsfj' }; 58 | 59 | return cudPubsub(event, context, callback) 60 | .then(() => { 61 | expect(callback.calledOnce).toBe(true); 62 | expect(callback.firstCall.args[0]).toBeInstanceOf(Error); 63 | }); 64 | }); 65 | 66 | it('should process one simple message', () => { 67 | const event = { data: encode([cudMessages[0]]) }; 68 | 69 | return cudPubsub(event, context, callback) 70 | .then(() => { 71 | expect(callback.calledOnce).toBe(true); 72 | expect(callback.firstCall.args[0]).toEqual(null); 73 | // [{"batch":true,"key":"create.node.Person.","sequence":0,"commands":1}] 74 | }); 75 | }); 76 | 77 | it('should process many messages', () => { 78 | const event = { data: encode(cudMessages) }; 79 | 80 | return cudPubsub(event, context, callback) 81 | .then(() => { 82 | expect(callback.calledOnce).toBe(true); 83 | expect(callback.firstCall.args[0]).toEqual(null); 84 | }); 85 | }); 86 | }); 87 | 88 | describe('HTTP', () => { 89 | it('should require a non-empty array', () => { 90 | // Mock ExpressJS 'req' and 'res' parameters 91 | const req = { body: null }; 92 | const res = test.mockResponse(); 93 | 94 | // Call tested function 95 | return cud(req, res) 96 | .then(() => { 97 | // Verify behavior of tested function 98 | expect(res.json.calledOnce).toBe(true); 99 | expect(res.status.firstCall.args[0]).toEqual(400); 100 | expect(res.json.firstCall.args[0].error) 101 | .toEqual('Required: a non-empty JSON array of CUD messages'); 102 | }); 103 | }); 104 | 105 | it('should process one simple message', () => { 106 | data = { op: 'create', type: 'node', id: 1 }; 107 | const req = { body: [cudMessages[0]] }; 108 | const res = test.mockResponse(); 109 | 110 | return cud(req, res) 111 | .then(() => { 112 | expect(res.json.calledOnce).toBe(true); 113 | expect(res.status.firstCall.args[0]).toEqual(200); 114 | const results = res.json.firstCall.args[0]; 115 | const batch0 = results[0]; 116 | expect(batch0.batch).toBe(true); 117 | expect(batch0.elements).toBe(1); 118 | }); 119 | }); 120 | 121 | it('should process many messages', () => { 122 | data = { op: 'create', type: 'node', id: 1 }; // faked result 123 | 124 | const req = { body: cudMessages[0] }; 125 | const res = test.mockResponse(); 126 | 127 | const a = cud(req, res) 128 | .then(() => { 129 | expect(res.json.calledOnce).toBe(true); 130 | expect(res.status.firstCall.args[0]).toEqual(200); 131 | expect(tx.run.callCount).toBe(cudMessages.length); 132 | }); 133 | }); 134 | }); 135 | }); -------------------------------------------------------------------------------- /services/http/customCypher.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment'); 2 | const gil = require('../../gil'); 3 | 4 | // This is the same as cypher, except the user has to define a CYPHER env var to bind the cypher statement 5 | // ahead of time. 6 | const cypher = (req, res) => { 7 | const input = req.body; 8 | 9 | if (!process.env.CYPHER) { 10 | throw new Error('You must define a pre-determined CYPHER query environment variable to use this endpoint'); 11 | } 12 | 13 | const sink = { 14 | cypher: process.env.CYPHER, 15 | batch: input, 16 | }; 17 | 18 | const strategy = new gil.CypherSink(sink); 19 | 20 | return new gil.DataSink([strategy]).run() 21 | .then(results => res.status(200).json(results)) 22 | .catch(err => res.status(500).json({ 23 | date: moment.utc().format(), 24 | error: `${err}`, 25 | stack: err.stack, 26 | })); 27 | }; 28 | 29 | module.exports = cypher; -------------------------------------------------------------------------------- /services/http/cypher.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment'); 2 | const gil = require('../../gil'); 3 | 4 | const cypher = (req, res) => { 5 | const input = req.body; 6 | 7 | const strategy = new gil.CypherSink(input); 8 | 9 | return new gil.DataSink([strategy]).run() 10 | .then(results => res.status(200).json(results)) 11 | .catch(err => res.status(500).json({ 12 | date: moment.utc().format(), 13 | error: `${err}`, 14 | stack: err.stack, 15 | })); 16 | }; 17 | 18 | module.exports = cypher; -------------------------------------------------------------------------------- /services/http/cypher.test.js: -------------------------------------------------------------------------------- 1 | const handlers = require('../../index'); 2 | const cypher = handlers.cypher; 3 | const cypherPubsub = handlers.cypherPubsub; 4 | const _ = require('lodash'); 5 | const test = require('../../test'); 6 | const gil = require('../../gil'); 7 | const sinon = require("sinon"); 8 | const moment = require('moment'); 9 | 10 | const testSinkBatch = { 11 | cypher: 'CREATE (f:Foo) f += event', 12 | batch : [ 13 | { x: 1 }, 14 | { x: 2 }, 15 | ], 16 | }; 17 | 18 | // CypherSink sends back results in "batch results". This validates one of 19 | // those response objects. 20 | const validateBatchObject = (batch, size) => { 21 | expect(batch.batch).toBe(true); 22 | expect(batch.elements).toBe(size); 23 | expect(batch.started).toBeTruthy(); 24 | expect(batch.finished).toBeTruthy(); 25 | 26 | const started = moment(batch.started); 27 | const finished = moment(batch.finished); 28 | 29 | // Started has to be before finished time; or it can sometimes be zero 30 | // since the test runs so fast. 31 | expect(started.diff(finished)).toBeLessThanOrEqual(0); 32 | }; 33 | 34 | describe('Cypher Sink Function', () => { 35 | let driver, session, tx, run, rec; 36 | let data; 37 | beforeEach(() => { 38 | rec = { 39 | get: field => data[field], 40 | }; 41 | 42 | run = sinon.stub(); 43 | run.returns(Promise.resolve({ 44 | records: [rec], 45 | })); 46 | 47 | tx = { run }; 48 | 49 | session = sinon.stub(); 50 | session.returns({ writeTransaction: f => f(tx) }); 51 | 52 | const getDriverStub = sinon.stub(gil.neo4j, 'getDriver'); 53 | getDriverStub.returns(Promise.resolve({ session })); 54 | }); 55 | 56 | afterEach(() => { 57 | gil.neo4j.getDriver.restore(); 58 | }); 59 | 60 | describe('Pubsub', () => { 61 | let callback, context; 62 | 63 | beforeEach(() => { 64 | callback = sinon.stub(); 65 | context = sinon.stub(); 66 | }); 67 | 68 | const encode = obj => Buffer.from(JSON.stringify(obj)).toString('base64'); 69 | 70 | it('should require a non-empty object', () => { 71 | const event = { data: encode([]) }; 72 | 73 | return cypherPubsub(event, context, callback) 74 | .then(() => { 75 | expect(callback.calledOnce).toBe(true); 76 | expect(callback.firstCall.args[0]).toBeInstanceOf(Error); 77 | }); 78 | }); 79 | 80 | it('should process a simple batch', () => { 81 | const event = { data: encode(testSinkBatch) }; 82 | 83 | return cypherPubsub(event, context, callback) 84 | .then(() => { 85 | expect(callback.calledOnce).toBe(true); 86 | const result = callback.firstCall.args[1]; 87 | 88 | console.log(result); 89 | expect(_.isArray(result)).toBe(true); 90 | expect(result.length).toBe(1); 91 | const first = result[0]; 92 | validateBatchObject(first, testSinkBatch.batch.length); 93 | }); 94 | }); 95 | }); 96 | 97 | describe('HTTP', () => { 98 | it('should require a non-empty object', () => { 99 | // Mock ExpressJS 'req' and 'res' parameters 100 | const req = { body: null }; 101 | const res = test.mockResponse(); 102 | 103 | // Call tested function 104 | return cypher(req, res) 105 | .then(() => { 106 | // Verify behavior of tested function 107 | expect(res.json.calledOnce).toBe(true); 108 | expect(res.status.firstCall.args[0]).toEqual(500); 109 | expect(''+res.json.firstCall.args[0].error) 110 | .toContain('must contain a valid'); 111 | }); 112 | }); 113 | 114 | it('should process a simple batch', () => { 115 | const req = { body: testSinkBatch }; 116 | const res = test.mockResponse(); 117 | 118 | return cypher(req, res) 119 | .then(() => { 120 | expect(res.json.calledOnce).toBe(true); 121 | expect(res.status.firstCall.args[0]).toEqual(200); 122 | const json = res.json.firstCall.args[0]; 123 | 124 | // Response will look something like this: 125 | // [{"batch": true, "elements": 2, "finished": "2020-06-11T14:10:54.155Z", "started": "2020-06-11T14:10:54.154Z"}] 126 | expect(_.isArray(json)).toBe(true); 127 | expect(json.length).toBe(1); 128 | const first = json[0]; 129 | validateBatchObject(first, req.body.batch.length); 130 | }); 131 | }); 132 | }); 133 | }); -------------------------------------------------------------------------------- /services/http/edge.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const moment = require('moment'); 3 | const neo4j = require('../../gil/driver'); 4 | const common = require('../common'); 5 | 6 | const VERSION = 1; 7 | const RESPOND_WITH_CONTENT = false; 8 | 9 | /** 10 | * Request must provide query params: 11 | * fromLabel 12 | * fromProp 13 | * fromVal 14 | * toLabel 15 | * toProp 16 | * toVal 17 | * relType (default: link) 18 | * 19 | * JSON body parameters will be used for the relationship properties. 20 | * @param {*} req 21 | * @param {*} res 22 | */ 23 | const edge = async (req, res) => { 24 | const requiredParams = [ 25 | 'fromLabel', 'fromProp', 'fromVal', 26 | 'toLabel', 'toProp', 'toVal', 'relType', 27 | ]; 28 | 29 | for(let i=0; i(b) 49 | RETURN r; 50 | `; 51 | 52 | const relMarkers = common.markers(); 53 | 54 | // Neo4j takes key/value props, not arbitrarily nested javascript objects, 55 | // so we convert. 56 | const relProps = neo4j.createNeo4jPropertiesFromObject( 57 | _.merge(_.cloneDeep(req.body), _.cloneDeep(relMarkers))); 58 | 59 | // Use regular request props, but override with the same markers 60 | // as the relationship has so we can correlate the two. 61 | const requestProps = _.merge(common.getRequestProps(req), 62 | _.cloneDeep(relMarkers)); 63 | 64 | const queryParams = _.merge( 65 | _.cloneDeep(req.query), 66 | { relProps, requestProps } 67 | ); 68 | 69 | const driver = await neo4j.getDriver(); 70 | const session = driver.session(); 71 | 72 | console.log('Running ', cypher, 'with', queryParams); 73 | return session.writeTransaction(tx => tx.run(cypher, queryParams)) 74 | .then(result => { 75 | if (RESPOND_WITH_CONTENT) { 76 | return res.status(200).json(result.records[0].get('r')); 77 | } 78 | 79 | return res.status(200).json('OK'); 80 | }) 81 | .catch(err => { 82 | return res.status(500).json({ 83 | date: moment.utc().format(), 84 | error: `${err}`, 85 | stack: err.stack, 86 | }); 87 | }); 88 | }; 89 | 90 | module.exports = edge; -------------------------------------------------------------------------------- /services/http/edge.test.js: -------------------------------------------------------------------------------- 1 | const edge = require('../../index').edge; 2 | const test = require('../../test'); 3 | const gil = require('../../gil'); 4 | const sinon = require("sinon"); 5 | const _ = require('lodash'); 6 | 7 | describe('Edge Function', () => { 8 | let driver, session, tx, run, rec; 9 | let data; 10 | 11 | beforeEach(() => { 12 | rec = { 13 | get: field => data[field], 14 | }; 15 | 16 | run = sinon.stub(); 17 | run.returns(Promise.resolve({ 18 | records: [rec], 19 | })); 20 | 21 | tx = { run }; 22 | 23 | session = sinon.stub(); 24 | session.returns({ 25 | close: sinon.stub(), 26 | writeTransaction: f => f(tx) 27 | }); 28 | 29 | const getDriverStub = sinon.stub(gil.neo4j, 'getDriver'); 30 | getDriverStub.returns(Promise.resolve({ session })); 31 | }); 32 | 33 | afterEach(() => { 34 | gil.neo4j.getDriver.restore(); 35 | }); 36 | 37 | const query = { 38 | fromLabel: 'Person', 39 | fromProp: 'id', 40 | fromVal: 1, 41 | toLabel: 'Person', 42 | toProp: 'id', 43 | toVal: 2, 44 | relType: 'KNOWS', 45 | }; 46 | 47 | ['fromLabel', 'fromProp', 'fromVal', 'toLabel', 'toProp', 'relType', 'toVal'].forEach(attr => { 48 | it(`should require query param ${attr}`, () => { 49 | const thisQuery = _.cloneDeep(query); 50 | delete(thisQuery[attr]); 51 | 52 | const req = { query: thisQuery, body: { rel: 'props' } }; 53 | const res = test.mockResponse(); 54 | 55 | return edge(req, res) 56 | .then(() => { 57 | expect(res.json.calledOnce).toBe(true); 58 | expect(res.status.firstCall.args[0]).toEqual(400); 59 | expect(res.json.firstCall.args[0].error).toBeTruthy(); 60 | }); 61 | }); 62 | }); 63 | 64 | it('should create a relationship', () => { 65 | const req = { query, body: { rel: 'props' } }; 66 | const res = test.mockResponse(); 67 | 68 | return edge(req, res) 69 | .then(() => { 70 | expect(res.json.calledOnce).toBe(true); 71 | expect(res.status.firstCall.args[0]).toEqual(200); 72 | }); 73 | }); 74 | 75 | }); -------------------------------------------------------------------------------- /services/http/node.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const moment = require('moment'); 3 | const neo4j = require('../../gil/driver'); 4 | const Integer = require('neo4j-driver/lib/integer.js'); 5 | const common = require('../common'); 6 | 7 | const node = async (req, res) => { 8 | // If user submitted a label, use that. 9 | const label = _.get(req.query, 'label') || _.get(req.params, 'label') || 'Entry'; 10 | 11 | // Neo4j takes key/value props, not arbitrarily nested javascript objects, 12 | // so we convert. 13 | const requestProps = common.getRequestProps(req); 14 | 15 | let records; 16 | 17 | if (_.isEmpty(req.body)) { 18 | records = [req.params]; 19 | } else if (_.isArray(req.body)) { 20 | records = req.body; 21 | } else if (_.isObject(req.body)) { 22 | records = [req.body]; 23 | } 24 | 25 | // Batch parameter to set in query. 26 | const batch = records.map(propSet => ({ props: neo4j.createNeo4jPropertiesFromObject(propSet) })); 27 | const driver = await neo4j.getDriver(); 28 | const session = driver.session(); 29 | 30 | const cypher = ` 31 | UNWIND $batch as input 32 | CREATE (p:\`${label}\`) 33 | SET p += input.props 34 | RETURN id(p) as id 35 | `; 36 | 37 | const queryParams = { 38 | batch, 39 | requestProps, 40 | method: req.method, 41 | }; 42 | 43 | return session.writeTransaction(tx => tx.run(cypher, queryParams)) 44 | .then(result => { 45 | const data = result.records 46 | .map(rec => rec.get('id')) 47 | .map(num => Integer.inSafeRange(num) ? Integer.toNumber(num) : Integer.toString(num)); 48 | return res.status(200).json(data); 49 | }) 50 | .catch(err => { 51 | return res.status(500).json({ 52 | date: moment.utc().format(), 53 | error: `${err}`, 54 | stack: err.stack, 55 | }); 56 | }) 57 | .finally(session.close); 58 | }; 59 | 60 | module.exports = node; -------------------------------------------------------------------------------- /services/http/node.test.js: -------------------------------------------------------------------------------- 1 | const node = require('./node'); 2 | const test = require('../../test'); 3 | const gil = require('../../gil'); 4 | const sinon = require("sinon"); 5 | 6 | describe('Node Function', () => { 7 | let driver, session, tx, run, rec; 8 | let data; 9 | 10 | beforeEach(() => { 11 | rec = { 12 | get: field => data[field], 13 | }; 14 | 15 | run = sinon.stub(); 16 | run.returns(Promise.resolve({ 17 | records: [rec], 18 | })); 19 | 20 | tx = { run }; 21 | 22 | session = sinon.stub(); 23 | session.returns({ 24 | close: sinon.stub(), 25 | writeTransaction: f => f(tx) 26 | }); 27 | 28 | const getDriverStub = sinon.stub(gil.neo4j, 'getDriver'); 29 | getDriverStub.returns(Promise.resolve({ session })); 30 | }); 31 | 32 | afterEach(() => { 33 | gil.neo4j.getDriver.restore(); 34 | }); 35 | 36 | it('should process many messages', () => { 37 | data = { id: 1 }; // faked result 38 | 39 | const req = { 40 | body: [ 41 | { label: 'Hello', props: { x: 1 } }, 42 | ], 43 | }; 44 | const res = test.mockResponse(); 45 | 46 | return node(req, res) 47 | .then(() => { 48 | expect(res.json.calledOnce).toBe(true); 49 | expect(res.status.firstCall.args[0]).toEqual(200); 50 | expect(tx.run.callCount).toBe(1); 51 | console.log(res.json.firstCall.args); 52 | expect(res.json.firstCall.args[0]).toEqual([data.id]); 53 | }); 54 | }); 55 | }); -------------------------------------------------------------------------------- /services/pubsub/cudPubsub.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const moment = require('moment'); 3 | const gil = require('../../gil'); 4 | 5 | // https://cloud.google.com/functions/docs/writing/background#function_parameters 6 | const cud = (pubSubEvent, context, callback) => { 7 | const records = JSON.parse(Buffer.from(pubSubEvent.data, 'base64').toString()); 8 | 9 | if (_.isNil(records) || !_.isArray(records) || _.isEmpty(records)) { 10 | return Promise.resolve(callback(new Error('Required: non-empty JSON array of CUD messages'))); 11 | } 12 | 13 | let commands; 14 | 15 | try { 16 | commands = records.map(rec => new gil.CUDCommand(rec)); 17 | } catch(e) { 18 | // This is going to be a CUD message formatting error 19 | return Promise.resolve(callback(e, JSON.stringify({ 20 | date: moment.utc().format(), 21 | error: `${e}`, 22 | }))); 23 | } 24 | 25 | const batches = gil.CUDBatch.batchCommands(commands); 26 | const sink = new gil.DataSink(batches); 27 | 28 | return sink.run() 29 | .then(results => callback(null, JSON.stringify(results))) 30 | .catch(err => callback(err)); 31 | }; 32 | 33 | module.exports = cud; -------------------------------------------------------------------------------- /services/pubsub/customCypherPubsub.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const gil = require('../../gil'); 3 | 4 | // https://cloud.google.com/functions/docs/writing/background#function_parameters 5 | // This is the same as cypherPubsub, but the difference is that the user has to pass in a hard-wired cypher 6 | // statement at create time, via a Cypher environment variable. 7 | const cypher = (pubSubEvent, context, callback) => { 8 | if (!process.env.CYPHER) { 9 | throw new Error('You must define a CYPHER env var in order to use this endpoint'); 10 | } 11 | 12 | const input = JSON.parse(Buffer.from(pubSubEvent.data, 'base64').toString()); 13 | const cypher = new gil.CypherSink({ 14 | cypher: process.env.CYPHER, 15 | batch: input, 16 | }); 17 | 18 | const sink = new gil.DataSink([cypher]); 19 | 20 | return sink.run() 21 | .then(results => callback(null, results)) 22 | .catch(err => callback(err)); 23 | }; 24 | 25 | module.exports = cypher; -------------------------------------------------------------------------------- /services/pubsub/cypherPubsub.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const gil = require('../../gil'); 3 | 4 | // https://cloud.google.com/functions/docs/writing/background#function_parameters 5 | const cypher = (pubSubEvent, context, callback) => { 6 | const input = JSON.parse(Buffer.from(pubSubEvent.data, 'base64').toString()); 7 | 8 | const cypher = new gil.CypherSink(input); 9 | const sink = new gil.DataSink([cypher]); 10 | 11 | return sink.run() 12 | .then(results => callback(null, results)) 13 | .catch(err => callback(err)); 14 | }; 15 | 16 | module.exports = cypher; -------------------------------------------------------------------------------- /test/cud-messages.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "op": "create", 4 | "type": "node", 5 | "labels": ["Person"], 6 | "properties": { 7 | "uuid": "A", 8 | "name": "Mark Bosc", 9 | "phone": "123-456-7899" 10 | } 11 | }, 12 | { 13 | "op": "merge", 14 | "type": "node", 15 | "labels": ["Person"], 16 | "ids": { "uuid": "A" }, 17 | "properties": { 18 | "dob": "2020-01-01", 19 | "age": 1 20 | } 21 | }, 22 | { 23 | "op": "merge", 24 | "type": "node", 25 | "labels": ["Person"], 26 | "ids": { "uuid": "B" }, 27 | "properties": { 28 | "uuid": "B", 29 | "name": "Mary Test", 30 | "phone": "123-456-7899" 31 | } 32 | }, 33 | { 34 | "op": "merge", 35 | "type": "relationship", 36 | "rel_type": "KNOWS", 37 | "from": { 38 | "labels": ["Person"], 39 | "ids": { "uuid": "A" } 40 | }, 41 | "to": { 42 | "labels": ["Person"], 43 | "ids": { "uuid": "B" } 44 | }, 45 | "properties": { 46 | "since": "2020-01-01" 47 | } 48 | } 49 | ] -------------------------------------------------------------------------------- /test/cypher-payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "cypher": "CREATE (i:Item) SET i += event", 3 | "batch": [ 4 | { "id": 1, "name": "Tube socks", "color": "black" }, 5 | { "id": 2, "name": "10gal aquarium", "color": "clear" }, 6 | { "id": 3, "name": "T-Shirt", "color": "red" } 7 | ] 8 | } -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | const sinon = require('sinon'); 2 | const uuid = require('uuid'); 3 | const moment = require('moment'); 4 | 5 | module.exports = { 6 | mockResponse: () => { 7 | const json = sinon.stub(); 8 | const status = sinon.stub(); 9 | status.returns({ json }); 10 | 11 | const res = { 12 | json, 13 | status, 14 | }; 15 | 16 | return res; 17 | }, 18 | 19 | generateCUDMessage: (op='create', type='node') => { 20 | const labels = ['Person', 'Job', 'Company']; 21 | const msg = { op, type }; 22 | 23 | const someProps = () => ({ 24 | id: Math.floor(Math.random() * 100000), 25 | uuid: uuid.v4(), 26 | x: Math.random(), 27 | when: moment.utc().toISOString(), 28 | }); 29 | 30 | const someKeys = () => ({ id: Math.floor(Math.random() * 100000) }); 31 | const randLabels = () => [labels[Math.floor(Math.random()*labels.length)]]; 32 | 33 | if (type === 'relationship') { 34 | msg.from = { 35 | labels: randLabels(), 36 | }; 37 | 38 | msg.to = { 39 | labels: randLabels(), 40 | }; 41 | } else { 42 | msg.labels = randLabels(); 43 | msg.ids = someKeys(); 44 | } 45 | 46 | msg.properties = someProps(); 47 | return msg; 48 | }, 49 | }; --------------------------------------------------------------------------------