├── .editorconfig ├── .gcloudignore ├── .github └── workflows │ └── semgrep.yml ├── .gitignore ├── LICENSE ├── README.md ├── cli ├── .gcloudignore ├── cli.js ├── cloudshell-security-events.md ├── cmds │ ├── account.js │ ├── buildConf.js │ ├── deploy.js │ ├── enableAPIs.js │ ├── serviceAcct.js │ ├── setEnv.js │ ├── setSchedule.js │ └── testRun.js ├── install.sh ├── package-lock.json ├── package.json ├── setup ├── templates │ ├── deploy.yml │ ├── enableAPIs.yml │ ├── serviceAcct.yml │ ├── setEnvPrompts.yml │ └── setSchedule.yml └── utils │ ├── debug.js │ ├── logger.js │ ├── paths.js │ └── testPermissions.js ├── logpush-to-bigquery ├── .gcloudignore ├── README.md ├── cloudshell.md ├── deploy.sh ├── index.js ├── package-lock.json ├── package.json ├── schema-firewall-events.json ├── schema-gateway-dns.json ├── schema-gateway-http.json ├── schema-http.json ├── schema-spectrum.json └── test.js └── security-events ├── .gcloudignore ├── README.md ├── assets.js ├── cflogs.js ├── findings.js ├── index.js ├── package-lock.json ├── package.json ├── static ├── colos.json ├── fields.json └── schema.json └── util.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | # Unix-style newlines with a newline ending every file 4 | [*] 5 | end_of_line = lf 6 | insert_final_newline = true 7 | indent_style = space 8 | indent_size = 2 9 | -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | -------------------------------------------------------------------------------- /.github/workflows/semgrep.yml: -------------------------------------------------------------------------------- 1 | 2 | on: 3 | pull_request: {} 4 | workflow_dispatch: {} 5 | push: 6 | branches: 7 | - main 8 | - master 9 | schedule: 10 | - cron: '0 0 * * *' 11 | name: Semgrep config 12 | jobs: 13 | semgrep: 14 | name: semgrep/ci 15 | runs-on: ubuntu-20.04 16 | env: 17 | SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} 18 | SEMGREP_URL: https://cloudflare.semgrep.dev 19 | SEMGREP_APP_URL: https://cloudflare.semgrep.dev 20 | SEMGREP_VERSION_CHECK_URL: https://cloudflare.semgrep.dev/api/check-version 21 | container: 22 | image: returntocorp/semgrep 23 | steps: 24 | - uses: actions/checkout@v3 25 | - run: semgrep ci 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/** 2 | 3 | # Serverless 4 | .serverless 5 | .env* 6 | tmp 7 | .coveralls.yml 8 | logs.js 9 | # Google 10 | keyfile.json 11 | 12 | # Logs 13 | *.log 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 22 | .grunt 23 | 24 | # node-waf configuration 25 | .lock-wscript 26 | 27 | # Compiled binary addons (http://nodejs.org/api/addons.html) 28 | build/Release 29 | 30 | # Dependency directory 31 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 32 | node_modules 33 | 34 | # IDE 35 | **/.idea 36 | 37 | # OS 38 | .vscode 39 | cloudflare-gcp.code-workspace 40 | 41 | security-events/.env.yml 42 | security-events/node_modules 43 | security-events/scc_key.json 44 | security-events/.DS_Store 45 | security-events/local.js 46 | 47 | cli/node_modules 48 | 49 | logpush-to-bigquery/.DS_Store 50 | logpush-to-bigquery/node_modules 51 | logpush-to-bigquery/local.js 52 | cli/dist 53 | cli/confs 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2017 Google Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | Copyright (c) 2017, CloudFlare. All rights reserved. 204 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 205 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 206 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 207 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 208 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 209 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloudflare + Google Cloud | Integrations 2 | Integrate Cloudflare Enterprise Log Push with BigQuery and Security Command Center on Google Cloud. 3 | 4 | * [Cloudflare Log Push to BigQuery](https://github.com/cloudflare/cloudflare-gcp/tree/master/logpush-to-bigquery) 5 | 6 | ---- 7 | -------------------------------------------------------------------------------- /cli/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | -------------------------------------------------------------------------------- /cli/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | exports = require('yargs') 4 | .commandDir('cmds') 5 | .demandCommand() 6 | .help() 7 | .argv 8 | -------------------------------------------------------------------------------- /cli/cloudshell-security-events.md: -------------------------------------------------------------------------------- 1 | # Cloudflare Security Events 2 | 3 | ## Enter the Cloud Shell subdirectory for the project you want to use for the Cloudflare SCC integration 4 | 5 | 6 | ## Set Cloudshell to the project where you'd like to deploy the integration 7 | ```sh 8 | gcloud config set project {{project-id}} 9 | ``` 10 | 11 | ## Install dependencies and CLI: 12 | Run: 13 | ```bash 14 | sh install.sh 15 | ``` 16 | 17 | ## Set Environment Variables and write deployment files 18 | Run: 19 | ```bash 20 | ./setup setEnv 21 | ``` 22 | 23 | ## Build configuration files: 24 | Run: 25 | ```bash 26 | ./setup buildConf 27 | ``` 28 | **Tip:** If you need to edit `security-events/.env.yml` in the future, use this command to update the configuration files before redeploying. 29 | 30 | ## Enable the necessary Cloud APIs to run the Cloudflare integration 31 | ```bash 32 | ./setup enableAPIs 33 | ``` 34 | 35 | ## Create Cloud Scheduler event (deployed via Pub/Sub) 36 | ```bash 37 | ./setup setSchedule 38 | ``` 39 | 40 | ## Create a service account key for SCC 41 | Run: 42 | ```bash 43 | ./setup getServiceAcctKey 44 | ``` 45 | 46 | **Tip:** This may throw some errors but if the final message succeeds, you're probably ok. Having issues? Make sure you're using the correct account: 47 | ```bash 48 | gcloud config set account MY_GCP_ACCT_EMAIL 49 | ``` 50 | 51 | ## Deploy integration 52 | ```bash 53 | ./setup deploy 54 | ``` 55 | 56 | 57 | ## Done! 58 | The configuration file, `.env.yml` can be modified here: 59 | ```sh 60 | cd cloudflare-gcp/security-events 61 | nano .env.yml 62 | ``` 63 | 64 | Then, to rebuild the necessary configuration files: 65 | ```sh 66 | cd cloudflare-gcp/cli 67 | ./setup buildConf 68 | ``` 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /cli/cmds/account.js: -------------------------------------------------------------------------------- 1 | const child = require('child_process') 2 | 3 | exports.command = 'account' 4 | 5 | exports.describe = 'Get or set current Google account configuration' 6 | 7 | exports.builder = { 8 | dir: { 9 | default: '.' 10 | } 11 | } 12 | 13 | exports.handler = (argv) => { 14 | if (argv.set) return child.execSync(`gcloud config set account ${argv.set}`).toString() 15 | return child.execSync(`gcloud config get-value account --format=object`).toString() 16 | } 17 | -------------------------------------------------------------------------------- /cli/cmds/buildConf.js: -------------------------------------------------------------------------------- 1 | const { info, success, err } = require('../utils/logger') 2 | const { baseDir, envDir, deploymentDir } = require('../utils/paths') 3 | const vinyl = require('vinyl-fs') 4 | const replace = require('gulp-frep') 5 | const fs = require('fs-extra') 6 | const del = require('del') 7 | 8 | require('env-yaml').config({ path: envDir }) 9 | 10 | exports.command = 'buildConf' 11 | 12 | exports.describe = 'Build or rebuild configuration files after changing .env.yml' 13 | 14 | exports.builder = { 15 | dir: { 16 | default: '.' 17 | } 18 | } 19 | 20 | exports.handler = (argv) => { 21 | const patterns = [ 22 | { 23 | pattern: /SERVICE_ACCOUNT/g, 24 | replacement: `${process.env.SERVICE_ACCOUNT}` 25 | }, 26 | { 27 | pattern: /PROJECT_ID/g, 28 | replacement: `${process.env.PROJECT_ID}` 29 | }, 30 | { 31 | pattern: /SOURCE_ID/g, 32 | replacement: `${process.env.SOURCE_ID}` 33 | }, 34 | { 35 | pattern: /BUCKET_NAME/g, 36 | replacement: `${process.env.BUCKET_NAME}` 37 | }, 38 | { 39 | pattern: /REGION/g, 40 | replacement: `${process.env.REGION}` 41 | }, 42 | { 43 | pattern: /DEPLOYMENT_DIR/g, 44 | replacement: `${deploymentDir}` 45 | }, 46 | { 47 | pattern: /BASE_DIR/g, 48 | replacement: `${baseDir}` 49 | }, 50 | { 51 | pattern: /GCLOUD_ORG/g, 52 | replacement: `${process.env.SOURCE_ID}`.split('/')[1] 53 | }, 54 | { 55 | pattern: /\*.*\s\*\s\*\s\*\s*\s\*/gi, 56 | replacement: `*/${process.env.INTERVAL.replace('m', '')} * * * *` 57 | } 58 | ] 59 | setTimeout(function () { 60 | try { 61 | vinyl.src([`templates/*.yml`, `!templates/setEnvPrompts.yml`]) 62 | .pipe(replace(patterns)) 63 | .pipe(vinyl.dest(`confs`)) 64 | } catch (e) { 65 | throw err(e) 66 | } finally { 67 | success(`\n\nService Account Key created and environment variables set. To modify this file, use \n\n$ nano ${envDir}\n\n${fs.readFileSync(envDir)}`) 68 | info(`Project: ${process.env.PROJECT_ID}, Org: ${process.env.GCLOUD_ORG}`) 69 | success(`Click next -->`) 70 | } 71 | }, 1500) 72 | } 73 | -------------------------------------------------------------------------------- /cli/cmds/deploy.js: -------------------------------------------------------------------------------- 1 | const yaml = { 2 | read: require('read-yaml') 3 | } 4 | const child = require('child_process') 5 | const { info, success, err } = require('../utils/logger') 6 | const { baseDir, deploymentDir, envDir } = require('../utils/paths') 7 | require('env-yaml').config({ path: envDir }) 8 | 9 | exports.command = 'deploy' 10 | 11 | exports.describe = 'Deploy Cloud Functions' 12 | 13 | exports.builder = { 14 | dir: { 15 | default: '.' 16 | } 17 | } 18 | 19 | exports.handler = async function deploy (argv) { 20 | const sh = cmd => child.execSync(cmd).toString() 21 | let file = yaml.read.sync(`${baseDir}/confs/deploy.yml`) 22 | console.log(file.create_bucket) 23 | try { 24 | sh(file.create_bucket.main) 25 | success(`Bucket created. Starting function deployment`) 26 | } catch (e) { 27 | info(`Bucket already created. Starting function deployment`) 28 | } finally { 29 | let cmdString = '' 30 | for (let [k, v] of Object.entries(file.deploy_function)) { 31 | cmdString += ` --${k}=${v}` 32 | } 33 | cmdString = 'gcloud beta functions deploy FirewallEventsToSecurityCenter' + cmdString 34 | cmdString = cmdString.trimRight().trimLeft() 35 | sh(cmdString) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /cli/cmds/enableAPIs.js: -------------------------------------------------------------------------------- 1 | const shP = require('exec-sh').promise 2 | const yaml = { 3 | read: require('read-yaml') 4 | } 5 | 6 | exports.command = 'enableAPIs' 7 | 8 | exports.describe = 'Enable APIs' 9 | 10 | exports.builder = { 11 | dir: { 12 | default: '.' 13 | } 14 | } 15 | 16 | exports.handler = async function enableAPIs () { 17 | const { info, success, err } = require('../utils/logger') 18 | const { baseDir } = require('../utils/paths') 19 | let cmds = yaml.read.sync(`${baseDir}/confs/enableAPIs.yml`) 20 | cmds = Object.values(cmds.apis) 21 | let i = 1 22 | const runCmds = cmds.map(async cmd => { 23 | let out 24 | try { 25 | out = await shP(`${cmd}`, false) 26 | } catch (e) { 27 | if (e instanceof TypeError) err(e) 28 | } 29 | return out 30 | }) 31 | 32 | for (const cmd of runCmds) { 33 | await cmd 34 | success(`${i++}/${cmds.length} .. ${cmds[i - 2]} succeeded`) 35 | if (i === runCmds.length) { 36 | setTimeout(() => { info('\nClick Next -->') }, 2000) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /cli/cmds/serviceAcct.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * 4 | * 5 | * 6 | * Retrieve service account key with sufficient permissions to use SCC integration 7 | * 8 | * 9 | * 10 | */ 11 | 12 | const yaml = { 13 | read: require('read-yaml') 14 | } 15 | const child = require('child_process') 16 | const { info, success, err } = require('../utils/logger') 17 | const { baseDir } = require('../utils/paths') 18 | 19 | exports.command = 'getServiceAcctKey' 20 | 21 | exports.describe = 'Retrieve new service account key' 22 | 23 | exports.builder = { 24 | dir: { 25 | default: '.' 26 | } 27 | } 28 | 29 | exports.handler = (argv) => { 30 | const sh = cmd => child.execSync(cmd).toString() 31 | let file = yaml.read.sync(`${baseDir}/confs/serviceAcct.yml`) 32 | console.log(file) 33 | if (argv.create) sh(file.create) 34 | for (let i = 0; i < file.add_binding.organization.roles.length; i++) { 35 | try { 36 | let iamBinding = (file.add_binding.organization.cmd[0]).replace('ROLE_ID', file.add_binding.organization.roles[i]) 37 | sh(iamBinding) 38 | success(file.add_binding.organization.roles[i]) 39 | } catch (e) { 40 | err(e) 41 | } 42 | } 43 | 44 | sh(file.download_key) 45 | info('\nClick Next -->') 46 | } 47 | -------------------------------------------------------------------------------- /cli/cmds/setEnv.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * 4 | * 5 | * 6 | * Setup environment variables and write to .env.yaml 7 | * 8 | * 9 | * 10 | */ 11 | 12 | const fs = require('fs-extra') 13 | const yargsInteractive = require('yargs-interactive') 14 | const bach = require('bach') 15 | const child = require('child_process') 16 | const chalkPipe = require('chalk-pipe') 17 | const del = require('del') 18 | const yaml = { 19 | read: require('read-yaml'), 20 | write: require('write-yaml') 21 | } 22 | const { info, success, err } = require('../utils/logger') 23 | const { baseDir, envDir, deploymentDir } = require('../utils/paths') 24 | 25 | exports.command = 'setEnv' 26 | 27 | exports.describe = 'Set environment variables' 28 | 29 | exports.builder = { 30 | dir: { 31 | default: '.' 32 | } 33 | } 34 | 35 | exports.handler = (argv) => { 36 | const GCP = { 37 | GCLOUD_ORG: '', 38 | PROJECT_ID: '', 39 | ENV () { 40 | try { 41 | this.PROJECT_ID = JSON.parse(child.execSync('gcloud config list --format "json(core.project)"').toString()).core.project 42 | this.GCLOUD_ORG = JSON.parse(child.execSync(`gcloud projects list --filter project_id=deep-presence-139721 --format=json`).toString())[0].parent.id 43 | yaml.write.sync(envDir, { PROJECT_ID: this.PROJECT_ID, GCLOUD_ORG: this.GCLOUD_ORG }) 44 | success(`Connected to ${this.PROJECT_ID} in org ${this.GCLOUD_ORG}`) 45 | } catch (e) { 46 | return err(e) 47 | } finally { 48 | const readEnv = yaml.read.sync(envDir) 49 | this.PROJECT_ID = readEnv.PROJECT_ID 50 | this.GCLOUD_ORG = readEnv.GCLOUD_ORG 51 | } 52 | } 53 | } 54 | 55 | function fn1 (cb) { 56 | del.sync([`deployment`, `confs/**`]) 57 | cb(null, 1) 58 | } 59 | 60 | function fn2 (cb) { 61 | GCP.ENV() 62 | cb(null, 1) 63 | } 64 | 65 | function fn3 (cb) { 66 | let prompts = yaml.read.sync(`${baseDir}/templates/setEnvPrompts.yml`) 67 | let options = { 68 | interactive: { default: true } 69 | } 70 | 71 | function * setExample () { 72 | yield `Example: Acme Corp` 73 | yield `Example: YQSn-xWAQiiEh9qM58wZNnyQS7FUdoqGIUAbrh7T` 74 | yield `Example: cloudflare-security-admin@${GCP.PROJECT_ID || 'PROJECT_ID'}@iam.gserviceaccount.com` 75 | yield `Example: organizations/12345678901/sources/112233445566778899abc` 76 | yield `Default: 10m` 77 | yield `Default: us-central1` 78 | } 79 | let ex = setExample() 80 | 81 | for (let prompt in prompts) { 82 | Reflect.set(options, `${prompt}`, prompts[prompt]) 83 | prompts[prompt].suffix = `\n` 84 | prompts[prompt].default = ex.next().value 85 | prompts[prompt].transformer = function (example) { 86 | return chalkPipe(example)(example) 87 | } 88 | } 89 | 90 | function writeEnv (answers) { 91 | let envs = { 92 | PROJECT_ID: `${GCP.PROJECT_ID}`, 93 | GCLOUD_ORG: GCP.GCLOUD_ORG, 94 | CREDENTIALS: `./scc_key.json`, 95 | CF_ORG_NAME: answers.CF_ORG_NAME_PROMPT.replace('Example: ', ''), 96 | API_KEY: answers.API_KEY_PROMPT.replace('Example: ', ''), 97 | SERVICE_ACCOUNT: answers.SERVICE_PROMPT.replace('Example: ', '').trim().replace(`\t`, ''), 98 | SOURCE_ID: answers.SRCID_PROMPT.replace('Example: ', '').trim().replace(`\t`, ''), 99 | INTERVAL: answers.INTERVAL_PROMPT.replace('Default: ', ''), 100 | REGION: answers.REGION_PROMPT.replace('Default: ', ''), 101 | BASE_DIR: baseDir, 102 | DEPLOYMENT_DIR: deploymentDir 103 | } 104 | 105 | yaml.write.sync(envDir, envs, { spaces: 2 }) 106 | success(`\n\nService Account Key created and environment variables set. To modify this file, use \n\n$ nano ${envDir}\n\n${fs.readFileSync(envDir)}`) 107 | // del.sync([`confs/setEnvPrompts`]) 108 | info(`Project: ${GCP.PROJECT_ID}, Org: ${GCP.GCLOUD_ORG}`) 109 | success(`Click next -->`) 110 | cb(null, 2) 111 | } 112 | 113 | yargsInteractive() 114 | .usage('$0 [args]') 115 | .interactive(options) 116 | .then(result => writeEnv(result)) 117 | .catch(e => { 118 | if (e) err(e) 119 | }) 120 | } 121 | 122 | return bach.settleSeries(fn1, fn2, fn3)() 123 | } 124 | -------------------------------------------------------------------------------- /cli/cmds/setSchedule.js: -------------------------------------------------------------------------------- 1 | const yaml = { 2 | read: require('read-yaml') 3 | } 4 | 5 | const child = require('child_process') 6 | 7 | exports.command = 'setSchedule' 8 | 9 | exports.describe = 'Set findings download schedule' 10 | 11 | exports.builder = { 12 | dir: { 13 | default: '.' 14 | } 15 | } 16 | 17 | exports.handler = (argv) => { 18 | const { info, success, err } = require('../utils/logger') 19 | const { baseDir } = require('../utils/paths') 20 | const sh = cmd => child.execSync(cmd).toString() 21 | const file = yaml.read.sync(`${baseDir}/confs/setSchedule.yml`) 22 | try { 23 | sh(file.set_cron) 24 | sh(file.set_pubsub_subscription) 25 | sh(file.set_pubsub_topic) 26 | success(`Scheduling job created.`) 27 | } catch (e) { 28 | err(e) 29 | } 30 | info('\nClick Next -->') 31 | } 32 | -------------------------------------------------------------------------------- /cli/cmds/testRun.js: -------------------------------------------------------------------------------- 1 | const child = require('child_process') 2 | 3 | exports.command = 'testRun' 4 | 5 | exports.describe = 'Manually run integration' 6 | 7 | exports.builder = { 8 | dir: { 9 | default: '.' 10 | } 11 | } 12 | 13 | exports.handler = (argv) => { 14 | return child.execSync(`cd ../security-events && node local.js`).toString() 15 | } 16 | -------------------------------------------------------------------------------- /cli/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | npm install && chmod +x setup 4 | -------------------------------------------------------------------------------- /cli/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudflare-security-events", 3 | "version": "0.1.31", 4 | "description": "Deliver Cloudflare logs to Google Cloud Security Command Center", 5 | "author": "Frank Taylor", 6 | "license": "MIT", 7 | "engines": { 8 | "node": "8.14.0" 9 | }, 10 | "preferGlobal": true, 11 | "repository": "shagamemnon/cloudflare-security-events", 12 | "scripts": { 13 | "enableAPIs": "node index.js enableAPIs", 14 | "setEnv": "node index.js setEnv", 15 | "deploy": "node index.js deploy", 16 | "getServiceAcctKey": "node index.js getServiceAcctKey", 17 | "serviceAcct": "node index.js getServiceAcctKey", 18 | "setSchedule": "node index.js setSchedule" 19 | }, 20 | "keywords": [ 21 | "cloudflare", 22 | "scc", 23 | "cscc", 24 | "security command center", 25 | "google cloud", 26 | "google cloud scc", 27 | "google", 28 | "securitys", 29 | "security events", 30 | "app" 31 | ], 32 | "main": "index.js", 33 | "files": [ 34 | "LICENSE", 35 | "cmds", 36 | "index.js", 37 | "templates", 38 | "utils", 39 | "cli.js" 40 | ], 41 | "bin": { 42 | "cfse": "./cli.js", 43 | "cloudflaregcp": "./cli.js" 44 | }, 45 | "dependencies": { 46 | "@google-cloud/asset": "^0.1.1", 47 | "@google-cloud/bigquery": "^2.0.5", 48 | "@google-cloud/pubsub": "^0.22.2", 49 | "@google-cloud/security-center": "^0.1.1", 50 | "@google-cloud/storage": "^2.3.4", 51 | "acorn": "^6.0.5", 52 | "bach": "^1.2.0", 53 | "chalk-pipe": "^2.0.0", 54 | "crypto-random-string": "^3.1.0", 55 | "del": "^3.0.0", 56 | "env-yaml": "^0.1.2", 57 | "envinfo": "^7.0.0", 58 | "exec-sh": "^0.3.2", 59 | "find-up": "^3.0.0", 60 | "fs-extra": "^7.0.1", 61 | "g": "^2.0.1", 62 | "global-dirs": "^0.1.1", 63 | "google-gax": "^0.23.0", 64 | "gulp-frep": "^0.1.3", 65 | "is-installed-globally": "^0.1.0", 66 | "liftoff": "^3.0.0", 67 | "prompts": "^2.0.0", 68 | "protobufjs": "^6.8.8", 69 | "quick-lru": "^2.0.0", 70 | "read-yaml": "^1.1.0", 71 | "validate-npm-package-name": "3.0.0", 72 | "vinyl-fs": "^3.0.3", 73 | "write-yaml": "^1.0.0", 74 | "yargs": "^12.0.5", 75 | "yargs-interactive": "^2.0.1" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /cli/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ./cli.js $1 4 | -------------------------------------------------------------------------------- /cli/templates/deploy.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Functions 3 | 4 | create_bucket: 5 | main: gsutil mb gs://cloudflare-scc-bucket-PROJECT_ID 6 | 7 | deploy_function: 8 | trigger-topic: cfscc 9 | env-vars-file: DEPLOYMENT_DIR/.env.yml 10 | region: REGION 11 | memory: 2048MB 12 | runtime: nodejs10 13 | service-account: PROJECT_ID@appspot.gserviceaccount.com 14 | stage-bucket: gs://cloudflare-scc-bucket-PROJECT_ID 15 | source: DEPLOYMENT_DIR 16 | timeout: 300 17 | -------------------------------------------------------------------------------- /cli/templates/enableAPIs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # APIs 3 | apis: 4 | - gcloud services enable servicemanagement.googleapis.com 5 | - gcloud services enable serviceusage.googleapis.com 6 | - gcloud services enable cloudfunctions.googleapis.com 7 | - gcloud services enable cloudresourcemanager.googleapis.com 8 | - gcloud services enable logging.googleapis.com 9 | - gcloud services enable pubsub.googleapis.com 10 | - gcloud services enable storage-component.googleapis.com 11 | - gcloud services enable securitycenter.googleapis.com 12 | -------------------------------------------------------------------------------- /cli/templates/serviceAcct.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Service Account 3 | 4 | list: gcloud iam service-accounts list --format="json[0](email)" --filter="email=SERVICE_ACCOUNT" 5 | 6 | create: gcloud iam service-accounts create SERVICE_ACCOUNT --display-name "Service Account for SCC" --format=json 7 | 8 | add_binding: 9 | organization: 10 | cmd: 11 | - gcloud organizations add-iam-policy-binding GCLOUD_ORG --member='serviceAccount:SERVICE_ACCOUNT' --role=ROLE_ID --format=json 12 | 13 | roles: 14 | - roles/bigquery.admin 15 | # - roles/cloudfunctions.serviceAgent 16 | - roles/securitycenter.admin 17 | - roles/storage.objectAdmin 18 | 19 | project: 20 | cmd: 21 | - gcloud projects add-iam-policy-binding PROJECT_ID --member='serviceAccount:SERVICE_ACCOUNT' --role=ROLE_ID --format=json 22 | 23 | roles: 24 | - roles/pubsub.admin 25 | - roles/browser 26 | 27 | download_key: gcloud iam service-accounts keys create DEPLOYMENT_DIR/scc_key.json --iam-account SERVICE_ACCOUNT 28 | -------------------------------------------------------------------------------- /cli/templates/setEnvPrompts.yml: -------------------------------------------------------------------------------- 1 | CF_ORG_NAME_PROMPT: 2 | type: input 3 | describe: 'What is the name of your Cloudflare organization?' 4 | 5 | API_KEY_PROMPT: 6 | type: input 7 | describe: 'Enter the Cloudflare API Token you created for Security Center (get a new token --> https://dash.cloudflare.com/profile/api-tokens):' 8 | 9 | SERVICE_PROMPT: 10 | type: input 11 | describe: 'Enter the service account id you created during the Google Cloud portion of the integration:' 12 | 13 | SRCID_PROMPT: 14 | type: input 15 | describe: 'Enter the source ID created for you during the Google Cloud signup:' 16 | 17 | INTERVAL_PROMPT: 18 | type: input 19 | describe: 'How often do you want Cloudflare to retrieve logs for you? Specify the interval in minutes. The minimum is 10 minutes; the maximum is 59 minutes:' 20 | 21 | REGION_PROMPT: 22 | type: input 23 | describe: '[Optional] Enter the Google Cloud Region where the Cloudflare Security Events integration should run:' 24 | -------------------------------------------------------------------------------- /cli/templates/setSchedule.yml: -------------------------------------------------------------------------------- 1 | set_pubsub_topic: gcloud pubsub topics create cfscc 2 | set_pubsub_subscription: gcloud pubsub subscriptions create --topic cfscc ping 3 | set_cron: gcloud beta scheduler jobs create pubsub cfscc --schedule="* * * * *" --topic="cfscc" --message-body="Retrieve CF logs for SCC" 4 | -------------------------------------------------------------------------------- /cli/utils/debug.js: -------------------------------------------------------------------------------- 1 | const envinfo = require('envinfo') 2 | 3 | exports = envinfo.run( 4 | { 5 | System: ['OS', 'CPU'], 6 | Binaries: ['Node', 'Yarn', 'npm'], 7 | Browsers: ['Chrome', 'Firefox', 'Safari'], 8 | npmPackages: ['styled-components', 'babel-plugin-styled-components'] 9 | }, 10 | { json: true, console: true, showNotFound: true } 11 | ) 12 | -------------------------------------------------------------------------------- /cli/utils/logger.js: -------------------------------------------------------------------------------- 1 | const { red, white, green, bold, blue } = require('kleur') 2 | 3 | module.exports.success = msg => console.log(`${green().bold(`[✔] `)}${white(msg)}`) 4 | module.exports.info = msg => console.log(`${blue(`[cse] `)}${white(msg)}`) 5 | module.exports.err = msg => console.log(`${red().bold(`[!] `)}${white(msg)}`) 6 | -------------------------------------------------------------------------------- /cli/utils/paths.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const findUp = require('find-up') 3 | 4 | // globalDirs.npm.binaries 5 | // findUp.sync(filename, [options]) 6 | 7 | const links = { 8 | get baseDir () { 9 | delete this.baseDir 10 | this.baseDir = path.join(findUp.sync('cloudflare-gcp'), 'cli') 11 | return this.baseDir 12 | }, 13 | get deploymentDir () { 14 | delete this.deploymentDir 15 | this.deploymentDir = path.join(findUp.sync('cloudflare-gcp'), 'security-events') 16 | return this.deploymentDir 17 | }, 18 | get envDir () { 19 | delete this.envDir 20 | this.envDir = path.join(findUp.sync('cloudflare-gcp'), 'security-events', '.env.yml') 21 | return this.envDir 22 | } 23 | } 24 | 25 | module.exports = links 26 | -------------------------------------------------------------------------------- /cli/utils/testPermissions.js: -------------------------------------------------------------------------------- 1 | 2 | // const CSE = require('../cse') 3 | // const { bqscc } = require('../index') 4 | 5 | // exports.command = 'scc' 6 | 7 | // exports.describe = 'Post findings from BigQuery to SCC' 8 | 9 | // exports.builder = {} 10 | 11 | // exports.handler = async function callCse (argv) { 12 | // const args = argv._[1] 13 | // if (args === 'post') return bqscc() 14 | // const cse = await CSE.init() 15 | // cse[`${args[1]}`]() 16 | // } 17 | -------------------------------------------------------------------------------- /logpush-to-bigquery/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | -------------------------------------------------------------------------------- /logpush-to-bigquery/README.md: -------------------------------------------------------------------------------- 1 | # Cloudflare Log Push ➜ Cloud Functions ➜ BigQuery 2 | 3 | **Prequisites:** 4 | 5 | - [Active Google Cloud account](https://cloud.google.com/free) 6 | - [Log Push enabled on Cloudflare](https://developers.cloudflare.com/logs/logpush/logpush-dashboard/) 7 | 8 | ### Automatic Install 9 | 10 | [![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/cloudflare/cloudflare-gcp&tutorial=cloudshell.md&cloudshell_working_dir=logpush-to-bigquery&cloudshell_open_in_editor=deploy.sh) 11 | 12 | ### Manual Install 13 | 14 | ```bash 15 | BRANCH="logstream-update" 16 | curl -LO "https://github.com/cloudflare/cloudflare-gcp/archive/refs/heads/$BRANCH.zip" && unzip "$BRANCH".zip && cd cloudflare-gcp-"$BRANCH"/logpush-to-biqquery 17 | ``` 18 | 19 | ```bash 20 | # Update the environment variables in deploy.sh 21 | BUCKET_NAME="" # required – The name of Google Cloud Storage bucket used for Cloudflare Logpush logs. 22 | DIRECTORY="/" # required – The name of the subdirectory in your bucket used for Cloudflare Logpush logs, # for example, "logs/". 23 | SCHEMA="" # optional - The schema based on the logs' source. schema-http.json is the default. Spectrum users will want to change this to "schema-spectrum.json" 24 | DATASET="" # optional – BigQuery dataset to write to. Will be created if necessary. 25 | TABLE="" # optional – BigQuery table to write to. Will be created if necessary. 26 | FN_NAME="" # optional - the name of your Cloud Function | default: gcsbq 27 | # optional - the name of the pubsub topic that will be published every minute 28 | TOPIC_NAME="" 29 | # Deploy to GCP 30 | sh ./deploy.sh 31 | ``` 32 | -------------------------------------------------------------------------------- /logpush-to-bigquery/cloudshell.md: -------------------------------------------------------------------------------- 1 | # Cloudflare Log Push 2 | 3 | ## Select project 4 | 5 | 6 | 7 | ## Set project ID 8 | 9 | ```sh 10 | gcloud config set project {{project-id}} 11 | ``` 12 | 13 | ## Update the environment variables in deploy.sh 14 | 15 | ```sh 16 | sudo nano deploy.sh 17 | ``` 18 | 19 | #### Variable reference 20 | 21 | ``` 22 | # required – The name of Google Cloud Storage bucket used for Cloudflare Logpush logs. 23 | BUCKET_NAME="" 24 | 25 | # required – The name of the subdirectory in your bucket used for Cloudflare Logpush logs, 26 | # for example, "logs/". If there is no subdirectory, use "/" 27 | DIRECTORY="/" 28 | 29 | # optional - specify a different schema. Spectrum users will want to change this to 30 | # schema-spectrum.json. 31 | SCHEMA="schema-http.json" 32 | 33 | # optional – BigQuery dataset to write to. Will be created if necessary. 34 | DATASET="" 35 | 36 | # optional – BigQuery table to write to. Will be created if necessary. 37 | TABLE="" 38 | 39 | # optional - the name of your Cloud Function. default: gcsbq 40 | FN_NAME="" 41 | 42 | # optional - the name of the pubsub topic that will be published every minute 43 | TOPIC_NAME="every_minute" 44 | 45 | ``` 46 | 47 | ## Deploy to GCP 48 | 49 | ```sh 50 | sh ./deploy.sh 51 | ``` 52 | 53 | ## Done! 54 | -------------------------------------------------------------------------------- /logpush-to-bigquery/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SCHEMA="schema-http.json" 4 | # The name of the subdirectory in your bucket used for Cloudflare Logpush logs, 5 | # for example, "logs/". If there is no subdirectory, use "" 6 | DIRECTORY="logs/" 7 | BUCKET_NAME="examplecom-logs" 8 | DATASET="cloudflare_logstream" 9 | TABLE="cloudflare_logs" 10 | REGION="us-central1" 11 | # You probably don't need to change these values: 12 | FN_NAME="cf-logs-to-bigquery" 13 | TOPIC_NAME="every_minute" 14 | 15 | # Create pubsub topic 16 | gcloud pubsub topics create $TOPIC_NAME 17 | # Create cron job 18 | gcloud scheduler jobs create pubsub cf_logs_cron --schedule="* * * * *" --topic=$TOPIC_NAME --message-body="60 seconds passed" 19 | # Deploy function 20 | gcloud functions deploy $FN_NAME \ 21 | --runtime nodejs12 \ 22 | --trigger-topic $TOPIC_NAME \ 23 | --region=$REGION \ 24 | --memory=1024MB \ 25 | --entry-point=runLoadJob \ 26 | --set-env-vars DATASET=$DATASET,TABLE=$TABLE,SCHEMA=$SCHEMA,BUCKET_NAME=$BUCKET_NAME,DIRECTORY=$DIRECTORY 27 | -------------------------------------------------------------------------------- /logpush-to-bigquery/index.js: -------------------------------------------------------------------------------- 1 | const { Storage } = require('@google-cloud/storage') 2 | const { BigQuery } = require('@google-cloud/bigquery') 3 | const { Logging } = require('@google-cloud/logging') 4 | const { DateTime } = require('luxon') 5 | 6 | const storage = new Storage() 7 | const bigquery = new BigQuery() 8 | const logging = new Logging() 9 | const log = logging.log('logpush-to-bigquery-sink') 10 | const bucket = storage.bucket(process.env.BUCKET_NAME) 11 | 12 | async function gcsbq (files) { 13 | const schema = require(`./${process.env.SCHEMA}`) 14 | const datasetId = process.env.DATASET 15 | const tableId = process.env.TABLE 16 | /* Configure the load job and ignore values undefined in schema */ 17 | const metadata = { 18 | sourceFormat: 'NEWLINE_DELIMITED_JSON', 19 | schema: { 20 | fields: schema 21 | }, 22 | ignoreUnknownValues: true 23 | } 24 | 25 | const addToTable = async tableId => { 26 | const dataset = await bigquery.dataset(datasetId).get({ autoCreate: true }) 27 | const table = await dataset[0].table(tableId).get({ autoCreate: true }) 28 | return table[0].createLoadJob(files, metadata) 29 | } 30 | 31 | try { 32 | return addToTable(tableId) 33 | } catch (e) { 34 | console.log(e) 35 | } 36 | } 37 | 38 | async function writeLog (logData) { 39 | logData = logData.reduce((acc, current) => [...acc, current.name], []) 40 | 41 | const metadata = { 42 | resource: { type: 'global' }, 43 | severity: 'INFO' 44 | } 45 | 46 | const entry = log.entry(metadata, logData) 47 | await log.write(entry) 48 | console.log( 49 | `Loaded to ${process.env.DATASET}.${process.env.TABLE}: ${logData}` 50 | ) 51 | } 52 | 53 | module.exports.runLoadJob = async function (message, context) { 54 | if (!context) { 55 | context = {} 56 | context.timestamp = new Date().toISOString() 57 | } 58 | context.timestamp = DateTime.fromISO(context.timestamp) 59 | 60 | const loadJobDeadline = context.timestamp 61 | .setZone('GMT') 62 | .minus({ minutes: 15 }) 63 | .startOf('minute') 64 | 65 | const [deadlineDate, deadlineDt] = [ 66 | loadJobDeadline.toFormat('yyyyMMdd'), 67 | loadJobDeadline.toFormat(`yyyyMMdd'T'HHmm`) 68 | ] 69 | 70 | let stackdriverEntry = [] 71 | 72 | let prefix = `${process.env.DIRECTORY}${deadlineDate}/${deadlineDt}`.trim() 73 | 74 | if (prefix.startsWith('/')) { 75 | prefix = prefix.slice(1) 76 | } 77 | 78 | try { 79 | let logFiles = await bucket.getFiles({ 80 | autoPaginate: false, 81 | maxResults: 5000, 82 | prefix 83 | }) 84 | 85 | logFiles = logFiles[0] 86 | 87 | if (logFiles.length < 1) { 88 | return console.log(`No new logs at ${deadlineDt}`) 89 | } 90 | 91 | await gcsbq(logFiles) 92 | await writeLog(logFiles) 93 | } catch (e) { 94 | console.log(e) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /logpush-to-bigquery/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sample-pubsub", 3 | "version": "0.0.1", 4 | "dependencies": { 5 | "@google-cloud/bigquery": "^5.5.0", 6 | "@google-cloud/logging": "^9.2.1", 7 | "@google-cloud/pubsub": "^0.18.0", 8 | "@google-cloud/storage": "^5.8.3", 9 | "luxon": "^1.26.0" 10 | }, 11 | "scripts": { 12 | "dev": "node -r dotenv/config local.js" 13 | }, 14 | "devDependencies": { 15 | "dotenv": "^8.2.0", 16 | "prettier": "^2.2.1" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /logpush-to-bigquery/schema-firewall-events.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Name": "Action", 4 | "type": "STRING", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "Name": "ClientASN", 9 | "type": "INTEGER", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "Name": "ClientASNDescription", 14 | "type": "STRING", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "Name": "ClientCountry", 19 | "type": "STRING", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "Name": "ClientIP", 24 | "type": "STRING", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "Name": "ClientIPClass", 29 | "type": "STRING", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "Name": "ClientRefererHost", 34 | "type": "STRING", 35 | "mode": "NULLABLE" 36 | }, 37 | { 38 | "Name": "ClientRefererPath", 39 | "type": "STRING", 40 | "mode": "NULLABLE" 41 | }, 42 | { 43 | "Name": "ClientRefererQuery", 44 | "type": "STRING", 45 | "mode": "NULLABLE" 46 | }, 47 | { 48 | "Name": "ClientRefererScheme", 49 | "type": "STRING", 50 | "mode": "NULLABLE" 51 | }, 52 | { 53 | "Name": "ClientRequestHost", 54 | "type": "STRING", 55 | "mode": "NULLABLE" 56 | }, 57 | { 58 | "Name": "ClientRequestMethod", 59 | "type": "STRING", 60 | "mode": "NULLABLE" 61 | }, 62 | { 63 | "Name": "ClientRequestPath", 64 | "type": "STRING", 65 | "mode": "NULLABLE" 66 | }, 67 | { 68 | "Name": "ClientRequestProtocol", 69 | "type": "STRING", 70 | "mode": "NULLABLE" 71 | }, 72 | { 73 | "Name": "ClientRequestQuery", 74 | "type": "STRING", 75 | "mode": "NULLABLE" 76 | }, 77 | { 78 | "Name": "ClientRequestScheme", 79 | "type": "STRING", 80 | "mode": "NULLABLE" 81 | }, 82 | { 83 | "Name": "ClientRequestUserAgent", 84 | "type": "STRING", 85 | "mode": "NULLABLE" 86 | }, 87 | { 88 | "Name": "Datetime", 89 | "type": "TIMESTAMP", 90 | "mode": "NULLABLE" 91 | }, 92 | { 93 | "Name": "EdgeColoCode", 94 | "type": "STRING", 95 | "mode": "NULLABLE" 96 | }, 97 | { 98 | "Name": "EdgeResponseStatus", 99 | "type": "INTEGER", 100 | "mode": "NULLABLE" 101 | }, 102 | { 103 | "Name": "Kind", 104 | "type": "STRING", 105 | "mode": "NULLABLE" 106 | }, 107 | { 108 | "Name": "MatchIndex", 109 | "type": "INTEGER", 110 | "mode": "NULLABLE" 111 | }, 112 | { 113 | "Name": "Metadata", 114 | "type": "STRUCT", 115 | "mode": "NULLABLE" 116 | }, 117 | { 118 | "Name": "OriginResponseStatus", 119 | "type": "INTEGER", 120 | "mode": "NULLABLE" 121 | }, 122 | { 123 | "Name": "OriginatorRayID", 124 | "type": "STRING", 125 | "mode": "NULLABLE" 126 | }, 127 | { 128 | "Name": "RayID", 129 | "type": "STRING", 130 | "mode": "NULLABLE" 131 | }, 132 | { 133 | "Name": "RuleID", 134 | "type": "STRING", 135 | "mode": "NULLABLE" 136 | }, 137 | { 138 | "Name": "Source", 139 | "type": "STRING", 140 | "mode": "NULLABLE" 141 | } 142 | ] 143 | -------------------------------------------------------------------------------- /logpush-to-bigquery/schema-gateway-dns.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "ColoID", 4 | "type": "INTEGER", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "name": "ColoName", 9 | "type": "STRING", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "name": "Datetime", 14 | "type": "STRING", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "name": "DeviceID", 19 | "type": "STRING", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "name": "DstIP", 24 | "type": "STRING", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "name": "DstPort", 29 | "type": "INTEGER", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "name": "Email", 34 | "type": "STRING", 35 | "mode": "NULLABLE" 36 | }, 37 | { 38 | "name": "Location", 39 | "type": "STRING", 40 | "mode": "NULLABLE" 41 | }, 42 | { 43 | "name": "MatchedCategoryIDs", 44 | "type": "INTEGER", 45 | "mode": "NULLABLE" 46 | }, 47 | { 48 | "name": "Policy", 49 | "type": "STRING", 50 | "mode": "NULLABLE" 51 | }, 52 | { 53 | "name": "PolicyID", 54 | "type": "STRING", 55 | "mode": "NULLABLE" 56 | }, 57 | { 58 | "name": "Protocol", 59 | "type": "STRING", 60 | "mode": "NULLABLE" 61 | }, 62 | { 63 | "name": "QueryCategoryIDs", 64 | "type": "INTEGER", 65 | "mode": "NULLABLE" 66 | }, 67 | { 68 | "name": "QueryNamestring", 69 | "type": "STRING", 70 | "mode": "NULLABLE" 71 | }, 72 | { 73 | "name": "QueryNameReversed", 74 | "type": "STRING", 75 | "mode": "NULLABLE" 76 | }, 77 | { 78 | "name": "QuerySize", 79 | "type": "INTEGER", 80 | "mode": "NULLABLE" 81 | }, 82 | { 83 | "name": "QueryType", 84 | "type": "STRING", 85 | "mode": "NULLABLE" 86 | }, 87 | { 88 | "name": "RData", 89 | "type": "STRING", 90 | "mode": "REPEATED" 91 | }, 92 | { 93 | "name": "ResolverDecision", 94 | "type": "STRING", 95 | "mode": "NULLABLE" 96 | }, 97 | { 98 | "name": "SrcIP", 99 | "type": "STRING", 100 | "mode": "NULLABLE" 101 | }, 102 | { 103 | "name": "SrcPort", 104 | "type": "INTEGER", 105 | "mode": "NULLABLE" 106 | }, 107 | { 108 | "name": "UserID", 109 | "type": "STRING", 110 | "mode": "NULLABLE" 111 | } 112 | ] 113 | -------------------------------------------------------------------------------- /logpush-to-bigquery/schema-gateway-http.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "AccountID", 4 | "type": "STRING", 5 | "MODE": "NULLABLE" 6 | }, 7 | { 8 | "name": "Action", 9 | "type": "STRING", 10 | "MODE": "NULLABLE" 11 | }, 12 | { 13 | "name": "BlockedFileHash", 14 | "type": "STRING", 15 | "MODE": "NULLABLE" 16 | }, 17 | { 18 | "name": "BlockedFileName", 19 | "type": "STRING", 20 | "MODE": "NULLABLE" 21 | }, 22 | { 23 | "name": "BlockedFileReason", 24 | "type": "STRING", 25 | "MODE": "NULLABLE" 26 | }, 27 | { 28 | "name": "BlockedFileSize", 29 | "type": "STRING", 30 | "MODE": "NULLABLE" 31 | }, 32 | { 33 | "name": "BlockedFileType", 34 | "type": "STRING", 35 | "MODE": "NULLABLE" 36 | }, 37 | { 38 | "name": "Datetime", 39 | "type": "STRING", 40 | "MODE": "NULLABLE" 41 | }, 42 | { 43 | "name": "DestinationIP", 44 | "type": "STRING", 45 | "MODE": "NULLABLE" 46 | }, 47 | { 48 | "name": "DestinationPort", 49 | "type": "STRING", 50 | "MODE": "NULLABLE" 51 | }, 52 | { 53 | "name": "DeviceID", 54 | "type": "STRING", 55 | "MODE": "NULLABLE" 56 | }, 57 | { 58 | "name": "DownloadedFileNames", 59 | "type": "STRING", 60 | "MODE": "REPEATABLE" 61 | }, 62 | { 63 | "name": "Email", 64 | "type": "STRING", 65 | "MODE": "NULLABLE" 66 | }, 67 | { 68 | "name": "HTTPHost", 69 | "type": "STRING", 70 | "MODE": "NULLABLE" 71 | }, 72 | { 73 | "name": "HTTPMethod", 74 | "type": "STRING", 75 | "MODE": "NULLABLE" 76 | }, 77 | { 78 | "name": "HTTPVersion", 79 | "type": "STRING", 80 | "MODE": "NULLABLE" 81 | }, 82 | { 83 | "name": "IsIsolated", 84 | "type": "BOOLEAN", 85 | "MODE": "NULLABLE" 86 | }, 87 | { 88 | "name": "PolicyID", 89 | "type": "STRING", 90 | "MODE": "NULLABLE" 91 | }, 92 | { 93 | "name": "Referer", 94 | "type": "STRING", 95 | "MODE": "NULLABLE" 96 | }, 97 | { 98 | "name": "RequestID", 99 | "type": "STRING", 100 | "MODE": "NULLABLE" 101 | }, 102 | { 103 | "name": "SourceIP", 104 | "type": "STRING", 105 | "MODE": "NULLABLE" 106 | }, 107 | { 108 | "name": "SourcePort", 109 | "type": "STRING", 110 | "MODE": "NULLABLE" 111 | }, 112 | { 113 | "name": "URL", 114 | "type": "STRING", 115 | "MODE": "NULLABLE" 116 | }, 117 | { 118 | "name": "UploadedFileNames", 119 | "type": "STRING", 120 | "mode": "REPEATABLE" 121 | }, 122 | { 123 | "name": "UserAgent", 124 | "type": "STRING", 125 | "MODE": "NULLABLE" 126 | }, 127 | { 128 | "name": "UserID", 129 | "type": "STRING", 130 | "MODE": "NULLABLE" 131 | } 132 | ] 133 | -------------------------------------------------------------------------------- /logpush-to-bigquery/schema-http.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "BotScore", 4 | "type": "INTEGER", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "name": "BotScoreSrc", 9 | "type": "STRING", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "name": "CacheCacheStatus", 14 | "type": "STRING", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "name": "CacheResponseBytes", 19 | "type": "INTEGER", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "name": "CacheResponseStatus", 24 | "type": "INTEGER", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "name": "CacheTieredFill", 29 | "type": "BOOLEAN", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "name": "ClientASN", 34 | "type": "INTEGER", 35 | "mode": "NULLABLE" 36 | }, 37 | { 38 | "name": "ClientCountry", 39 | "type": "STRING", 40 | "mode": "NULLABLE" 41 | }, 42 | { 43 | "name": "ClientDeviceType", 44 | "type": "STRING", 45 | "mode": "NULLABLE" 46 | }, 47 | { 48 | "name": "ClientIP", 49 | "type": "STRING", 50 | "mode": "NULLABLE" 51 | }, 52 | { 53 | "name": "ClientIPClass", 54 | "type": "STRING", 55 | "mode": "NULLABLE" 56 | }, 57 | { 58 | "name": "ClientMTLSAuthCertFingerprint", 59 | "type": "STRING", 60 | "mode": "NULLABLE" 61 | }, 62 | { 63 | "name": "ClientMTLSAuthStatus", 64 | "type": "STRING", 65 | "mode": "NULLABLE" 66 | }, 67 | { 68 | "name": "ClientRequestBytes", 69 | "type": "INTEGER", 70 | "mode": "NULLABLE" 71 | }, 72 | { 73 | "name": "ClientRequestHost", 74 | "type": "STRING", 75 | "mode": "NULLABLE" 76 | }, 77 | { 78 | "name": "ClientRequestMethod", 79 | "type": "STRING", 80 | "mode": "NULLABLE" 81 | }, 82 | { 83 | "name": "ClientRequestPath", 84 | "type": "STRING", 85 | "mode": "NULLABLE" 86 | }, 87 | { 88 | "name": "ClientRequestProtocol", 89 | "type": "STRING", 90 | "mode": "NULLABLE" 91 | }, 92 | { 93 | "name": "ClientRequestReferer", 94 | "type": "STRING", 95 | "mode": "NULLABLE" 96 | }, 97 | { 98 | "name": "ClientRequestScheme", 99 | "type": "STRING", 100 | "mode": "NULLABLE" 101 | }, 102 | { 103 | "name": "ClientRequestSource", 104 | "type": "STRING", 105 | "mode": "NULLABLE" 106 | }, 107 | { 108 | "name": "ClientRequestURI", 109 | "type": "STRING", 110 | "mode": "NULLABLE" 111 | }, 112 | { 113 | "name": "ClientRequestUserAgent", 114 | "type": "STRING", 115 | "mode": "NULLABLE" 116 | }, 117 | { 118 | "name": "ClientSrcPort", 119 | "type": "INTEGER", 120 | "mode": "NULLABLE" 121 | }, 122 | { 123 | "name": "ClientSSLCipher", 124 | "type": "STRING", 125 | "mode": "NULLABLE" 126 | }, 127 | { 128 | "name": "ClientSSLProtocol", 129 | "type": "STRING", 130 | "mode": "NULLABLE" 131 | }, 132 | { 133 | "name": "ClientTCPRTTMs", 134 | "type": "INTEGER", 135 | "mode": "NULLABLE" 136 | }, 137 | { 138 | "name": "ClientXRequestedWith", 139 | "type": "STRING", 140 | "mode": "NULLABLE" 141 | }, 142 | { 143 | "name": "EdgeCFConnectingO2O", 144 | "type": "BOOLEAN", 145 | "mode": "NULLABLE" 146 | }, 147 | { 148 | "name": "EdgeColoCode", 149 | "type": "STRING", 150 | "mode": "NULLABLE" 151 | }, 152 | { 153 | "name": "EdgeColoID", 154 | "type": "INTEGER", 155 | "mode": "NULLABLE" 156 | }, 157 | { 158 | "name": "EdgeEndTimestamp", 159 | "type": "TIMESTAMP", 160 | "mode": "NULLABLE" 161 | }, 162 | { 163 | "name": "EdgePathingOp", 164 | "type": "STRING", 165 | "mode": "NULLABLE" 166 | }, 167 | { 168 | "name": "EdgePathingSrc", 169 | "type": "STRING", 170 | "mode": "NULLABLE" 171 | }, 172 | { 173 | "name": "EdgePathingStatus", 174 | "type": "STRING", 175 | "mode": "NULLABLE" 176 | }, 177 | { 178 | "name": "EdgeRateLimitAction", 179 | "type": "STRING", 180 | "mode": "NULLABLE" 181 | }, 182 | { 183 | "name": "EdgeRateLimitID", 184 | "type": "INTEGER", 185 | "mode": "NULLABLE" 186 | }, 187 | { 188 | "name": "EdgeRequestHost", 189 | "type": "STRING", 190 | "mode": "NULLABLE" 191 | }, 192 | { 193 | "name": "EdgeResponseBodyBytes", 194 | "type": "INTEGER", 195 | "mode": "NULLABLE" 196 | }, 197 | { 198 | "name": "EdgeResponseBytes", 199 | "type": "INTEGER", 200 | "mode": "NULLABLE" 201 | }, 202 | { 203 | "name": "EdgeResponseCompressionRatio", 204 | "type": "FLOAT", 205 | "mode": "NULLABLE" 206 | }, 207 | { 208 | "name": "EdgeResponseContentType", 209 | "type": "STRING", 210 | "mode": "NULLABLE" 211 | }, 212 | { 213 | "name": "EdgeResponseStatus", 214 | "type": "INTEGER", 215 | "mode": "NULLABLE" 216 | }, 217 | { 218 | "name": "EdgeServerIP", 219 | "type": "STRING", 220 | "mode": "NULLABLE" 221 | }, 222 | { 223 | "name": "EdgeStartTimestamp", 224 | "type": "TIMESTAMP", 225 | "mode": "NULLABLE" 226 | }, 227 | { 228 | "name": "EdgeTimeToFirstByteMs", 229 | "type": "INTEGER", 230 | "mode": "NULLABLE" 231 | }, 232 | { 233 | "name": "FirewallMatchesActions", 234 | "type": "STRING", 235 | "mode": "REPEATED" 236 | }, 237 | { 238 | "name": "FirewallMatchesRuleIDs", 239 | "type": "STRING", 240 | "mode": "REPEATED" 241 | }, 242 | { 243 | "name": "FirewallMatchesSources", 244 | "type": "STRING", 245 | "mode": "REPEATED" 246 | }, 247 | { 248 | "name": "OriginDNSResponseTimeMs", 249 | "type": "INTEGER", 250 | "mode": "NULLABLE" 251 | }, 252 | { 253 | "name": "OriginIP", 254 | "type": "STRING", 255 | "mode": "NULLABLE" 256 | }, 257 | { 258 | "name": "OriginRequestHeaderSendDurationMs", 259 | "type": "INTEGER", 260 | "mode": "NULLABLE" 261 | }, 262 | { 263 | "name": "OriginResponseBytes", 264 | "type": "INTEGER", 265 | "mode": "NULLABLE" 266 | }, 267 | { 268 | "name": "OriginResponseDurationMs", 269 | "type": "INTEGER", 270 | "mode": "NULLABLE" 271 | }, 272 | { 273 | "name": "OriginResponseHeaderReceiveDurationMs", 274 | "type": "INTEGER", 275 | "mode": "NULLABLE" 276 | }, 277 | { 278 | "name": "OriginResponseHTTPExpires", 279 | "type": "STRING", 280 | "mode": "NULLABLE" 281 | }, 282 | { 283 | "name": "OriginResponseHTTPLastModified", 284 | "type": "STRING", 285 | "mode": "NULLABLE" 286 | }, 287 | { 288 | "name": "OriginResponseStatus", 289 | "type": "INTEGER", 290 | "mode": "NULLABLE" 291 | }, 292 | { 293 | "name": "OriginResponseTime", 294 | "type": "INTEGER", 295 | "mode": "NULLABLE" 296 | }, 297 | { 298 | "name": "OriginSSLProtocol", 299 | "type": "STRING", 300 | "mode": "NULLABLE" 301 | }, 302 | { 303 | "name": "OriginTCPHandshakeDurationMs", 304 | "type": "INTEGER", 305 | "mode": "NULLABLE" 306 | }, 307 | { 308 | "name": "OriginTLSHandshakeDurationMs", 309 | "type": "INTEGER", 310 | "mode": "NULLABLE" 311 | }, 312 | { 313 | "name": "ParentRayID", 314 | "type": "STRING", 315 | "mode": "NULLABLE" 316 | }, 317 | { 318 | "name": "RayID", 319 | "type": "STRING", 320 | "mode": "NULLABLE" 321 | }, 322 | { 323 | "name": "SecurityLevel", 324 | "type": "STRING", 325 | "mode": "NULLABLE" 326 | }, 327 | { 328 | "name": "SmartRouteColoID", 329 | "type": "INTEGER", 330 | "mode": "NULLABLE" 331 | }, 332 | { 333 | "name": "UpperTierColoID", 334 | "type": "INTEGER", 335 | "mode": "NULLABLE" 336 | }, 337 | { 338 | "name": "WAFAction", 339 | "type": "STRING", 340 | "mode": "NULLABLE" 341 | }, 342 | { 343 | "name": "WAFFlags", 344 | "type": "INTEGER", 345 | "mode": "NULLABLE" 346 | }, 347 | { 348 | "name": "WAFMatchedVar", 349 | "type": "STRING", 350 | "mode": "NULLABLE" 351 | }, 352 | { 353 | "name": "WAFProfile", 354 | "type": "STRING", 355 | "mode": "NULLABLE" 356 | }, 357 | { 358 | "name": "WAFRuleID", 359 | "type": "STRING", 360 | "mode": "NULLABLE" 361 | }, 362 | { 363 | "name": "WAFRuleMessage", 364 | "type": "STRING", 365 | "mode": "NULLABLE" 366 | }, 367 | { 368 | "name": "WorkerCPUTime", 369 | "type": "INTEGER", 370 | "mode": "NULLABLE" 371 | }, 372 | { 373 | "name": "WorkerStatus", 374 | "type": "STRING", 375 | "mode": "NULLABLE" 376 | }, 377 | { 378 | "name": "WorkerSubrequest", 379 | "type": "BOOLEAN", 380 | "mode": "NULLABLE" 381 | }, 382 | { 383 | "name": "WorkerSubrequestCount", 384 | "type": "INTEGER", 385 | "mode": "NULLABLE" 386 | }, 387 | { 388 | "name": "ZoneID", 389 | "type": "INTEGER", 390 | "mode": "NULLABLE" 391 | }, 392 | { 393 | "name": "ZoneName", 394 | "type": "STRING", 395 | "mode": "NULLABLE" 396 | } 397 | ] -------------------------------------------------------------------------------- /logpush-to-bigquery/schema-spectrum.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Application", 4 | "type": "STRING", 5 | "MODE": "NULLABLE" 6 | }, 7 | { 8 | "name": "ClientAsn", 9 | "type": "INTEGER", 10 | "MODE": "NULLABLE" 11 | }, 12 | { 13 | "name": "ClientBytes", 14 | "type": "INTEGER", 15 | "MODE": "NULLABLE" 16 | }, 17 | { 18 | "name": "ClientCountry", 19 | "type": "STRING", 20 | "MODE": "NULLABLE" 21 | }, 22 | { 23 | "name": "ClientIP", 24 | "type": "STRING", 25 | "MODE": "NULLABLE" 26 | }, 27 | { 28 | "name": "ClientMatchedIpFirewall", 29 | "type": "STRING", 30 | "MODE": "NULLABLE" 31 | }, 32 | { 33 | "name": "ClientPort", 34 | "type": "INTEGER", 35 | "MODE": "NULLABLE" 36 | }, 37 | { 38 | "name": "ClientProto", 39 | "type": "STRING", 40 | "MODE": "NULLABLE" 41 | }, 42 | { 43 | "name": "ClientTcpRtt", 44 | "type": "INTEGER", 45 | "MODE": "NULLABLE" 46 | }, 47 | { 48 | "name": "ClientTlsCipher", 49 | "type": "STRING", 50 | "MODE": "NULLABLE" 51 | }, 52 | { 53 | "name": "ClientTlsClientHelloServerName", 54 | "type": "STRING", 55 | "MODE": "NULLABLE" 56 | }, 57 | { 58 | "name": "ClientTlsProtocol", 59 | "type": "STRING", 60 | "MODE": "NULLABLE" 61 | }, 62 | { 63 | "name": "ClientTlsStatus", 64 | "type": "STRING", 65 | "MODE": "NULLABLE" 66 | }, 67 | { 68 | "name": "ColoCode", 69 | "type": "STRING", 70 | "MODE": "NULLABLE" 71 | }, 72 | { 73 | "name": "ConnectTimestamp", 74 | "type": "STRING", 75 | "MODE": "NULLABLE" 76 | }, 77 | { 78 | "name": "DisconnectTimestamp", 79 | "type": "STRING", 80 | "MODE": "NULLABLE" 81 | }, 82 | { 83 | "name": "Event", 84 | "type": "STRING", 85 | "MODE": "NULLABLE" 86 | }, 87 | { 88 | "name": "IpFirewall", 89 | "type": "BOOLEAN", 90 | "MODE": "NULLABLE" 91 | }, 92 | { 93 | "name": "OriginBytes", 94 | "type": "INTEGER", 95 | "MODE": "NULLABLE" 96 | }, 97 | { 98 | "name": "OriginIP", 99 | "type": "STRING", 100 | "MODE": "NULLABLE" 101 | }, 102 | { 103 | "name": "OriginPort", 104 | "type": "INTEGER", 105 | "MODE": "NULLABLE" 106 | }, 107 | { 108 | "name": "OriginProto", 109 | "type": "STRING", 110 | "MODE": "NULLABLE" 111 | }, 112 | { 113 | "name": "OriginTcpRtt", 114 | "type": "INTEGER", 115 | "MODE": "NULLABLE" 116 | }, 117 | { 118 | "name": "OriginTlsCipher", 119 | "type": "STRING", 120 | "MODE": "NULLABLE" 121 | }, 122 | { 123 | "name": "OriginTlsFingerprint", 124 | "type": "STRING", 125 | "MODE": "NULLABLE" 126 | }, 127 | { 128 | "name": "OriginTlsMode", 129 | "type": "STRING", 130 | "MODE": "NULLABLE" 131 | }, 132 | { 133 | "name": "OriginTlsProtocol", 134 | "type": "STRING", 135 | "MODE": "NULLABLE" 136 | }, 137 | { 138 | "name": "OriginTlsStatus", 139 | "type": "STRING", 140 | "MODE": "NULLABLE" 141 | }, 142 | { 143 | "name": "ProxyProtocol", 144 | "type": "STRING", 145 | "MODE": "NULLABLE" 146 | }, 147 | { 148 | "name": "Status", 149 | "type": "INTEGER", 150 | "MODE": "NULLABLE" 151 | }, 152 | { 153 | "name": "Timestamp", 154 | "type": "TIMESTAMP", 155 | "MODE": "NULLABLE" 156 | } 157 | ] 158 | -------------------------------------------------------------------------------- /logpush-to-bigquery/test.js: -------------------------------------------------------------------------------- 1 | const {BigQuery} = require('@google-cloud/bigquery'); 2 | const bigquery = new BigQuery(); 3 | const dataset = bigquery.dataset('cloudflare_logs'); 4 | 5 | const table = dataset.table('recent_events'); 6 | 7 | 8 | 9 | module.exports = table.getMetadata().then((data) => { 10 | console.log(data[0].numRows, data[1]) 11 | return data 12 | }).catch(e => console.log(e)) 13 | -------------------------------------------------------------------------------- /security-events/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | config 18 | config.js 19 | -------------------------------------------------------------------------------- /security-events/README.md: -------------------------------------------------------------------------------- 1 | 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 2 | 3 | Cloudflare GCP Security Events has been deprecated. No more contributions will be accepted. The master branch will be replaced by examples for integrating Cloudflare logs with Security Command Center 4 | 5 | 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 6 | # SCC Documentation - Beta v0.2 7 | 8 | ## Prerequisites 9 | Before starting, please ensure that [Cloudflare Log retention is enabled](https://api.cloudflare.com/#logs-received-get-log-retention-flag). Note that this setting is different from Logpush and is off by default. Log retention can only be enabled by an admin in your Cloudflare account. If you need to enable log retention, we recommend this shell script: https://gist.github.com/shagamemnon/f3aecce00e192cfd9282dc7dc2bd1ee8 10 | 11 | ## How it works 12 | ![diagram.jpg](https://storage.franktaylor.io/scc_diagram.jpg) 13 | 14 | ## Setup 15 | 1. Go to [https://dash.cloudflare.com/profile/api-tokens](https://dash.cloudflare.com/profile/api-tokens) 16 | 2. Click **Create Token** and check **Start with a Template**. Select the **Read all resources** option: 17 | 18 | ![](https://storage.franktaylor.io/d06cef5527f329e519553f649b3a76e219f2c9d6/CleanShot%202020-01-22%20at%2003.27.31.png) 19 | 20 | 3. Once the template is visible, you can remove permissions for the zones/accounts you don't want to use with Google Security Command Center 21 | 4. Copy the token to your clipboard or keep the browser tab open 22 | 5. [Enable Cloudflare as a security source](https://console.cloud.google.com/security/command-center/source-registration;partnerId=cloudflare;solutionId=cloudflare-security-events) in Google Cloud Security Command Center. Leave this browser tab open as well 23 | 24 | ## Install 25 | [![Open in Cloud Shell](https://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/cloudflare/cloudflare-gcp&tutorial=cloudshell-security-events.md&cloudshell_working_dir=cli&cloudshell_print=install.sh) 26 | 27 | 28 | ## Updating your configuration 29 | 30 | 31 | ```sh 32 | # The configuration file, `.env.yml` is generated during the Cloud Shell session. Once created, you can find it here: 33 | cat ~/cloudflare-gcp/security-events/.env.yml 34 | 35 | # If you change `.env.yml` you need to rebuild the configuration files before deploying 36 | cd ~/cloudflare-gcp/cli 37 | ./setup buildConf 38 | 39 | # Retrieve new service account key based on env variable in .env.yml 40 | cd ~/cloudflare-gcp/cli 41 | ./setup getServiceAcctKey 42 | 43 | # Configure and update cron schedule 44 | nano ~/cloudflare-gcp/cli/confs/setSchedule.yml 45 | ./setup setSchedule 46 | 47 | # Deploy new configuration 48 | cd ~/cloudflare-gcp/cli 49 | ./setup deploy 50 | ``` 51 | -------------------------------------------------------------------------------- /security-events/assets.js: -------------------------------------------------------------------------------- 1 | require('env-yaml').config() 2 | const { Storage } = require('@google-cloud/storage') 3 | const securityCenter = require('@google-cloud/security-center') 4 | const { getJson, lru } = require('./util') 5 | 6 | const sccClient = new securityCenter.SecurityCenterClient({ 7 | keyFilename: 'scc_key.json' 8 | }) 9 | const storage = new Storage() 10 | const bucket = storage.bucket(process.env.BUCKET_NAME || `cloudflare-scc-bucket-${process.env.PROJECT_ID}`) 11 | 12 | let assets = { 13 | Cloudflare: {}, 14 | Google: {}, 15 | Shared: {}, 16 | file: {}, 17 | 18 | set accountId (token) { 19 | lru.account.set('id', token) 20 | }, 21 | 22 | get accountId () { 23 | if (lru.account.get('id')) { 24 | if (lru.account.get('id').length > 6) { 25 | return lru.account.get('id') 26 | } 27 | } 28 | return null 29 | }, 30 | 31 | set zoneIds (arr) { 32 | lru.zones.set(arr[0], arr[1]) 33 | }, 34 | 35 | get zoneIds () { 36 | return Array.from(lru.zones.values()) 37 | }, 38 | 39 | getLbHosts: async function () { 40 | try { 41 | let response = await getJson({ 42 | path: `/zones/${process.env.ZONE_ID}/load_balancers` 43 | }) 44 | 45 | let hostPoolPairs = {} 46 | for (const host of response.result) { 47 | hostPoolPairs[`${host.name}`] = [].concat(...[ 48 | Object.values(host.default_pools), 49 | Object.values(host.region_pools), 50 | Object.values(host.pop_pools), 51 | host.fallback_pool 52 | ])[0] 53 | } 54 | 55 | for (let [host, pool] of Object.entries(hostPoolPairs)) { 56 | let poolIp = await getJson({ 57 | path: `/accounts/${this.accountId}/load_balancers/pools/${pool}` 58 | }) 59 | this.Cloudflare[`${host}`] = poolIp.result.origins[0].address 60 | } 61 | } catch (e) { 62 | return 'caught' 63 | } 64 | }, 65 | 66 | getDnsRecords: async function () { 67 | try { 68 | for (const zoneId of this.zoneIds) { 69 | console.log(zoneId) 70 | let records = ['A', 'AAAA', 'CNAME'].map(async recordType => { 71 | let response = await getJson({ 72 | params: { 73 | type: recordType, 74 | match: 'any', 75 | per_page: '100', 76 | order: 'proxied' 77 | }, 78 | path: `/zones/${zoneId}/dns_records` 79 | }) 80 | if (response && response.result) { 81 | for (const host of response.result) { 82 | this.Cloudflare[`${host.name}`] = host.content 83 | } 84 | } else { 85 | console.log('Request failed') 86 | } 87 | }) 88 | 89 | for (const record of records) { 90 | await record 91 | } 92 | } 93 | } catch (e) { 94 | return 'caught' 95 | } 96 | }, 97 | 98 | getZoneIds: async function () { 99 | try { 100 | if (this.zoneIds && this.zoneIds.length > 0) { 101 | console.log('Got zone IDs from memory') 102 | return this.zoneIds 103 | } 104 | let response = await getJson({ 105 | params: { 106 | 'account.id': await this.getAccountId() 107 | }, 108 | path: `/zones` 109 | }) 110 | console.log('Retrieving zones ...') 111 | for (const zone of response.result) { 112 | this.zoneIds = [`${zone.name}`, zone.id] 113 | } 114 | 115 | return this.zoneIds 116 | } catch (e) { 117 | throw console.log(e) 118 | } 119 | }, 120 | 121 | getAccountId: async function () { 122 | if (this.accountId && this.accountId !== '') { 123 | console.log('Got acct ID from memory') 124 | return this.accountId 125 | } 126 | try { 127 | let response = await getJson({ 128 | params: { 129 | per_page: '50' 130 | }, 131 | path: `/accounts` 132 | }) 133 | 134 | for (const account of response.result) { 135 | if (account.name === process.env.CF_ORG_NAME) { 136 | this.accountId = account.id 137 | console.log(this.accountId) 138 | return this.accountId 139 | } 140 | } 141 | } catch (e) { 142 | return 'caught' 143 | } 144 | }, 145 | 146 | getAllHosts: async function () { 147 | try { 148 | let hosts = Promise.all([ 149 | this.getLbHosts(), 150 | this.getDnsRecords() 151 | ]) 152 | return hosts 153 | } catch (e) { 154 | console.log('not all cloudflare resources retrieved') 155 | return 'caught' 156 | } 157 | }, 158 | 159 | getGoogleResources: async function () { 160 | try { 161 | let resources = await sccClient.listAssets({ 162 | parent: sccClient.organizationSettingsPath(process.env.SOURCE_ID), 163 | filter: `security_center_properties.resource_name:"address" OR security_center_properties.resource_name:"storage"` 164 | }) 165 | console.log(resources) 166 | resources = (await resources)[0] 167 | 168 | let resourceId 169 | 170 | for (const resource of resources) { 171 | switch (resource.asset.securityCenterProperties.resourceType) { 172 | case 'google.compute.Address': 173 | resourceId = `${resource.asset.resourceProperties.address.stringValue}` 174 | this.Google[`${resource.asset.name}`] = resourceId 175 | break 176 | 177 | case 'google.cloud.storage.Bucket': 178 | resourceId = `${resource.asset.securityCenterProperties.resourceName}` 179 | resourceId = `${resourceId.split('/').pop()}` 180 | this.Google[`${resource.asset.name}`] = resourceId 181 | break 182 | 183 | default: 184 | break 185 | } 186 | } 187 | } catch (e) { 188 | return e 189 | } 190 | }, 191 | 192 | updateAssetsFile: async function (filename = 'assets.json') { 193 | try { 194 | await bucket.get({ autoCreate: true }) 195 | this.file = bucket.file(filename) 196 | for (const [host, origin] of Object.entries(this.Cloudflare)) { 197 | await Object.entries(this.Google).map(([sccAssetId, ip]) => { 198 | if (ip === origin) { 199 | this.Shared[`${host}`] = sccAssetId 200 | } 201 | }) 202 | } 203 | 204 | let contents = {} 205 | Object.assign(contents, this.Cloudflare, this.Google, this.Shared) 206 | contents = JSON.stringify(contents, null, 2) 207 | this.file.save(contents, function (err) { 208 | if (!err) { 209 | console.log(`assets.json written to ${process.env.BUCKET_NAME || `cloudflare-scc-bucket-${process.env.PROJECT_ID}`}`) 210 | } else { 211 | console.log(err) 212 | } 213 | }) 214 | } catch (e) { 215 | return 'caught' 216 | } 217 | }, 218 | 219 | getAssetsFile: async function () { 220 | try { 221 | let file = await this.file.download() 222 | let assetIds = JSON.parse(file[0]) 223 | return assetIds 224 | } catch (e) { 225 | return e 226 | } 227 | } 228 | } 229 | 230 | // (async () => { console.log(await Assets.getGoogleResources()) })() 231 | 232 | module.exports = assets 233 | -------------------------------------------------------------------------------- /security-events/cflogs.js: -------------------------------------------------------------------------------- 1 | 'use-strict' 2 | 3 | require('env-yaml').config() 4 | 5 | const { getJson, getDate } = require('./util') 6 | const { findings } = require('./findings') 7 | const assets = require('./assets') 8 | const ndjsonParser = require('ndjson-parse') 9 | 10 | const cfLogs = { 11 | async firewallEvents ({ zone }) { 12 | try { 13 | let res = await getJson({ 14 | params: { 15 | kind: 'firewall', 16 | limit: '1000', 17 | mode: '!=whitelist', 18 | since: getDate({ minutes: process.env.INTERVAL }), 19 | until: getDate({ minutes: 0 }) 20 | }, 21 | path: `/zones/${zone}/security/events` 22 | }) 23 | return res 24 | } catch (e) { 25 | console.log(`Error`, JSON.stringify(e, null, 2)) 26 | } 27 | }, 28 | 29 | async elsEvents ({ zone }) { 30 | try { 31 | let res = await getJson({ 32 | params: { 33 | fields: 'ClientASN,EdgePathingOp,ClientCountry,ClientDeviceType,ClientIP,ClientRequestBytes,ClientRequestHost,ClientRequestMethod,ClientRequestProtocol,ClientRequestReferer,ClientRequestURI,ClientRequestUserAgent,ClientSrcPort,ClientSSLCipher,ClientSSLProtocol,EdgeColoID,EdgeEndTimestamp,EdgePathingSrc,EdgePathingStatus,EdgeResponseBytes,EdgeResponseContentType,EdgeResponseStatus,EdgeStartTimestamp,OriginIP,OriginResponseStatus,RayID,WAFAction,WAFFlags,WAFMatchedVar,WAFProfile,WAFRuleID,WAFRuleMessage,FirewallMatchesRuleIDs,FirewallMatchesSources,FirewallMatchesActions', 34 | start: getDate({ minutes: parseInt(process.env.INTERVAL, 10) + 6 }), 35 | end: getDate({ minutes: 5 }) 36 | }, 37 | path: `/zones/${zone}/logs/received`, 38 | stream: true 39 | }) 40 | 41 | const logs = ndjsonParser(res) 42 | 43 | for (const log of logs) { 44 | try { 45 | let evt = await log 46 | let isSecurityEvent = [ 47 | 'log', 48 | 'simulate', 49 | 'drop', 50 | 'challenge', 51 | 'jschallenge', 52 | 'connectionClose' 53 | ].some(evtType => evt.FirewallMatchesActions.includes(evtType)) 54 | if (evt.FirewallMatchesActions && Array.isArray(evt.FirewallMatchesActions)) { 55 | if (isSecurityEvent) { 56 | console.log(log.RayID) 57 | findings.format.els(log) 58 | } 59 | } 60 | } catch (e) { 61 | continue 62 | } 63 | } 64 | } catch (e) { 65 | console.log(`Error`, e) 66 | } 67 | }, 68 | 69 | async getElsEvents () { 70 | const elsLogs = assets.zoneIds.map(async zoneId => { 71 | const elsEvents = await this.elsEvents({ zone: zoneId }) 72 | return elsEvents 73 | }) 74 | 75 | for (const latestFindings of elsLogs) { 76 | let findingsArr = await findings 77 | if (!findingsArr.result) { 78 | console.log('no new logs') 79 | continue 80 | } 81 | findingsArr.result.map(async finding => findings.format.els(latestFindings, assets.Shared[`${finding.host}`])) 82 | } 83 | }, 84 | 85 | async getFwEvents () { 86 | const qraphQLLogs = assets.zoneIds.map(async zoneId => { 87 | const firewallEvents = await this.firewallEvents({ zone: zoneId }) 88 | return firewallEvents 89 | }) 90 | 91 | for (const latestFindings of qraphQLLogs) { 92 | let findingsArr = await findings 93 | if (!findingsArr.result) { 94 | console.log('no new logs') 95 | continue 96 | } 97 | findingsArr.result.map(async finding => findings.format.qraphQL(latestFindings, assets.Shared[`${finding.host}`])) 98 | } 99 | } 100 | } 101 | 102 | module.exports = cfLogs 103 | -------------------------------------------------------------------------------- /security-events/findings.js: -------------------------------------------------------------------------------- 1 | require('env-yaml').config() 2 | 3 | const securityCenter = require('@google-cloud/security-center') 4 | const sccClient = new securityCenter.SecurityCenterClient({ 5 | keyFilename: 'scc_key.json' 6 | }) 7 | const { getColo } = require('./util') 8 | 9 | const format = { 10 | async qraphQL (log, asset) { 11 | if ('err' in log) { 12 | return console.log(log) 13 | } 14 | 15 | let category 16 | if ('source' in log && 'rule_id' in log) { 17 | category = `${log.source} - ${log.rule_id}` 18 | } else if ('source' in log) { 19 | category = log.source 20 | } else { 21 | category = 'Cloudflare' 22 | } 23 | 24 | if (!asset) { 25 | asset = log.host 26 | } 27 | 28 | let eventTime = new Date(log.occurred_at) 29 | console.log(`Logging finding ... ${process.env.SOURCE_ID}/findings/${log.ray_id}`) 30 | 31 | let finding = { 32 | updateMask: { 33 | paths: ['event_time', 'source_properties'] 34 | }, 35 | name: `${process.env.SOURCE_ID.replace('\t', '')}/findings/${log.ray_id}`, 36 | externalUri: `https://dash.cloudflare.com/firewall`, 37 | state: 'ACTIVE', 38 | resourceName: asset, 39 | category: category, 40 | eventTime: { 41 | seconds: Math.floor(eventTime.getTime() / 1000), 42 | nanos: (eventTime.getTime() % 1000) * 1e6 43 | }, 44 | sourceProperties: { 45 | Action: { 46 | stringValue: log.action, 47 | kind: 'stringValue' 48 | }, 49 | Source: { 50 | stringValue: log.source, 51 | kind: 'stringValue' 52 | }, 53 | Country: { 54 | stringValue: log.country, 55 | kind: 'stringValue' 56 | }, 57 | Protocol: { 58 | stringValue: log.proto, 59 | kind: 'stringValue' 60 | }, 61 | Method: { 62 | stringValue: log.method, 63 | kind: 'stringValue' 64 | }, 65 | Host: { 66 | stringValue: log.host, 67 | kind: 'stringValue' 68 | }, 69 | Agent: { 70 | stringValue: log.ua, 71 | kind: 'stringValue' 72 | }, 73 | Path: { 74 | stringValue: log.uri, 75 | kind: 'stringValue' 76 | }, 77 | Location: { 78 | stringValue: getColo(log.EdgeColoID), 79 | kind: 'stringValue' 80 | }, 81 | Rule: { 82 | stringValue: log.rule_id, 83 | kind: 'stringValue' 84 | } 85 | } 86 | } 87 | await sccClient.updateFinding({ finding }) 88 | await sccClient.updateSecurityMarks({ 89 | securityMarks: { 90 | name: `${finding.name}/securityMarks`, 91 | marks: { host: `${log.host}` } 92 | } 93 | }) 94 | }, 95 | 96 | async els (log) { 97 | if ('err' in log) { 98 | return console.log(log) 99 | } 100 | 101 | console.log(`Logging finding ... ${process.env.SOURCE_ID}/findings/${log.RayID}`) 102 | 103 | let finding = { 104 | updateMask: { 105 | paths: ['source_properties'] 106 | }, 107 | name: `${process.env.SOURCE_ID.replace('\t', '')}/findings/${log.RayID}`, 108 | externalUri: `https://dash.cloudflare.com/firewall`, 109 | state: 'ACTIVE', 110 | category: `firewallRules - ${log.FirewallMatchesRuleIDs[0]}`, 111 | sourceProperties: { 112 | Action: { 113 | stringValue: log.EdgePathingStatus, 114 | kind: 'stringValue' 115 | }, 116 | Status: { 117 | stringValue: `${log.EdgeResponseStatus}`, 118 | kind: 'stringValue' 119 | }, 120 | Host: { 121 | stringValue: log.ClientRequestHost, 122 | kind: 'stringValue' 123 | }, 124 | URI: { 125 | stringValue: `${log.ClientRequestMethod} ${log.ClientRequestURI}`, 126 | kind: 'stringValue' 127 | }, 128 | Country: { 129 | stringValue: log.ClientCountry.toUpperCase(), 130 | kind: 'stringValue' 131 | }, 132 | Location: { 133 | stringValue: getColo(log.EdgeColoID), 134 | kind: 'stringValue' 135 | }, 136 | ClientIP: { 137 | stringValue: log.ClientIP, 138 | kind: 'stringValue' 139 | }, 140 | ClientASN: { 141 | stringValue: log.ClientASN, 142 | kind: 'stringValue' 143 | }, 144 | Device: { 145 | stringValue: log.ClientDeviceType, 146 | kind: 'stringValue' 147 | }, 148 | EdgePathingSignature: { 149 | stringValue: `${log.EdgePathingStatus} ${log.EdgePathingSrc}` 150 | }, 151 | ClientRequestBytes: { 152 | stringValue: log.ClientRequestBytes, 153 | kind: 'stringValue' 154 | }, 155 | ClientSSLCipher: { 156 | stringValue: log.ClientSSLCipher, 157 | kind: 'stringValue' 158 | }, 159 | UA: { 160 | stringValue: log.ClientRequestUserAgent, 161 | kind: 'stringValue' 162 | }, 163 | Referer: { 164 | stringValue: log.ClientRequestReferer, 165 | kind: 'stringValue' 166 | } 167 | }, 168 | securityMarks: { 169 | OriginIP: log.OriginIP 170 | } 171 | } 172 | 173 | switch (true) { 174 | case log.EdgeResponseStatus === 429: 175 | finding.category = 'Block: Rate Limit' 176 | break 177 | 178 | case log.WAFRuleMessage && log.WAFRuleMessage !== 'undefined': 179 | finding.category = log.WAFRuleMessage 180 | finding.sourceProperties.WAFAction = { 181 | stringValue: log.WAFAction, 182 | kind: 'stringValue' 183 | } 184 | finding.sourceProperties.WAFProfile = { 185 | stringValue: log.WAFProfile, 186 | kind: 'stringValue' 187 | } 188 | finding.sourceProperties.Action = { 189 | stringValue: log.WAFAction, 190 | kind: 'stringValue' 191 | } 192 | break 193 | 194 | default: 195 | break 196 | } 197 | 198 | finding.eventTime = { 199 | seconds: Number.parseInt(`${Date.now().toString().slice(0, 10)}`), 200 | nanos: Number.parseInt(`${Date.now().toString().slice(0, 9)}`) 201 | } 202 | await sccClient.updateFinding({ finding }) 203 | await sccClient.updateSecurityMarks({ 204 | securityMarks: { 205 | name: `${finding.name}/securityMarks`, 206 | marks: { host: `${log.ClientRequestHost}` } 207 | } 208 | }) 209 | } 210 | } 211 | 212 | module.exports.findings = { format } 213 | -------------------------------------------------------------------------------- /security-events/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | require('env-yaml').config() 4 | 5 | const assets = require('./assets') 6 | const cfLogs = require('./cflogs') 7 | 8 | exports.FirewallEventsToSecurityCenter = async function (pubSubMessage, context) { 9 | try { 10 | if (pubSubMessage && 'data' in pubSubMessage) { 11 | console.log('Received pubsub trigger. Starting ...') 12 | } 13 | const sequence = (arr, input) => { 14 | arr.reduce( 15 | (promiseChain, currentFunction) => promiseChain.then(currentFunction), 16 | Promise.resolve(input) 17 | ) 18 | } 19 | 20 | let pipeline = await sequence([ 21 | await assets.getZoneIds(), 22 | // await assets.getLbHosts() 23 | await assets.getDnsRecords(), 24 | await assets.getGoogleResources(), 25 | await assets.updateAssetsFile(), 26 | await assets.getAssetsFile(), 27 | await cfLogs.getElsEvents(), 28 | await cfLogs.getFwEvents() 29 | ], 10) 30 | console.log(pipeline) 31 | } catch (e) { 32 | console.log(e) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /security-events/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudflare-to-scc", 3 | "version": "0.1.15", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@google-cloud/common": { 8 | "version": "0.32.1", 9 | "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", 10 | "integrity": "sha512-bLdPzFvvBMtVkwsoBtygE9oUm3yrNmPa71gvOgucYI/GqvNP2tb6RYsDHPq98kvignhcgHGDI5wyNgxaCo8bKQ==", 11 | "requires": { 12 | "@google-cloud/projectify": "^0.3.3", 13 | "@google-cloud/promisify": "^0.4.0", 14 | "@types/request": "^2.48.1", 15 | "arrify": "^2.0.0", 16 | "duplexify": "^3.6.0", 17 | "ent": "^2.2.0", 18 | "extend": "^3.0.2", 19 | "google-auth-library": "^3.1.1", 20 | "pify": "^4.0.1", 21 | "retry-request": "^4.0.0", 22 | "teeny-request": "^3.11.3" 23 | }, 24 | "dependencies": { 25 | "agent-base": { 26 | "version": "4.3.0", 27 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", 28 | "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", 29 | "requires": { 30 | "es6-promisify": "^5.0.0" 31 | } 32 | }, 33 | "debug": { 34 | "version": "3.2.6", 35 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", 36 | "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", 37 | "requires": { 38 | "ms": "^2.1.1" 39 | } 40 | }, 41 | "gaxios": { 42 | "version": "1.8.4", 43 | "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", 44 | "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", 45 | "requires": { 46 | "abort-controller": "^3.0.0", 47 | "extend": "^3.0.2", 48 | "https-proxy-agent": "^2.2.1", 49 | "node-fetch": "^2.3.0" 50 | } 51 | }, 52 | "gcp-metadata": { 53 | "version": "1.0.0", 54 | "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", 55 | "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", 56 | "requires": { 57 | "gaxios": "^1.0.2", 58 | "json-bigint": "^0.3.0" 59 | } 60 | }, 61 | "google-auth-library": { 62 | "version": "3.1.2", 63 | "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", 64 | "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", 65 | "requires": { 66 | "base64-js": "^1.3.0", 67 | "fast-text-encoding": "^1.0.0", 68 | "gaxios": "^1.2.1", 69 | "gcp-metadata": "^1.0.0", 70 | "gtoken": "^2.3.2", 71 | "https-proxy-agent": "^2.2.1", 72 | "jws": "^3.1.5", 73 | "lru-cache": "^5.0.0", 74 | "semver": "^5.5.0" 75 | } 76 | }, 77 | "google-p12-pem": { 78 | "version": "1.0.4", 79 | "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", 80 | "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", 81 | "requires": { 82 | "node-forge": "^0.8.0", 83 | "pify": "^4.0.0" 84 | } 85 | }, 86 | "gtoken": { 87 | "version": "2.3.3", 88 | "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", 89 | "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", 90 | "requires": { 91 | "gaxios": "^1.0.4", 92 | "google-p12-pem": "^1.0.0", 93 | "jws": "^3.1.5", 94 | "mime": "^2.2.0", 95 | "pify": "^4.0.0" 96 | } 97 | }, 98 | "https-proxy-agent": { 99 | "version": "2.2.4", 100 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", 101 | "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==", 102 | "requires": { 103 | "agent-base": "^4.3.0", 104 | "debug": "^3.1.0" 105 | } 106 | }, 107 | "jwa": { 108 | "version": "1.4.1", 109 | "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", 110 | "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", 111 | "requires": { 112 | "buffer-equal-constant-time": "1.0.1", 113 | "ecdsa-sig-formatter": "1.0.11", 114 | "safe-buffer": "^5.0.1" 115 | } 116 | }, 117 | "jws": { 118 | "version": "3.2.2", 119 | "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", 120 | "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", 121 | "requires": { 122 | "jwa": "^1.4.1", 123 | "safe-buffer": "^5.0.1" 124 | } 125 | }, 126 | "node-forge": { 127 | "version": "0.8.5", 128 | "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", 129 | "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" 130 | }, 131 | "semver": { 132 | "version": "5.7.1", 133 | "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", 134 | "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" 135 | } 136 | } 137 | }, 138 | "@google-cloud/paginator": { 139 | "version": "0.2.0", 140 | "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-0.2.0.tgz", 141 | "integrity": "sha512-2ZSARojHDhkLvQ+CS32K+iUhBsWg3AEw+uxtqblA7xoCABDyhpj99FPp35xy6A+XlzMhOSrHHaxFE+t6ZTQq0w==", 142 | "requires": { 143 | "arrify": "^1.0.1", 144 | "extend": "^3.0.1", 145 | "split-array-stream": "^2.0.0", 146 | "stream-events": "^1.0.4" 147 | }, 148 | "dependencies": { 149 | "arrify": { 150 | "version": "1.0.1", 151 | "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", 152 | "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" 153 | } 154 | } 155 | }, 156 | "@google-cloud/projectify": { 157 | "version": "0.3.3", 158 | "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", 159 | "integrity": "sha512-7522YHQ4IhaafgSunsFF15nG0TGVmxgXidy9cITMe+256RgqfcrfWphiMufW+Ou4kqagW/u3yxwbzVEW3dk2Uw==" 160 | }, 161 | "@google-cloud/promisify": { 162 | "version": "0.4.0", 163 | "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", 164 | "integrity": "sha512-4yAHDC52TEMCNcMzVC8WlqnKKKq+Ssi2lXoUg9zWWkZ6U6tq9ZBRYLHHCRdfU+EU9YJsVmivwGcKYCjRGjnf4Q==" 165 | }, 166 | "@google-cloud/security-center": { 167 | "version": "2.3.1", 168 | "resolved": "https://registry.npmjs.org/@google-cloud/security-center/-/security-center-2.3.1.tgz", 169 | "integrity": "sha512-R0/SaK/8Vetm3L/eKj23888DB6lTFaSUlmmCCG9S04ko9/OyLRCW1uMguHOiVHcXypz5EQcDBL5ggpVD8GMdLw==", 170 | "requires": { 171 | "google-gax": "^1.7.5", 172 | "protobufjs": "^6.8.0" 173 | } 174 | }, 175 | "@google-cloud/storage": { 176 | "version": "2.5.0", 177 | "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-2.5.0.tgz", 178 | "integrity": "sha512-q1mwB6RUebIahbA3eriRs8DbG2Ij81Ynb9k8hMqTPkmbd8/S6Z0d6hVvfPmnyvX9Ej13IcmEYIbymuq/RBLghA==", 179 | "requires": { 180 | "@google-cloud/common": "^0.32.0", 181 | "@google-cloud/paginator": "^0.2.0", 182 | "@google-cloud/promisify": "^0.4.0", 183 | "arrify": "^1.0.0", 184 | "async": "^2.0.1", 185 | "compressible": "^2.0.12", 186 | "concat-stream": "^2.0.0", 187 | "date-and-time": "^0.6.3", 188 | "duplexify": "^3.5.0", 189 | "extend": "^3.0.0", 190 | "gcs-resumable-upload": "^1.0.0", 191 | "hash-stream-validation": "^0.2.1", 192 | "mime": "^2.2.0", 193 | "mime-types": "^2.0.8", 194 | "onetime": "^5.1.0", 195 | "pumpify": "^1.5.1", 196 | "snakeize": "^0.1.0", 197 | "stream-events": "^1.0.1", 198 | "teeny-request": "^3.11.3", 199 | "through2": "^3.0.0", 200 | "xdg-basedir": "^3.0.0" 201 | }, 202 | "dependencies": { 203 | "arrify": { 204 | "version": "1.0.1", 205 | "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", 206 | "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" 207 | } 208 | } 209 | }, 210 | "@grpc/grpc-js": { 211 | "version": "0.6.15", 212 | "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-0.6.15.tgz", 213 | "integrity": "sha512-BFK5YMu8JILedibo0nr3NYM0ZC5hCZuXtzk10wEUp3d3pH11PjdvTfN1yEJ0VsfBY5Gtp3WOQ+t7Byq0NzH/iQ==", 214 | "requires": { 215 | "semver": "^6.2.0" 216 | }, 217 | "dependencies": { 218 | "semver": { 219 | "version": "6.3.0", 220 | "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", 221 | "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" 222 | } 223 | } 224 | }, 225 | "@grpc/proto-loader": { 226 | "version": "0.5.3", 227 | "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.5.3.tgz", 228 | "integrity": "sha512-8qvUtGg77G2ZT2HqdqYoM/OY97gQd/0crSG34xNmZ4ZOsv3aQT/FQV9QfZPazTGna6MIoyUd+u6AxsoZjJ/VMQ==", 229 | "requires": { 230 | "lodash.camelcase": "^4.3.0", 231 | "protobufjs": "^6.8.6" 232 | } 233 | }, 234 | "@protobufjs/aspromise": { 235 | "version": "1.1.2", 236 | "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", 237 | "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" 238 | }, 239 | "@protobufjs/base64": { 240 | "version": "1.1.2", 241 | "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", 242 | "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" 243 | }, 244 | "@protobufjs/codegen": { 245 | "version": "2.0.4", 246 | "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", 247 | "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" 248 | }, 249 | "@protobufjs/eventemitter": { 250 | "version": "1.1.0", 251 | "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", 252 | "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" 253 | }, 254 | "@protobufjs/fetch": { 255 | "version": "1.1.0", 256 | "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", 257 | "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", 258 | "requires": { 259 | "@protobufjs/aspromise": "^1.1.1", 260 | "@protobufjs/inquire": "^1.1.0" 261 | } 262 | }, 263 | "@protobufjs/float": { 264 | "version": "1.0.2", 265 | "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", 266 | "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" 267 | }, 268 | "@protobufjs/inquire": { 269 | "version": "1.1.0", 270 | "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", 271 | "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" 272 | }, 273 | "@protobufjs/path": { 274 | "version": "1.1.2", 275 | "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", 276 | "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" 277 | }, 278 | "@protobufjs/pool": { 279 | "version": "1.1.0", 280 | "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", 281 | "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" 282 | }, 283 | "@protobufjs/utf8": { 284 | "version": "1.1.0", 285 | "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", 286 | "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" 287 | }, 288 | "@types/caseless": { 289 | "version": "0.12.2", 290 | "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", 291 | "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==" 292 | }, 293 | "@types/fs-extra": { 294 | "version": "8.0.1", 295 | "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.0.1.tgz", 296 | "integrity": "sha512-J00cVDALmi/hJOYsunyT52Hva5TnJeKP5yd1r+mH/ZU0mbYZflR0Z5kw5kITtKTRYMhm1JMClOFYdHnQszEvqw==", 297 | "requires": { 298 | "@types/node": "*" 299 | } 300 | }, 301 | "@types/long": { 302 | "version": "4.0.1", 303 | "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.1.tgz", 304 | "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==" 305 | }, 306 | "@types/node": { 307 | "version": "10.17.13", 308 | "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.13.tgz", 309 | "integrity": "sha512-pMCcqU2zT4TjqYFrWtYHKal7Sl30Ims6ulZ4UFXxI4xbtQqK/qqKwkDoBFCfooRqqmRu9vY3xaJRwxSh673aYg==" 310 | }, 311 | "@types/request": { 312 | "version": "2.48.4", 313 | "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.4.tgz", 314 | "integrity": "sha512-W1t1MTKYR8PxICH+A4HgEIPuAC3sbljoEVfyZbeFJJDbr30guDspJri2XOaM2E+Un7ZjrihaDi7cf6fPa2tbgw==", 315 | "requires": { 316 | "@types/caseless": "*", 317 | "@types/node": "*", 318 | "@types/tough-cookie": "*", 319 | "form-data": "^2.5.0" 320 | } 321 | }, 322 | "@types/tough-cookie": { 323 | "version": "2.3.6", 324 | "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.6.tgz", 325 | "integrity": "sha512-wHNBMnkoEBiRAd3s8KTKwIuO9biFtTf0LehITzBhSco+HQI0xkXZbLOD55SW3Aqw3oUkHstkm5SPv58yaAdFPQ==" 326 | }, 327 | "abort-controller": { 328 | "version": "3.0.0", 329 | "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", 330 | "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", 331 | "requires": { 332 | "event-target-shim": "^5.0.0" 333 | } 334 | }, 335 | "agent-base": { 336 | "version": "5.1.1", 337 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", 338 | "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" 339 | }, 340 | "argparse": { 341 | "version": "1.0.10", 342 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", 343 | "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", 344 | "requires": { 345 | "sprintf-js": "~1.0.2" 346 | } 347 | }, 348 | "arrify": { 349 | "version": "2.0.1", 350 | "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", 351 | "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" 352 | }, 353 | "async": { 354 | "version": "2.6.3", 355 | "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", 356 | "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", 357 | "requires": { 358 | "lodash": "^4.17.14" 359 | } 360 | }, 361 | "asynckit": { 362 | "version": "0.4.0", 363 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 364 | "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" 365 | }, 366 | "base64-js": { 367 | "version": "1.3.1", 368 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", 369 | "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" 370 | }, 371 | "bignumber.js": { 372 | "version": "7.2.1", 373 | "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", 374 | "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" 375 | }, 376 | "buffer-equal-constant-time": { 377 | "version": "1.0.1", 378 | "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", 379 | "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" 380 | }, 381 | "buffer-from": { 382 | "version": "1.1.1", 383 | "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", 384 | "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" 385 | }, 386 | "combined-stream": { 387 | "version": "1.0.8", 388 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 389 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 390 | "requires": { 391 | "delayed-stream": "~1.0.0" 392 | } 393 | }, 394 | "compressible": { 395 | "version": "2.0.18", 396 | "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", 397 | "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", 398 | "requires": { 399 | "mime-db": ">= 1.43.0 < 2" 400 | } 401 | }, 402 | "concat-stream": { 403 | "version": "2.0.0", 404 | "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", 405 | "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", 406 | "requires": { 407 | "buffer-from": "^1.0.0", 408 | "inherits": "^2.0.3", 409 | "readable-stream": "^3.0.2", 410 | "typedarray": "^0.0.6" 411 | }, 412 | "dependencies": { 413 | "readable-stream": { 414 | "version": "3.5.0", 415 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.5.0.tgz", 416 | "integrity": "sha512-gSz026xs2LfxBPudDuI41V1lka8cxg64E66SGe78zJlsUofOg/yqwezdIcdfwik6B4h8LFmWPA9ef9X3FiNFLA==", 417 | "requires": { 418 | "inherits": "^2.0.3", 419 | "string_decoder": "^1.1.1", 420 | "util-deprecate": "^1.0.1" 421 | } 422 | } 423 | } 424 | }, 425 | "configstore": { 426 | "version": "4.0.0", 427 | "resolved": "https://registry.npmjs.org/configstore/-/configstore-4.0.0.tgz", 428 | "integrity": "sha512-CmquAXFBocrzaSM8mtGPMM/HiWmyIpr4CcJl/rgY2uCObZ/S7cKU0silxslqJejl+t/T9HS8E0PUNQD81JGUEQ==", 429 | "requires": { 430 | "dot-prop": "^4.1.0", 431 | "graceful-fs": "^4.1.2", 432 | "make-dir": "^1.0.0", 433 | "unique-string": "^1.0.0", 434 | "write-file-atomic": "^2.0.0", 435 | "xdg-basedir": "^3.0.0" 436 | } 437 | }, 438 | "core-util-is": { 439 | "version": "1.0.2", 440 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 441 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" 442 | }, 443 | "crypto-random-string": { 444 | "version": "1.0.0", 445 | "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", 446 | "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=" 447 | }, 448 | "date-and-time": { 449 | "version": "0.6.3", 450 | "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-0.6.3.tgz", 451 | "integrity": "sha512-lcWy3AXDRJOD7MplwZMmNSRM//kZtJaLz4n6D1P5z9wEmZGBKhJRBIr1Xs9KNQJmdXPblvgffynYji4iylUTcA==" 452 | }, 453 | "debug": { 454 | "version": "4.1.1", 455 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", 456 | "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", 457 | "requires": { 458 | "ms": "^2.1.1" 459 | } 460 | }, 461 | "delayed-stream": { 462 | "version": "1.0.0", 463 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 464 | "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" 465 | }, 466 | "dot-prop": { 467 | "version": "4.2.0", 468 | "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", 469 | "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==", 470 | "requires": { 471 | "is-obj": "^1.0.0" 472 | } 473 | }, 474 | "dotenv": { 475 | "version": "8.2.0", 476 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", 477 | "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" 478 | }, 479 | "duplexify": { 480 | "version": "3.7.1", 481 | "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", 482 | "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", 483 | "requires": { 484 | "end-of-stream": "^1.0.0", 485 | "inherits": "^2.0.1", 486 | "readable-stream": "^2.0.0", 487 | "stream-shift": "^1.0.0" 488 | } 489 | }, 490 | "ecdsa-sig-formatter": { 491 | "version": "1.0.11", 492 | "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", 493 | "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", 494 | "requires": { 495 | "safe-buffer": "^5.0.1" 496 | } 497 | }, 498 | "end-of-stream": { 499 | "version": "1.4.4", 500 | "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", 501 | "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", 502 | "requires": { 503 | "once": "^1.4.0" 504 | } 505 | }, 506 | "ent": { 507 | "version": "2.2.0", 508 | "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", 509 | "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=" 510 | }, 511 | "env-yaml": { 512 | "version": "0.1.2", 513 | "resolved": "https://registry.npmjs.org/env-yaml/-/env-yaml-0.1.2.tgz", 514 | "integrity": "sha512-OCoideEIX5zlpMytuukSK9i/57AdVluhwgTev+nlSxYR1foOeZ726zhCcHQpTDDdwozgPPrXZdnaCMRsF1kn9g==", 515 | "requires": { 516 | "js-yaml": "^3.8.4" 517 | } 518 | }, 519 | "es6-promise": { 520 | "version": "4.2.8", 521 | "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", 522 | "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" 523 | }, 524 | "es6-promisify": { 525 | "version": "5.0.0", 526 | "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", 527 | "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", 528 | "requires": { 529 | "es6-promise": "^4.0.3" 530 | } 531 | }, 532 | "esprima": { 533 | "version": "4.0.1", 534 | "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", 535 | "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" 536 | }, 537 | "event-target-shim": { 538 | "version": "5.0.1", 539 | "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", 540 | "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" 541 | }, 542 | "extend": { 543 | "version": "3.0.2", 544 | "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", 545 | "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" 546 | }, 547 | "fast-text-encoding": { 548 | "version": "1.0.0", 549 | "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", 550 | "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==" 551 | }, 552 | "form-data": { 553 | "version": "2.5.1", 554 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", 555 | "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", 556 | "requires": { 557 | "asynckit": "^0.4.0", 558 | "combined-stream": "^1.0.6", 559 | "mime-types": "^2.1.12" 560 | } 561 | }, 562 | "gaxios": { 563 | "version": "2.2.2", 564 | "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-2.2.2.tgz", 565 | "integrity": "sha512-fzttYsjvZxCaN+bQK7FtAMgoIlPtHkMwlz7vHD+aNRcU7I7gHgnp6hvGJksoo+dO1TDxaog+dSBycbYhHIStaA==", 566 | "requires": { 567 | "abort-controller": "^3.0.0", 568 | "extend": "^3.0.2", 569 | "https-proxy-agent": "^4.0.0", 570 | "is-stream": "^2.0.0", 571 | "node-fetch": "^2.3.0" 572 | } 573 | }, 574 | "gcp-metadata": { 575 | "version": "3.3.0", 576 | "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-3.3.0.tgz", 577 | "integrity": "sha512-uO3P/aByOQmoDu5bOYBODHmD1oDCZw7/R8SYY0MdmMQSZVEmeTSxmiM1vwde+YHYSpkaQnAAMAIZuOqLvgfp/Q==", 578 | "requires": { 579 | "gaxios": "^2.1.0", 580 | "json-bigint": "^0.3.0" 581 | } 582 | }, 583 | "gcs-resumable-upload": { 584 | "version": "1.1.0", 585 | "resolved": "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-1.1.0.tgz", 586 | "integrity": "sha512-uBz7uHqp44xjSDzG3kLbOYZDjxxR/UAGbB47A0cC907W6yd2LkcyFDTHg+bjivkHMwiJlKv4guVWcjPCk2zScg==", 587 | "requires": { 588 | "abort-controller": "^2.0.2", 589 | "configstore": "^4.0.0", 590 | "gaxios": "^1.5.0", 591 | "google-auth-library": "^3.0.0", 592 | "pumpify": "^1.5.1", 593 | "stream-events": "^1.0.4" 594 | }, 595 | "dependencies": { 596 | "abort-controller": { 597 | "version": "2.0.3", 598 | "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-2.0.3.tgz", 599 | "integrity": "sha512-EPSq5wr2aFyAZ1PejJB32IX9Qd4Nwus+adnp7STYFM5/23nLPBazqZ1oor6ZqbH+4otaaGXTlC8RN5hq3C8w9Q==", 600 | "requires": { 601 | "event-target-shim": "^5.0.0" 602 | } 603 | }, 604 | "agent-base": { 605 | "version": "4.3.0", 606 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", 607 | "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", 608 | "requires": { 609 | "es6-promisify": "^5.0.0" 610 | } 611 | }, 612 | "debug": { 613 | "version": "3.2.6", 614 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", 615 | "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", 616 | "requires": { 617 | "ms": "^2.1.1" 618 | } 619 | }, 620 | "gaxios": { 621 | "version": "1.8.4", 622 | "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", 623 | "integrity": "sha512-BoENMnu1Gav18HcpV9IleMPZ9exM+AvUjrAOV4Mzs/vfz2Lu/ABv451iEXByKiMPn2M140uul1txXCg83sAENw==", 624 | "requires": { 625 | "abort-controller": "^3.0.0", 626 | "extend": "^3.0.2", 627 | "https-proxy-agent": "^2.2.1", 628 | "node-fetch": "^2.3.0" 629 | }, 630 | "dependencies": { 631 | "abort-controller": { 632 | "version": "3.0.0", 633 | "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", 634 | "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", 635 | "requires": { 636 | "event-target-shim": "^5.0.0" 637 | } 638 | } 639 | } 640 | }, 641 | "gcp-metadata": { 642 | "version": "1.0.0", 643 | "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", 644 | "integrity": "sha512-Q6HrgfrCQeEircnNP3rCcEgiDv7eF9+1B+1MMgpE190+/+0mjQR8PxeOaRgxZWmdDAF9EIryHB9g1moPiw1SbQ==", 645 | "requires": { 646 | "gaxios": "^1.0.2", 647 | "json-bigint": "^0.3.0" 648 | } 649 | }, 650 | "google-auth-library": { 651 | "version": "3.1.2", 652 | "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", 653 | "integrity": "sha512-cDQMzTotwyWMrg5jRO7q0A4TL/3GWBgO7I7q5xGKNiiFf9SmGY/OJ1YsLMgI2MVHHsEGyrqYnbnmV1AE+Z6DnQ==", 654 | "requires": { 655 | "base64-js": "^1.3.0", 656 | "fast-text-encoding": "^1.0.0", 657 | "gaxios": "^1.2.1", 658 | "gcp-metadata": "^1.0.0", 659 | "gtoken": "^2.3.2", 660 | "https-proxy-agent": "^2.2.1", 661 | "jws": "^3.1.5", 662 | "lru-cache": "^5.0.0", 663 | "semver": "^5.5.0" 664 | } 665 | }, 666 | "google-p12-pem": { 667 | "version": "1.0.4", 668 | "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", 669 | "integrity": "sha512-SwLAUJqUfTB2iS+wFfSS/G9p7bt4eWcc2LyfvmUXe7cWp6p3mpxDo6LLI29MXdU6wvPcQ/up298X7GMC5ylAlA==", 670 | "requires": { 671 | "node-forge": "^0.8.0", 672 | "pify": "^4.0.0" 673 | } 674 | }, 675 | "gtoken": { 676 | "version": "2.3.3", 677 | "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", 678 | "integrity": "sha512-EaB49bu/TCoNeQjhCYKI/CurooBKkGxIqFHsWABW0b25fobBYVTMe84A8EBVVZhl8emiUdNypil9huMOTmyAnw==", 679 | "requires": { 680 | "gaxios": "^1.0.4", 681 | "google-p12-pem": "^1.0.0", 682 | "jws": "^3.1.5", 683 | "mime": "^2.2.0", 684 | "pify": "^4.0.0" 685 | } 686 | }, 687 | "https-proxy-agent": { 688 | "version": "2.2.4", 689 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", 690 | "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==", 691 | "requires": { 692 | "agent-base": "^4.3.0", 693 | "debug": "^3.1.0" 694 | } 695 | }, 696 | "jwa": { 697 | "version": "1.4.1", 698 | "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", 699 | "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", 700 | "requires": { 701 | "buffer-equal-constant-time": "1.0.1", 702 | "ecdsa-sig-formatter": "1.0.11", 703 | "safe-buffer": "^5.0.1" 704 | } 705 | }, 706 | "jws": { 707 | "version": "3.2.2", 708 | "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", 709 | "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", 710 | "requires": { 711 | "jwa": "^1.4.1", 712 | "safe-buffer": "^5.0.1" 713 | } 714 | }, 715 | "node-forge": { 716 | "version": "0.8.5", 717 | "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", 718 | "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" 719 | }, 720 | "semver": { 721 | "version": "5.7.1", 722 | "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", 723 | "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" 724 | } 725 | } 726 | }, 727 | "google-auth-library": { 728 | "version": "5.9.1", 729 | "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-5.9.1.tgz", 730 | "integrity": "sha512-927NlfjqRLrkok2JRBap4iYqe5+bsdtaotwO1JSAAxoQmji+Xmf+pFLzB+52C8ovtfycwFXbvC9f/SB4gmQCXw==", 731 | "requires": { 732 | "arrify": "^2.0.0", 733 | "base64-js": "^1.3.0", 734 | "fast-text-encoding": "^1.0.0", 735 | "gaxios": "^2.1.0", 736 | "gcp-metadata": "^3.3.0", 737 | "gtoken": "^4.1.0", 738 | "jws": "^4.0.0", 739 | "lru-cache": "^5.0.0" 740 | } 741 | }, 742 | "google-gax": { 743 | "version": "1.13.0", 744 | "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-1.13.0.tgz", 745 | "integrity": "sha512-MSDPDz+eK8X6XHkb2C1DGPRX50RZBLeSnXChV5P6ojLkc1zSfII8OWGdxREBw8/izvRJLQ5XnuSk1ylLB1BKfQ==", 746 | "requires": { 747 | "@grpc/grpc-js": "^0.6.12", 748 | "@grpc/proto-loader": "^0.5.1", 749 | "@types/fs-extra": "^8.0.1", 750 | "@types/long": "^4.0.0", 751 | "abort-controller": "^3.0.0", 752 | "duplexify": "^3.6.0", 753 | "google-auth-library": "^5.0.0", 754 | "is-stream-ended": "^0.1.4", 755 | "lodash.at": "^4.6.0", 756 | "lodash.has": "^4.5.2", 757 | "node-fetch": "^2.6.0", 758 | "protobufjs": "^6.8.8", 759 | "retry-request": "^4.0.0", 760 | "semver": "^7.0.0", 761 | "walkdir": "^0.4.0" 762 | } 763 | }, 764 | "google-p12-pem": { 765 | "version": "2.0.4", 766 | "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-2.0.4.tgz", 767 | "integrity": "sha512-S4blHBQWZRnEW44OcR7TL9WR+QCqByRvhNDZ/uuQfpxywfupikf/miba8js1jZi6ZOGv5slgSuoshCWh6EMDzg==", 768 | "requires": { 769 | "node-forge": "^0.9.0" 770 | } 771 | }, 772 | "graceful-fs": { 773 | "version": "4.2.3", 774 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", 775 | "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" 776 | }, 777 | "gtoken": { 778 | "version": "4.1.4", 779 | "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-4.1.4.tgz", 780 | "integrity": "sha512-VxirzD0SWoFUo5p8RDP8Jt2AGyOmyYcT/pOUgDKJCK+iSw0TMqwrVfY37RXTNmoKwrzmDHSk0GMT9FsgVmnVSA==", 781 | "requires": { 782 | "gaxios": "^2.1.0", 783 | "google-p12-pem": "^2.0.0", 784 | "jws": "^4.0.0", 785 | "mime": "^2.2.0" 786 | } 787 | }, 788 | "hash-stream-validation": { 789 | "version": "0.2.2", 790 | "resolved": "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.2.tgz", 791 | "integrity": "sha512-cMlva5CxWZOrlS/cY0C+9qAzesn5srhFA8IT1VPiHc9bWWBLkJfEUIZr7MWoi89oOOGmpg8ymchaOjiArsGu5A==", 792 | "requires": { 793 | "through2": "^2.0.0" 794 | }, 795 | "dependencies": { 796 | "through2": { 797 | "version": "2.0.5", 798 | "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", 799 | "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", 800 | "requires": { 801 | "readable-stream": "~2.3.6", 802 | "xtend": "~4.0.1" 803 | } 804 | } 805 | } 806 | }, 807 | "https-proxy-agent": { 808 | "version": "4.0.0", 809 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", 810 | "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", 811 | "requires": { 812 | "agent-base": "5", 813 | "debug": "4" 814 | } 815 | }, 816 | "imurmurhash": { 817 | "version": "0.1.4", 818 | "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", 819 | "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=" 820 | }, 821 | "inherits": { 822 | "version": "2.0.4", 823 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 824 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" 825 | }, 826 | "is-obj": { 827 | "version": "1.0.1", 828 | "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", 829 | "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" 830 | }, 831 | "is-stream": { 832 | "version": "2.0.0", 833 | "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", 834 | "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" 835 | }, 836 | "is-stream-ended": { 837 | "version": "0.1.4", 838 | "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", 839 | "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==" 840 | }, 841 | "isarray": { 842 | "version": "1.0.0", 843 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 844 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" 845 | }, 846 | "js-yaml": { 847 | "version": "3.13.1", 848 | "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", 849 | "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", 850 | "requires": { 851 | "argparse": "^1.0.7", 852 | "esprima": "^4.0.0" 853 | } 854 | }, 855 | "json-bigint": { 856 | "version": "0.3.0", 857 | "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", 858 | "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", 859 | "requires": { 860 | "bignumber.js": "^7.0.0" 861 | } 862 | }, 863 | "json-stringify-safe": { 864 | "version": "5.0.1", 865 | "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", 866 | "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" 867 | }, 868 | "jwa": { 869 | "version": "2.0.0", 870 | "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", 871 | "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", 872 | "requires": { 873 | "buffer-equal-constant-time": "1.0.1", 874 | "ecdsa-sig-formatter": "1.0.11", 875 | "safe-buffer": "^5.0.1" 876 | } 877 | }, 878 | "jws": { 879 | "version": "4.0.0", 880 | "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", 881 | "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", 882 | "requires": { 883 | "jwa": "^2.0.0", 884 | "safe-buffer": "^5.0.1" 885 | } 886 | }, 887 | "lodash": { 888 | "version": "4.17.15", 889 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", 890 | "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" 891 | }, 892 | "lodash.at": { 893 | "version": "4.6.0", 894 | "resolved": "https://registry.npmjs.org/lodash.at/-/lodash.at-4.6.0.tgz", 895 | "integrity": "sha1-k83OZk8KGZTqM9181A4jr9EbD/g=" 896 | }, 897 | "lodash.camelcase": { 898 | "version": "4.3.0", 899 | "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", 900 | "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=" 901 | }, 902 | "lodash.has": { 903 | "version": "4.5.2", 904 | "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", 905 | "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=" 906 | }, 907 | "long": { 908 | "version": "4.0.0", 909 | "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", 910 | "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" 911 | }, 912 | "lru-cache": { 913 | "version": "5.1.1", 914 | "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", 915 | "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", 916 | "requires": { 917 | "yallist": "^3.0.2" 918 | } 919 | }, 920 | "make-dir": { 921 | "version": "1.3.0", 922 | "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", 923 | "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", 924 | "requires": { 925 | "pify": "^3.0.0" 926 | }, 927 | "dependencies": { 928 | "pify": { 929 | "version": "3.0.0", 930 | "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", 931 | "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" 932 | } 933 | } 934 | }, 935 | "mime": { 936 | "version": "2.4.4", 937 | "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", 938 | "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" 939 | }, 940 | "mime-db": { 941 | "version": "1.43.0", 942 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", 943 | "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" 944 | }, 945 | "mime-types": { 946 | "version": "2.1.26", 947 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", 948 | "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", 949 | "requires": { 950 | "mime-db": "1.43.0" 951 | } 952 | }, 953 | "mimic-fn": { 954 | "version": "2.1.0", 955 | "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", 956 | "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" 957 | }, 958 | "minimist": { 959 | "version": "1.2.0", 960 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", 961 | "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" 962 | }, 963 | "moment": { 964 | "version": "2.24.0", 965 | "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", 966 | "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" 967 | }, 968 | "ms": { 969 | "version": "2.1.2", 970 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 971 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" 972 | }, 973 | "ndjson": { 974 | "version": "1.5.0", 975 | "resolved": "https://registry.npmjs.org/ndjson/-/ndjson-1.5.0.tgz", 976 | "integrity": "sha1-rmA7NrE0vOw0e0UkIrC/mNWDLsg=", 977 | "requires": { 978 | "json-stringify-safe": "^5.0.1", 979 | "minimist": "^1.2.0", 980 | "split2": "^2.1.0", 981 | "through2": "^2.0.3" 982 | }, 983 | "dependencies": { 984 | "through2": { 985 | "version": "2.0.5", 986 | "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", 987 | "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", 988 | "requires": { 989 | "readable-stream": "~2.3.6", 990 | "xtend": "~4.0.1" 991 | } 992 | } 993 | } 994 | }, 995 | "ndjson-parse": { 996 | "version": "1.0.4", 997 | "resolved": "https://registry.npmjs.org/ndjson-parse/-/ndjson-parse-1.0.4.tgz", 998 | "integrity": "sha512-xwglvz2dMbxvX4NAVKnww8xEJ4kp4+CKVseQQdtkA79yI3abPqyBYqk6A6HvNci5oS0cUsSHheMEV1c+9MWlEw==" 999 | }, 1000 | "node-fetch": { 1001 | "version": "2.6.0", 1002 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", 1003 | "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" 1004 | }, 1005 | "node-forge": { 1006 | "version": "0.9.1", 1007 | "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", 1008 | "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" 1009 | }, 1010 | "once": { 1011 | "version": "1.4.0", 1012 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 1013 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 1014 | "requires": { 1015 | "wrappy": "1" 1016 | } 1017 | }, 1018 | "onetime": { 1019 | "version": "5.1.0", 1020 | "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", 1021 | "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", 1022 | "requires": { 1023 | "mimic-fn": "^2.1.0" 1024 | } 1025 | }, 1026 | "pify": { 1027 | "version": "4.0.1", 1028 | "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", 1029 | "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" 1030 | }, 1031 | "process-nextick-args": { 1032 | "version": "2.0.1", 1033 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", 1034 | "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" 1035 | }, 1036 | "protobufjs": { 1037 | "version": "6.8.8", 1038 | "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", 1039 | "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", 1040 | "requires": { 1041 | "@protobufjs/aspromise": "^1.1.2", 1042 | "@protobufjs/base64": "^1.1.2", 1043 | "@protobufjs/codegen": "^2.0.4", 1044 | "@protobufjs/eventemitter": "^1.1.0", 1045 | "@protobufjs/fetch": "^1.1.0", 1046 | "@protobufjs/float": "^1.0.2", 1047 | "@protobufjs/inquire": "^1.1.0", 1048 | "@protobufjs/path": "^1.1.2", 1049 | "@protobufjs/pool": "^1.1.0", 1050 | "@protobufjs/utf8": "^1.1.0", 1051 | "@types/long": "^4.0.0", 1052 | "@types/node": "^10.1.0", 1053 | "long": "^4.0.0" 1054 | } 1055 | }, 1056 | "pump": { 1057 | "version": "2.0.1", 1058 | "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", 1059 | "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", 1060 | "requires": { 1061 | "end-of-stream": "^1.1.0", 1062 | "once": "^1.3.1" 1063 | } 1064 | }, 1065 | "pumpify": { 1066 | "version": "1.5.1", 1067 | "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", 1068 | "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", 1069 | "requires": { 1070 | "duplexify": "^3.6.0", 1071 | "inherits": "^2.0.3", 1072 | "pump": "^2.0.0" 1073 | } 1074 | }, 1075 | "quick-lru": { 1076 | "version": "2.0.0", 1077 | "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-2.0.0.tgz", 1078 | "integrity": "sha512-DqOtZziv7lDjEyuqyVQacRciAwMCEjTNrLYCHYEIIgjcE/tLEpBF82hiDIwCjRnEL9/hY2GJxA0T8ZvYvVVSSA==" 1079 | }, 1080 | "readable-stream": { 1081 | "version": "2.3.7", 1082 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", 1083 | "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", 1084 | "requires": { 1085 | "core-util-is": "~1.0.0", 1086 | "inherits": "~2.0.3", 1087 | "isarray": "~1.0.0", 1088 | "process-nextick-args": "~2.0.0", 1089 | "safe-buffer": "~5.1.1", 1090 | "string_decoder": "~1.1.1", 1091 | "util-deprecate": "~1.0.1" 1092 | } 1093 | }, 1094 | "retry-request": { 1095 | "version": "4.1.1", 1096 | "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", 1097 | "integrity": "sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ==", 1098 | "requires": { 1099 | "debug": "^4.1.1", 1100 | "through2": "^3.0.1" 1101 | } 1102 | }, 1103 | "safe-buffer": { 1104 | "version": "5.1.2", 1105 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 1106 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" 1107 | }, 1108 | "semver": { 1109 | "version": "7.1.1", 1110 | "resolved": "https://registry.npmjs.org/semver/-/semver-7.1.1.tgz", 1111 | "integrity": "sha512-WfuG+fl6eh3eZ2qAf6goB7nhiCd7NPXhmyFxigB/TOkQyeLP8w8GsVehvtGNtnNmyboz4TgeK40B1Kbql/8c5A==" 1112 | }, 1113 | "signal-exit": { 1114 | "version": "3.0.2", 1115 | "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", 1116 | "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" 1117 | }, 1118 | "snakeize": { 1119 | "version": "0.1.0", 1120 | "resolved": "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz", 1121 | "integrity": "sha1-EMCI2LWOsHazIpu1oE4jLOEmQi0=" 1122 | }, 1123 | "split-array-stream": { 1124 | "version": "2.0.0", 1125 | "resolved": "https://registry.npmjs.org/split-array-stream/-/split-array-stream-2.0.0.tgz", 1126 | "integrity": "sha512-hmMswlVY91WvGMxs0k8MRgq8zb2mSen4FmDNc5AFiTWtrBpdZN6nwD6kROVe4vNL+ywrvbCKsWVCnEd4riELIg==", 1127 | "requires": { 1128 | "is-stream-ended": "^0.1.4" 1129 | } 1130 | }, 1131 | "split2": { 1132 | "version": "2.2.0", 1133 | "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz", 1134 | "integrity": "sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==", 1135 | "requires": { 1136 | "through2": "^2.0.2" 1137 | }, 1138 | "dependencies": { 1139 | "through2": { 1140 | "version": "2.0.5", 1141 | "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", 1142 | "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", 1143 | "requires": { 1144 | "readable-stream": "~2.3.6", 1145 | "xtend": "~4.0.1" 1146 | } 1147 | } 1148 | } 1149 | }, 1150 | "sprintf-js": { 1151 | "version": "1.0.3", 1152 | "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", 1153 | "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" 1154 | }, 1155 | "stream-events": { 1156 | "version": "1.0.5", 1157 | "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", 1158 | "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", 1159 | "requires": { 1160 | "stubs": "^3.0.0" 1161 | } 1162 | }, 1163 | "stream-shift": { 1164 | "version": "1.0.1", 1165 | "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", 1166 | "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" 1167 | }, 1168 | "string_decoder": { 1169 | "version": "1.1.1", 1170 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 1171 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 1172 | "requires": { 1173 | "safe-buffer": "~5.1.0" 1174 | } 1175 | }, 1176 | "stubs": { 1177 | "version": "3.0.0", 1178 | "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", 1179 | "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls=" 1180 | }, 1181 | "teeny-request": { 1182 | "version": "3.11.3", 1183 | "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", 1184 | "integrity": "sha512-CKncqSF7sH6p4rzCgkb/z/Pcos5efl0DmolzvlqRQUNcpRIruOhY9+T1FsIlyEbfWd7MsFpodROOwHYh2BaXzw==", 1185 | "requires": { 1186 | "https-proxy-agent": "^2.2.1", 1187 | "node-fetch": "^2.2.0", 1188 | "uuid": "^3.3.2" 1189 | }, 1190 | "dependencies": { 1191 | "agent-base": { 1192 | "version": "4.3.0", 1193 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", 1194 | "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", 1195 | "requires": { 1196 | "es6-promisify": "^5.0.0" 1197 | } 1198 | }, 1199 | "debug": { 1200 | "version": "3.2.6", 1201 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", 1202 | "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", 1203 | "requires": { 1204 | "ms": "^2.1.1" 1205 | } 1206 | }, 1207 | "https-proxy-agent": { 1208 | "version": "2.2.4", 1209 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", 1210 | "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==", 1211 | "requires": { 1212 | "agent-base": "^4.3.0", 1213 | "debug": "^3.1.0" 1214 | } 1215 | } 1216 | } 1217 | }, 1218 | "through2": { 1219 | "version": "3.0.1", 1220 | "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", 1221 | "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", 1222 | "requires": { 1223 | "readable-stream": "2 || 3" 1224 | } 1225 | }, 1226 | "typedarray": { 1227 | "version": "0.0.6", 1228 | "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", 1229 | "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" 1230 | }, 1231 | "unique-string": { 1232 | "version": "1.0.0", 1233 | "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-1.0.0.tgz", 1234 | "integrity": "sha1-nhBXzKhRq7kzmPizOuGHuZyuwRo=", 1235 | "requires": { 1236 | "crypto-random-string": "^1.0.0" 1237 | } 1238 | }, 1239 | "util-deprecate": { 1240 | "version": "1.0.2", 1241 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 1242 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 1243 | }, 1244 | "uuid": { 1245 | "version": "3.4.0", 1246 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", 1247 | "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" 1248 | }, 1249 | "walkdir": { 1250 | "version": "0.4.1", 1251 | "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.4.1.tgz", 1252 | "integrity": "sha512-3eBwRyEln6E1MSzcxcVpQIhRG8Q1jLvEqRmCZqS3dsfXEDR/AhOF4d+jHg1qvDCpYaVRZjENPQyrVxAkQqxPgQ==" 1253 | }, 1254 | "wrappy": { 1255 | "version": "1.0.2", 1256 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 1257 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 1258 | }, 1259 | "write-file-atomic": { 1260 | "version": "2.4.3", 1261 | "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", 1262 | "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", 1263 | "requires": { 1264 | "graceful-fs": "^4.1.11", 1265 | "imurmurhash": "^0.1.4", 1266 | "signal-exit": "^3.0.2" 1267 | } 1268 | }, 1269 | "xdg-basedir": { 1270 | "version": "3.0.0", 1271 | "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz", 1272 | "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=" 1273 | }, 1274 | "xtend": { 1275 | "version": "4.0.2", 1276 | "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", 1277 | "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" 1278 | }, 1279 | "yallist": { 1280 | "version": "3.1.1", 1281 | "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", 1282 | "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" 1283 | } 1284 | } 1285 | } 1286 | -------------------------------------------------------------------------------- /security-events/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudflare-to-scc", 3 | "version": "0.1.15", 4 | "description": "Deliver Cloudflare logs to Google Cloud Security Command Center", 5 | "author": "Frank Taylor", 6 | "license": "MIT", 7 | "engines": { 8 | "node": "10.15.3" 9 | }, 10 | "main": "index.js", 11 | "keywords": [ 12 | "cloudflare", 13 | "scc", 14 | "cscc", 15 | "security command center", 16 | "google cloud", 17 | "google cloud scc", 18 | "google", 19 | "security", 20 | "security events", 21 | "app" 22 | ], 23 | "dependencies": { 24 | "@google-cloud/security-center": "^2.2.1", 25 | "@google-cloud/storage": "^2.3.4", 26 | "dotenv": "^8.2.0", 27 | "env-yaml": "^0.1.2", 28 | "moment": "^2.24.0", 29 | "ndjson": "^1.5.0", 30 | "ndjson-parse": "^1.0.4", 31 | "node-fetch": "^2.6.0", 32 | "quick-lru": "^2.0.0" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /security-events/static/fields.json: -------------------------------------------------------------------------------- 1 | { 2 | "EdgePathingStatus": { 3 | "Asnum": "ASN", 4 | "ctry": "IP Country", 5 | "captchaNew": "Failed CAPTCHA", 6 | "l7ddos": "DDoS", 7 | "ip": "IP", 8 | "ipr24": "IP /24", 9 | "ipr16": "IP /16", 10 | "ip6": "IPv6", 11 | "ip6r64": "IP /64", 12 | "ip6r48": "IP /48", 13 | "ip6r32": "IP /32", 14 | "rate_limit": "HTTP rate limit exceeded", 15 | "ua": "Block: Banned User Agent", 16 | "zl": "Block: IP Zone Lockdown", 17 | "reserved_ip": "DNS record points to local or disallowed IP", 18 | "reserved_ip6": "DNS record points to local or disallowed IP", 19 | "bad_host": "Invalid Host Header sent with request", 20 | "fint": "Cloudflare Administrative Ban", 21 | "no_existing_host": "Zone not present or active on Cloudflare" 22 | }, 23 | "EdgePathingSrc": { 24 | "bic": "Block: Request failed Browser Integrity Check", 25 | "hot": "Block: Hot Linking", 26 | "protect": "Block: DDoS", 27 | "filterBasedFirewall": "Firewall Rules" 28 | }, 29 | "WAFAction": null, 30 | "WAFProfile": "unknown", 31 | "EdgeRateLimitID": [ 32 | 0, 33 | 1, 34 | 2, 35 | 3 36 | ], 37 | "EdgePathingOp": [ 38 | 0, 39 | 1, 40 | 2, 41 | 3, 42 | 4 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /security-events/static/schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "CacheCacheStatus", 4 | "type": "STRING", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "name": "CacheResponseBytes", 9 | "type": "INTEGER", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "name": "CacheResponseStatus", 14 | "type": "INTEGER", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "name": "CacheTieredFill", 19 | "type": "BOOLEAN", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "name": "ClientASN", 24 | "type": "INTEGER", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "name": "ClientCountry", 29 | "type": "STRING", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "name": "ClientDeviceType", 34 | "type": "STRING", 35 | "mode": "NULLABLE" 36 | }, 37 | { 38 | "name": "ClientIP", 39 | "type": "STRING", 40 | "mode": "NULLABLE" 41 | }, 42 | { 43 | "name": "ClientIPClass", 44 | "type": "STRING", 45 | "mode": "NULLABLE" 46 | }, 47 | { 48 | "name": "ClientRequestBytes", 49 | "type": "INTEGER", 50 | "mode": "NULLABLE" 51 | }, 52 | { 53 | "name": "ClientRequestHost", 54 | "type": "STRING", 55 | "mode": "NULLABLE" 56 | }, 57 | { 58 | "name": "ClientRequestMethod", 59 | "type": "STRING", 60 | "mode": "NULLABLE" 61 | }, 62 | { 63 | "name": "ClientRequestPath", 64 | "type": "STRING", 65 | "mode": "NULLABLE" 66 | }, 67 | { 68 | "name": "ClientRequestProtocol", 69 | "type": "STRING", 70 | "mode": "NULLABLE" 71 | }, 72 | { 73 | "name": "ClientRequestReferer", 74 | "type": "STRING", 75 | "mode": "NULLABLE" 76 | }, 77 | { 78 | "name": "ClientRequestURI", 79 | "type": "STRING", 80 | "mode": "NULLABLE" 81 | }, 82 | { 83 | "name": "ClientRequestUserAgent", 84 | "type": "STRING", 85 | "mode": "NULLABLE" 86 | }, 87 | { 88 | "name": "ClientSrcPort", 89 | "type": "INTEGER", 90 | "mode": "NULLABLE" 91 | }, 92 | { 93 | "name": "ClientSSLCipher", 94 | "type": "STRING", 95 | "mode": "NULLABLE" 96 | }, 97 | { 98 | "name": "ClientSSLProtocol", 99 | "type": "STRING", 100 | "mode": "NULLABLE" 101 | }, 102 | { 103 | "name": "EdgeColoID", 104 | "type": "INTEGER", 105 | "mode": "NULLABLE" 106 | }, 107 | { 108 | "name": "EdgeEndTimestamp", 109 | "type": "TIMESTAMP", 110 | "mode": "NULLABLE" 111 | }, 112 | { 113 | "name": "EdgePathingOp", 114 | "type": "STRING", 115 | "mode": "NULLABLE" 116 | }, 117 | { 118 | "name": "EdgePathingSrc", 119 | "type": "STRING", 120 | "mode": "NULLABLE" 121 | }, 122 | { 123 | "name": "EdgePathingStatus", 124 | "type": "STRING", 125 | "mode": "NULLABLE" 126 | }, 127 | { 128 | "name": "EdgeRateLimitAction", 129 | "type": "STRING", 130 | "mode": "NULLABLE" 131 | }, 132 | { 133 | "name": "EdgeRateLimitID", 134 | "type": "INTEGER", 135 | "mode": "NULLABLE" 136 | }, 137 | { 138 | "name": "EdgeRequestHost", 139 | "type": "STRING", 140 | "mode": "NULLABLE" 141 | }, 142 | { 143 | "name": "EdgeResponseBytes", 144 | "type": "INTEGER", 145 | "mode": "NULLABLE" 146 | }, 147 | { 148 | "name": "EdgeResponseCompressionRatio", 149 | "type": "FLOAT", 150 | "mode": "NULLABLE" 151 | }, 152 | { 153 | "name": "EdgeResponseContentType", 154 | "type": "STRING", 155 | "mode": "NULLABLE" 156 | }, 157 | { 158 | "name": "EdgeResponseStatus", 159 | "type": "INTEGER", 160 | "mode": "NULLABLE" 161 | }, 162 | { 163 | "name": "EdgeServerIP", 164 | "type": "STRING", 165 | "mode": "NULLABLE" 166 | }, 167 | { 168 | "name": "EdgeStartTimestamp", 169 | "type": "TIMESTAMP", 170 | "mode": "NULLABLE" 171 | }, 172 | { 173 | "name": "OriginIP", 174 | "type": "STRING", 175 | "mode": "NULLABLE" 176 | }, 177 | { 178 | "name": "OriginResponseBytes", 179 | "type": "INTEGER", 180 | "mode": "NULLABLE" 181 | }, 182 | { 183 | "name": "OriginResponseHTTPExpires", 184 | "type": "STRING", 185 | "mode": "NULLABLE" 186 | }, 187 | { 188 | "name": "OriginResponseHTTPLastModified", 189 | "type": "STRING", 190 | "mode": "NULLABLE" 191 | }, 192 | { 193 | "name": "OriginResponseStatus", 194 | "type": "INTEGER", 195 | "mode": "NULLABLE" 196 | }, 197 | { 198 | "name": "OriginResponseTime", 199 | "type": "INTEGER", 200 | "mode": "NULLABLE" 201 | }, 202 | { 203 | "name": "OriginSSLProtocol", 204 | "type": "STRING", 205 | "mode": "NULLABLE" 206 | }, 207 | { 208 | "name": "RayID", 209 | "type": "STRING", 210 | "mode": "NULLABLE" 211 | }, 212 | { 213 | "name": "ParentRayID", 214 | "type": "STRING", 215 | "mode": "NULLABLE" 216 | }, 217 | { 218 | "name": "SecurityLevel", 219 | "type": "STRING", 220 | "mode": "NULLABLE" 221 | }, 222 | { 223 | "name": "WAFAction", 224 | "type": "STRING", 225 | "mode": "NULLABLE" 226 | }, 227 | { 228 | "name": "WAFFlags", 229 | "type": "INTEGER", 230 | "mode": "NULLABLE" 231 | }, 232 | { 233 | "name": "WAFMatchedVar", 234 | "type": "STRING", 235 | "mode": "NULLABLE" 236 | }, 237 | { 238 | "name": "WAFProfile", 239 | "type": "STRING", 240 | "mode": "NULLABLE" 241 | }, 242 | { 243 | "name": "WAFRuleID", 244 | "type": "STRING", 245 | "mode": "NULLABLE" 246 | }, 247 | { 248 | "name": "WAFRuleMessage", 249 | "type": "STRING", 250 | "mode": "NULLABLE" 251 | }, 252 | { 253 | "name": "WorkerCPUTime", 254 | "type": "INTEGER", 255 | "mode": "NULLABLE" 256 | }, 257 | { 258 | "name": "WorkerStatus", 259 | "type": "STRING", 260 | "mode": "NULLABLE" 261 | }, 262 | { 263 | "name": "WorkerSubrequest", 264 | "type": "BOOLEAN", 265 | "mode": "NULLABLE" 266 | }, 267 | { 268 | "name": "WorkerSubrequestCount", 269 | "type": "INTEGER", 270 | "mode": "NULLABLE" 271 | }, 272 | { 273 | "name": "ZoneID", 274 | "type": "INTEGER", 275 | "mode": "NULLABLE" 276 | } 277 | ] 278 | -------------------------------------------------------------------------------- /security-events/util.js: -------------------------------------------------------------------------------- 1 | require('env-yaml').config() 2 | 3 | const LRU = require('quick-lru') 4 | const moment = require('moment') 5 | const colos = require('./static/colos.json') 6 | const request = require('node-fetch') 7 | const { URLSearchParams } = require('url') 8 | 9 | const Lru = new LRU({ 10 | maxSize: 300 11 | }) 12 | 13 | const lruCacheHandler = { 14 | get (target, prop, receiver) { 15 | return Reflect.get(...arguments) 16 | }, 17 | set (obj, prop, value) { 18 | return Reflect.set(...arguments) 19 | } 20 | } 21 | 22 | const lru = { 23 | account: new Proxy(Lru, lruCacheHandler), 24 | colos: new Proxy(Lru, lruCacheHandler), 25 | assets: new Proxy(Lru, lruCacheHandler), 26 | zones: new Proxy(Lru, lruCacheHandler) 27 | } 28 | 29 | const getJson = async ({ params, path, body = null, stream = false }, method = 'GET') => { 30 | let headers = { 31 | 'Content-Type': 'application/json', 32 | 'Authorization': process.env.API_KEY.startsWith('Bearer') ? process.env.API_KEY : `Bearer ${process.env.API_KEY}` 33 | } 34 | if (params) { 35 | params = new URLSearchParams(params).toString() 36 | params = '?' + decodeURIComponent(params) 37 | } else { 38 | params = '' 39 | } 40 | try { 41 | let uri = `https://api.cloudflare.com/client/v4${path}${params}`.trimEnd() 42 | let options = { 43 | headers: headers, 44 | method: method 45 | } 46 | if (body) { 47 | options.body = body 48 | } 49 | let response = await request(uri, options) 50 | if (!response.ok) { 51 | throw console.log('') 52 | } 53 | 54 | if (stream) { 55 | return response.text() 56 | } 57 | return response.json() 58 | } catch (e) { 59 | return JSON.stringify({ err: e }) 60 | } 61 | } 62 | 63 | const getDate = ({ minutes = null }) => { 64 | let m = moment() 65 | m.startOf('minute') 66 | if (typeof minutes === 'string') minutes = minutes.replace('m', '') 67 | if (minutes) m.subtract(parseInt(minutes, 10), 'minutes') 68 | const formatted = m.toISOString().replace('.000', '') 69 | console.log(formatted) 70 | return formatted 71 | } 72 | 73 | const getColo = (edgeColoID) => { 74 | let id = Number.parseInt(edgeColoID, 10) 75 | let inlru = lru.colos.has(edgeColoID) 76 | 77 | if (!inlru) { 78 | if (edgeColoID <= 172) { 79 | lru.colos.set(edgeColoID, String(colos[id].colo_alias)) 80 | return lru.colos.get(edgeColoID) 81 | } 82 | 83 | const inChina = colos.slice(172).findIndex(colo => colo.colo_id === edgeColoID) 84 | 85 | if (inChina > -1) { 86 | lru.colos.set(edgeColoID, String(colos[id].colo_alias)) 87 | return lru.colos.get(edgeColoID) 88 | } 89 | 90 | return 'San Francisco, CA' 91 | } 92 | return lru.colos.get(edgeColoID) 93 | } 94 | 95 | module.exports.getColo = getColo 96 | module.exports.getDate = getDate 97 | module.exports.getJson = getJson 98 | module.exports.lru = lru 99 | --------------------------------------------------------------------------------