├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── Pipfile ├── README.md ├── api.py ├── bucket.py ├── config.dev.yml ├── config.prod.yml ├── cron.py ├── models └── v1 │ ├── asset_groups │ └── asset_group.py │ ├── assets │ └── asset.py │ ├── indicators │ └── indicator.py │ └── services │ └── service.py ├── package-lock.json ├── package.json ├── serverless.yml ├── tests ├── conftest.py └── test_rest.py └── utils ├── auth.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Distribution / packaging 2 | .Python 3 | env/ 4 | build/ 5 | develop-eggs/ 6 | dist/ 7 | downloads/ 8 | eggs/ 9 | .eggs/ 10 | lib/ 11 | lib64/ 12 | parts/ 13 | sdist/ 14 | var/ 15 | *.egg-info/ 16 | .installed.cfg 17 | *.egg 18 | Pipfile.lock 19 | 20 | 21 | # Serverless directories 22 | .serverless 23 | node_modules/ 24 | 25 | #vscode 26 | settings.json 27 | .cache 28 | 29 | #cruft 30 | tests/__pycache__ 31 | .pytest_cache 32 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Participation Guidelines 2 | 3 | This repository is governed by Mozilla's code of conduct and etiquette guidelines. 4 | For more details, please read the 5 | [Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). 6 | 7 | ## How to Report 8 | For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. 9 | 10 | 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License, version 2.0 2 | 3 | 1. Definitions 4 | 5 | 1.1. "Contributor" 6 | 7 | means each individual or legal entity that creates, contributes to the 8 | creation of, or owns Covered Software. 9 | 10 | 1.2. "Contributor Version" 11 | 12 | means the combination of the Contributions of others (if any) used by a 13 | Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | 17 | means Covered Software of a particular Contributor. 18 | 19 | 1.4. "Covered Software" 20 | 21 | means Source Code Form to which the initial Contributor has attached the 22 | notice in Exhibit A, the Executable Form of such Source Code Form, and 23 | Modifications of such Source Code Form, in each case including portions 24 | thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | a. that the initial Contributor has attached the notice described in 30 | Exhibit B to the Covered Software; or 31 | 32 | b. that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the terms of 34 | a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | 38 | means any form of the work other than Source Code Form. 39 | 40 | 1.7. "Larger Work" 41 | 42 | means a work that combines Covered Software with other material, in a 43 | separate file or files, that is not Covered Software. 44 | 45 | 1.8. "License" 46 | 47 | means this document. 48 | 49 | 1.9. "Licensable" 50 | 51 | means having the right to grant, to the maximum extent possible, whether 52 | at the time of the initial grant or subsequently, any and all of the 53 | rights conveyed by this License. 54 | 55 | 1.10. "Modifications" 56 | 57 | means any of the following: 58 | 59 | a. any file in Source Code Form that results from an addition to, 60 | deletion from, or modification of the contents of Covered Software; or 61 | 62 | b. any new file in Source Code Form that contains any Covered Software. 63 | 64 | 1.11. "Patent Claims" of a Contributor 65 | 66 | means any patent claim(s), including without limitation, method, 67 | process, and apparatus claims, in any patent Licensable by such 68 | Contributor that would be infringed, but for the grant of the License, 69 | by the making, using, selling, offering for sale, having made, import, 70 | or transfer of either its Contributions or its Contributor Version. 71 | 72 | 1.12. "Secondary License" 73 | 74 | means either the GNU General Public License, Version 2.0, the GNU Lesser 75 | General Public License, Version 2.1, the GNU Affero General Public 76 | License, Version 3.0, or any later versions of those licenses. 77 | 78 | 1.13. "Source Code Form" 79 | 80 | means the form of the work preferred for making modifications. 81 | 82 | 1.14. "You" (or "Your") 83 | 84 | means an individual or a legal entity exercising rights under this 85 | License. For legal entities, "You" includes any entity that controls, is 86 | controlled by, or is under common control with You. For purposes of this 87 | definition, "control" means (a) the power, direct or indirect, to cause 88 | the direction or management of such entity, whether by contract or 89 | otherwise, or (b) ownership of more than fifty percent (50%) of the 90 | outstanding shares or beneficial ownership of such entity. 91 | 92 | 93 | 2. License Grants and Conditions 94 | 95 | 2.1. Grants 96 | 97 | Each Contributor hereby grants You a world-wide, royalty-free, 98 | non-exclusive license: 99 | 100 | a. under intellectual property rights (other than patent or trademark) 101 | Licensable by such Contributor to use, reproduce, make available, 102 | modify, display, perform, distribute, and otherwise exploit its 103 | Contributions, either on an unmodified basis, with Modifications, or 104 | as part of a Larger Work; and 105 | 106 | b. under Patent Claims of such Contributor to make, use, sell, offer for 107 | sale, have made, import, and otherwise transfer either its 108 | Contributions or its Contributor Version. 109 | 110 | 2.2. Effective Date 111 | 112 | The licenses granted in Section 2.1 with respect to any Contribution 113 | become effective for each Contribution on the date the Contributor first 114 | distributes such Contribution. 115 | 116 | 2.3. Limitations on Grant Scope 117 | 118 | The licenses granted in this Section 2 are the only rights granted under 119 | this License. No additional rights or licenses will be implied from the 120 | distribution or licensing of Covered Software under this License. 121 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 122 | Contributor: 123 | 124 | a. for any code that a Contributor has removed from Covered Software; or 125 | 126 | b. for infringements caused by: (i) Your and any other third party's 127 | modifications of Covered Software, or (ii) the combination of its 128 | Contributions with other software (except as part of its Contributor 129 | Version); or 130 | 131 | c. under Patent Claims infringed by Covered Software in the absence of 132 | its Contributions. 133 | 134 | This License does not grant any rights in the trademarks, service marks, 135 | or logos of any Contributor (except as may be necessary to comply with 136 | the notice requirements in Section 3.4). 137 | 138 | 2.4. Subsequent Licenses 139 | 140 | No Contributor makes additional grants as a result of Your choice to 141 | distribute the Covered Software under a subsequent version of this 142 | License (see Section 10.2) or under the terms of a Secondary License (if 143 | permitted under the terms of Section 3.3). 144 | 145 | 2.5. Representation 146 | 147 | Each Contributor represents that the Contributor believes its 148 | Contributions are its original creation(s) or it has sufficient rights to 149 | grant the rights to its Contributions conveyed by this License. 150 | 151 | 2.6. Fair Use 152 | 153 | This License is not intended to limit any rights You have under 154 | applicable copyright doctrines of fair use, fair dealing, or other 155 | equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under 169 | the terms of this License. You must inform recipients that the Source 170 | Code Form of the Covered Software is governed by the terms of this 171 | License, and how they can obtain a copy of this License. You may not 172 | attempt to alter or restrict the recipients' rights in the Source Code 173 | Form. 174 | 175 | 3.2. Distribution of Executable Form 176 | 177 | If You distribute Covered Software in Executable Form then: 178 | 179 | a. such Covered Software must also be made available in Source Code Form, 180 | as described in Section 3.1, and You must inform recipients of the 181 | Executable Form how they can obtain a copy of such Source Code Form by 182 | reasonable means in a timely manner, at a charge no more than the cost 183 | of distribution to the recipient; and 184 | 185 | b. You may distribute such Executable Form under the terms of this 186 | License, or sublicense it under different terms, provided that the 187 | license for the Executable Form does not attempt to limit or alter the 188 | recipients' rights in the Source Code Form under this License. 189 | 190 | 3.3. Distribution of a Larger Work 191 | 192 | You may create and distribute a Larger Work under terms of Your choice, 193 | provided that You also comply with the requirements of this License for 194 | the Covered Software. If the Larger Work is a combination of Covered 195 | Software with a work governed by one or more Secondary Licenses, and the 196 | Covered Software is not Incompatible With Secondary Licenses, this 197 | License permits You to additionally distribute such Covered Software 198 | under the terms of such Secondary License(s), so that the recipient of 199 | the Larger Work may, at their option, further distribute the Covered 200 | Software under the terms of either this License or such Secondary 201 | License(s). 202 | 203 | 3.4. Notices 204 | 205 | You may not remove or alter the substance of any license notices 206 | (including copyright notices, patent notices, disclaimers of warranty, or 207 | limitations of liability) contained within the Source Code Form of the 208 | Covered Software, except that You may alter any license notices to the 209 | extent required to remedy known factual inaccuracies. 210 | 211 | 3.5. Application of Additional Terms 212 | 213 | You may choose to offer, and to charge a fee for, warranty, support, 214 | indemnity or liability obligations to one or more recipients of Covered 215 | Software. However, You may do so only on Your own behalf, and not on 216 | behalf of any Contributor. You must make it absolutely clear that any 217 | such warranty, support, indemnity, or liability obligation is offered by 218 | You alone, and You hereby agree to indemnify every Contributor for any 219 | liability incurred by such Contributor as a result of warranty, support, 220 | indemnity or liability terms You offer. You may include additional 221 | disclaimers of warranty and limitations of liability specific to any 222 | jurisdiction. 223 | 224 | 4. Inability to Comply Due to Statute or Regulation 225 | 226 | If it is impossible for You to comply with any of the terms of this License 227 | with respect to some or all of the Covered Software due to statute, 228 | judicial order, or regulation then You must: (a) comply with the terms of 229 | this License to the maximum extent possible; and (b) describe the 230 | limitations and the code they affect. Such description must be placed in a 231 | text file included with all distributions of the Covered Software under 232 | this License. Except to the extent prohibited by statute or regulation, 233 | such description must be sufficiently detailed for a recipient of ordinary 234 | skill to be able to understand it. 235 | 236 | 5. Termination 237 | 238 | 5.1. The rights granted under this License will terminate automatically if You 239 | fail to comply with any of its terms. However, if You become compliant, 240 | then the rights granted under this License from a particular Contributor 241 | are reinstated (a) provisionally, unless and until such Contributor 242 | explicitly and finally terminates Your grants, and (b) on an ongoing 243 | basis, if such Contributor fails to notify You of the non-compliance by 244 | some reasonable means prior to 60 days after You have come back into 245 | compliance. Moreover, Your grants from a particular Contributor are 246 | reinstated on an ongoing basis if such Contributor notifies You of the 247 | non-compliance by some reasonable means, this is the first time You have 248 | received notice of non-compliance with this License from such 249 | Contributor, and You become compliant prior to 30 days after Your receipt 250 | of the notice. 251 | 252 | 5.2. If You initiate litigation against any entity by asserting a patent 253 | infringement claim (excluding declaratory judgment actions, 254 | counter-claims, and cross-claims) alleging that a Contributor Version 255 | directly or indirectly infringes any patent, then the rights granted to 256 | You by any and all Contributors for the Covered Software under Section 257 | 2.1 of this License shall terminate. 258 | 259 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 260 | license agreements (excluding distributors and resellers) which have been 261 | validly granted by You or Your distributors under this License prior to 262 | termination shall survive termination. 263 | 264 | 6. Disclaimer of Warranty 265 | 266 | Covered Software is provided under this License on an "as is" basis, 267 | without warranty of any kind, either expressed, implied, or statutory, 268 | including, without limitation, warranties that the Covered Software is free 269 | of defects, merchantable, fit for a particular purpose or non-infringing. 270 | The entire risk as to the quality and performance of the Covered Software 271 | is with You. Should any Covered Software prove defective in any respect, 272 | You (not any Contributor) assume the cost of any necessary servicing, 273 | repair, or correction. This disclaimer of warranty constitutes an essential 274 | part of this License. No use of any Covered Software is authorized under 275 | this License except under this disclaimer. 276 | 277 | 7. Limitation of Liability 278 | 279 | Under no circumstances and under no legal theory, whether tort (including 280 | negligence), contract, or otherwise, shall any Contributor, or anyone who 281 | distributes Covered Software as permitted above, be liable to You for any 282 | direct, indirect, special, incidental, or consequential damages of any 283 | character including, without limitation, damages for lost profits, loss of 284 | goodwill, work stoppage, computer failure or malfunction, or any and all 285 | other commercial damages or losses, even if such party shall have been 286 | informed of the possibility of such damages. This limitation of liability 287 | shall not apply to liability for death or personal injury resulting from 288 | such party's negligence to the extent applicable law prohibits such 289 | limitation. Some jurisdictions do not allow the exclusion or limitation of 290 | incidental or consequential damages, so this exclusion and limitation may 291 | not apply to You. 292 | 293 | 8. Litigation 294 | 295 | Any litigation relating to this License may be brought only in the courts 296 | of a jurisdiction where the defendant maintains its principal place of 297 | business and such litigation shall be governed by laws of that 298 | jurisdiction, without reference to its conflict-of-law provisions. Nothing 299 | in this Section shall prevent a party's ability to bring cross-claims or 300 | counter-claims. 301 | 302 | 9. Miscellaneous 303 | 304 | This License represents the complete agreement concerning the subject 305 | matter hereof. If any provision of this License is held to be 306 | unenforceable, such provision shall be reformed only to the extent 307 | necessary to make it enforceable. Any law or regulation which provides that 308 | the language of a contract shall be construed against the drafter shall not 309 | be used to construe this License against a Contributor. 310 | 311 | 312 | 10. Versions of the License 313 | 314 | 10.1. New Versions 315 | 316 | Mozilla Foundation is the license steward. Except as provided in Section 317 | 10.3, no one other than the license steward has the right to modify or 318 | publish new versions of this License. Each version will be given a 319 | distinguishing version number. 320 | 321 | 10.2. Effect of New Versions 322 | 323 | You may distribute the Covered Software under the terms of the version 324 | of the License under which You originally received the Covered Software, 325 | or under the terms of any subsequent version published by the license 326 | steward. 327 | 328 | 10.3. Modified Versions 329 | 330 | If you create software not governed by this License, and you want to 331 | create a new license for such software, you may create and use a 332 | modified version of this License if you rename the license and remove 333 | any references to the name of the license steward (except to note that 334 | such modified license differs from this License). 335 | 336 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 337 | Licenses If You choose to distribute Source Code Form that is 338 | Incompatible With Secondary Licenses under the terms of this version of 339 | the License, the notice described in Exhibit B of this License must be 340 | attached. 341 | 342 | Exhibit A - Source Code Form License Notice 343 | 344 | This Source Code Form is subject to the 345 | terms of the Mozilla Public License, v. 346 | 2.0. If a copy of the MPL was not 347 | distributed with this file, You can 348 | obtain one at 349 | http://mozilla.org/MPL/2.0/. 350 | 351 | If it is not possible or desirable to put the notice in a particular file, 352 | then You may include the notice in a location (such as a LICENSE file in a 353 | relevant directory) where a recipient would be likely to look for such a 354 | notice. 355 | 356 | You may add additional accurate notices of copyright ownership. 357 | 358 | Exhibit B - "Incompatible With Secondary Licenses" Notice 359 | 360 | This Source Code Form is "Incompatible 361 | With Secondary Licenses", as defined by 362 | the Mozilla Public License, v. 2.0. 363 | 364 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | marshmallow = "==2.15.4" 8 | dynamorm = "*" 9 | "boto3" = "*" 10 | flask = "*" 11 | flask-restplus = "*" 12 | flask-cors = "*" 13 | schematics = "*" 14 | python-jose-cryptodome = "*" 15 | everett = "*" 16 | pyyaml = "*" 17 | six = "*" 18 | requests = "*" 19 | credstash = "*" 20 | gspread = "*" 21 | oauth2client = "*" 22 | 23 | [dev-packages] 24 | pytest="*" 25 | requests="*" 26 | 27 | [requires] 28 | python_version = "3.7" 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | service-map 2 | =========== 3 | 4 | Service map is a set of micro serverless services that marry 5 | - Services 6 | - Their risk levels 7 | - Indicators about the security posture of the service 8 | - Various control metrics 9 | 10 | into an ongoing risk score for each service based on what we know about it. 11 | 12 | ## Architecture 13 | The goal is to use as little servers, containers, etc as possible and rely on serverless tech such as lambda and dynamodb to focus on the business logic rather than the infra. 14 | 15 | 16 | ## Deployment 17 | First setup creds in credstash for the functions to use: 18 | The cron function needs oauth creds as per the gspread docs: 19 | https://gspread.readthedocs.io/en/latest/oauth2.html 20 | 21 | Store these in credstash as a json blob: 22 | ``` 23 | credstash --profile devadmin put serviceapi.gdrive @yourfilename.json app=serviceapi 24 | ``` 25 | where devadmin is the name of your aws profile in ~/.aws/config specifying where you will be deploying. 26 | 27 | Then deploy: 28 | 29 | ``` 30 | pipenv shell 31 | sls deploy 32 | 33 | ``` 34 | 35 | ## Testing 36 | Get credstash creds for oidc auth: 37 | ``` 38 | credstash --profile devadmin get serviceapi.oidc_client_secret app=serviceapi 39 | credstash --profile devadmin get serviceapi.oidc_client_id app=serviceapi 40 | ``` 41 | 42 | Set env variables: 43 | ``` 44 | export API_URL="https://serviceapi.security.allizom.org" 45 | export CLIENT_ID = value from above 46 | export CLIENT_SECRET = value from above 47 | ``` 48 | and run pytest -------------------------------------------------------------------------------- /api.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask import Flask, jsonify 3 | from flask_cors import cross_origin 4 | from flask_restplus import Resource, Api 5 | from werkzeug.exceptions import HTTPException, default_exceptions 6 | from models.v1.indicators.indicator import api as indicator_api 7 | from models.v1.assets.asset import api as asset_api 8 | from models.v1.asset_groups.asset_group import api as asset_group_api 9 | from models.v1.services.service import api as service_api 10 | from utils.utils import get_config 11 | 12 | logger = logging.getLogger(__name__) 13 | CONFIG = get_config() 14 | 15 | 16 | app = Flask(__name__) 17 | api = Api(app, doc=False) 18 | api.add_namespace(indicator_api) 19 | api.add_namespace(asset_api) 20 | api.add_namespace(asset_group_api) 21 | api.add_namespace(service_api) 22 | 23 | 24 | @api.route("/status") 25 | class status(Resource): 26 | @api.doc("a klingon test/status endpoint") 27 | def get(self): 28 | body = {"message": "Qapla'!"} 29 | return jsonify(body) 30 | 31 | 32 | @api.errorhandler(HTTPException) 33 | def handle_error(e): 34 | code = 500 35 | if isinstance(e, HTTPException): 36 | code = e.code 37 | return jsonify(message=str(e)), code 38 | 39 | 40 | # flask routes errors by specificity of the handler 41 | # so explicitly route all errors to the handler above 42 | # https://stackoverflow.com/a/29332131 43 | for ex in default_exceptions: 44 | app.register_error_handler(ex, handle_error) 45 | 46 | if __name__ == "__main__": 47 | app.run() 48 | -------------------------------------------------------------------------------- /bucket.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from models.v1.assets.asset import Asset 3 | from models.v1.asset_groups.asset_group import AssetGroup 4 | from models.v1.services.service import Service 5 | 6 | 7 | class iRule: 8 | def __init__(self, ruletype, action, tokens=[]): 9 | self.ruletype = ruletype 10 | self.action = action 11 | self.tokens = tokens 12 | 13 | 14 | def event(event, context): 15 | """ 16 | Triggered by s3 events, object create and remove 17 | """ 18 | # Sample event: 19 | # 20 | # _event = {'Records': [{'eventVersion': '2.0', 'eventSource': 'aws:s3', 'awsRegion': 'us-east-1', 21 | # 'eventTime': '2017-11-25T23:57:38.988Z', 'eventName': 'ObjectCreated:Put', 22 | # 'userIdentity': {'principalId': 'AWS:AROAJWJG5IVL3URF4WKKK:su-xx-test-create'}, 23 | # 'requestParameters': {'sourceIPAddress': '75.82.111.45'}, 24 | # 'responseElements': {'x-amz-request-id': '9E39B8F9A3D22C83', 25 | # 'x-amz-id-2': 'GiWcmOHnxnxOJa64k5rkgTsiiwo+JOR3p2DvuQ6txQXl9jC0jNhO+gbDwwP/3WKAl4oPbVZsTE4='}, 26 | # 's3': {'s3SchemaVersion': '1.0', 'configurationId': 'dad7b639-0cd8-4e47-a2ae-91cc5bf866c8', 27 | # 'bucket': {'name': 'su-xx', 'ownerIdentity': {'principalId': 'AEZOG5WRKFUM2'}, 28 | # 'arn': 'arn:aws:s3:::su-xx'}, 29 | # 'object': {'key': 'test/bbc498ea-d23b-11e7-af42-2a31486da301', 'size': 11060, 30 | # 'eTag': 'd50cb2e8d7ad6768d46b3d47ba9b241e', 31 | # 'sequencer': '005A1A0372C5A1D292'}}}]} 32 | 33 | s3 = boto3.resource("s3") 34 | print("event: {}".format(event)) 35 | for record in event["Records"]: 36 | print(record["eventName"]) 37 | print(record["s3"]["object"]["key"]) 38 | bucket = record["s3"]["bucket"]["name"] 39 | key = record["s3"]["object"]["key"] 40 | # download as a file for easier line by line parsing 41 | s3.meta.client.download_file(bucket, key, "/tmp/{}".format(key)) 42 | # parsing the interlink.rules 43 | rules = [] 44 | for rule in open("/tmp/{}".format(key)): 45 | 46 | rule = rule.strip() 47 | # ignore junk/cr/lr 48 | if len(rule) <= 1: 49 | continue 50 | tokens = rule.split(" ") 51 | # ignore comments 52 | if len(tokens) < 2 or "#" in tokens[0]: 53 | continue 54 | 55 | if ( 56 | len(tokens) > 2 57 | and tokens[0] in ["add", "remove"] 58 | and tokens[1] == "assetgroup" 59 | ): 60 | # add/remove an asset group 61 | # ex: add assetgroup mana-production description goes here 62 | rules.append(iRule("assetgroup", tokens[0], tokens)) 63 | 64 | elif ( 65 | len(tokens) >= 5 66 | and tokens[0] == "assetgroup" 67 | and tokens[1] == "matches" 68 | and tokens[3] == "link" 69 | and tokens[4] == "service" 70 | ): 71 | # link a service to an asset group 72 | # ex: assetgroup matches officeadminhosts-production link service Admin hosts 73 | # since services can have spaces in the name 74 | # make sure the last token is the complete service name 75 | ruletokens = [] 76 | ruletokens.append(tokens[0]) 77 | ruletokens.append(tokens[1]) 78 | ruletokens.append(tokens[2]) 79 | ruletokens.append(tokens[3]) 80 | ruletokens.append(tokens[4]) 81 | ruletokens.append(" ".join(tokens[5::])) 82 | rules.append(iRule("assetgroupLinkService", "link", ruletokens)) 83 | 84 | elif ( 85 | len(tokens) == 6 86 | and tokens[0] == "asset" 87 | and tokens[1] == "matches" 88 | and tokens[3] == "link" 89 | and tokens[4] == "assetgroup" 90 | ): 91 | # link an asset to an asset group 92 | # ex: asset matches mana link assetgroup mana-production 93 | rules.append(iRule("assetLinkAssetgroup", "link", tokens)) 94 | 95 | elif ( 96 | len(tokens) >= 6 97 | and tokens[0] == "asset" 98 | and tokens[1] == "matches" 99 | and tokens[3] == "ownership" 100 | ): 101 | # link an asset to an owner 102 | # ex: asset matches mana1.webapp.mdc1.mozilla.com ownership webops it 103 | # remove simple regex escapes 104 | tokens[2] = ( 105 | tokens[2].replace("\\d+", "").replace("\\d", "").replace("\\", "") 106 | ) 107 | rules.append(iRule("assetOwnership", "link", tokens)) 108 | 109 | elif len(tokens) > 2 and tokens[0] == "service" and tokens[1] == "mask": 110 | # mask a service/rra from being reported as active 111 | # ex: service mask SSO OKTA 112 | rules.append(iRule("maskService", "mask", tokens)) 113 | elif len(tokens) > 2 and tokens[0] == "service" and tokens[1] == "add": 114 | # add a service/rra if it doesn't exist 115 | # ex: service add Mozdef 116 | rules.append(iRule("addService", "add", tokens)) 117 | else: 118 | print("unparsed rule {0}", rule) 119 | 120 | for rule in rules: 121 | # debug 122 | print(rule.ruletype, rule.action, rule.tokens) 123 | if rule.ruletype == "addService": 124 | # add a service 125 | # handy if an RRA doesn't exist 126 | # add unless it already exists 127 | service = None 128 | service_matches = [ 129 | s for s in Service.scan(name__eq=" ".join(rule.tokens[2::])) 130 | ] 131 | if len(service_matches): 132 | service = service_matches[0] 133 | else: 134 | service = Service(name=" ".join(rule.tokens[2::])) 135 | service.save() 136 | 137 | if rule.ruletype == "maskService": 138 | # set masked to true for this service 139 | # to weed out things that are deprecated, template docs, etc 140 | for service in Service.scan(name__eq=" ".join(rule.tokens[2::])): 141 | service.masked = True 142 | service.save() 143 | 144 | if rule.ruletype == "assetgroup": 145 | if rule.tokens[0] == "add": 146 | # rule will be formatted like: 147 | # add assetgroup reference a reference group of assets 148 | # add unless it already exists 149 | asset_group = None 150 | asset_groups = [a for a in AssetGroup.scan(name__eq=rule.tokens[2])] 151 | if len(asset_groups): 152 | asset_group = asset_groups[0] 153 | else: 154 | asset_group = AssetGroup(name=rule.tokens[2]) 155 | # add any description 156 | if len(rule.tokens) > 3: 157 | asset_group.description = " ".join(rule.tokens[3::]) 158 | asset_group.save() 159 | 160 | if rule.ruletype == "assetgroupLinkService": 161 | # rule will be formated like: 162 | # assetgroup matches reference link service Reference_Service 163 | # find the asset group 164 | asset_group = None 165 | asset_groups = [a for a in AssetGroup.scan(name__eq=rule.tokens[2])] 166 | if len(asset_groups): 167 | asset_group = asset_groups[0] 168 | # find the service 169 | services = [a for a in Service.scan(name__eq=rule.tokens[-1])] 170 | if len(services): 171 | service_id = services[0].id 172 | asset_group.service_id = service_id 173 | asset_group.save() 174 | 175 | if rule.ruletype == "assetOwnership": 176 | # rule will be formatted like: 177 | # asset matches www.reference.com ownership referencegroup it 178 | # team is generally a subgroup of operator 179 | for asset in Asset.scan(asset_identifier__contains=rule.tokens[2]): 180 | print("updating: {}".format(asset.asset_identifier)) 181 | asset.team = rule.tokens[4] 182 | asset.operator = rule.tokens[5] 183 | asset.save() 184 | 185 | if rule.ruletype == "assetLinkAssetgroup": 186 | # rule will be formatted like: 187 | # asset matches reference link assetgroup reference 188 | asset_group = rule.tokens[-1] 189 | asset_group_id = None 190 | asset_identifier = rule.tokens[2] 191 | # get the group id 192 | groups = AssetGroup.scan(name__eq=asset_group) 193 | for group in groups: 194 | # update all matching assets to be in this group 195 | # by adding them to the asset_group.assets list 196 | assets = Asset.scan( 197 | asset_identifier__contains=asset_identifier 198 | ).recursive() 199 | for asset in assets: 200 | if group.assets: 201 | group.assets.append(asset.id) 202 | else: 203 | group.assets = [asset.id] 204 | group.save() 205 | -------------------------------------------------------------------------------- /config.dev.yml: -------------------------------------------------------------------------------- 1 | DOMAIN_NAME: serviceapi.security.allizom.org 2 | ZONE_ID: Z13VQZ50081YZU 3 | CERTIFICATE_ARN: arn:aws:acm:us-west-2:656532927350:certificate/436a9e26-6fc9-496d-a383-53b7c144e21a 4 | AUTH0_URL: auth-dev.mozilla.auth0.com 5 | AUDIENCE: https://serviceapi.security.allizom.org/ 6 | ENVIRONMENT: dev 7 | REGION: us-west-2 8 | KMSGUID: 6762fa27-618c-4c0b-83d4-ca42ffac454f 9 | RISKS_BUCKET_NAME: infosec-risk-data-dev 10 | RISKS_KEY_NAME: risks.json -------------------------------------------------------------------------------- /config.prod.yml: -------------------------------------------------------------------------------- 1 | DOMAIN_NAME: serviceapi.security.mozilla.org 2 | ZONE_ID: ZBALOKPGJTQW 3 | CERTIFICATE_ARN: arn:aws:acm:us-west-2:371522382791:certificate/77e8a1bd-8f66-455d-99a3-261c64c661ec 4 | AUTH0_URL: auth.mozilla.auth0.com 5 | AUDIENCE: https://serviceapi.security.mozilla.org/ 6 | ENVIRONMENT: prod 7 | REGION: us-west-2 8 | KMSGUID: 7e581152-11be-44eb-9867-ea25afff8981 9 | RISKS_BUCKET_NAME: infosec-risk-data 10 | RISKS_KEY_NAME: risks.json -------------------------------------------------------------------------------- /cron.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import credstash 3 | import gspread 4 | import json 5 | import os 6 | from oauth2client.service_account import ServiceAccountCredentials 7 | from oauth2client import file, client, tools 8 | from models.v1.services.service import Service 9 | from models.v1.assets.asset import Asset 10 | from models.v1.asset_groups.asset_group import AssetGroup 11 | from models.v1.indicators.indicator import Indicator 12 | 13 | 14 | def event(event, context): 15 | # print('event: {}'.format(event)) 16 | risk_scores = {"MAXIMUM": 5, "HIGH": 4, "MEDIUM": 3, "LOW": 2, "UNKNOWN": 1} 17 | # get our gdrive creds 18 | # and auth to google 19 | gcreds_json = credstash.getSecret( 20 | name="serviceapi.gdrive", context={"app": "serviceapi"}, region="us-east-1" 21 | ) 22 | scopes = [ 23 | "https://www.googleapis.com/auth/drive.metadata.readonly", 24 | "https://www.googleapis.com/auth/drive.file ", 25 | "https://www.googleapis.com/auth/drive", 26 | ] 27 | credentials = ServiceAccountCredentials.from_json_keyfile_dict( 28 | json.loads(gcreds_json), scopes 29 | ) 30 | gs = gspread.authorize(credentials) 31 | 32 | # get rras 33 | rras = gs.open("Mozilla Information Security Risk Register").worksheet("RRA3") 34 | heading_keys = [] 35 | for r in range(1, rras.row_count): 36 | if r == 1: 37 | row_keys = rras.row_values(r) 38 | for key in row_keys: 39 | # lowercase and underscore the keys to fields 40 | heading_keys.append(key.lower().replace(" ", "_")) 41 | 42 | else: 43 | row = rras.row_values(r) 44 | if len(row) == 0: 45 | break 46 | else: 47 | try: 48 | service_dict = dict(zip(heading_keys, row)) 49 | # empty strings can't be stored in dynamodb 50 | # convert any empty string to None 51 | for key, value in service_dict.items(): 52 | if not value: 53 | service_dict[key] = None 54 | # determine the risk score for a service 55 | if service_dict["highest_risk_impact"]: 56 | service_dict["score"] = risk_scores[ 57 | service_dict["highest_risk_impact"].strip().upper() 58 | ] 59 | else: 60 | service_dict["score"] = 0 61 | if "recommendations" in service_dict: 62 | # adjust the score if the recomendations outweigh the risk score 63 | if int(service_dict["recommendations"]) > service_dict["score"]: 64 | service_dict["score"] = ( 65 | int(service_dict["recommendations"]) 66 | - service_dict["score"] 67 | ) 68 | # find the service or create it 69 | # matching on name and link 70 | services = [ 71 | s 72 | for s in Service.scan( 73 | name__eq=service_dict["name"], link__eq=service_dict["link"] 74 | ) 75 | ] 76 | if len(services): 77 | # found one, update it 78 | service = services[0] 79 | service.update(**service_dict) 80 | else: 81 | # create a service 82 | service = Service.new_from_raw(service_dict) 83 | service.save() 84 | except Exception as e: 85 | message = {"exception": "{}".format(e)} 86 | print(message, service_dict) 87 | continue 88 | 89 | # score assets for risk based on their indicators: 90 | # get all assets, letting dynamorm do paging 91 | assets = Asset.scan(id__exists=True).recursive() 92 | for asset in assets: 93 | # recalc the score 94 | asset.score = 0 95 | indicators = Indicator.scan(asset_id__eq=asset.id) 96 | for indicator in indicators: 97 | if "likelihood_indicator" in indicator.to_dict(): 98 | if not "score" in asset.to_dict(): 99 | asset.score = risk_scores[ 100 | indicator.likelihood_indicator.strip().upper() 101 | ] 102 | else: 103 | asset.score = max( 104 | asset.score, 105 | risk_scores[indicator.likelihood_indicator.strip().upper()], 106 | ) 107 | asset.save() 108 | 109 | # write risks.json out for the heatmap 110 | # risks structure 111 | risks = {"services": []} 112 | risks["services"] = [ 113 | s.to_dict() for s in Service.scan(id__exists=True, masked__eq=False).recursive() 114 | ] 115 | for service in risks["services"]: 116 | if not "highest_risk_impact" in service: 117 | # the score is not set by an RRA, but by it's assets 118 | # reset it to zero to get current asset rollup 119 | service["score"] = 0 120 | # add asset groups for this service 121 | service["assetgroups"] = [ 122 | a.to_dict() 123 | for a in AssetGroup.scan( 124 | service_id__eq=service["id"], assets__exists=True 125 | ).recursive() 126 | ] 127 | 128 | for ag in service["assetgroups"]: 129 | # do we have assets? 130 | if "assets" in ag: 131 | # add assets for this asset group 132 | # they are stored in dynamo as just the ID 133 | # so replace the ID with the full record 134 | ag["assetids"] = ag["assets"] 135 | ag["assets"] = [] 136 | for assetid in ag["assetids"]: 137 | assets = [ 138 | a.to_dict() for a in Asset.scan(id__eq=assetid).recursive() 139 | ] 140 | for a in assets: 141 | # add indicators for this asset 142 | a["indicators"] = [ 143 | i.to_dict() 144 | for i in Indicator.scan(asset_id__eq=a["id"]).recursive() 145 | ] 146 | # finally append the asset with indicators to the asset group 147 | ag["assets"].append(a) 148 | # does this asset increase the service score? 149 | service["score"] = max(service["score"], a["score"]) 150 | 151 | # risks['assets'] = [a.to_dict() for a in Asset.scan(id__exists=True).recursive()] 152 | # risks['assetgroups'] = [a.to_dict() for a in AssetGroup.scan(id__exists=True).recursive()] 153 | # risks['indicators'] = [i.to_dict() for i in Indicator.scan(id__exists=True).recursive()] 154 | s3 = boto3.resource("s3") 155 | s3object = s3.Object(os.environ["RISKS_BUCKET_NAME"], os.environ["RISKS_KEY_NAME"]) 156 | s3object.put(Body=(bytes(json.dumps(risks).encode("UTF-8")))) 157 | -------------------------------------------------------------------------------- /models/v1/asset_groups/asset_group.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | from utils.utils import randuuid 5 | from flask import jsonify, request 6 | from dynamorm import DynaModel 7 | from dynamorm.indexes import GlobalIndex, ProjectKeys, ProjectAll 8 | from dynamorm.exceptions import ValidationError 9 | from flask_restplus import Namespace, Resource 10 | from datetime import datetime, timezone 11 | from schematics.models import Model 12 | from schematics.types import StringType as String, IntType as Number 13 | from schematics.types import ( 14 | DateTimeType, 15 | ModelType, 16 | BooleanType, 17 | BaseType, 18 | DictType, 19 | ListType, 20 | PolyModelType, 21 | ) 22 | from utils.auth import requires_auth 23 | 24 | api = Namespace( 25 | "asset_group", 26 | description="list asset groups (created through interllink.rules)", 27 | path="/api/v1/asset-group", 28 | ) 29 | 30 | 31 | class AssetGroup(DynaModel): 32 | class Table: 33 | name = "{env}-AssetGroups".format(env=os.environ.get("ENVIRONMENT", "dev")) 34 | hash_key = "id" 35 | 36 | resource_kwargs = {"region_name": os.environ.get("REGION", "us-west-2")} 37 | read = 5 38 | write = 5 39 | 40 | class Schema: 41 | id = String(default=randuuid) 42 | service_id = String() 43 | timestamp_utc = String(default=datetime.now(timezone.utc).isoformat()) 44 | name = String(required=True) 45 | description = String() 46 | assets = ListType(BaseType) 47 | 48 | 49 | # create table if needed 50 | inittable = AssetGroup(name="init") 51 | if not inittable.Table.exists: 52 | inittable.Table.create_table(wait=True) 53 | 54 | 55 | @api.route("/status") 56 | class status(Resource): 57 | @api.doc("a klingon test/status endpoint") 58 | def get(self): 59 | body = {"message": "Qapla'!"} 60 | return jsonify(body) 61 | 62 | 63 | # asset group creation is handled through the interlink.rules file uploaded to the s3 bucket 64 | # so no endpoint for creation/deletion 65 | @api.route("s/") 66 | @api.route("s/", defaults={"name": None}) 67 | class search(Resource): 68 | @api.doc( 69 | "/ partial or full asset group id to return all matches for this word/term" 70 | ) 71 | @requires_auth 72 | def get(self, name): 73 | try: 74 | asset_groups = [] 75 | 76 | if name is not None: 77 | for asset_group in AssetGroup.scan(name__contains=name): 78 | asset_groups.append(asset_group.to_dict()) 79 | else: 80 | for asset_group in AssetGroup.scan(name__exists=True): 81 | asset_groups.append(asset_group.to_dict()) 82 | 83 | return json.dumps(asset_group), 200 84 | except Exception as e: 85 | message = {"exception": "{}".format(e)} 86 | return json.dumps(message), 500 87 | 88 | 89 | @api.route("/") 90 | class specific(Resource): 91 | @api.doc("get /asset/uuid to retrieve a single asset group") 92 | @requires_auth 93 | def get(self, uuid): 94 | try: 95 | asset_groups = [] 96 | if uuid is not None: 97 | for asset_group in AssetGroup.scan(id__eq=uuid): 98 | asset_groups.append(asset_group.to_dict()) 99 | return json.dumps(asset_groups), 200 100 | except Exception as e: 101 | message = {"exception": "{}".format(e)} 102 | return json.dumps(message), 500 103 | -------------------------------------------------------------------------------- /models/v1/assets/asset.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | from utils.utils import randuuid 5 | from flask import jsonify, request 6 | from dynamorm import DynaModel 7 | from dynamorm.indexes import GlobalIndex, ProjectKeys, ProjectAll 8 | from dynamorm.exceptions import ValidationError 9 | from flask_restplus import Namespace, Resource 10 | from datetime import datetime, timezone 11 | from schematics.models import Model 12 | from schematics.types import StringType as String, IntType as Number 13 | from schematics.types import ( 14 | DateTimeType, 15 | ModelType, 16 | BooleanType, 17 | BaseType, 18 | DictType, 19 | ListType, 20 | PolyModelType, 21 | ) 22 | from utils.auth import requires_auth 23 | 24 | api = Namespace( 25 | "asset", description="create, list, update, delete asset", path="/api/v1/asset" 26 | ) 27 | 28 | 29 | class Asset(DynaModel): 30 | class Table: 31 | name = "{env}-Assets".format(env=os.environ.get("ENVIRONMENT", "dev")) 32 | hash_key = "id" 33 | 34 | resource_kwargs = {"region_name": os.environ.get("REGION", "us-west-2")} 35 | read = 5 36 | write = 5 37 | 38 | class Schema: 39 | id = String(default=randuuid) 40 | asset_type = String(required=True) 41 | asset_identifier = String(required=True) 42 | team = String() 43 | operator = String() 44 | zone = String(required=True) 45 | timestamp_utc = String(default=datetime.now(timezone.utc).isoformat()) 46 | description = String() 47 | score = Number(default=0) 48 | 49 | 50 | # create table if needed 51 | inittable = Asset(asset_type="init", asset_identifier="init", zone="init") 52 | if not inittable.Table.exists: 53 | inittable.Table.create_table(wait=True) 54 | 55 | 56 | @api.route("/status") 57 | class status(Resource): 58 | @api.doc("a klingon test/status endpoint") 59 | def get(self): 60 | body = {"message": "Qapla'!"} 61 | return jsonify(body) 62 | 63 | 64 | @api.route("s/") 65 | @api.route("s/", defaults={"identifier": None}) 66 | class search(Resource): 67 | @api.doc( 68 | "/ partial or full asset identifier to return all matches for this word/term" 69 | ) 70 | @requires_auth 71 | def get(self, identifier): 72 | try: 73 | assets = [] 74 | 75 | if identifier is not None: 76 | for asset in Asset.scan(asset_identifier__contains=identifier): 77 | assets.append(asset.to_dict()) 78 | else: 79 | for asset in Asset.scan(asset_identifier__exists=True): 80 | assets.append(asset.to_dict()) 81 | 82 | return json.dumps(assets), 200 83 | except Exception as e: 84 | message = {"exception": "{}".format(e)} 85 | return json.dumps(message), 500 86 | 87 | 88 | @api.route("/") 89 | class remove(Resource): 90 | @api.doc("get /asset/uuid to retrieve a single asset") 91 | @requires_auth 92 | def get(self, uuid): 93 | try: 94 | assets = [] 95 | if uuid is not None: 96 | for asset in Asset.scan(id__eq=uuid): 97 | assets.append(asset.to_dict()) 98 | return json.dumps(assets), 200 99 | except Exception as e: 100 | message = {"exception": "{}".format(e)} 101 | return json.dumps(message), 500 102 | 103 | @api.doc("delete /asset/uuid to remove an entry and it's indicators") 104 | @requires_auth 105 | def delete(self, uuid): 106 | from models.v1.indicators.indicator import Indicator 107 | 108 | try: 109 | assets = [] 110 | 111 | if uuid is not None: 112 | for asset in Asset.scan(id__eq=uuid): 113 | assets.append(asset.to_dict()) 114 | 115 | for indicator in Indicator.scan(asset_id__eq=uuid): 116 | indicator.delete() 117 | asset.delete() 118 | 119 | return json.dumps(assets), 200 120 | except Exception as e: 121 | message = {"exception": "{}".format(e)} 122 | return json.dumps(message), 500 123 | -------------------------------------------------------------------------------- /models/v1/indicators/indicator.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | from utils.utils import randuuid 5 | from flask import jsonify, request 6 | from dynamorm import DynaModel 7 | from dynamorm.indexes import GlobalIndex, ProjectKeys, ProjectAll 8 | from dynamorm.exceptions import ValidationError 9 | from flask_restplus import Namespace, Resource 10 | from datetime import datetime, timezone 11 | from schematics.models import Model 12 | from schematics.types import StringType as String, IntType as Number 13 | from schematics.types import ( 14 | DateTimeType, 15 | ModelType, 16 | BooleanType, 17 | BaseType, 18 | DictType, 19 | ListType, 20 | PolyModelType, 21 | ) 22 | from utils.auth import requires_auth 23 | 24 | api = Namespace( 25 | "indicator", 26 | description="create, list, update, delete indicator", 27 | path="/api/v1/indicator", 28 | ) 29 | 30 | # supporting Models for the details portion of the indicator 31 | class VulnerabilitySummary(Model): 32 | coverage = BooleanType() 33 | maximum = Number(default=0) 34 | high = Number(default=0) 35 | medium = Number(default=0) 36 | low = Number(default=0) 37 | 38 | 39 | class ObservatoryScore(Model): 40 | grade = BaseType(serialize_when_none=True) 41 | tests = ListType(BaseType) 42 | 43 | 44 | class DastVulnerabilities(Model): 45 | findings = ListType(BaseType) 46 | 47 | 48 | # helper function to figure out which 'details' we are working with: 49 | def claim_func(field, data): 50 | """ 51 | figure out which schema to use for the 'details' field 52 | """ 53 | if "coverage" in data: 54 | return VulnerabilitySummary 55 | elif "tests" in data: 56 | return ObservatoryScore 57 | elif "findings" in data: 58 | return DastVulnerabilities 59 | 60 | 61 | class Indicator(DynaModel): 62 | class Table: 63 | name = "{env}-Indicators".format(env=os.environ.get("ENVIRONMENT", "dev")) 64 | hash_key = "id" 65 | range_key = "asset_id" 66 | 67 | resource_kwargs = {"region_name": os.environ.get("REGION", "us-west-2")} 68 | read = 5 69 | write = 5 70 | 71 | # class ByAsset(GlobalIndex): 72 | # name = 'indicators-by-asset' 73 | # hash_key = 'asset_id' 74 | # range_key = 'description' 75 | # projection = ProjectAll() 76 | # read = 1 77 | # write = 1 78 | 79 | class Schema: 80 | id = String(default=randuuid) 81 | asset_id = String(required=False) 82 | timestamp_utc = String(default=datetime.now(timezone.utc).isoformat()) 83 | description = String() 84 | event_source_name = String() 85 | likelihood_indicator = String() 86 | details = PolyModelType( 87 | [ObservatoryScore, VulnerabilitySummary, DastVulnerabilities], 88 | claim_function=claim_func, 89 | ) 90 | 91 | 92 | # create table if needed 93 | inittable = Indicator(asset_id="init") 94 | if not inittable.Table.exists: 95 | inittable.Table.create_table(wait=True) 96 | 97 | 98 | @api.route("/status") 99 | class status(Resource): 100 | @api.doc("a klingon test/status endpoint") 101 | def get(self): 102 | body = {"message": "Qapla'!"} 103 | return jsonify(body) 104 | 105 | 106 | @api.route("", "/") 107 | class create(Resource): 108 | @api.doc( 109 | "post route to create a new indicator provided an asset's uuid or an asset_identifier (hostname)" 110 | ) 111 | @requires_auth 112 | def post(self): 113 | from models.v1.assets.asset import Asset 114 | 115 | try: 116 | post_data = request.get_json(force=True) 117 | try: 118 | # were we given an asset_id? 119 | asset_id = "" 120 | if "asset_id" not in post_data.keys(): 121 | # find the asset or create it 122 | assets = [ 123 | a 124 | for a in Asset.scan( 125 | asset_identifier__eq=post_data["asset_identifier"] 126 | ) 127 | ] 128 | if len(assets): 129 | asset_id = assets[0].id 130 | else: 131 | # create an asset 132 | asset = Asset.new_from_raw( 133 | { 134 | "asset_type": post_data["asset_type"], 135 | "asset_identifier": post_data["asset_identifier"], 136 | "zone": post_data["zone"], 137 | } 138 | ) 139 | asset.save() 140 | asset_id = asset.id 141 | post_data["asset_id"] = asset_id 142 | else: 143 | # asset_id was included in the post data, lets make sure it's valid 144 | assets = [a for a in Asset.scan(id__eq=post_data["asset_id"])] 145 | if len(assets): 146 | asset_id = assets[0].id 147 | else: 148 | raise ValueError("invalid asset_id, no matching asset found") 149 | 150 | # let dynamorn/marshmallow validate the data 151 | indicator = Indicator.new_from_raw(post_data) 152 | # find/remove any previous indicators of this type 153 | # to retain only the most recent indicator 154 | for oldindicator in Indicator.scan( 155 | asset_id__eq=asset_id, 156 | event_source_name__eq=indicator.event_source_name, 157 | ): 158 | oldindicator.delete() 159 | indicator.save() 160 | return json.dumps(indicator.to_dict()) 161 | except ValidationError as e: 162 | # api.abort(code=400,message=jsonify(e.errors)) 163 | return json.dumps(e.errors), 400 164 | 165 | except Exception as e: 166 | message = {"exception": "{}".format(e)} 167 | # api.abort(code=500,message=jsonify(message)) 168 | return json.dumps(message), 500 169 | 170 | 171 | # endpoint /indicators 172 | @api.route("/") 173 | @api.route("s", defaults={"id": None}) 174 | class list(Resource): 175 | @api.doc( 176 | "get /indicators to get all entries, hit /indicator/ for any partial or full UUID match" 177 | ) 178 | @requires_auth 179 | def get(self, id): 180 | try: 181 | indicators = [] 182 | if id is None: 183 | # return everything 184 | 185 | for indicator in Indicator.scan(id__exists=True): 186 | indicators.append(indicator.to_dict()) 187 | else: 188 | for indicator in Indicator.scan(id__contains=id): 189 | indicators.append(indicator.to_dict()) 190 | 191 | return json.dumps(indicators), 200 192 | except Exception as e: 193 | message = {"exception": "{}".format(e)} 194 | return json.dumps(message), 500 195 | 196 | @api.doc("delete /indicator/uuid to remove an entry") 197 | @requires_auth 198 | def delete(self, id): 199 | try: 200 | indicators = [] 201 | 202 | if id is not None: 203 | 204 | for indicator in Indicator.scan(id__eq=id): 205 | indicators.append(indicator.to_dict()) 206 | indicator.delete() 207 | 208 | return json.dumps(indicators), 200 209 | except Exception as e: 210 | message = {"exception": "{}".format(e)} 211 | return json.dumps(message), 500 212 | 213 | 214 | @api.route("s/") 215 | @api.route("s/", defaults={"identifier": None}) 216 | class search(Resource): 217 | @api.doc( 218 | "/ partial or full asset identifier to return all matches for this word/term" 219 | ) 220 | @requires_auth 221 | def get(self, identifier): 222 | from models.v1.assets.asset import Asset 223 | 224 | try: 225 | assets = [] 226 | indicators = [] 227 | 228 | if identifier is not None: 229 | for asset in Asset.scan(asset_identifier__contains=identifier): 230 | 231 | indicators.clear() 232 | for indicator in Indicator.scan(asset_id__eq=asset.id): 233 | indicators.append(indicator.to_dict()) 234 | 235 | returnAsset = asset.to_dict() 236 | returnAsset["indicators"] = indicators 237 | 238 | assets.append(returnAsset) 239 | 240 | return json.dumps(assets), 200 241 | except Exception as e: 242 | message = {"exception": "{}".format(e)} 243 | return json.dumps(message), 500 244 | -------------------------------------------------------------------------------- /models/v1/services/service.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | from utils.utils import randuuid 5 | from flask import jsonify, request 6 | from dynamorm import DynaModel 7 | from dynamorm.indexes import GlobalIndex, ProjectKeys, ProjectAll 8 | from dynamorm.exceptions import ValidationError 9 | from flask_restplus import Namespace, Resource 10 | from datetime import datetime, timezone 11 | from schematics.models import Model 12 | from schematics.types import StringType as String, IntType as Number 13 | from schematics.types import ( 14 | DateTimeType, 15 | ModelType, 16 | BooleanType, 17 | BaseType, 18 | DictType, 19 | ListType, 20 | PolyModelType, 21 | ) 22 | from utils.auth import requires_auth 23 | 24 | 25 | api = Namespace( 26 | "service", 27 | description="list services (creation handled through rra interface)", 28 | path="/api/v1/service", 29 | ) 30 | 31 | 32 | class Service(DynaModel): 33 | class Table: 34 | name = "{env}-Services".format(env=os.environ.get("ENVIRONMENT", "dev")) 35 | hash_key = "id" 36 | 37 | resource_kwargs = {"region_name": os.environ.get("REGION", "us-west-2")} 38 | read = 5 39 | write = 5 40 | 41 | class Schema: 42 | id = String(default=randuuid) 43 | timestamp_utc = String(default=datetime.now(timezone.utc).isoformat()) # 44 | name = String(required=True) 45 | link = String(required=False) 46 | masked = BooleanType(default=False) 47 | service_owner = String() 48 | director = String() 49 | service_data_classification = String() 50 | highest_risk_impact = String() 51 | recommendations = Number() 52 | highest_recommendation = String() 53 | creation_date = String() 54 | modification_date = String() 55 | score = Number(default=0) 56 | 57 | 58 | # create table if needed 59 | inittable = Service(name="init") 60 | if not inittable.Table.exists: 61 | inittable.Table.create_table(wait=True) 62 | 63 | 64 | @api.route("/status") 65 | class status(Resource): 66 | @api.doc("a klingon test/status endpoint") 67 | @requires_auth 68 | def get(self): 69 | try: 70 | body = {"message": "Qapla'!"} 71 | return jsonify(body) 72 | except Exception as e: 73 | message = {"exception": "{}".format(e)} 74 | return json.dumps(message), 500 75 | 76 | 77 | # service creation is handles through reading RRAs 78 | # so no endpoint for creation/deletion 79 | @api.route("s/") 80 | @api.route("s/", defaults={"name": None}) 81 | class search(Resource): 82 | @api.doc( 83 | "/ partial or full service name to return all matches for this word/term" 84 | ) 85 | @requires_auth 86 | def get(self, name): 87 | try: 88 | services = [] 89 | 90 | if name is not None: 91 | for service in Service.scan(name__contains=name): 92 | services.append(service.to_dict()) 93 | else: 94 | for service in Service.scan(name__exists=True): 95 | services.append(service.to_dict()) 96 | 97 | return json.dumps(services), 200 98 | except Exception as e: 99 | message = {"exception": "{}".format(e)} 100 | return json.dumps(message), 500 101 | 102 | 103 | @api.route("/") 104 | class specific(Resource): 105 | @api.doc("get /asset/uuid to retrieve a single asset group") 106 | @requires_auth 107 | def get(self, uuid): 108 | try: 109 | services = [] 110 | if uuid is not None: 111 | for service in Service.scan(id__eq=uuid): 112 | services.append(service.to_dict()) 113 | return json.dumps(services), 200 114 | except Exception as e: 115 | message = {"exception": "{}".format(e)} 116 | return json.dumps(message), 500 117 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "servicemapper", 3 | "version": "0.1.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@iarna/toml": { 8 | "version": "2.2.3", 9 | "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.3.tgz", 10 | "integrity": "sha512-FmuxfCuolpLl0AnQ2NHSzoUKWEJDFl63qXjzdoWBVyFCXzMGm1spBzk7LeHNoVCiWCF7mRVms9e6jEV9+MoPbg==", 11 | "dev": true 12 | }, 13 | "ansi-styles": { 14 | "version": "3.2.1", 15 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", 16 | "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", 17 | "dev": true, 18 | "requires": { 19 | "color-convert": "^1.9.0" 20 | } 21 | }, 22 | "appdirectory": { 23 | "version": "0.1.0", 24 | "resolved": "https://registry.npmjs.org/appdirectory/-/appdirectory-0.1.0.tgz", 25 | "integrity": "sha1-62yBYyDnsqsW9e2ZfyjYIF31Y3U=", 26 | "dev": true 27 | }, 28 | "async": { 29 | "version": "1.5.2", 30 | "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", 31 | "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", 32 | "dev": true 33 | }, 34 | "atob": { 35 | "version": "2.1.2", 36 | "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", 37 | "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" 38 | }, 39 | "aws-sdk": { 40 | "version": "2.543.0", 41 | "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.543.0.tgz", 42 | "integrity": "sha512-ABHsA4W7LLYnTBCgtbTt0NF7+66nMCtdcJuAn30+RGzF0bOw3lCguqzootFrMlIOXklddQFJ9kKTXd+p/+uCVQ==", 43 | "dev": true, 44 | "requires": { 45 | "buffer": "4.9.1", 46 | "events": "1.1.1", 47 | "ieee754": "1.1.13", 48 | "jmespath": "0.15.0", 49 | "querystring": "0.2.0", 50 | "sax": "1.2.1", 51 | "url": "0.10.3", 52 | "uuid": "3.3.2", 53 | "xml2js": "0.4.19" 54 | } 55 | }, 56 | "balanced-match": { 57 | "version": "1.0.0", 58 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 59 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", 60 | "dev": true 61 | }, 62 | "base64-js": { 63 | "version": "1.3.1", 64 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", 65 | "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", 66 | "dev": true 67 | }, 68 | "bluebird": { 69 | "version": "3.7.0", 70 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.0.tgz", 71 | "integrity": "sha512-aBQ1FxIa7kSWCcmKHlcHFlT2jt6J/l4FzC7KcPELkOJOsPOb/bccdhmIrKDfXhwFrmc7vDoDrrepFvGqjyXGJg==", 72 | "dev": true 73 | }, 74 | "brace-expansion": { 75 | "version": "1.1.11", 76 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 77 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 78 | "dev": true, 79 | "requires": { 80 | "balanced-match": "^1.0.0", 81 | "concat-map": "0.0.1" 82 | } 83 | }, 84 | "buffer": { 85 | "version": "4.9.1", 86 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", 87 | "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", 88 | "dev": true, 89 | "requires": { 90 | "base64-js": "^1.0.2", 91 | "ieee754": "^1.1.4", 92 | "isarray": "^1.0.0" 93 | } 94 | }, 95 | "chalk": { 96 | "version": "2.4.2", 97 | "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", 98 | "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", 99 | "dev": true, 100 | "requires": { 101 | "ansi-styles": "^3.2.1", 102 | "escape-string-regexp": "^1.0.5", 103 | "supports-color": "^5.3.0" 104 | } 105 | }, 106 | "color-convert": { 107 | "version": "1.9.3", 108 | "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", 109 | "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", 110 | "dev": true, 111 | "requires": { 112 | "color-name": "1.1.3" 113 | } 114 | }, 115 | "color-name": { 116 | "version": "1.1.3", 117 | "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", 118 | "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", 119 | "dev": true 120 | }, 121 | "concat-map": { 122 | "version": "0.0.1", 123 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 124 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", 125 | "dev": true 126 | }, 127 | "core-util-is": { 128 | "version": "1.0.2", 129 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 130 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", 131 | "dev": true 132 | }, 133 | "escape-string-regexp": { 134 | "version": "1.0.5", 135 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 136 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", 137 | "dev": true 138 | }, 139 | "events": { 140 | "version": "1.1.1", 141 | "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", 142 | "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", 143 | "dev": true 144 | }, 145 | "fs-extra": { 146 | "version": "7.0.1", 147 | "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", 148 | "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", 149 | "dev": true, 150 | "requires": { 151 | "graceful-fs": "^4.1.2", 152 | "jsonfile": "^4.0.0", 153 | "universalify": "^0.1.0" 154 | } 155 | }, 156 | "fs.realpath": { 157 | "version": "1.0.0", 158 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 159 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 160 | "dev": true 161 | }, 162 | "glob": { 163 | "version": "7.1.4", 164 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", 165 | "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", 166 | "dev": true, 167 | "requires": { 168 | "fs.realpath": "^1.0.0", 169 | "inflight": "^1.0.4", 170 | "inherits": "2", 171 | "minimatch": "^3.0.4", 172 | "once": "^1.3.0", 173 | "path-is-absolute": "^1.0.0" 174 | } 175 | }, 176 | "glob-all": { 177 | "version": "3.1.0", 178 | "resolved": "https://registry.npmjs.org/glob-all/-/glob-all-3.1.0.tgz", 179 | "integrity": "sha1-iRPd+17hrHgSZWJBsD1SF8ZLAqs=", 180 | "dev": true, 181 | "requires": { 182 | "glob": "^7.0.5", 183 | "yargs": "~1.2.6" 184 | } 185 | }, 186 | "graceful-fs": { 187 | "version": "4.2.2", 188 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz", 189 | "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==", 190 | "dev": true 191 | }, 192 | "has-flag": { 193 | "version": "3.0.0", 194 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", 195 | "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", 196 | "dev": true 197 | }, 198 | "hasbin": { 199 | "version": "1.2.3", 200 | "resolved": "https://registry.npmjs.org/hasbin/-/hasbin-1.2.3.tgz", 201 | "integrity": "sha1-eMWSaJPIAhXCtWiuH9P8q3omlrA=", 202 | "dev": true, 203 | "requires": { 204 | "async": "~1.5" 205 | } 206 | }, 207 | "ieee754": { 208 | "version": "1.1.13", 209 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", 210 | "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", 211 | "dev": true 212 | }, 213 | "immediate": { 214 | "version": "3.0.6", 215 | "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", 216 | "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", 217 | "dev": true 218 | }, 219 | "inflight": { 220 | "version": "1.0.6", 221 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 222 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 223 | "dev": true, 224 | "requires": { 225 | "once": "^1.3.0", 226 | "wrappy": "1" 227 | } 228 | }, 229 | "inherits": { 230 | "version": "2.0.4", 231 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 232 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", 233 | "dev": true 234 | }, 235 | "is-wsl": { 236 | "version": "2.1.1", 237 | "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.1.1.tgz", 238 | "integrity": "sha512-umZHcSrwlDHo2TGMXv0DZ8dIUGunZ2Iv68YZnrmCiBPkZ4aaOhtv7pXJKeki9k3qJ3RJr0cDyitcl5wEH3AYog==", 239 | "dev": true 240 | }, 241 | "isarray": { 242 | "version": "1.0.0", 243 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 244 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", 245 | "dev": true 246 | }, 247 | "jmespath": { 248 | "version": "0.15.0", 249 | "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", 250 | "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", 251 | "dev": true 252 | }, 253 | "jsonfile": { 254 | "version": "4.0.0", 255 | "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", 256 | "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", 257 | "dev": true, 258 | "requires": { 259 | "graceful-fs": "^4.1.6" 260 | } 261 | }, 262 | "jszip": { 263 | "version": "3.2.2", 264 | "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.2.2.tgz", 265 | "integrity": "sha512-NmKajvAFQpbg3taXQXr/ccS2wcucR1AZ+NtyWp2Nq7HHVsXhcJFR8p0Baf32C2yVvBylFWVeKf+WI2AnvlPhpA==", 266 | "dev": true, 267 | "requires": { 268 | "lie": "~3.3.0", 269 | "pako": "~1.0.2", 270 | "readable-stream": "~2.3.6", 271 | "set-immediate-shim": "~1.0.1" 272 | } 273 | }, 274 | "lie": { 275 | "version": "3.3.0", 276 | "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", 277 | "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", 278 | "dev": true, 279 | "requires": { 280 | "immediate": "~3.0.5" 281 | } 282 | }, 283 | "lodash": { 284 | "version": "4.17.15", 285 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", 286 | "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", 287 | "dev": true 288 | }, 289 | "lodash.get": { 290 | "version": "4.4.2", 291 | "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", 292 | "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", 293 | "dev": true 294 | }, 295 | "lodash.set": { 296 | "version": "4.3.2", 297 | "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", 298 | "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=", 299 | "dev": true 300 | }, 301 | "lodash.uniqby": { 302 | "version": "4.7.0", 303 | "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", 304 | "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=", 305 | "dev": true 306 | }, 307 | "lodash.values": { 308 | "version": "4.3.0", 309 | "resolved": "https://registry.npmjs.org/lodash.values/-/lodash.values-4.3.0.tgz", 310 | "integrity": "sha1-o6bCsOvsxcLLocF+bmIP6BtT00c=", 311 | "dev": true 312 | }, 313 | "minimatch": { 314 | "version": "3.0.4", 315 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 316 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 317 | "dev": true, 318 | "requires": { 319 | "brace-expansion": "^1.1.7" 320 | } 321 | }, 322 | "minimist": { 323 | "version": "0.1.0", 324 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.1.0.tgz", 325 | "integrity": "sha1-md9lelJXTCHJBXSX33QnkLK0wN4=", 326 | "dev": true 327 | }, 328 | "once": { 329 | "version": "1.4.0", 330 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 331 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 332 | "dev": true, 333 | "requires": { 334 | "wrappy": "1" 335 | } 336 | }, 337 | "pako": { 338 | "version": "1.0.10", 339 | "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.10.tgz", 340 | "integrity": "sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw==", 341 | "dev": true 342 | }, 343 | "path-is-absolute": { 344 | "version": "1.0.1", 345 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 346 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", 347 | "dev": true 348 | }, 349 | "process-nextick-args": { 350 | "version": "2.0.1", 351 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", 352 | "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", 353 | "dev": true 354 | }, 355 | "punycode": { 356 | "version": "1.3.2", 357 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", 358 | "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", 359 | "dev": true 360 | }, 361 | "querystring": { 362 | "version": "0.2.0", 363 | "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", 364 | "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", 365 | "dev": true 366 | }, 367 | "readable-stream": { 368 | "version": "2.3.6", 369 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", 370 | "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", 371 | "dev": true, 372 | "requires": { 373 | "core-util-is": "~1.0.0", 374 | "inherits": "~2.0.3", 375 | "isarray": "~1.0.0", 376 | "process-nextick-args": "~2.0.0", 377 | "safe-buffer": "~5.1.1", 378 | "string_decoder": "~1.1.1", 379 | "util-deprecate": "~1.0.1" 380 | } 381 | }, 382 | "rimraf": { 383 | "version": "2.7.1", 384 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", 385 | "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", 386 | "dev": true, 387 | "requires": { 388 | "glob": "^7.1.3" 389 | } 390 | }, 391 | "safe-buffer": { 392 | "version": "5.1.2", 393 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 394 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", 395 | "dev": true 396 | }, 397 | "sax": { 398 | "version": "1.2.1", 399 | "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", 400 | "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=", 401 | "dev": true 402 | }, 403 | "serverless-domain-manager": { 404 | "version": "3.3.0", 405 | "resolved": "https://registry.npmjs.org/serverless-domain-manager/-/serverless-domain-manager-3.3.0.tgz", 406 | "integrity": "sha512-oGZDNzI/Y15QusC/E+Zm6VaXoAwnbQnm6e6wyXPFDi1Sqoeo8xarKNj0fZODhEf4DKggAqxVfkiXjhuC8yZu0Q==", 407 | "dev": true, 408 | "requires": { 409 | "aws-sdk": "^2.490.0", 410 | "chalk": "^2.4.1" 411 | } 412 | }, 413 | "serverless-python-requirements": { 414 | "version": "5.0.0", 415 | "resolved": "https://registry.npmjs.org/serverless-python-requirements/-/serverless-python-requirements-5.0.0.tgz", 416 | "integrity": "sha512-WAruPzTQjI4aIsoSegT6l2O+VHXxqjxhHeBiCdorLxBPoCq1GJJrHr8WDczANX14Xgo9+qN8g3V7FxgIzV5jKg==", 417 | "dev": true, 418 | "requires": { 419 | "@iarna/toml": "^2.2.3", 420 | "appdirectory": "^0.1.0", 421 | "bluebird": "^3.0.6", 422 | "fs-extra": "^7.0.0", 423 | "glob-all": "^3.1.0", 424 | "is-wsl": "^2.0.0", 425 | "jszip": "^3.1.0", 426 | "lodash.get": "^4.4.2", 427 | "lodash.set": "^4.3.2", 428 | "lodash.uniqby": "^4.0.0", 429 | "lodash.values": "^4.3.0", 430 | "rimraf": "^2.6.2", 431 | "sha256-file": "1.0.0", 432 | "shell-quote": "^1.6.1" 433 | } 434 | }, 435 | "serverless-wsgi": { 436 | "version": "1.7.3", 437 | "resolved": "https://registry.npmjs.org/serverless-wsgi/-/serverless-wsgi-1.7.3.tgz", 438 | "integrity": "sha512-LKzAbLBFam9m7Iu9t5/Oa0AVfd9qRDRNKkrG35h2enlourDebooia3p638bu0nCOqs4VTQ2IqdBlv8devjFYGQ==", 439 | "dev": true, 440 | "requires": { 441 | "bluebird": "^3.5.5", 442 | "fs-extra": "^8.1.0", 443 | "hasbin": "^1.2.3", 444 | "lodash": "^4.17.15" 445 | }, 446 | "dependencies": { 447 | "fs-extra": { 448 | "version": "8.1.0", 449 | "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", 450 | "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", 451 | "dev": true, 452 | "requires": { 453 | "graceful-fs": "^4.2.0", 454 | "jsonfile": "^4.0.0", 455 | "universalify": "^0.1.0" 456 | } 457 | } 458 | } 459 | }, 460 | "set-immediate-shim": { 461 | "version": "1.0.1", 462 | "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", 463 | "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", 464 | "dev": true 465 | }, 466 | "sha256-file": { 467 | "version": "1.0.0", 468 | "resolved": "https://registry.npmjs.org/sha256-file/-/sha256-file-1.0.0.tgz", 469 | "integrity": "sha512-nqf+g0veqgQAkDx0U2y2Tn2KWyADuuludZTw9A7J3D+61rKlIIl9V5TS4mfnwKuXZOH9B7fQyjYJ9pKRHIsAyg==", 470 | "dev": true 471 | }, 472 | "shell-quote": { 473 | "version": "1.7.2", 474 | "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.2.tgz", 475 | "integrity": "sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg==", 476 | "dev": true 477 | }, 478 | "string_decoder": { 479 | "version": "1.1.1", 480 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 481 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 482 | "dev": true, 483 | "requires": { 484 | "safe-buffer": "~5.1.0" 485 | } 486 | }, 487 | "supports-color": { 488 | "version": "5.5.0", 489 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", 490 | "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", 491 | "dev": true, 492 | "requires": { 493 | "has-flag": "^3.0.0" 494 | } 495 | }, 496 | "universalify": { 497 | "version": "0.1.2", 498 | "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", 499 | "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", 500 | "dev": true 501 | }, 502 | "url": { 503 | "version": "0.10.3", 504 | "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", 505 | "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", 506 | "dev": true, 507 | "requires": { 508 | "punycode": "1.3.2", 509 | "querystring": "0.2.0" 510 | } 511 | }, 512 | "util-deprecate": { 513 | "version": "1.0.2", 514 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 515 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", 516 | "dev": true 517 | }, 518 | "uuid": { 519 | "version": "3.3.2", 520 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", 521 | "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", 522 | "dev": true 523 | }, 524 | "wrappy": { 525 | "version": "1.0.2", 526 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 527 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", 528 | "dev": true 529 | }, 530 | "xml2js": { 531 | "version": "0.4.19", 532 | "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", 533 | "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", 534 | "dev": true, 535 | "requires": { 536 | "sax": ">=0.6.0", 537 | "xmlbuilder": "~9.0.1" 538 | } 539 | }, 540 | "xmlbuilder": { 541 | "version": "9.0.7", 542 | "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", 543 | "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", 544 | "dev": true 545 | }, 546 | "yargs": { 547 | "version": "1.2.6", 548 | "resolved": "https://registry.npmjs.org/yargs/-/yargs-1.2.6.tgz", 549 | "integrity": "sha1-nHtKgv1dWVsr8Xq23MQxNUMv40s=", 550 | "dev": true, 551 | "requires": { 552 | "minimist": "^0.1.0" 553 | } 554 | } 555 | } 556 | } 557 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "servicemapper", 3 | "description": "", 4 | "version": "0.1.0", 5 | "dependencies": { 6 | "atob": "^2.1.2" 7 | }, 8 | "devDependencies": { 9 | "serverless-domain-manager": "^3.3.0", 10 | "serverless-python-requirements": "^5.0.0", 11 | "serverless-wsgi": "^1.7.3" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: servicemapper 2 | resources: 3 | Description: > 4 | A set of micro serverless services that marry services, their risk levels, 5 | indicators about the security posture of the service and various control 6 | metrics into an ongoing risk score for each service based on what we know 7 | about it. https://github.com/mozilla/service-map 8 | custom: 9 | s3_bucket: mozilla-service-map-${opt:stage, self:provider.stage} 10 | s3_bucket_arn: arn:aws:s3:::${self:custom.s3_bucket} 11 | s3_risk_bucket: ${file(config.${self:provider.stage}.yml):RISKS_BUCKET_NAME} 12 | s3_risk_bucket_arn: arn:aws:s3:::${self:custom.s3_risk_bucket} 13 | pythonRequirements: 14 | usePipenv: true 15 | dockerizePip: true 16 | wsgi: 17 | app: api.app 18 | packRequirements: false 19 | customDomain: 20 | domainName: ${file(config.${self:provider.stage}.yml):DOMAIN_NAME} 21 | stage: ${self:provider.stage} 22 | createRoute53Record: true 23 | hostedZoneId: ${file(config.${self:provider.stage}.yml):ZONE_ID} 24 | certificateArn: ${file(config.${self:provider.stage}.yml):CERTIFICATE_ARN} 25 | endpointType: "regional" 26 | enabled: true 27 | logRetentionInDays: 30 28 | 29 | provider: 30 | name: aws 31 | runtime: python3.7 32 | stage: ${opt:stage,'dev'} 33 | region: us-west-2 34 | environment: 35 | INDICATOR_TABLE: ${opt:stage, self:provider.stage}-Indicators 36 | ASSET_TABLE: ${opt:stage, self:provider.stage}-Assets 37 | ASSET_GROUP_TABLE: ${opt:stage, self:provider.stage}-AssetGroups 38 | SERVICE_TABLE: ${opt:stage, self:provider.stage}-Services 39 | ENVIRONMENT: ${self:provider.stage} 40 | REGION: ${opt:region, self:provider.region} 41 | CONFIGFILE: config.${self:provider.stage}.yml 42 | RISKS_BUCKET_NAME: ${file(config.${self:provider.stage}.yml):RISKS_BUCKET_NAME} 43 | RISKS_KEY_NAME: ${file(config.${self:provider.stage}.yml):RISKS_KEY_NAME} 44 | iamRoleStatements: 45 | - Effect: Allow 46 | Action: 47 | - acm:ListCertificates 48 | Resource: "*" 49 | - Effect: Allow 50 | Action: 51 | - route53:ChangeResourceRecordSets 52 | - route53:GetHostedZone 53 | - route53:ListResourceRecordSets 54 | Resource: "arn:aws:route53:::hostedzone/${self:custom.customDomain.hostedZoneId}" 55 | - Effect: Allow 56 | Action: 57 | - route53:ListHostedZones 58 | Resource: "*" 59 | - Effect: Allow 60 | Action: 61 | - cloudfront:UpdateDistribution 62 | Resource: "*" 63 | - Effect: Allow 64 | Action: 65 | - apigateway:POST 66 | Resource: "arn:aws:apigateway:${self:provider.region}::/domainnames" 67 | - Effect: Allow 68 | Action: 69 | - apigateway:GET 70 | - apigateway:DELETE 71 | Resource: "arn:aws:apigateway:${self:provider.region}::/domainnames/*" 72 | - Effect: Allow 73 | Action: 74 | - apigateway:POST 75 | Resource: "arn:aws:apigateway:${self:provider.region}::/domainnames/*" 76 | - Effect: Allow 77 | Action: 78 | - dynamodb:Query 79 | - dynamodb:Scan 80 | - dynamodb:GetItem 81 | - dynamodb:BatchGetItem 82 | - dynamodb:PutItem 83 | - dynamodb:UpdateItem 84 | - dynamodb:DeleteItem 85 | - dynamodb:DescribeTable 86 | - dynamodb:CreateTable 87 | - dynamodb:UpdateTable 88 | - dynamodb:GetRecords 89 | - dynamodb:BatchWriteItem 90 | - dynamodb:DescribeLimits 91 | - dynamodb:UpdateTimeToLive 92 | Resource: "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/${self:provider.environment.INDICATOR_TABLE}" 93 | - Effect: Allow 94 | Action: 95 | - dynamodb:Query 96 | - dynamodb:Scan 97 | - dynamodb:GetItem 98 | - dynamodb:BatchGetItem 99 | - dynamodb:PutItem 100 | - dynamodb:UpdateItem 101 | - dynamodb:DeleteItem 102 | - dynamodb:DescribeTable 103 | - dynamodb:CreateTable 104 | - dynamodb:UpdateTable 105 | - dynamodb:GetRecords 106 | - dynamodb:BatchWriteItem 107 | - dynamodb:DescribeLimits 108 | - dynamodb:UpdateTimeToLive 109 | Resource: "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/${self:provider.environment.ASSET_TABLE}" 110 | - Effect: Allow 111 | Action: 112 | - dynamodb:Query 113 | - dynamodb:Scan 114 | - dynamodb:GetItem 115 | - dynamodb:BatchGetItem 116 | - dynamodb:PutItem 117 | - dynamodb:UpdateItem 118 | - dynamodb:DeleteItem 119 | - dynamodb:DescribeTable 120 | - dynamodb:CreateTable 121 | - dynamodb:UpdateTable 122 | - dynamodb:GetRecords 123 | - dynamodb:BatchWriteItem 124 | - dynamodb:DescribeLimits 125 | - dynamodb:UpdateTimeToLive 126 | Resource: "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/${self:provider.environment.ASSET_GROUP_TABLE}" 127 | - Effect: Allow 128 | Action: 129 | - dynamodb:Query 130 | - dynamodb:Scan 131 | - dynamodb:GetItem 132 | - dynamodb:BatchGetItem 133 | - dynamodb:PutItem 134 | - dynamodb:UpdateItem 135 | - dynamodb:DeleteItem 136 | - dynamodb:DescribeTable 137 | - dynamodb:CreateTable 138 | - dynamodb:UpdateTable 139 | - dynamodb:GetRecords 140 | - dynamodb:BatchWriteItem 141 | - dynamodb:DescribeLimits 142 | - dynamodb:UpdateTimeToLive 143 | Resource: "arn:aws:dynamodb:${opt:region, self:provider.region}:*:table/${self:provider.environment.SERVICE_TABLE}" 144 | - Effect: Allow 145 | Action: 146 | - dynamodb:GetItem 147 | - dynamodb:Query 148 | - dynamodb:Scan 149 | Resource: "arn:aws:dynamodb:us-east-1:*:table/credential-store" 150 | - Effect: Allow 151 | Action: 152 | - kms:Decrypt 153 | Resource: "arn:aws:kms:us-east-1:*:key/${file(config.${self:provider.stage}.yml):KMSGUID}" 154 | - Effect: Allow 155 | Action: 156 | - dynamodb:ListTables 157 | - dynamodb:DescribeLimits 158 | - dynamodb:DescribeReservedCapacity 159 | - dynamodb:DescribeReservedCapacityOfferings 160 | Resource: "*" 161 | - Effect: Allow 162 | Action: 163 | - s3:* 164 | Resource: 165 | - ${self:custom.s3_bucket_arn}/* 166 | - ${self:custom.s3_risk_bucket_arn}/* 167 | 168 | 169 | functions: 170 | api: 171 | handler: wsgi.handler 172 | timeout: 60 173 | events: 174 | - http: ANY / 175 | - http: ANY {proxy+} 176 | 177 | bucket: 178 | handler: bucket.event 179 | timeout: 600 180 | events: 181 | - s3: 182 | bucket: ${self:custom.s3_bucket} 183 | event: s3:ObjectCreated:* 184 | cron: 185 | handler: cron.event 186 | timeout: 600 187 | events: 188 | - schedule: rate(1 hour) 189 | 190 | plugins: 191 | - serverless-python-requirements 192 | - serverless-wsgi 193 | - serverless-domain-manager 194 | - serverless-plugin-log-retention 195 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def pytest_configure(): 5 | pytest.testvalues = dict() 6 | pytest.testvalues['asset_id'] = None 7 | 8 | def pytest_runtest_makereport(item, call): 9 | if "incremental" in item.keywords: 10 | if call.excinfo is not None: 11 | parent = item.parent 12 | parent._previousfailed = item 13 | 14 | 15 | def pytest_runtest_setup(item): 16 | if "incremental" in item.keywords: 17 | previousfailed = getattr(item.parent, "_previousfailed", None) 18 | if previousfailed is not None: 19 | pytest.xfail("previous test failed (%s)" % previousfailed.name) -------------------------------------------------------------------------------- /tests/test_rest.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import requests 5 | import pytest 6 | 7 | # export a API_URL environment varialble to be something like: 8 | # API_URL="https://something.execute-api.us-west-2.amazonaws.com/dev/" 9 | API_URL = os.environ.get("API_URL", None) 10 | AUTH0_URL = os.environ.get( 11 | "AUTH0_URL", "https://auth-dev.mozilla.auth0.com/oauth/token" 12 | ) 13 | CLIENT_ID = os.environ.get("CLIENT_ID", None) 14 | CLIENT_SECRET = os.environ.get("CLIENT_SECRET", None) 15 | 16 | if API_URL is None: 17 | pytest.fail("Missing API_URL environment variable") 18 | 19 | if CLIENT_ID is None: 20 | pytest.fail("Missing CLIENT_ID environment variable") 21 | 22 | if CLIENT_SECRET is None: 23 | pytest.fail("Missing CLIENT_SECRET environment variable") 24 | 25 | 26 | @pytest.mark.incremental 27 | class TestEnvironment(object): 28 | def test_api_url_environtment_variable(self): 29 | assert API_URL is not None 30 | 31 | def test_client_id_environment_variable(self): 32 | assert CLIENT_ID is not None 33 | 34 | def test_client_secret_environment_variable(self): 35 | assert CLIENT_SECRET is not None 36 | 37 | 38 | if not API_URL.endswith("/"): 39 | API_URL = API_URL + "/" 40 | 41 | # get api key: 42 | r = requests.post( 43 | AUTH0_URL, 44 | headers={"content-type": "application/json"}, 45 | data=json.dumps( 46 | { 47 | "grant_type": "client_credentials", 48 | "client_id": CLIENT_ID, 49 | "client_secret": CLIENT_SECRET, 50 | "audience": API_URL, 51 | } 52 | ), 53 | ) 54 | access_token = r.json()["access_token"] 55 | 56 | 57 | class TestStatus(object): 58 | def test_api_status(self): 59 | r = requests.get("{}status".format(API_URL)) 60 | assert r.json()["message"] == "Qapla'!" 61 | 62 | def test_asset_status(self): 63 | r = requests.get("{}api/v1/asset/status".format(API_URL)) 64 | assert r.json()["message"] == "Qapla'!" 65 | 66 | def test_indicator_status(self): 67 | r = requests.get("{}api/v1/indicator/status".format(API_URL)) 68 | assert r.json()["message"] == "Qapla'!" 69 | 70 | def test_asset_group_status(self): 71 | r = requests.get("{}api/v1/asset-group/status".format(API_URL)) 72 | assert r.json()["message"] == "Qapla'!" 73 | 74 | def test_service_status(self): 75 | r = requests.get( 76 | "{}api/v1/service/status".format(API_URL), 77 | headers={"Authorization": "Bearer {}".format(access_token)}, 78 | ) 79 | assert r.json()["message"] == "Qapla'!" 80 | 81 | 82 | class TestMissing(object): 83 | def test_nonexistent_asset(self): 84 | r = requests.get( 85 | "{}api/v1/assets/hereisathingthatshouldnotexist".format(API_URL), 86 | headers={"Authorization": "Bearer {}".format(access_token)}, 87 | ) 88 | result = json.loads(r.json()) 89 | assert len(result) == 0 90 | 91 | def test_nonexistent_indicator(self): 92 | r = requests.get( 93 | "{}api/v1/indicators/hereisathingthatshouldnotexist".format(API_URL), 94 | headers={"Authorization": "Bearer {}".format(access_token)}, 95 | ) 96 | result = json.loads(r.json()) 97 | assert len(result) == 0 98 | 99 | def test_nonexistent_asset_group(self): 100 | r = requests.get( 101 | "{}api/v1/asset-group/hereisathingthatshouldnotexist".format(API_URL), 102 | headers={"Authorization": "Bearer {}".format(access_token)}, 103 | ) 104 | result = json.loads(r.json()) 105 | assert len(result) == 0 106 | 107 | @pytest.mark.incremental 108 | class TestAsset(object): 109 | def __init__(self): 110 | self.asset_id = None 111 | 112 | def test_adding_asset_through_scanapi_indicator(self): 113 | r = requests.post( 114 | "{}api/v1/indicator".format(API_URL), 115 | headers={"Authorization": "Bearer {}".format(access_token)}, 116 | data=json.dumps( 117 | { 118 | "asset_identifier": "pytest.testing.com", 119 | "asset_type": "website", 120 | "zone": "pytest", 121 | "description": "scanapi vulnerability result", 122 | "event_source_name": "scanapi", 123 | "likelihood_indicator": "high", 124 | "details": { 125 | "coverage": True, 126 | "maximum": 0, 127 | "high": 1, 128 | "medium": 6, 129 | "low": 8, 130 | }, 131 | } 132 | ), 133 | ) 134 | 135 | print(r.json()) 136 | result = json.loads(r.json()) 137 | self.asset_id = result["asset_id"] 138 | print("Test created asset_id: {}".format(self.asset_id)) 139 | assert self.asset_id is not None 140 | 141 | def test_adding_ZAP_scan_indicator(self): 142 | r = requests.post( 143 | "{}api/v1/indicator".format(API_URL), 144 | headers={"Authorization": "Bearer {}".format(access_token)}, 145 | data=json.dumps( 146 | { 147 | "asset_type": "website", 148 | "asset_identifier": "pytest.testing.com", 149 | "zone": "pytest", 150 | "description": "ZAP DAST scan", 151 | "event_source_name": "ZAP DAST scan", 152 | "likelihood_indicator": "medium", 153 | "details": { 154 | "findings": [ 155 | { 156 | "name": "Cookie No HttpOnly Flag", 157 | "site": "pytest.testing.com", 158 | "likelihood_indicator": "low", 159 | }, 160 | { 161 | "name": "Cross-Domain Javascript Source File Inclusion", 162 | "site": "pytest.testing.com", 163 | "likelihood_indicator": "low", 164 | }, 165 | { 166 | "name": "CSP scanner: script-src unsafe-inline", 167 | "site": "pytest.testing.com", 168 | "likelihood_indicator": "medium", 169 | }, 170 | ] 171 | }, 172 | } 173 | ), 174 | ) 175 | print(r.json()) 176 | result = json.loads(r.json()) 177 | assert self.asset_id == result["asset_id"] 178 | 179 | def test_adding_observatory_indicator(self): 180 | r = requests.post( 181 | "{}api/v1/indicator".format(API_URL), 182 | headers={"Authorization": "Bearer {}".format(access_token)}, 183 | data=json.dumps( 184 | { 185 | "asset_type": "website", 186 | "asset_identifier": "pytest.testing.com", 187 | "zone": "pytest", 188 | "description": "Mozilla Observatory scan", 189 | "event_source_name": "Mozilla Observatory", 190 | "likelihood_indicator": "medium", 191 | "details": { 192 | "grade": "F", 193 | "tests": [ 194 | {"name": "Content security policy", "pass": False}, 195 | {"name": "Cookies", "pass": True}, 196 | {"name": "HTTP Public Key Pinning", "pass": True}, 197 | {"name": "X-Frame-Options", "pass": False}, 198 | {"name": "Cross-origin Resource Sharing", "pass": True}, 199 | ], 200 | }, 201 | } 202 | ), 203 | ) 204 | print(r.json()) 205 | result = json.loads(r.json()) 206 | assert self.asset_id == result["asset_id"] 207 | 208 | def test_adding_observatory_api_indicator(self): 209 | # api endpoints can have troublesome data types (grade: null, pass;null) 210 | # Fields with None as the value aren't stored in the dynamo table: 211 | # https://github.com/NerdWalletOSS/dynamorm/issues/57 212 | # so the schema needs to be watching to add missing fields back in as null in json, None in python 213 | r = requests.post( 214 | "{}api/v1/indicator".format(API_URL), 215 | headers={"Authorization": "Bearer {}".format(access_token)}, 216 | data=json.dumps( 217 | { 218 | "asset_type": "website", 219 | "asset_identifier": "pytest.testing.com", 220 | "zone": "pytest", 221 | "description": "Mozilla Observatory scan", 222 | "event_source_name": "Mozilla Observatory", 223 | "likelihood_indicator": "medium", 224 | "details": { 225 | "grade": None, 226 | "tests": [ 227 | {"name": "Content security policy", "pass": None}, 228 | {"name": "Cookies", "pass": True}, 229 | {"name": "HTTP Public Key Pinning", "pass": True}, 230 | {"name": "X-Frame-Options", "pass": None}, 231 | {"name": "Cross-origin Resource Sharing", "pass": True}, 232 | ], 233 | }, 234 | } 235 | ), 236 | ) 237 | print(r.json()) 238 | result = json.loads(r.json()) 239 | assert self.asset_id == result["asset_id"] 240 | assert result["details"]["grade"] is None 241 | 242 | def test_retrieving_asset(self): 243 | assert self.asset_id is not None 244 | print("retrieving asset with id: {}".format(self.asset_id)) 245 | r = requests.get( 246 | "{}api/v1/asset/{}".format(API_URL, self.asset_id), 247 | headers={"Authorization": "Bearer {}".format(access_token)}, 248 | ) 249 | result = json.loads(r.json()) 250 | print(r.json()) 251 | assert result[0]["id"] == self.asset_id 252 | 253 | def test_removing_asset(self): 254 | assert self.asset_id is not None 255 | r = requests.delete( 256 | "{}api/v1/asset/{}".format(API_URL, self.asset_id), 257 | headers={"Authorization": "Bearer {}".format(access_token)}, 258 | ) 259 | print(r.json()) 260 | assert len(r.json()) > 1 261 | -------------------------------------------------------------------------------- /utils/auth.py: -------------------------------------------------------------------------------- 1 | import json 2 | from jose import jwt 3 | from six.moves.urllib.request import urlopen 4 | from functools import wraps 5 | from flask import Flask, request, jsonify, _request_ctx_stack, abort 6 | from utils.utils import get_config 7 | 8 | CONFIG = get_config() 9 | 10 | AUTH0_DOMAIN = CONFIG("AUTH0_URL") 11 | API_AUDIENCE = CONFIG("AUDIENCE") 12 | ALGORITHMS = ["RS256"] 13 | 14 | # from https://auth0.com/docs/quickstart/backend/python/01-authorization#validate-access-tokens 15 | 16 | 17 | def get_token_auth_header(): 18 | """Obtains the Access Token from the Authorization Header 19 | """ 20 | auth = request.headers.get("Authorization", None) 21 | if not auth: 22 | abort(401, "Authorization header is expected") 23 | 24 | parts = auth.split() 25 | 26 | if parts[0].lower() != "bearer": 27 | abort(401, "Authorization header must start with Bearer") 28 | elif len(parts) == 1: 29 | abort(401, "Invalid header, token not found") 30 | elif len(parts) > 2: 31 | abort(401, 'Authorization header must be in the form of "Bearer token"') 32 | 33 | token = parts[1] 34 | return token 35 | 36 | 37 | def requires_auth(f): 38 | """Determines if the Access Token is valid 39 | """ 40 | 41 | @wraps(f) 42 | def decorated(*args, **kwargs): 43 | token = get_token_auth_header() 44 | jsonurl = urlopen("https://" + AUTH0_DOMAIN + "/.well-known/jwks.json") 45 | jwks = json.loads(jsonurl.read()) 46 | unverified_header = jwt.get_unverified_header(token) 47 | rsa_key = {} 48 | for key in jwks["keys"]: 49 | if key["kid"] == unverified_header["kid"]: 50 | rsa_key = { 51 | "kty": key["kty"], 52 | "kid": key["kid"], 53 | "use": key["use"], 54 | "n": key["n"], 55 | "e": key["e"], 56 | } 57 | if rsa_key: 58 | try: 59 | payload = jwt.decode( 60 | token, 61 | rsa_key, 62 | algorithms=ALGORITHMS, 63 | audience=API_AUDIENCE, 64 | issuer="https://" + AUTH0_DOMAIN + "/", 65 | ) 66 | except jwt.ExpiredSignatureError: 67 | abort(401, "Authorization token is expired") 68 | except jwt.JWTClaimsError: 69 | abort( 70 | 401, 71 | "Authorization claim is incorrect, please check audience and issuer", 72 | ) 73 | except Exception: 74 | abort(401, "Authorization header cannot be parsed") 75 | _request_ctx_stack.top.current_user = payload 76 | return f(*args, **kwargs) 77 | else: 78 | abort(401, "Authorization error, unable to find appropriate key") 79 | 80 | return decorated 81 | -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | import yaml 4 | from everett.manager import ConfigManager, ConfigDictEnv, ConfigOSEnv 5 | 6 | 7 | def get_config(): 8 | """ 9 | Environment/yml config vars: 10 | API_AUDIENCE 11 | AUTHO_URL 12 | """ 13 | # load our config file (if any) 14 | conf = yaml.load(open(os.environ.get("CONFIGFILE", "/dev/null"))) 15 | if conf is None: 16 | conf = dict() 17 | 18 | return ConfigManager([ConfigOSEnv(), ConfigDictEnv(conf)]) 19 | 20 | 21 | def randuuid(): 22 | return str(uuid.uuid4()) 23 | --------------------------------------------------------------------------------