├── .gitignore ├── LICENSE ├── README.md ├── anchore-inline.sh ├── clair-scanner-run.sh ├── clean.sh ├── dmscan.py ├── ecr-push-scanner.sh ├── get-images.sh ├── images ├── cli-list.png ├── cli-main-s.png ├── dashboard-s.png ├── heatmap-s.png ├── xl-repair-confirmation-s.png └── xl-repair-prompt-s.png ├── install-scanners.sh ├── nvd-cache.db ├── nvd_cache.py ├── requirements.txt ├── run-tests.sh ├── scanner_plugin.py ├── scanners.yml └── snyk-scanner.sh /.gitignore: -------------------------------------------------------------------------------- 1 | output/ 2 | test-data/ 3 | anchore-reports/ 4 | .idea/ 5 | 6 | *.json 7 | *.iml 8 | 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | *$py.class 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | pip-wheel-metadata/ 32 | share/python-wheels/ 33 | *.egg-info/ 34 | .installed.cfg 35 | *.egg 36 | MANIFEST 37 | 38 | # PyInstaller 39 | # Usually these files are written by a python script from a template 40 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 41 | *.manifest 42 | *.spec 43 | 44 | # Installer logs 45 | pip-log.txt 46 | pip-delete-this-directory.txt 47 | 48 | # Unit test / coverage reports 49 | htmlcov/ 50 | .tox/ 51 | .nox/ 52 | .coverage 53 | .coverage.* 54 | .cache 55 | nosetests.xml 56 | coverage.xml 57 | *.cover 58 | *.py,cover 59 | .hypothesis/ 60 | .pytest_cache/ 61 | 62 | # Translations 63 | *.mo 64 | *.pot 65 | 66 | # Django stuff: 67 | *.log 68 | local_settings.py 69 | db.sqlite3 70 | db.sqlite3-journal 71 | 72 | # Flask stuff: 73 | instance/ 74 | .webassets-cache 75 | 76 | # Scrapy stuff: 77 | .scrapy 78 | 79 | # Sphinx documentation 80 | docs/_build/ 81 | 82 | # PyBuilder 83 | target/ 84 | 85 | # Jupyter Notebook 86 | .ipynb_checkpoints 87 | 88 | # IPython 89 | profile_default/ 90 | ipython_config.py 91 | 92 | # pyenv 93 | .python-version 94 | 95 | # pipenv 96 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 97 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 98 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 99 | # install all needed dependencies. 100 | #Pipfile.lock 101 | 102 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 103 | __pypackages__/ 104 | 105 | # Celery stuff 106 | celerybeat-schedule 107 | celerybeat.pid 108 | 109 | # SageMath parsed files 110 | *.sage.py 111 | 112 | # Environments 113 | .env 114 | .venv 115 | env/ 116 | venv/ 117 | ENV/ 118 | env.bak/ 119 | venv.bak/ 120 | 121 | # Spyder project settings 122 | .spyderproject 123 | .spyproject 124 | 125 | # Rope project settings 126 | .ropeproject 127 | 128 | # mkdocs documentation 129 | /site 130 | 131 | # mypy 132 | .mypy_cache/ 133 | .dmypy.json 134 | dmypy.json 135 | 136 | # Pyre type checker 137 | .pyre/ 138 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021 John Sotiropoulos 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dmscan - docker-multi-scan 2 | A multi-scanner utility for docker images. It drives Clair, Anchore, Trivy, Snyk, Grype, AWS ECR push on scan. These scanners produce varying results, sometimes hit and miss; dmscan helps you review the results easily and decide how to address them. 3 | 4 | *dmscan* run the scans for all registered scanners and then consolidates the results in an excel spreadsheet report, which generates on the fly to help you better understand the results. The generated spreadsheet contains: 5 | 6 | - **Summary tables and charts** by scanner and severity with scanning times, and unique vulnerabilities and CVEs. For non-CVE vulnerabilities, dmscan will do an internet look up to find the referenced CVEs which uses to consolidate results. If no matching cve is found the original identifier (eg NWG-XYZ) is kept. 7 | 8 | ![summary dashboard](images/dashboard-s.png) 9 | 10 | - A **Vulnerability and Component Heatmap** which allows you to see at a glance what has been found and what has been ignored. Vulnerabilities have clickable urls to the security advisories and you can hover on the url to see the description and packages affected. 11 | 12 | ![heatmap](images/heatmap-s.png) 13 | 14 | - **Vulnerability and Component totals** for each scanner 15 | 16 | - **"Normalised results"** with the same column headers, severities mapped to use the same rates. The mapping is configurable and the default mapping 1) groups Informational, and Negligible with Low 2) Defcon1 to Critical and 3) Adds a number to make sorting easier. Normalised results have autofilter on by default and conditional formatting to colour code severities. 17 | 18 | - **Original results**, generated by the scanners 19 | 20 | # Setup # 21 | 22 | ## Prerequisites ## 23 | _dmscan_ requires 24 | 1. Linux/MacOS shell 25 | 2. Docker CLI 26 | 3. Python 3.7 or later 27 | 4. AWS CLI v2, only if you are going to include AWS ECR scans. In that case you will need to setup your AWS CL2 with a valid Access & Secret key for an account with ECR push and repo creation rights 28 | 5. A snyk account and token, if you are going to include snyk in the scans 29 | 30 | if you don't want to use AWS or Snyk then skip their pre-reqs and once the dmscan is isntalled, change the scanners.yml file to disable these two scanners. for more information see [here](#customising-dmscan) 31 | 32 | ##### It has been used/tested on #### 33 | - Ubuntu 18 and 20 34 | - MacOS 35 | - EC2 Linux on AWS 36 | - WSL2 on Windows 10 pro 37 | 38 | ## Installing the scanners 39 | You can manually install the scanners or clone this repository from github and **use the ./install-scanners.sh to install them in one go**: 40 | 41 | ``` 42 | sudo ./install-scanners.sh 43 | ``` 44 | (you sould run it as sudo otherwise some installs - snyk for instance - will fail) 45 | 46 | The script should already have execution permissions. If not, or you receive an permission denied error, give it execution permissions with 47 | ``` 48 | chmod +x install-scanners.sh 49 | ``` 50 | 51 | 52 | Once they are installed, you need to logon to snyk.io and find your token (under settings) then create an environment variable in your profile file (~/.profile, ~/.zprofile etc) as 53 | ``` 54 | export SNYK_TOKEN= 55 | ``` 56 | ## Installing dmscan 57 | Install the python packages required by running from the command line and in the project's directory (eg docker-multiscan) 58 | ``` 59 | pip install -r requirements.txt 60 | ``` 61 | 62 | or if your system uses pip3 as the version of pip for python 3 63 | ``` 64 | pip3 install -r requirements.txt 65 | ``` 66 | (you can find out if your pip is for Python 2.x or 3.x with pip --version) 67 | 68 | 69 | ## Using dmscan 70 | You can run dmscan using 71 | ``` 72 | python ./dmscan.py 73 | ``` 74 | 75 | or run it as a script 76 | ``` 77 | ./dmscan.py 78 | ``` 79 | 80 | To run it as a script, Linux/WSL2 users should create a symlink to python3 in their /usr/local/bin i.e. 81 | ``` 82 | sudo ln -s /usr/bin/python3 /usr/local/bin/python3 83 | ``` 84 | 85 | dmscan.py should already have executable permissions, but if you get a permission denied error, give it (and the dependent scripts) execution permissions with 86 | 87 | ``` 88 | chmod +x *.sh 89 | chmod +x dmscan.py 90 | ``` 91 | 92 | Because of the dependencies on other included shell scripts, you should only run dmscan under the project's directory. 93 | 94 | 95 | Running it with any parameters it will show the same screen as if you run it with -h or --help parameter 96 | 97 | ![cli](images/cli-main-s.png) 98 | 99 | You can list the registered scanners (in scanners.yml) by running 100 | ./dmscan.py -l 101 | 102 | ![cli list](images/cli-list.png) 103 | 104 | ### Running a multiscan 105 | To run a full multiscan you need to use 106 | ``` 107 | ./dmscan.py -i 108 | ``` 109 | where image name can include the image tag eg my/image:7.0.1 110 | If you don't specify tag latest will be assumed, and the image name will be reported as my/image:latest 111 | 112 | Please note that the first run, anchore-inline will take a longer time than usual as the script pulls a 6+ GB docker image. 113 | 114 | dmscan will display statistics and create a **spreadsheet report under the output subdirectory** (which it will create if it does not exist). The spreadhseet name will contain a timestamp so that you have all the scan results, rather than overwriting them. 115 | 116 | Normally dmscan suprsesses the output of each scanner. If you want to see detailed output run it as 117 | ``` 118 | ./dmscan.py -i -v 119 | ``` 120 | where -v is verbose mode. 121 | 122 | dmscan maintains a sql lite database with a cache of all the cve records from NVDs NIST which is used primarily for CSVS ratings. If you are not interested in these severity ratings, or you are off line you can disable this feature by running dmscan as 123 | ``` 124 | ./dmscan.py -i -of 125 | ``` 126 | Sometimes you'd want to run a quick scan for a subset of registered scanners. Instead of modifying the scanners.yml file, you can pass the -s or --scanners command line parameter 127 | This is a comma delimited list of ids for scanners to include. The numbers can be found when running ./dmscanner.py -l 128 | 129 | For instance if you only wanted to run a scan for snyk, grype, and trivy you'd run 130 | ``` 131 | ./dmscan.py -i -s 1,5,6 132 | ``` 133 | Invalid scanner ids will be ignored and a warning will be displayed. 134 | 135 | All options except -l and -h can be combined. e.g. 136 | ``` 137 | ./dmscan.py -i -s 1,5,6 -v -of 138 | ``` 139 | 140 | ### Customising dmscan 141 | scanners.yml is the configuration file driving dmscan and you can customise the following: 142 | 1. **disable a scanner** (e.g. if you don't use AWS ECR). you can do that by entering under the scanner name 143 | enabled: 144 | False 145 | You can re-enable a scanner by changing False to True or completely remove the enabled entry. 146 | 2. **change the level of logging** (*debug, info, warn, error, critical*) 147 | 3. **change the severity mappings** of each scanner to a common list 148 | 4. **customize the configuration of a scanner** under plugins. each scanner has its own plugin section 149 | 5. **define a new scanner** 150 | 151 | ## Known issues 152 | 1. Currently, when you first open the generated spreadsheet, excel prompts you to repair the workbook. 153 | 2. Error handling needs improvement. 154 | 3. Clair-scanner will fail on a system running PostgreSQL on the default port (5432). Please either stop temporarily your postgres service or reconfigure it to run on a different port. 155 | Issue #1 relates to the generated Urls, and the issue is being investigated. Until a fix is submitted, just say yes 156 | 157 | ![repair prompt](images/xl-repair-prompt-s.png) 158 | 159 | The repair is done and a confirmation of the repair is shown 160 | 161 | ![repair confirmation](images/xl-repair-confirmation-s.png) 162 | 163 | Save the repaired workbook. 164 | 165 | For issue #2, if you see 0 results from a scanner run the script with the individual scanner id (using the -s param) and verbose mode -v 166 | 167 | ## Where can I get help? 168 | Raise an issue in GitHub 169 | 170 | ## Can I contribute a fix or improvement? 171 | dmscan belongs to those who use it and want it improved. You encouraged and welcome to contribute fixes or improvements using pull requests. 172 | 173 | (c) John Sotiropoulos 2021. This software is licenced under Apache License 2.0 and is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. The full license notice can be found in the LICENSE file. 174 | -------------------------------------------------------------------------------- /anchore-inline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | IMAGE=$1 3 | curl -s https://ci-tools.anchore.io/inline_scan-latest | bash -s -- -r -t 1000 $IMAGE 4 | anchore_container=$(docker ps -a | grep inline-anchore-engine|cut -d" " -f1) 5 | docker stop $anchore_container 2>/dev/null 6 | docker rm $anchore_container 2>/dev/null 7 | -------------------------------------------------------------------------------- /clair-scanner-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | REPORT_FILE=$1 3 | CLAIR_SERVER=$2 4 | IMAGE=$3 5 | 6 | if [[ "$OSTYPE" == "darwin"* ]]; then 7 | LOCALHOST=host.docker.internal 8 | else 9 | LOCALHOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+') 10 | fi 11 | echo $LOCALHOST 12 | echo starting clair containers... 13 | docker start clair-db 2>/dev/null 14 | docker start clair 2>/dev/null 15 | 16 | echo scanning image 17 | rm -f clair.json 18 | echo "clair-scanner $REPORT_FILE $CLAIR_SERVER --ip=$LOCALHOST $IMAGE" 19 | clair-scanner $REPORT_FILE $CLAIR_SERVER --ip=$LOCALHOST $IMAGE 20 | 21 | echo stopping clair containers... 22 | docker stop clair-db 2>/dev/null 23 | docker stop clair 2>/dev/null 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /clean.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/clean.sh -------------------------------------------------------------------------------- /dmscan.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import logging 3 | import argparse 4 | import sys 5 | import re 6 | import os 7 | from _datetime import datetime 8 | 9 | import pandas as pd 10 | import yaml 11 | from pandas.api.types import is_numeric_dtype 12 | from scanner_plugin import ScannerPlugin 13 | import xlsxwriter.utility as xlsutil 14 | 15 | 16 | def config(yaml_filename): 17 | scanner_config = None 18 | with open(yaml_filename) as file: 19 | scanner_config = yaml.load(file, Loader=yaml.FullLoader) 20 | return scanner_config 21 | 22 | 23 | levels = { 24 | 'critical': logging.CRITICAL, 25 | 'error': logging.ERROR, 26 | 'warn': logging.WARNING, 27 | 'info': logging.INFO, 28 | 'debug': logging.DEBUG 29 | } 30 | logging_level = logging.INFO 31 | config = config('scanners.yml') 32 | if 'logging' in config: 33 | if 'level' in config['logging']: 34 | logging_level = levels[str(config['logging']['level']).lower()] 35 | log_format = '%(asctime)s.%(msecs)03d %(levelname)s] %(message)s' 36 | 37 | #logging.basicConfig(format=log_format, datefmt='%Y-%m-%d,%H:%M:%S', level=logging_level) 38 | file_handler = logging.FileHandler("dmscan.log-{}".format(str(datetime.now()))) 39 | 40 | logging.basicConfig(format=log_format, 41 | datefmt='%Y-%m-%d,%H:%M:%S', 42 | handlers=[logging.StreamHandler(),file_handler], 43 | level=logging_level) 44 | 45 | 46 | not_found_string = '-' 47 | columns = config['columns'] 48 | severities = config['severities'] 49 | severities_summaries = severities.copy() 50 | severities_summaries.append(not_found_string) 51 | severity_indices = {k: v + 1 for v, k in enumerate(reversed(severities))} 52 | severity_reverse_idx = dict([(value, key) for key, value in severity_indices.items()]) 53 | 54 | severity_mappings = config['severity-mappings'] 55 | 56 | cols = ['cve'] 57 | scanners = list(config['plugins'].keys()) 58 | cols.extend(scanners) 59 | cves = pd.DataFrame(columns=cols) 60 | 61 | cols = ['component'] 62 | cols.extend(scanners) 63 | components = pd.DataFrame(columns=cols) 64 | 65 | 66 | def calculate_composite_severity_rate(data_row): 67 | result = "" 68 | severity_rates = [] 69 | for item in data_row: 70 | if type(item) != str: 71 | severity_rates.append(item) 72 | severity_rates.sort(reverse=True) 73 | result = result.join(map(str, severity_rates)) 74 | return int(result) 75 | 76 | 77 | def merge_aggregates(aggregates, aggregate_name, merge_fields=None, fill_na_value=0): 78 | scanners = list(aggregates.keys()) 79 | if merge_fields is None: 80 | merge_fields = aggregate_name 81 | summary_df = pd.DataFrame() 82 | for name in scanners: 83 | df = aggregates[name] 84 | if type(df) is not pd.DataFrame: 85 | df = pd.DataFrame(df) 86 | df.rename(columns={aggregate_name: name}, inplace=True) 87 | if summary_df.empty: 88 | summary_df = df 89 | else: 90 | summary_df = pd.merge(summary_df, df, on=merge_fields, how="outer") 91 | for col in summary_df.columns: 92 | summary_df[col].fillna(fill_na_value, inplace=True) 93 | if is_numeric_dtype(summary_df[col]): 94 | summary_df[col] = summary_df[col].astype(int) 95 | return summary_df 96 | 97 | 98 | def populate_totals(scanner, totals, scan_time, group_by_severity_results, unique_vulns, unique_cves, 99 | unique_components): 100 | totals_row = [scanner, scan_time, unique_vulns, unique_cves, unique_components] 101 | for col in severities: 102 | if col in group_by_severity_results.index: 103 | totals_row.append(group_by_severity_results[col]) 104 | else: 105 | totals_row.append(0) 106 | 107 | totals.loc[len(totals.index)] = totals_row 108 | 109 | 110 | def slugify(value, allow_unicode=False): 111 | """ 112 | adapted from https://github.com/django/django/blob/master/django/utils/text.py 113 | Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated 114 | dashes to single dashes. Remove characters that aren't alphanumerics, 115 | underscores, or hyphens. Convert to lowercase. Also strip leading and 116 | trailing whitespace, dashes, and underscores. 117 | """ 118 | value = str(value) 119 | value = value.replace(":", "_") 120 | value = value.replace("/", "_") 121 | value = re.sub(r'[^\w\s-]', '', value.lower()) 122 | return re.sub(r'[-\s]+', '-', value).strip('-_') 123 | 124 | 125 | def get_valid_value(series): 126 | result = "" 127 | for value in series: 128 | if pd.isnull(value) or value == "": 129 | continue 130 | else: 131 | result = value 132 | break 133 | return result 134 | 135 | 136 | def cve_link(cve, df): 137 | url = "" 138 | hint = "" 139 | try: 140 | url = df[df.cve == '{}'.format(cve)].link.iloc[0] 141 | description = get_valid_value(df[df.cve == '{}'.format(cve)].description) 142 | components = sorted(set(df[df.cve == '{}'.format(cve)].component.to_list())) 143 | hint = " ".join(components) + "\n" + description 144 | except Exception as ex: 145 | logging.error(cve + str(ex)) 146 | url = "https://www.google.com/search?q=" + cve 147 | return url, hint 148 | 149 | 150 | def write_sheet(writer, ref_data, df, prefix, name, header_format, format_values, format_styles): 151 | worksheet_name = '{}{}'.format(prefix, name) 152 | df.to_excel(writer, sheet_name=worksheet_name, index=False) 153 | # Get the dimensions of the dataframe 154 | (max_row, max_col) = df.shape 155 | cols = df.columns 156 | if max_col > 0: 157 | worksheet = writer.sheets[worksheet_name] 158 | # Make the columns wider for clarity 159 | i = 0 160 | for col in cols: 161 | values = df[col].map(str).to_list() 162 | values.append(col) 163 | width = len(max(values, key=len)) 164 | if width < 20: 165 | width = width + 2 166 | else: 167 | width = len(col) + 2 168 | worksheet.set_column(i, i, width) 169 | i += 1 170 | worksheet.set_row(0, None, header_format) 171 | # Set the autofilter 172 | worksheet.autofilter(0, 0, max_row, max_col - 1) 173 | i = 0 174 | for value in format_values: 175 | worksheet.conditional_format(0, 0, max_row, max_col - 1, 176 | {'type': 'cell', 177 | 'criteria': '=', 178 | 'value': '"' + value + '"', 179 | 'format': format_styles[i]}) 180 | i += 1 181 | if name == 'totals': 182 | # scan time 183 | # Create a new chart object. 184 | scan_chart = writer.book.add_chart({'type': 'bar'}) 185 | vuln_chart = writer.book.add_chart({'type': 'column'}) 186 | # Add a series to the chart. 187 | for row in range(1, max_row + 1): 188 | scan_chart.add_series({ 189 | 'name': [worksheet_name, row, 0, row, 0], 190 | 'values': [worksheet_name, row, 1, row, 1]}) 191 | 192 | scan_chart.set_x_axis({'name': 'seconds'}) 193 | scan_chart.set_title({'name': 'Scan Time'}) 194 | scan_chart.width = 700 195 | 196 | for col in range(5, max_col): 197 | vuln_chart.add_series({ 198 | 'categories': [worksheet_name, 1, 0, max_row, 0], 199 | 'name': [worksheet_name, 0, col, 0, col], 200 | 'fill': {'color': severity_colors[col - 5]}, 201 | 'values': [worksheet_name, 1, col, max_row, col]}) 202 | vuln_chart.set_title({'name': 'Vulnerabilities'}) 203 | vuln_chart.width = 700 204 | # Insert the chart into the worksheet. 205 | cell = xlsutil.xl_rowcol_to_cell(max_row + 1, 0) 206 | worksheet.insert_chart(cell, vuln_chart) 207 | cell = xlsutil.xl_rowcol_to_cell(22, 0) 208 | worksheet.insert_chart(cell, scan_chart) 209 | elif name == 'vulnerability heatmap': 210 | for row in range(1, max_row + 1): 211 | cve = df['cve'].iloc[row - 1] 212 | url, hint = cve_link(cve, ref_data) 213 | cell = xlsutil.xl_rowcol_to_cell(row, 0) 214 | worksheet.write_url(cell, str(url), string=cve, tip=hint) 215 | 216 | 217 | severity_colors = ["#b85c00", "#ff420e", "#ffd428", "#579d1c", '#999999'] 218 | 219 | 220 | def save_to_excel(image_name, totals, normalised, original): 221 | name = slugify(image_name) 222 | created = datetime.now().strftime("%Y-%m-%d.%H%M%S") 223 | severities_format = [] 224 | all_data = pd.DataFrame() 225 | for plugin in selected_plugins: 226 | if plugin not in active_plugins: 227 | continue 228 | all_data = all_data.append(normalised[plugin]) 229 | 230 | with pd.ExcelWriter('output/{}-{}.xlsx'.format(name, created), engine='xlsxwriter') as writer: 231 | bold_format = writer.book.add_format({'bold': True}) 232 | critital_format = writer.book.add_format({'bg_color': '#b85c00', 233 | 'font_color': '#1c1c1c'}) 234 | severities_format.append(critital_format) 235 | high_format = writer.book.add_format({'bg_color': '#ff420e', 236 | 'font_color': '#1c1c1c'}) 237 | severities_format.append(high_format) 238 | medium_format = writer.book.add_format({'bg_color': '#ffd428', 239 | 'font_color': '#1c1c1c'}) 240 | severities_format.append(medium_format) 241 | low_format = writer.book.add_format({'bg_color': '#579d1c', 242 | 'font_color': '#1c1c1c'}) 243 | severities_format.append(low_format) 244 | unknown_format = writer.book.add_format({'bg_color': '#999999', 245 | 'font_color': '#1c1c1c'}) 246 | severities_format.append(unknown_format) 247 | 248 | notfound_format = writer.book.add_format({'bg_color': '#dee6ef', 249 | 'font_color': '#dee6ef', 250 | 'italic': True, 251 | 'align': 'center', 252 | 'font_size': 9}) 253 | severities_format.append(notfound_format) 254 | for key in totals: 255 | write_sheet(writer, all_data, totals[key], "", key, bold_format, severities_summaries, severities_format) 256 | for key in normalised: 257 | if not normalised[key].empty: 258 | write_sheet(writer, all_data, normalised[key], "normalised-", key, bold_format, severities_summaries, 259 | severities_format) 260 | for key in original: 261 | if not original[key].empty: 262 | write_sheet(writer, None, original[key], "original-", key, bold_format, severities_summaries, 263 | severities_format) 264 | 265 | 266 | def aggregate_dataframe(aggregates, column): 267 | df = pd.DataFrame(aggregates) 268 | aggregate_name = df.columns[0] 269 | df.rename(columns={aggregate_name: "total"}, inplace=True) 270 | df.reset_index(inplace=True, level=df.index.names) 271 | return df 272 | 273 | 274 | active_plugins = [] 275 | active_plugins_idx = [] 276 | selected_plugins = [] 277 | 278 | 279 | def get_active_plugins(config): 280 | plugins = [] 281 | plugins_idx = [] 282 | i = 1 283 | for plugin in config: 284 | if is_plugin_enabled(config[plugin]): 285 | plugins.append(plugin) 286 | plugins_idx.append(i) 287 | i += 1 288 | return plugins, plugins_idx 289 | 290 | 291 | def scan(args): 292 | image = args.image 293 | verbose = args.verbose 294 | offline = args.offline 295 | results = {} 296 | originals = {} 297 | all = {} 298 | cve_summary = {} 299 | components_summary = {} 300 | cve_summary_by_severity = {} 301 | components_summary_by_severity = {} 302 | severity_maps = {} 303 | descriptions = {} 304 | cves = pd.DataFrame() 305 | components = pd.DataFrame() 306 | cve_totals_by_severity = pd.DataFrame() 307 | components_heatmap = pd.DataFrame() 308 | cols = ['SCANNER', 'scan time', 'vulnerabilities', 'CVEs', 'components'] 309 | cols.extend(severities) 310 | totals_df = pd.DataFrame(columns=cols) 311 | if not os.path.exists('output'): 312 | os.makedirs('output') 313 | 314 | for plugin in selected_plugins: 315 | if plugin not in active_plugins: 316 | continue 317 | logging.info('scanning with {}'.format(plugin)) 318 | scanner = ScannerPlugin(plugin, config['plugins'][plugin], columns, severity_mappings, verbose=verbose, 319 | offline=offline) 320 | results[plugin], originals[plugin] = scanner.scan(image) 321 | logging.info('summary for {} scan by {}'.format(image, plugin)) 322 | scan_time = scanner.scan_time() 323 | if results[plugin].empty: 324 | logging.info('No vulnerabilities found!') 325 | # do the zeros with an iteration of severities 326 | totals_df.loc[len(totals_df.index)] = [plugin, scan_time, 0, 0, 0, 0, 0, 0, 0, 0] 327 | else: 328 | # snyk seems to have a lot of duplicate rows in json results 329 | results[plugin].drop_duplicates(inplace=True) 330 | severity_totals = results[plugin].groupby('severity').severity.count() 331 | logging.info(severity_totals) 332 | unique_vulns = results[plugin].vulnerability.drop_duplicates().shape[0] 333 | unique_cves = results[plugin].cve.drop_duplicates().shape[0] 334 | unique_components = results[plugin].component.drop_duplicates().shape[0] 335 | populate_totals(plugin, totals_df, scan_time, severity_totals, unique_vulns, unique_cves, unique_components) 336 | 337 | cve_summary[plugin] = aggregate_dataframe( 338 | results[plugin].groupby(["cve", "cssv_v2_severity", "cssv_v3_severity"]).cve.count(), 'cve') 339 | components_summary[plugin] = aggregate_dataframe( 340 | results[plugin].groupby(["component", "cssv_v2_severity", "cssv_v3_severity"]).component.count(), 341 | 'component') 342 | severity_map = pd.DataFrame() 343 | severity_map['cve'] = results[plugin]['cve'] 344 | severity_map['component'] = results[plugin]['component'] 345 | severity_map['severity'] = results[plugin]['severity'] 346 | # severity_map['description'] = results[plugin]['description'] 347 | severity_map['severity_index'] = severity_map.severity.map(severity_indices) 348 | 349 | # severity_map_df = aggregate_dataframe( 350 | # severity_map.groupby(['cve']).severity_index.max().map(severity_reverse_idx), 'cve') 351 | logging.info("creating the severity map") 352 | severity_map_df = aggregate_dataframe( 353 | severity_map.groupby(['cve']).severity_index.max(), 'cve') 354 | 355 | severity_maps[plugin] = severity_map_df 356 | 357 | # cve_summary_by_severity[plugin] = aggregate_dataframe( 358 | # results[plugin].groupby(['cve', 'severity']).cve.count(), 'cve') 359 | # components_summary_by_severity[plugin] = aggregate_dataframe( 360 | # results[plugin].groupby(['component', 'severity']).component.count(), 'component') 361 | logging.info("creating the severity map for components") 362 | severity_map_df = aggregate_dataframe( 363 | severity_map.groupby(['component']).severity_index.max(), 'cve') 364 | components_summary_by_severity[plugin] = severity_map_df 365 | if not results: 366 | logging.info('no scan results were produced with the current configuration. please check that the selected scanners are enabled') 367 | sys.exit(0) 368 | cve_severities = merge_aggregates(severity_maps, 'total', 369 | merge_fields=['cve'], fill_na_value=0) 370 | logging.info("merging component's heatmap") 371 | components_heatmap = merge_aggregates(components_summary_by_severity, 'total', 372 | merge_fields=['component'], fill_na_value=0) 373 | 374 | logging.info("merging severity map") 375 | cves = merge_aggregates(cve_summary, 'total', 376 | merge_fields=['cve', 'cssv_v2_severity', 'cssv_v3_severity']) 377 | logging.info("merging severity map for components") 378 | components = merge_aggregates(components_summary, 'total', 379 | merge_fields=['component', 'cssv_v2_severity', 'cssv_v3_severity']) 380 | # cve_totals_by_severity = merge_aggregates(cve_summary_by_severity, 'total', merge_fields=['cve', 'severity']) 381 | logging.info("generating totals") 382 | all['totals'] = totals_df 383 | all['vulnerability heatmap'] = format_severities_map(cve_severities) 384 | all['component heatmap'] = format_severities_map(components_heatmap) 385 | all['components'] = components 386 | all['vulnerabities'] = cves 387 | logging.info("saving to excel") 388 | save_to_excel(image, all, results, originals) 389 | 390 | 391 | ## if col is vulnerability or cve 392 | ## 393 | 394 | def format_severities_map(df): 395 | # df.loc[:,'severity index']=df.sum(numeric_only=True, axis=1) 396 | df['severity index'] = 0 397 | df['severity index'] = df.apply(calculate_composite_severity_rate, axis=1) 398 | map_name = df.columns[0] 399 | df.sort_values(by=['severity index', map_name], inplace=True, ascending=False) 400 | map_dict = severity_reverse_idx.copy() 401 | map_dict[0] = not_found_string 402 | for scanner in df.columns[1:-1]: 403 | df[scanner] = df[scanner].map(map_dict) 404 | return df 405 | 406 | 407 | def selected_plugins(selected_str): 408 | result = [] 409 | selected_idx = map(int(selected_str)) 410 | 411 | 412 | def is_plugin_enabled(param): 413 | result = True 414 | if 'enabled' in param: 415 | result = param['enabled'] 416 | return result 417 | 418 | 419 | def filter_selected(selected): 420 | result = [] 421 | plugins_list = list(config['plugins'].keys()) 422 | for i in selected: 423 | idx = int(i)-1 424 | result.append(plugins_list[idx]) 425 | return result 426 | 427 | 428 | if __name__ == "__main__": 429 | # create parser 430 | parser = argparse.ArgumentParser() 431 | # Adding optional argument 432 | parser.add_argument("-l", "--list", action="store_true", 433 | help="lists registered scanners - can only be used on its own. any other options will be ignored when -l is specified") 434 | parser.add_argument("-i", "--image", 435 | help="The image to scan (e.g owasp/benchmark,owasp/benchmark:latest, owasp/benchmark:1)") 436 | parser.add_argument("-v", "--verbose", action="store_true", 437 | help="Shows all scanner output, shows only summary if ommited") 438 | parser.add_argument("-of", "--offline", action="store_true", 439 | help="don't update nvd cache severities scores, lookup and update if ommited") 440 | parser.add_argument("-s", "--scanners", 441 | help=" optional. scanners to include in the scan. all (default) or specific id for scanners as found in with the -l command e.g. 3,5 to use only 3rd and 5th registered scanner in the scan") 442 | # parse the arguments 443 | args = parser.parse_args() 444 | if (len(sys.argv) == 1) or ((args.list is None) and (args.image is None)): 445 | print( 446 | "Missing options. you should either specify -l to list registered scanners, or -i for a docker image to scan ", 447 | file=sys.stderr) 448 | parser.print_help(sys.stderr) 449 | sys.exit(1) 450 | 451 | active_plugins, active_plugins_idx = get_active_plugins(config['plugins']) 452 | if args.list: 453 | i = 1 454 | print('registered scanners') 455 | for plugin in config['plugins']: 456 | enabled = is_plugin_enabled(config['plugins'][plugin]) 457 | print('{} {} - Enabled:{}'.format(i, plugin, enabled)) 458 | i += 1 459 | sys.exit(0) 460 | if args.image: 461 | if ":" not in args.image: 462 | args.image += ":latest" 463 | selected_plugins = list(config['plugins'].keys()) 464 | if args.scanners: 465 | selected = args.scanners.split(",") 466 | try: 467 | selected_plugins = filter_selected(selected) 468 | except: 469 | logging.warning('invalid scanner options. will be ignored') 470 | scan(args) 471 | -------------------------------------------------------------------------------- /ecr-push-scanner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # ./ecr-push-scan.sh 272208797173 eu-west-1 vulnerable/powersploit:latest 4 | 5 | AWS_ACCOUNT=$1 6 | AWS_REGION=$2 7 | FULL_IMAGE_NAME=$3 8 | arrIN=(${FULL_IMAGE_NAME//:/ }) 9 | echo $FULL_IMAGE_NAME 10 | AWS_ECR_REPO=${arrIN[0]} 11 | BUILD_TAG=${arrIN[1]} 12 | echo image name $AWS_ECR_REPO 13 | echo image tag $BUILD_TAG 14 | rm -f ecr.json 15 | aws ecr describe-repositories --repository-names ${AWS_ECR_REPO} --region ${AWS_REGION} || aws ecr create-repository --repository-name ${AWS_ECR_REPO} --image-scanning-configuration scanOnPush=true --region ${AWS_REGION} 16 | docker tag $AWS_ECR_REPO:$BUILD_TAG $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com/$AWS_ECR_REPO:$BUILD_TAG 17 | aws ecr get-login-password --region $AWS_REGION | \ 18 | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com && \ 19 | docker push $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com/$AWS_ECR_REPO:$BUILD_TAG && \ 20 | aws ecr wait image-scan-complete \ 21 | --repository-name $AWS_ECR_REPO \ 22 | --image-id imageTag=$BUILD_TAG \ 23 | --region $AWS_REGION && \ 24 | VULNS=$(aws ecr describe-image-scan-findings \ 25 | --repository-name $AWS_ECR_REPO \ 26 | --image-id imageTag=$BUILD_TAG \ 27 | --region $AWS_REGION \ 28 | --query imageScanFindings \ 29 | --output json) 30 | 31 | echo $VULNS > ecr.json 32 | echo results saved in ecr.json 33 | 34 | -------------------------------------------------------------------------------- /get-images.sh: -------------------------------------------------------------------------------- 1 | docker pull alpine 2 | docker pull debian 3 | docker pull ubuntu 4 | docker pull mcr.microsoft.com/dotnet/runtime 5 | docker tag mcr.microsoft.com/dotnet/runtime:latest dotnet/runtime:latest 6 | docker pull openjdk 7 | docker pull python 8 | docker pull node 9 | docker pull php 10 | docker pull ruby 11 | docker pull bkimminich/juice-shop 12 | docker pull docker owasp/benchmark 13 | docker pull jsotiro/cve-2020-9402 -------------------------------------------------------------------------------- /images/cli-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/images/cli-list.png -------------------------------------------------------------------------------- /images/cli-main-s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/images/cli-main-s.png -------------------------------------------------------------------------------- /images/dashboard-s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/images/dashboard-s.png -------------------------------------------------------------------------------- /images/heatmap-s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/images/heatmap-s.png -------------------------------------------------------------------------------- /images/xl-repair-confirmation-s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/images/xl-repair-confirmation-s.png -------------------------------------------------------------------------------- /images/xl-repair-prompt-s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/images/xl-repair-prompt-s.png -------------------------------------------------------------------------------- /install-scanners.sh: -------------------------------------------------------------------------------- 1 | echo installing snyk 2 | npm install -g snyk 3 | echo installing grype and trivy 4 | if [[ "$OSTYPE" == "linux-gnu"* ]]; then 5 | sudo curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin 6 | sudo curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.16.0 7 | elif [[ "$OSTYPE" == "darwin"* ]]; then 8 | # Mac OSX 9 | curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin 10 | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.16.0 11 | fi 12 | 13 | echo getting the docker images needed by clair-scanner ... 14 | #docker pull arminc/clair-db:latest 15 | docker pull arminc/clair-local-scan 16 | docker run -d --name clair-db arminc/clair-db:latest 17 | docker run -p 6060:6060 --link clair-db:postgres -d --name clair arminc/clair-local-scan:latest 18 | docker stop clair 19 | docker stop clair-db 20 | 21 | echo installing clair-scanner ... 22 | 23 | if [[ "$OSTYPE" == "linux-gnu"* ]]; then 24 | wget https://github.com/arminc/clair-scanner/releases/download/v12/clair-scanner_linux_amd64 -O clair-scanner 25 | chmod +x clair-scanner 26 | sudo mv clair-scanner /usr/local/bin 27 | elif [[ "$OSTYPE" == "darwin"* ]]; then 28 | # Mac OSX 29 | curl -L https://github.com/arminc/clair-scanner/releases/download/v12/clair-scanner_darwin_amd64 --output clair-scanner 30 | chmod +x clair-scanner 31 | mv clair-scanner /usr/local/bin 32 | fi 33 | 34 | echo all scanners installed 35 | echo run pip/pip3 install -r requirements.txt before your run dmscan.py 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /nvd-cache.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsotiro/docker-multiscan/ab5d462541156376032640b6a7f9614d04863bcb/nvd-cache.db -------------------------------------------------------------------------------- /nvd_cache.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import requests 4 | from sqlalchemy import create_engine 5 | import pandas as pd 6 | import logging 7 | # import utils 8 | from datetime import datetime, timedelta 9 | 10 | 11 | class NvdCache: 12 | __instance = None 13 | 14 | @staticmethod 15 | def get_instance(): 16 | if NvdCache.__instance is None: 17 | __instance = NvdCache() 18 | return NvdCache.__instance 19 | 20 | 21 | def __init__(self): 22 | self.dbname = 'nvd-cache.db' 23 | self.engine = create_engine('sqlite:///' + self.dbname, echo=False) 24 | self.sqlite_connection = self.engine.raw_connection() 25 | self.sqlite_table = "vulnerabilities" 26 | self.cache_ttl = timedelta(days=30) 27 | self.items_found_max = 20 28 | self.items_found = 1 29 | 30 | if self.table_exists(): 31 | sql = "select * from {}".format(self.sqlite_table) 32 | self.df = pd.read_sql(sql, self.sqlite_connection, index_col='cve', parse_dates='last_updated') 33 | else: 34 | self.df = pd.DataFrame(columns={'details', 'last_updated'}) 35 | 36 | def read_cve_list(self, filename): 37 | self.df = pd.read_csv('cves-list.txt', header=None, names=['cve']) 38 | self.df.drop_duplicates(inplace=True) 39 | self.df['details'] = '' 40 | self.df['last_updated'] = pd.Timestamp.min 41 | self.df.set_index('cve', inplace=True) 42 | 43 | def create_from_list(self): 44 | if not self.table_exists(): 45 | self.df = self.df.apply(self.import_df_entry, axis=1) 46 | self.df.to_sql(self.sqlite_table, self.sqlite_connection) 47 | 48 | def create_for_cve_list(self, filename): 49 | self.read_cve_list(filename) 50 | self.create_from_list() 51 | 52 | def has_expired(self, timestamp): 53 | now = datetime.now() 54 | return timestamp < now - self.cache_ttl 55 | 56 | def get_updated_entry(self, cve): 57 | url = "https://services.nvd.nist.gov/rest/json/cve/1.0/{}".format(cve) 58 | try: 59 | json_response = requests.get(url).json() 60 | except Exception as e: 61 | json_response = '' 62 | logging.error(e) 63 | return json_response 64 | 65 | def get_item(self, cve, offline=False): 66 | found = False 67 | expired = False 68 | result = "" 69 | if cve in self.df.index: 70 | item = self.df.loc[cve] 71 | found = True 72 | # duplicates in the db 73 | if len(item.shape) > 1: 74 | item = item[item['last_updated'] == item['last_updated'].max()].iloc[0] 75 | if self.has_expired(item['last_updated']): 76 | expired = True 77 | else: 78 | result = json.loads(item['details']) 79 | 80 | if (not found or expired) and not offline : 81 | result = self.get_item_from_nvd(cve) 82 | self.add_item(cve, json.dumps(result)) 83 | return result 84 | 85 | def get_item_from_nvd(self, cve): 86 | json_response = self.get_updated_entry(cve) 87 | if type(json_response) is str: 88 | return json.loads(json_response) 89 | else: 90 | return json_response 91 | 92 | def add_item(self, cve, details): 93 | cursor = self.sqlite_connection.cursor() 94 | last_updated = datetime.now() 95 | cursor.execute('''INSERT INTO {} (cve, details, last_updated) 96 | VALUES(?,?,?)'''.format(self.sqlite_table), (cve, details, last_updated)) 97 | self.sqlite_connection.commit() 98 | self.df.loc[cve] = {'details': details, 'last_updated': last_updated} 99 | logging.info('cve {} was written to the database and added to the cache '.format(cve)) 100 | 101 | def import_df_entry(self, item): 102 | if ((item['details'] == '') or self.has_expired(item['last_updated'])): 103 | # and (self.items_found <= self.items_found_max): 104 | json_response = self.get_updated_entry(item.name) 105 | item['details'] = json.dumps(json_response) 106 | item['last_updated'] = datetime.now() 107 | self.items_found += 1 108 | logging.info("{} items updated ".format(self.items_found)) 109 | return item 110 | 111 | def table_exists(self, create_ifnot=False): 112 | cursor = self.sqlite_connection.cursor() 113 | cursor.execute(''' SELECT count(name) FROM sqlite_master WHERE type='table' AND name= ? ''', 114 | [self.sqlite_table]) 115 | # if the count is 1, then table exists 116 | result = cursor.fetchone()[0] == 1 117 | return result 118 | 119 | def close(self): 120 | self.sqlite_connection.close() 121 | 122 | 123 | if __name__ == "__main__": 124 | nvd_cache = NvdCache() 125 | nvd_cache.create_for_cve_list('cves-list.txt') 126 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | pandas 3 | XlsxWriter 4 | PyYAML 5 | SQLAlchemy 6 | -------------------------------------------------------------------------------- /run-tests.sh: -------------------------------------------------------------------------------- 1 | dmscan -i alpine 2 | dmscan -i debian 3 | dmscan -i ubuntu 4 | dmscan -i dotnet/runtime 5 | dmscan -i openjdk 6 | dmscan -i python 7 | dmscan -i node 8 | dmscan -i php 9 | dmscan -i ruby 10 | dmscan -i bkimminich/juice-shop 11 | dmscan -i owasp/benchmark 12 | dmscan -i jsotiro/cve-2020-9402 -------------------------------------------------------------------------------- /scanner_plugin.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | import subprocess 5 | import datetime 6 | import pandas as pd 7 | import requests 8 | from nvd_cache import NvdCache 9 | import logging 10 | 11 | 12 | def safe_get_key(dict, key): 13 | if key in dict: 14 | return dict[key] 15 | else: 16 | return None 17 | 18 | 19 | cves_dict = {} 20 | 21 | 22 | class ScannerPlugin: 23 | def __init__(self, plugin, plugin_config, columns, severity_mappings, verbose=False, offline=False): 24 | self.name = plugin 25 | self.config = plugin_config 26 | self.command_line = plugin_config['command_line'] 27 | self.resultsRoot = plugin_config['results_root'] 28 | self.columnMappings = plugin_config['mappings'] 29 | self.output_file = safe_get_key(plugin_config, 'output_file') 30 | self.flatten_key_value_pairs = safe_get_key(plugin_config, 'flatten_key_value_pairs') 31 | self.severity_mappings = severity_mappings 32 | self.columns = columns 33 | self.cve_cache = NvdCache() 34 | self.timeout_in_secs = 60 35 | self.started = None 36 | self.finished = None 37 | self.failed = False 38 | self.verbose = verbose 39 | self.offline = offline 40 | 41 | def scan_time(self): 42 | scan_time = (self.finished - self.started).total_seconds() 43 | return scan_time 44 | 45 | def scan_image(self, command_line_params): 46 | scanner = subprocess.Popen(command_line_params, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 47 | self.started = datetime.datetime.now() 48 | stdout, stderr = scanner.communicate() 49 | stdout = stdout.decode('utf-8') 50 | stderr = stderr.decode('utf-8') 51 | logging.info("scanner command was executed: {}".format(command_line_params)) 52 | if self.verbose: 53 | logging.info(stdout) 54 | logging.error(stderr) 55 | if self.output_file: 56 | logging.info("looking for output file: {}".format(self.output_file)) 57 | start = datetime.datetime.now() 58 | elapsed_secs = 0 59 | while not os.path.isfile(self.output_file) and elapsed_secs <= self.timeout_in_secs: 60 | now = datetime.datetime.now() 61 | elapsed_secs = (now - start).total_seconds() 62 | self.failed = elapsed_secs > self.timeout_in_secs 63 | if not self.failed: 64 | logging.info("file {} found. opening file".format(self.output_file)) 65 | json_file = open(self.output_file, ) 66 | logging.info("reading json from file") 67 | json_result = json.load(json_file) 68 | logging.info("cleaning up file") 69 | os.remove(self.output_file) 70 | logging.info("cleaning up completed") 71 | else: 72 | json_result = {"error": "waiting for results file timed-out"} 73 | logging.error(json_result) 74 | else: 75 | if stderr != "": 76 | json_result = {"error": stderr} 77 | else: 78 | json_result = json.loads(stdout) 79 | self.finished = datetime.datetime.now() 80 | return json_result 81 | 82 | def flatten_list_to_first_item(self, item): 83 | if type(item) is list: 84 | if len(item) > 0: 85 | return str(item[0]) 86 | else: 87 | return "" 88 | else: 89 | return item 90 | 91 | def cve_from_reference(self, vuld_id, url): 92 | cve = vuld_id 93 | try: 94 | html = requests.get(url).content 95 | result = re.search("CVE-\d{4}-\d*", str(html), flags=re.IGNORECASE) 96 | if result: 97 | cve = result[0] 98 | except Exception as e: 99 | msg = "error in retrieving cve from link " + str(e) 100 | logging.error(msg) 101 | cve = vuld_id 102 | return cve 103 | 104 | def cvss_severities(self, cve): 105 | cssv_v2 = "" 106 | cssv_v3 = "" 107 | try: 108 | cve_details = self.cve_cache.get_item(cve, offline=self.offline) 109 | if "result" in cve_details: 110 | if "CVE_Items" in cve_details["result"]: 111 | if "impact" in cve_details["result"]["CVE_Items"][0]: 112 | if "baseMetricV2" in cve_details["result"]["CVE_Items"][0]["impact"]: 113 | cssv_v2 = cve_details["result"]["CVE_Items"][0]["impact"]["baseMetricV2"]["severity"] 114 | if "baseMetricV3" in cve_details["result"]["CVE_Items"][0]["impact"]: 115 | cssv_v3 = cve_details["result"]["CVE_Items"][0]["impact"]["baseMetricV3"]["cvssV3"]["baseSeverity"] 116 | except Exception as ex: 117 | msg = cve + str(ex) 118 | logging.error(msg) 119 | return cssv_v2, cssv_v3 120 | 121 | def replace_GSHA_withCVE(self, item): 122 | if item.vulnerability.startswith('CVE') or pd.isna(item.link): 123 | item['cve'] = item.vulnerability 124 | else: 125 | item['cve'] = self.cve_from_reference(item.vulnerability, item.link) 126 | cssv_v2 = "" 127 | cssv_v3 = "" 128 | if str(item['cve']).lower().startswith("cve"): 129 | cssv_v2, cssv_v3 = self.cvss_severities(item['cve']) 130 | item['cssv_v2_severity'] = cssv_v2 131 | item['cssv_v3_severity'] = cssv_v3 132 | 133 | return item 134 | 135 | def normalize_results(self, df): 136 | if self.columnMappings: 137 | df.rename(columns=self.columnMappings, inplace=True) 138 | for col in df.columns: 139 | if col not in self.columns: 140 | df.drop([col], axis='columns', inplace=True) 141 | else: 142 | df[col] = df[col].map(self.flatten_list_to_first_item, na_action='ignore') 143 | 144 | # df['vulnerability'] = df['vulnerability'].map(self.flatten_list_to_first_item, na_action='ignore') 145 | 146 | df['cve'] = df['vulnerability'] 147 | 148 | # df['link'] = df['link'].map(self.flatten_list_to_first_item, na_action='ignore') 149 | df['severity'] = df['severity'].str.upper() 150 | df['severity'] = df['severity'].map(self.severity_mappings) 151 | df['cssv_v2_severity'] = df['severity'] 152 | df['cssv_v3_severity'] = df['severity'] 153 | 154 | df = df.apply(self.replace_GSHA_withCVE, axis=1) 155 | return df 156 | 157 | def parsed_image_name(self, full_image_name): 158 | name = "" 159 | author = "" 160 | tag = "" 161 | if ":" in full_image_name: 162 | temp = full_image_name.split(":") 163 | main = temp[0] 164 | tag = temp[1] 165 | else: 166 | tag = "latest" 167 | main = full_image_name 168 | if "/" in main: 169 | temp_name = main.split("/") 170 | author = temp_name[0] 171 | name = temp_name[1] 172 | else: 173 | name = main 174 | return author, name, tag 175 | 176 | ''' 177 | secure version of eval to allow the dynamic evaluation of filenames 178 | but nothing else 179 | ''' 180 | 181 | def eval_expression(self, input_string, image_author, image_name, image_tag): 182 | allowed_names = {"format": format, "image_author": image_author, 183 | "image_name": image_name, 184 | "image_tag": image_tag} 185 | code = compile(input_string, "", "eval") 186 | for name in code.co_names: 187 | if name not in allowed_names: 188 | raise NameError(f"Use of {name} not allowed") 189 | return eval(code, {"__builtins__" : {}}, allowed_names) 190 | 191 | def transpose_key_value_pairs_to_named_keys(self, json, root, pair_parent, key_name, value_name): 192 | json_array = json[root] 193 | for element in json_array: 194 | for attribute in element[pair_parent]: 195 | element[attribute[key_name]] = attribute[value_name] 196 | 197 | def unpack_json_values(self, json, root, json_parent): 198 | json_array = json[root] 199 | for element in json_array: 200 | i = 0 201 | for node in element[json_parent]: 202 | for key in node.keys(): 203 | name = json_parent + "." + key 204 | if i > 0: 205 | name = name + "_" + str(i) 206 | element[name] = node[key] 207 | i += 1 208 | 209 | ''' 210 | happens before we transform the json results to a dataframe 211 | and helps with flattening json into tabular form 212 | ''' 213 | 214 | def pre_process_json(self, json_results): 215 | if self.flatten_key_value_pairs: 216 | self.transpose_key_value_pairs_to_named_keys( 217 | json_results, 218 | self.resultsRoot, 219 | self.flatten_key_value_pairs['below'], 220 | self.flatten_key_value_pairs['key_name'], 221 | self.flatten_key_value_pairs['value_name']) 222 | if 'unpack_json' in self.config: 223 | for item in self.config['unpack_json']: 224 | self.unpack_json_values(json_results, self.resultsRoot, item) 225 | 226 | ''' 227 | we transform the json report to a dataframe 228 | and consolidate columns and values so that all 229 | scanner results have the same reference names 230 | ''' 231 | 232 | def preprocess_dataframe(self, json_results): 233 | try: 234 | logging.info('normalising data') 235 | results = pd.json_normalize(json_results, record_path=self.resultsRoot) 236 | except KeyError as e: 237 | logging.info('normalising data failed with {} because of array, going up to 1st array element '.format(self.resultsRoot)) 238 | json_subtree = json_results[0][self.resultsRoot] 239 | results = pd.json_normalize(json_subtree) 240 | original = results.copy() 241 | if not results.empty: 242 | self.normalize_results(results) 243 | return results, original 244 | 245 | def scan(self, image): 246 | results = pd.DataFrame() 247 | original = pd.DataFrame() 248 | cmd = self.command_line 249 | cmd.append(image) 250 | image_author, image_name, image_tag = self.parsed_image_name(image) 251 | if self.output_file: 252 | self.output_file = self.eval_expression(self.output_file, image_author, image_name, image_tag) 253 | json_results = self.scan_image(cmd) 254 | logging.info('scan completed') 255 | if not "error" in json_results: 256 | logging.info('pre-processing json data') 257 | self.pre_process_json(json_results) 258 | results, original = self.preprocess_dataframe(json_results) 259 | logging.info('data preprocessed and transformed ') 260 | logging.info('results data {} '.format(results.shape)) 261 | logging.info('original data {} '.format(original.shape)) 262 | return results, original 263 | -------------------------------------------------------------------------------- /scanners.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: debug 3 | 4 | columns: 5 | - vulnerability 6 | - severity 7 | - description 8 | - link 9 | - component 10 | - version 11 | - fixedInVersion 12 | - cssv_v2_severity 13 | - cssv_v3_severity 14 | 15 | severities: 16 | - 5.CRITICAL 17 | - 4.HIGH 18 | - 3.MEDIUM 19 | - 2.LOW 20 | - 1.UNKNOWN 21 | severity-mappings: 22 | UNDEFINED: 1.UNKNOWN 23 | UNKNOWN: 1.UNKNOWN 24 | NEGLIGIBLE: 2.LOW 25 | INFORMATIONAL: 2.LOW 26 | LOW: 2.LOW 27 | MEDIUM: 3.MEDIUM 28 | HIGH: 4.HIGH 29 | CRITICAL: 5.CRITICAL 30 | DEFCON1: 5.CRITICAL 31 | plugins: 32 | snyk: 33 | enabled: 34 | True 35 | command_line: 36 | - ./snyk-scanner.sh 37 | results_root: 38 | vulnerabilities 39 | mappings: 40 | packageName: component 41 | identifiers.CVE: vulnerability 42 | nearestFixedInVersion: fixedInVersion 43 | references.url: link 44 | unpack_json: 45 | - references 46 | aws-ecr: 47 | enabled: 48 | False 49 | output_file: 50 | "\"ecr.json\"" 51 | command_line: 52 | - ./ecr-push-scanner.sh 53 | - "272208797173" 54 | - eu-west-1 55 | results_root: 56 | findings 57 | mappings: 58 | package_name: component 59 | package_version: version 60 | name: vulnerability 61 | uri: link 62 | flatten_key_value_pairs: 63 | below: attributes 64 | key_name: key 65 | value_name: value 66 | clair-scanner: 67 | output_file: 68 | "\"clair.json\"" 69 | command_line: 70 | - ./clair-scanner-run.sh 71 | - --report=clair.json 72 | - --clair=http://127.0.0.1:6060 73 | results_root: 74 | vulnerabilities 75 | mappings: 76 | featurename: component 77 | featureversion: version 78 | fixedby: fixedInVersion 79 | anchore-inline: 80 | output_file: 81 | "\"anchore-reports/{}_{}-vuln.json\".format(image_name,image_tag)" 82 | command_line: 83 | - ./anchore-inline.sh 84 | results_root: 85 | vulnerabilities 86 | mappings: 87 | vuln: vulnerability 88 | package: component 89 | package_version: version 90 | fix: fixedInVersion 91 | nvd_data: description 92 | url: link 93 | grype: 94 | command_line: 95 | - grype 96 | - --output 97 | - json 98 | results_root: 99 | matches 100 | mappings: 101 | vulnerability.id: vulnerability 102 | vulnerability.severity: severity 103 | artifact.name: component 104 | artifact.version: version 105 | vulnerability.fixedInVersion: fixedInVersion 106 | vulnerability.description: description 107 | vulnerability.links: link 108 | trivy: 109 | command_line: 110 | - trivy 111 | - -f 112 | - json 113 | - -q 114 | results_root: 115 | Vulnerabilities 116 | mappings: 117 | VulnerabilityID: vulnerability 118 | Severity: severity 119 | PkgName: component 120 | InstalledVersion: version 121 | FixedVersion: fixedInVersion 122 | Description: description 123 | PrimaryURL: link 124 | -------------------------------------------------------------------------------- /snyk-scanner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #echo $NVM_BIN 3 | #echo $SNYK_TOKEN 4 | #echo " " 5 | 6 | if [[ "$DMSCAN_ENV" == "DEV" ]]; then 7 | $NVM_BIN/node $NVM_BIN/snyk auth $SNYK_TOKEN >/dev/null 2>&1 8 | $NVM_BIN/node $NVM_BIN/snyk container test $1 --json 9 | else 10 | snyk auth $SNYK_TOKEN >/dev/null 2>&1 11 | snyk container test $1 --json 12 | fi 13 | --------------------------------------------------------------------------------