├── .gitignore ├── ExploreAll ├── ExploreImage ├── README.md ├── classify_data.sh ├── cleanup.sh ├── install.sh ├── requirements.txt └── utils ├── ImagesScanner.py ├── Log.py ├── TriageBlockerAndCritical.py ├── __init__.py └── docker_explorer_transparent_v6.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | explore_* 132 | 133 | ### VisualStudioCode ### 134 | .vscode/* 135 | 136 | ### VisualStudioCode Patch ### 137 | # Ignore all local history of files 138 | .history -------------------------------------------------------------------------------- /ExploreAll: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import subprocess 4 | import argparse 5 | import os 6 | 7 | from requests import request 8 | from multiprocessing import Process 9 | from time import sleep 10 | from utils.ImagesScanner import scan_image 11 | from utils.Log import getLogger 12 | LOG = getLogger(__name__) 13 | 14 | def scan_dockerhub(target:str, proc_quant:int, page:int, end:int, page_size:int, order_by_pull_count:str): 15 | url= f"https://hub.docker.com/v2/search/repositories?query={target}&page={page}&page_size={page_size}" 16 | 17 | if order_by_pull_count: 18 | if order_by_pull_count=='ASC': url+="&ordering=pull_count" 19 | else: url+="&ordering=-pull_count" 20 | 21 | headers = { 22 | "Accept": "application/json", 23 | "Search-Version": "v3", 24 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0", 25 | "Connection": "close", 26 | "Accept-Encoding": "gzip, deflate", 27 | "Accept-Language": "en-US,en;q=0.5", 28 | "Content-Type": "application/json" 29 | } 30 | 31 | processes=list() 32 | while end is None or page<=end: 33 | response = request("GET", url, headers=headers) 34 | response= response.json() 35 | images_count= response["count"] 36 | LOG.debug(f"Found {images_count} images. ") 37 | 38 | index=1 39 | for image_data in response["results"]: 40 | 41 | while len(processes) >= proc_quant: 42 | for proc in processes: 43 | if not proc.is_alive(): 44 | LOG.debug("removed process") 45 | processes.remove(proc) 46 | sleep(5) 47 | 48 | LOG.debug(f"Image {index} page {page}") 49 | 50 | if len(processes) < proc_quant: 51 | p = Process(target=scan_image, args=(image_data["repo_name"],tmp_path,whispers_config, whispers_output, whispers_timeout,)) 52 | p.start() 53 | processes.append(p) 54 | index+=1 55 | 56 | page=page+1 57 | url=response.get('next') 58 | if url is None or len(url)==0: 59 | LOG.debug(f"Reached end of result") 60 | break 61 | 62 | return 63 | 64 | if __name__ == '__main__': 65 | 66 | parser = argparse.ArgumentParser(description='Docker Images Scanner.\nThis script receives a keyword and it starts to scan all matching DockerHub images.') 67 | requiredGroup = parser.add_argument_group('Required arguments') 68 | requiredGroup.add_argument('-t','--target', 69 | help='Keyword to search in dockerhub', 70 | required = True) 71 | parser.add_argument('-c','--config', 72 | help="Whispers custom config filepath. By default will use Docker Explorer whispers config.", 73 | default='../whispers/whispers/config.yml' 74 | ) 75 | parser.add_argument('-o','--output', 76 | help="Directory where to store matching files. Will use ./ by default.", 77 | default=None 78 | ) 79 | parser.add_argument('--tmp', 80 | help="Temporary path to dump the filesystem and perform the scan. Default is /tmp", 81 | default='/tmp/' 82 | ) 83 | parser.add_argument("--page_size", 84 | help= 'Size of the Dockerhub API page. Default 100.', 85 | type= int, 86 | default=100) 87 | parser.add_argument("--timeout", 88 | help= 'Timeout in minutes for scan engine execution per image. Default 45 minutes.', 89 | type= int, 90 | default=45) 91 | parser.add_argument("--order-by-pull-count", 92 | help= 'Order by the amount of pull counts of the image. The lesser pulls, the newer the image.', 93 | nargs='?', 94 | choices=['ASC', 'DESC'], 95 | default=None) 96 | parser.add_argument("-p", "--processes", 97 | help= 'Amount of parallel processes. Default is 4', 98 | type= int, 99 | default=4) 100 | parser.add_argument("--start", 101 | help= 'Start page', 102 | type= int, 103 | default=1) 104 | parser.add_argument("--end", 105 | help= 'End page', 106 | type= int, 107 | default=None) 108 | options= parser.parse_args() 109 | 110 | 111 | tmp_path= options.tmp 112 | whispers_config= options.config 113 | LOG.debug(f"Using whisper config {os.path.abspath(whispers_config)}") 114 | whispers_timeout=options.timeout*60 115 | whispers_output= options.output if options.output is not None else os.getcwd() 116 | 117 | LOG.debug(f"Using output path {whispers_output}") 118 | 119 | try: 120 | scan_dockerhub(options.target, proc_quant=options.processes, page=options.start, end=options.end, page_size=options.page_size, order_by_pull_count=options.order_by_pull_count) 121 | except KeyboardInterrupt: 122 | # mkdir = subprocess.run(f"rm -rf {tmp_path}explore_*", shell=True, stdout=subprocess.PIPE, text=True, check=True) 123 | pass -------------------------------------------------------------------------------- /ExploreImage: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import logging 5 | import os 6 | 7 | from utils.ImagesScanner import scan_image 8 | from multiprocessing import Process 9 | from time import sleep 10 | from utils.Log import getLogger 11 | LOG = getLogger(__name__) 12 | 13 | 14 | if __name__ == '__main__': 15 | 16 | parser = argparse.ArgumentParser(description='Scan a single image or a given list of DockerHub images') 17 | parser.add_argument('-i','--image', 18 | help='Image to scan in dockerhub in the format repository/image_name. It will be used the latest version by default.', 19 | default=None) 20 | parser.add_argument('-f','--file', 21 | help='File with list of images to scan in the format repository/image_name. It will be used the latest version by default.', 22 | default= None) 23 | parser.add_argument('-o','--output', 24 | help="Directory where to store matching files. Will use ./ by default.", 25 | default=None 26 | ) 27 | parser.add_argument('-c','--config', 28 | help="Whispers custom config filepath. By default will use Docker Explorer whispers config.", 29 | default='../whispers/whispers/config.yml' 30 | ) 31 | parser.add_argument('--tmp', 32 | help="Temporary path to dump the filesystem and perform the scan. Default is /tmp", 33 | default='/tmp/' 34 | ) 35 | parser.add_argument("--timeout", 36 | help= 'Timeout in minutes for scan engine execution per image. Default 45 minutes.', 37 | type= int, 38 | default=45) 39 | parser.add_argument("-p", "--processes", 40 | help= 'Amount of parallel processes. Default is 4.', 41 | type= int, 42 | default=4) 43 | options= parser.parse_args() 44 | 45 | if options.image is None and options.file is None: 46 | print("Provide either an image or file\n") 47 | 48 | proc_quant= options.processes 49 | tmp_path= options.tmp 50 | whispers_config= options.config 51 | LOG.debug(f"Using whisper config {os.path.abspath(whispers_config)}") 52 | whispers_timeout=options.timeout*60 53 | whispers_output= options.output if options.output is not None else os.getcwd() 54 | LOG.debug(f"Using output path {whispers_output}") 55 | 56 | try: 57 | if options.image: 58 | scan_image(options.image,tmp_path,whispers_config, whispers_output, whispers_timeout) 59 | else: 60 | images_list= open(options.file,"r") 61 | processes=list() 62 | for line in images_list.readlines(): 63 | if len(processes) < proc_quant: 64 | p = Process(target=scan_image, args=(line.strip(),tmp_path,whispers_config, whispers_output, whispers_timeout,)) 65 | p.start() 66 | processes.append(p) 67 | 68 | while len(processes) >= proc_quant: 69 | for proc in processes: 70 | if not proc.is_alive(): 71 | processes.remove(proc) 72 | sleep(5) 73 | for proc in processes: 74 | proc.join() 75 | except KeyboardInterrupt: 76 | pass 77 | except Exception as e: 78 | logging.exception(f"Error: {e}") 79 | 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | # Docker Images Explorer 6 | 7 | This tool scans (for the moment) Dockerhub images that match a given keyword in order to find "forgotten" secrets. The scan engine used is a modified fork from [Whispers](https://github.com/Skyscanner/whispers). 8 | 9 | Using the tool, I found numerous AWS credentials, SSH private keys, databases, API keys, etc. It’s an interesting tool to add to the bug hunter / pentester arsenal, not only for the possibility of finding secrets, but for fingerprinting an organization. If you are a DevOps or Security Engineer, you might want to integrate the scan engine to your CI/CD for your Docker images. 10 | 11 | ## Prerequisites: 12 | 13 | * Python3 14 | * Pip3 15 | * Docker 16 | 17 | 18 | ## Installation (tested in Ubuntu and macOS) 19 | 20 | Since the execution of this tool can take a long time and it's very CPU demanding, it's recommended to run it in a VPS running Ubuntu (I'm working on a Terraform template to ease the setup). To prepare the environment for Docker Explorer (install whispers and Python requirements), run the following commands: 21 | 22 | ``` 23 | git clone https://www.github.com/matiassequeira/docker_explorer 24 | cd docker_explorer 25 | chmod +x install.sh 26 | export PATH=$(pwd):$PATH 27 | ./install.sh 28 | ``` 29 | 30 | ## Usage 31 | 32 | It's recommended to run this tool in a fresh new directory in order to keep the tool separate from the output. Momentarily, there are two scripts you can execute according to your needs: 33 | 34 | ### ExploreAll 35 | 36 | ``` 37 | ❯ ./ExploreAll -h 38 | usage: ExploreAll [-h] -t TARGET [-c CONFIG] [-o OUTPUT] [--tmp TMP] [--page_size PAGE_SIZE] [--timeout TIMEOUT] 39 | [--order-by-pull-count [{ASC,DESC}]] [-p PROCESSES] [--start START] [--end END] 40 | 41 | Docker Images Scanner. This script receives a keyword and it starts to scan all matching DockerHub images. 42 | 43 | optional arguments: 44 | -h, --help show this help message and exit 45 | -c CONFIG, --config CONFIG 46 | Whispers custom config filepath. By default will use Docker Explorer whispers config. 47 | -o OUTPUT, --output OUTPUT 48 | Directory where to store matching files. Will use ./ by default. 49 | --tmp TMP Temporary path to dump the filesystem and perform the scan. Default is /tmp 50 | --page_size PAGE_SIZE 51 | Size of the Dockerhub API page. Default 100. 52 | --timeout TIMEOUT Timeout in minutes for scan engine execution per image. Default 45 minutes. 53 | --order-by-pull-count [{ASC,DESC}] 54 | Order by the amount of pull counts of the image. The lesser pulls, the newer the image. 55 | -p PROCESSES, --processes PROCESSES 56 | Amount of parallel processes. Default is 4 57 | --start START Start page 58 | --end END End page 59 | 60 | Required arguments: 61 | -t TARGET, --target TARGET 62 | Keyword to search in dockerhub 63 | ``` 64 | 65 | An example of this command that scans all the images matching the keyword **aws** is: 66 | 67 | ``` 68 | ./ExploreAll -t aws 69 | ``` 70 | 71 | If you're using a VPS with Ubuntu, I recommend to leave it running in background with the `nohup` (no hangup) command and redirect the logs to a file that you can `cat` to track progress: 72 | 73 | ``` 74 | nohup ExploreAll -t aws &> out.txt & 75 | tail -f out.txt 76 | ``` 77 | 78 | ### ExploreImage 79 | 80 | ``` 81 | ❯ ./ExploreImage -h 82 | usage: ExploreImage [-h] [-i IMAGE] [-f FILE] [-o OUTPUT] [-c CONFIG] [--tmp TMP] [--timeout TIMEOUT] [-p PROCESSES] 83 | 84 | Scan a single image or a given list of DockerHub images 85 | 86 | optional arguments: 87 | -h, --help show this help message and exit 88 | -i IMAGE, --image IMAGE 89 | Image to scan in dockerhub in the format repository/image_name. It will be used the latest version by 90 | default. 91 | -f FILE, --file FILE File with list of images to scan in the format repository/image_name. It will be used the latest version 92 | by default. 93 | -o OUTPUT, --output OUTPUT 94 | Directory where to store matching files. Will use ./ by default. 95 | -c CONFIG, --config CONFIG 96 | Whispers custom config filepath. By default will use Docker Explorer whispers config. 97 | --tmp TMP Temporary path to dump the filesystem and perform the scan. Default is /tmp 98 | --timeout TIMEOUT Timeout in minutes for scan engine execution per image. Default 45 minutes. 99 | -p PROCESSES, --processes PROCESSES 100 | Amount of parallel processes. Default is 4. 101 | ``` 102 | 103 | An example of this command to scan a specific image is: 104 | 105 | ``` 106 | ./ExploreImage -i repository/image_name 107 | ``` 108 | 109 | ## Data triage 110 | 111 | In case you scanned a decent amount of images and obtained many findings, you might need to triage the data. For this, I've developed a few scripts which I recommend to take a look at first. 112 | 113 | To run the triage you need to locate your output directory and run: 114 | 115 | ``` 116 | cd YOUR_docker_explorer_output 117 | classify_data.sh 118 | ``` 119 | 120 | After this, you'll see that a `./triaged` directory was created. Within this directory, there are two important files: `triaged_blocker.txt` and `triaged_critical.txt`, as well as the potential files with secrets separated in folders. 121 | 122 | ## Finished? Cleanup your disk! 123 | 124 | If you're done scanning your Docker images and **saved your results**, you might need to delete some files / directories / containers / images, for which I have a script with a few commands that I **recommend to check first**. The cleanup script should be run in the directory where it's stored the output of your scan. After this, your containers, images, and files in the current directory will re removed: 125 | 126 | ``` 127 | cleanup.sh 128 | ``` 129 | 130 | ## TODO 131 | * Integrate with quay.io 132 | * ExploreImage: Add functionality to explore older version of an image 133 | * Add plugin to whispers: Run string command on binaries and report strings that meet certain entropy and extra conditions. 134 | * Allow scanning a repository profile / list of profiles 135 | -------------------------------------------------------------------------------- /classify_data.sh: -------------------------------------------------------------------------------- 1 | find . -maxdepth 1 -name "*.log" -type f -exec cat {} + | grep "CRITICAL" >> critical.txt 2 | find . -maxdepth 1 -name "*.log" -type f -exec cat {} + | grep "BLOCKER" >> blocker.txt 3 | $(dirname $0)/utils/TriageBlockerAndCritical.py 4 | sudo chown -R "$(whoami)" triaged -------------------------------------------------------------------------------- /cleanup.sh: -------------------------------------------------------------------------------- 1 | docker stop $(docker ps -a -q) 2 | docker rm $(docker ps -a -q) 3 | docker rmi $(docker images -q) -f 4 | sudo rm -rf /tmp/explore_* 5 | sudo rm -rf explore_* triaged/ logs/ 6 | rm *.zip out.txt whispers.log blocker.txt critical.txt -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | pip3 install -r requirements.txt 2 | chmod +x ExploreAll ExploreImage install.sh cleanup.sh utils/TriageBlockerAndCritical.py 3 | cd ../ && git clone https://github.com/matiassequeira/whispers 4 | cd whispers && make install -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | docker -------------------------------------------------------------------------------- /utils/ImagesScanner.py: -------------------------------------------------------------------------------- 1 | import json 2 | import subprocess 3 | import docker 4 | import time 5 | import os 6 | 7 | from sys import platform 8 | from requests import request 9 | from utils.Log import getLogger 10 | LOG = getLogger(__name__) 11 | 12 | def scan_image(image_name:str, tmp_path:str, whispers_config:str, whispers_output:str, whispers_timeout:int): 13 | LOG.debug(f"Image: {image_name}") 14 | container_name= "explore_"+image_name.replace("/", "_") 15 | 16 | latest_version= get_image_latest_version(image_name) 17 | 18 | try: 19 | client = docker.from_env() 20 | 21 | LOG.debug(f"Pull image: {image_name}:{latest_version}") 22 | image= client.images.pull(f"{image_name}:{latest_version}") 23 | 24 | LOG.debug(f"Create container: {image_name}") 25 | container= client.containers.create(image=f"{image_name}:{latest_version}", command="fake_command", name=container_name) 26 | 27 | LOG.debug(f"Export fs: {image_name}") 28 | tmp_dump= os.path.join(tmp_path, container_name) 29 | export = subprocess.run(f"docker export {container_name} -o {tmp_dump}.tar", shell=True, stdout=subprocess.PIPE, text=True) 30 | 31 | LOG.debug(f"Remove container: {image_name}") 32 | container.remove() 33 | 34 | LOG.debug(f"Untar: {image_name}") 35 | mkdir = subprocess.run(f"mkdir {tmp_dump}", shell=True, stdout=subprocess.PIPE, text=True, check=True) 36 | untar = subprocess.run(f"tar -xf {tmp_dump}.tar -C {tmp_dump}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) 37 | 38 | except Exception as e: 39 | LOG.debug(f"Error with image: {image_name}: {e}", exc_info=True) 40 | exit(0) 41 | 42 | LOG.debug(f"Whispers: {image_name}") 43 | start = time.time() 44 | try: 45 | output_dir= os.path.join(whispers_output, container_name) 46 | 47 | if platform == "linux" or platform == "linux2": 48 | # In linux it can be used the shell command `timeout` to limit Whispers execution 49 | if whispers_config: 50 | whispers = subprocess.run(f"timeout {whispers_timeout} whispers -d {output_dir} -c {whispers_config} {tmp_dump}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) 51 | else: 52 | whispers = subprocess.run(f"timeout {whispers_timeout} whispers -d {output_dir} {tmp_dump}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) 53 | else: 54 | if whispers_config: 55 | whispers = subprocess.run(f"whispers -d {output_dir} -c {whispers_config} {tmp_dump}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) 56 | else: 57 | whispers = subprocess.run(f"whispers -d {output_dir} {tmp_dump}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) 58 | 59 | except subprocess.TimeoutExpired as e: 60 | LOG.debug(f"Timeout: {image_name}") 61 | 62 | elapsed = (time.time() - start) 63 | LOG.debug(f"Whispers {image_name} execution took {elapsed/60} minutes") 64 | 65 | if len(whispers.stderr)>0: 66 | LOG.debug(f"Whispers {image_name} error: {whispers.stderr}") 67 | 68 | if len(whispers.stdout)>0: 69 | f = open(f"./{container_name}.log", "w") 70 | f.write(whispers.stdout) 71 | f.close 72 | 73 | mkdir = subprocess.run(f"rm -rf {tmp_dump}", shell=True, stdout=subprocess.PIPE, text=True, check=True) 74 | mkdir = subprocess.run(f"rm {tmp_dump}.tar", shell=True, stdout=subprocess.PIPE, text=True, check=True) 75 | client.images.remove(f"{image_name}:{latest_version}") 76 | return 77 | 78 | def get_image_latest_version(image:str): 79 | url = f"https://hub.docker.com:443/v2/repositories/{image}/tags/?page_size=25&page=1" 80 | payload = "" 81 | headers = { 82 | "Cookie": "optimizelyEndUserId=oeu1588552838182r0.7987033509302466; _gcl_au=1.1.1697460598.1588552839; _biz_uid=80332eb0e3694249d5118a8282f5858a; _biz_nA=48; _biz_pendingA=%5B%22m%2Fipv%3F_biz_r%3Dhttps%253A%252F%252Fhub.docker.com%252Fr%252Foracle%252Fweblogic-kubernetes-operator%26_biz_h%3D802059049%26_biz_u%3D80332eb0e3694249d5118a8282f5858a%26_biz_s%3D10d4b9%26_biz_l%3Dhttps%253A%252F%252Fhub.docker.com%252Fr%252Foracle%252Fweblogic-kubernetes-operator%252Ftags%26_biz_t%3D1590280334154%26_biz_i%3Doracle%252Fweblogic-kubernetes-operator%2520-%2520Docker%2520Hub%26_biz_n%3D47%26rnd%3D232513%22%5D; ajs_user_id=null; ajs_group_id=null; _ga=GA1.2.1787025428.1588552840; ajs_anonymous_id=%22883ea900-5ad3-4d7d-ab20-8859a44938b4%22; _fbp=fb.1.1588552840330.690238236; _mkto_trk=id:929-FJL-178&token:_mch-docker.com-1588552840335-25829; _biz_flagsA=%7B%22Version%22%3A1%2C%22Mkto%22%3A%221%22%2C%22XDomain%22%3A%221%22%7D; NPS_383366e9_last_seen=1588552845194; dwf_banner=True; _biz_sid=10d4b9; _gid=GA1.2.261156126.1590278909; _gat=1", 83 | "Accept": "application/json", 84 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:76.0) Gecko/20100101 Firefox/76.0", 85 | "Connection": "close", 86 | "Host": "hub.docker.com", 87 | "Accept-Encoding": "gzip, deflate", 88 | "Accept-Language": "en-US,en;q=0.5" 89 | } 90 | response = request("GET", url, data=payload, headers=headers) 91 | response= json.loads(response.text) 92 | return response.get("results")[0].get("name") -------------------------------------------------------------------------------- /utils/Log.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logging.config 3 | import traceback 4 | import os 5 | 6 | logger=None 7 | level= logging.DEBUG 8 | os.makedirs("logs", exist_ok=True) 9 | logging.config.dictConfig( 10 | { 11 | "version": 1, 12 | "disable_existing_loggers": True, 13 | "formatters": { 14 | "default": { 15 | "class": "logging.Formatter", 16 | "style": "{", 17 | "datefmt": "%Y-%m-%d %H:%M", 18 | "format": "[{asctime:s}] {message:s}" 19 | } 20 | }, 21 | "handlers": { 22 | "console": { 23 | "class": "logging.StreamHandler", 24 | "formatter": "default", 25 | "level": level 26 | }, 27 | "file": { 28 | "level": level, 29 | "class": "logging.handlers.WatchedFileHandler", 30 | "formatter": "default", 31 | "filename": "logs/docker_explorer.log", 32 | "mode": "a", 33 | "encoding": "utf-8" 34 | } 35 | }, 36 | 'root': {'handlers': ('console', 'file')} 37 | } 38 | ) 39 | 40 | def getLogger(name): 41 | global logger 42 | if logger is None: 43 | logger = logging.getLogger(name) 44 | logger.setLevel(level) 45 | return logger -------------------------------------------------------------------------------- /utils/TriageBlockerAndCritical.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import re 5 | import argparse 6 | import os 7 | import shutil 8 | 9 | cwd= os.getcwd() 10 | triaged_path= os.path.join(cwd, "triaged") 11 | 12 | def triage_critical_file(in_file_name, out_file_name): 13 | if out_file_name is None: 14 | out_file_name= f'triaged_{in_file_name}' 15 | 16 | in_file= open(f'../{in_file_name}',"r") 17 | out_file= open(out_file_name,"w") 18 | discarded_elements_file= open(f'discarded_{in_file_name}',"w") 19 | critical_elements=list() 20 | private_keys=list() 21 | discarded_elements=list() 22 | for line in in_file.readlines(): 23 | is_private_key=False 24 | discard=False 25 | try: 26 | line= json.loads(line) 27 | value= line.get('value') 28 | if is_example(line.get('file')): discard= True 29 | elif line.get('message') in ['Password']: 30 | if len(value) <= 10: discard= True 31 | # At least a number and a letter 32 | elif not bool(re.match('^(?=.*[a-zA-Z])(?=.*[0-9])', value)): discard= True 33 | elif ' ' in value: discard= True 34 | elif 'PASSWORD' in value.upper(): discard= True 35 | elif value.upper().startswith('/USR/'): discard= True 36 | elif value.upper().startswith('INCLUDE/'): discard= True 37 | elif value.upper().startswith('/TMP/'): discard= True 38 | elif value.upper().startswith('/HOME'): discard= True 39 | elif value.upper()=='[WSO2CARBON]': discard= True 40 | elif line.get('message') == 'Private key': 41 | if len(value) < 300: discard= True 42 | if '>' in value or '<' in value or ':' in value: discard= True 43 | is_private_key=True 44 | elif line.get('message') == 'Private SSH key file': 45 | is_private_key=True 46 | 47 | if discard: 48 | discarded_elements.append(line) 49 | continue 50 | elif is_private_key: 51 | private_keys.append(line) 52 | else: 53 | critical_elements.append(line) 54 | 55 | store_triaged_file(line.get('file')) 56 | 57 | except Exception as e: 58 | print(f"Error triaging critical: {e}. Line: {line}\n") 59 | 60 | critical_elements= remove_duplicated_key_values(critical_elements) 61 | for i in critical_elements: 62 | out_file.write(json.dumps(i)+'\n') 63 | for i in private_keys: 64 | out_file.write(json.dumps(i)+'\n') 65 | for i in discarded_elements: 66 | discarded_elements_file.write(json.dumps(i)+'\n') 67 | 68 | in_file.close() 69 | out_file.close() 70 | discarded_elements_file.close() 71 | 72 | def split_path_by_dir(path): 73 | path = os.path.normpath(path) 74 | return path.split(os.sep) 75 | 76 | def store_triaged_file(entire_file_path): 77 | split_path = split_path_by_dir(entire_file_path) 78 | explorer_dir= split_path[2] 79 | file_name= split_path[-1] 80 | 81 | original_file_path = os.path.join(cwd, explorer_dir, file_name) 82 | copied_file_path = os.path.join(triaged_path, explorer_dir) 83 | 84 | # os.makedirs(os.path.dirname(copied_file_path), exist_ok=True) 85 | os.makedirs(copied_file_path, exist_ok=True) 86 | shutil.copy(original_file_path, copied_file_path) 87 | 88 | def is_example(entire_file_path): 89 | path = os.path.normpath(entire_file_path) 90 | split_path = path.split(os.sep) 91 | file_name= split_path[-1] 92 | if 'EXAMPLE' in file_name.upper(): 93 | return True 94 | else: 95 | return False 96 | 97 | def triage_blocker_file(in_file_name, out_file_name): 98 | if out_file_name is None: 99 | out_file_name= f'triaged_{in_file_name}' 100 | 101 | in_file= open('../'+in_file_name,"r") 102 | out_file= open(out_file_name,"w") 103 | discarded_elements_file= open(f'discarded_{in_file_name}',"w") 104 | blocker_elements=list() 105 | discarded_elements=list() 106 | for line in in_file.readlines(): 107 | discard=False 108 | try: 109 | line= json.loads(line) 110 | value= line.get('value') 111 | 112 | if line.get('message') in ['AWS Access Key ID', 'AWS Secret Access Key', 'AWS Session Token']: 113 | if 'EXAMPLE' in value: discard= True 114 | if line.get('message') in ['AWS Access Key ID']: 115 | if value in ['AKIAJKAUQVHU6X4CODDQ']: discard= True 116 | if line.get('message') in ['AWS Secret Access Key']: 117 | if value in ['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', 'CustomerManagedDatastoreS3StorageSummary', \ 118 | 'Scte35SpliceInsertScheduleActionSettings', 'AwsS3BucketServerSideEncryptionByDefault', \ 119 | 'ExportEC2InstanceRecommendationsResponse', 'c1dCell2matInconsistentSizesOfLocalParts', \ 120 | 'c2dbcBuildFromReplicatedUnsupportedClass', 'transformPointsPackedMatrixInvalidSize3d', \ 121 | 'LWMSecondDegreePolynomialRequires6Points', 'HPCServer2008SOFailedToGenerateSOAInputs', \ 122 | 'CompressionWithJPEG2000CodecLosslessMode', 'mapBlkInputQuadrilateralVerticesR1c1r4c4', \ 123 | 'InvalidColumnsEnteredInRow0numberinteger', 'ImplicitSelectorDimBasedNumIterMismatch1', \ 124 | 'CordicATan2UnSignedInputWLGreaterThan125', 'GPUAccelerated2DCanvasImageBufferCreated', \ 125 | 'replaceApiregistrationV1APIServiceStatus', 'watchAppsV1DaemonSetListForAllNamespaces', \ 126 | 'watchAppsV1beta2NamespacedReplicaSetList', 'createAuthorizationV1SubjectAccessReview', \ 127 | 'listBatchV2alpha1CronJobForAllNamespaces', 'replaceRbacAuthorizationV1NamespacedRole', \ 128 | 'ExtensionsV1beta1RollingUpdateDeployment', 'R53RResolverEndpointIpAddressAssociation', \ 129 | 'EC2SecurityGroupToClientVpnTargetNetwork', 'EC2SecurityGroupToClientVpnTargetNetwork', \ 130 | 'seeAdminOrderProductOptionValueDropdown1', 'connectCoreV1GetNamespacedPodPortforward', \ 131 | 'watchAppsV1beta1NamespacedDeploymentList', 'deleteExtensionsV1beta1PodSecurityPolicy', \ 132 | 'patchRbacAuthorizationV1beta1ClusterRole', 'watchSettingsV1alpha1NamespacedPodPreset', \ 133 | 'watchCoreV1EndpointsListForAllNamespaces', 'readAppsV1beta1NamespacedDeploymentScale', \ 134 | 'getRbacAuthorizationV1alpha1APIResources', 'watchSchedulingV1alpha1PriorityClassList', \ 135 | 'deleteExtensionsV1beta1NamespacedIngress', 'watchRbacAuthorizationV1beta1ClusterRole', \ 136 | 'createAppsV1NamespacedControllerRevision', 'readBatchV2alpha1NamespacedCronJobStatus', \ 137 | 'readAppsV1beta2NamespacedDeploymentScale', 'aaw1AZ7+OlEEy6FrXkGy0oP2p4BvE/Eeg0L9ucmj']: discard= True 138 | elif value.upper().startswith('/USR/'): discard= True 139 | elif value.upper().startswith('INCLUDE/'): discard= True 140 | elif value.upper().startswith('V1BETA1A/SUBSCRIPTIONS/'): discard= True 141 | elif value.upper().startswith('GOOGLE'): discard= True 142 | elif value.upper().startswith('BUILD'): discard= True 143 | elif value.upper().startswith('RECURSIVE'): discard= True 144 | elif value.upper().startswith('DATADEP'): discard= True 145 | elif value.upper().startswith('ASSERT'): discard= True 146 | elif value.upper().startswith('MATCODE'): discard= True 147 | elif value.upper().startswith('WAITFOR'): discard= True 148 | elif value.upper().startswith('J2EE'): discard= True 149 | elif value.upper().startswith('BEASAML'): discard= True 150 | if line.get('message') in ['Azure Data']: 151 | if ('HTTPS://GOLANGROCKSONAZURE') in line.get('key').upper(): discard= True 152 | elif len(value) <= 10: discard= True 153 | # At least a number and a letter 154 | elif not bool(re.match('^(?=.*[a-zA-Z])(?=.*[0-9])', value)): discard= True 155 | elif value.upper().endswith('PREVIEW') : discard= True 156 | elif value in ['AzDataFactoryV2IntegrationRuntimeUpgrade']: discard= True 157 | elif value.upper().startswith('X86_64'): discard= True 158 | 159 | if discard: 160 | discarded_elements.append(line) 161 | continue 162 | else: 163 | blocker_elements.append(line) 164 | 165 | store_triaged_file(line.get('file')) 166 | except Exception as e: 167 | print(f"Error triaging blocker: {e}. Line: {line}\n") 168 | 169 | blocker_elements= remove_duplicated_key_values(blocker_elements) 170 | for i in blocker_elements: 171 | out_file.write(json.dumps(i)+'\n') 172 | for i in discarded_elements: 173 | discarded_elements_file.write(json.dumps(i)+'\n') 174 | 175 | in_file.close() 176 | out_file.close() 177 | discarded_elements_file.close() 178 | 179 | def remove_duplicated_key_values(elements): 180 | deduplicated_list= elements.copy() 181 | for elem in elements: 182 | skipped_once=False 183 | elem_split_path = split_path_by_dir(elem.get('file')) 184 | for elem2 in deduplicated_list: 185 | elem2_split_path = split_path_by_dir(elem2.get('file')) 186 | # If it's the same image, key, and value remove it 187 | if elem_split_path[2]==elem2_split_path[2] and elem['key']==elem2['key'] and elem['value']==elem2['value']: 188 | if skipped_once: 189 | deduplicated_list.remove(elem2) 190 | else: 191 | skipped_once=True 192 | return deduplicated_list 193 | 194 | if __name__ == '__main__': 195 | 196 | # parser = argparse.ArgumentParser(description='Dockerhub Explorer') 197 | # requiredGroup = parser.add_argument_group('Required arguments') 198 | # requiredGroup.add_argument('-f','--file', 199 | # help='File to triage', 200 | # required = True) 201 | # parser.add_argument('-o','--output', 202 | # help= 'Output file name', 203 | # default=None) 204 | # options= parser.parse_args() 205 | 206 | if not os.path.exists(triaged_path): 207 | os.makedirs(triaged_path) 208 | os.chdir(triaged_path) 209 | 210 | triage_critical_file('critical.txt', None) 211 | triage_blocker_file('blocker.txt', None) 212 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matiassequeira/docker_explorer/e257cb6f9e171cb6b0218edd8868cda8b2c350ba/utils/__init__.py -------------------------------------------------------------------------------- /utils/docker_explorer_transparent_v6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matiassequeira/docker_explorer/e257cb6f9e171cb6b0218edd8868cda8b2c350ba/utils/docker_explorer_transparent_v6.png --------------------------------------------------------------------------------