├── src └── robot_api │ ├── __init__.py │ ├── configs │ ├── ansible_inventory │ └── default_config.json │ ├── api │ ├── __init__.py │ ├── ansible.py │ ├── upload.py │ ├── dockerize.py │ ├── web_resources.py │ └── aggregation.py │ ├── docker_buildfiles │ ├── Dockerfile.Aquatone.tmp │ ├── Dockerfile.Subfinder.tmp │ ├── Dockerfile.Aquatone.Flyover.tmp │ ├── Dockerfile.Subbrute.tmp │ ├── Dockerfile.PDList.tmp │ ├── Dockerfile.Massdns.tmp │ ├── Dockerfile.Sublist3r.tmp │ ├── Dockerfile.Nmap.tmp │ ├── Dockerfile.Turbolist3r.tmp │ ├── Dockerfile.Gowitness.Screenshot.tmp │ ├── Dockerfile.CT.tmp │ ├── Dockerfile.CTFR.tmp │ ├── Dockerfile.Altdns.tmp │ ├── Dockerfile.Spyse.tmp │ ├── Dockerfile.Anubis.tmp │ ├── Dockerfile.Webscreenshot.Screenshot.tmp │ ├── Dockerfile.Knock.tmp │ ├── Dockerfile.Amass.tmp │ ├── Dockerfile.Nmap.Screenshot.tmp │ ├── Dockerfile.Eyewitness.tmp │ ├── Dockerfile.HTTPScreenshot.tmp │ └── Dockerfile.Reconng.tmp │ ├── ansible_plays │ ├── eyewitness_play.yml │ ├── webscreenshot_play.yml │ └── httpscreenshot_play.yml │ ├── config.py │ ├── parse.py │ └── cli.py ├── MANIFEST.in ├── docs ├── images │ ├── demo.gif │ ├── serve.gif │ ├── dumpdb.gif │ ├── inspect.gif │ ├── output.gif │ └── upload_slack.gif ├── source │ ├── modules.rst │ ├── setup.rst │ ├── drrobot.rst │ ├── tests.rst │ ├── index.rst │ ├── robot_api.rst │ ├── src.rst │ ├── robot_api.api.rst │ ├── tests.unit.rst │ └── conf.py ├── Makefile └── make.bat ├── LICENSES ├── LICENSE.mattermostpythondrive.txt ├── LICENSE.pipenv.txt ├── LICENSE.aquatone.txt ├── LICENSE.shodanpython.txt ├── LICENSE.dockerpy.txt └── LICENSE.docker.txt ├── Makefile ├── open-docs.py ├── setup.cfg ├── requirements.txt ├── docker-compose.yml ├── INSTALL ├── Pipfile ├── setup.py ├── .gitignore ├── LICENSE ├── .travis.yml ├── readmes ├── usage.md └── config.md ├── CHANGELOG.md └── README.md /src/robot_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include src * 2 | -------------------------------------------------------------------------------- /src/robot_api/configs/ansible_inventory: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/images/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/dr_robot/HEAD/docs/images/demo.gif -------------------------------------------------------------------------------- /docs/images/serve.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/dr_robot/HEAD/docs/images/serve.gif -------------------------------------------------------------------------------- /docs/images/dumpdb.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/dr_robot/HEAD/docs/images/dumpdb.gif -------------------------------------------------------------------------------- /docs/images/inspect.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/dr_robot/HEAD/docs/images/inspect.gif -------------------------------------------------------------------------------- /docs/images/output.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/dr_robot/HEAD/docs/images/output.gif -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | src 2 | === 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | robot_api 8 | -------------------------------------------------------------------------------- /docs/images/upload_slack.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/dr_robot/HEAD/docs/images/upload_slack.gif -------------------------------------------------------------------------------- /LICENSES/LICENSE.mattermostpythondrive.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandialabs/dr_robot/HEAD/LICENSES/LICENSE.mattermostpythondrive.txt -------------------------------------------------------------------------------- /docs/source/setup.rst: -------------------------------------------------------------------------------- 1 | setup module 2 | ============ 3 | 4 | .. automodule:: setup 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/source/drrobot.rst: -------------------------------------------------------------------------------- 1 | drrobot module 2 | ============== 3 | 4 | .. automodule:: drrobot 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: docs 2 | 3 | .PHONY: clean 4 | clean: 5 | find . -name "*.pyc" -delete 6 | 7 | .PHONY: docs 8 | docs: 9 | sphinx-apidoc -f -o docs/source src/ && cd docs && make html && cd .. 10 | -------------------------------------------------------------------------------- /open-docs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import webbrowser 3 | 4 | url = "file://%s/docs/build/html/index.html" 5 | url = url % (os.path.dirname(os.path.abspath(__file__))) 6 | webbrowser.open(url, new=0, autoraise=True) 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test = pytest 3 | 4 | [bumpversion] 5 | commit = True 6 | tag = True 7 | current_version = 1.0.0 8 | 9 | [bumpversion:file:setup.py] 10 | 11 | [bumpversion:file:src/__init__.py] 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/robot_api/api/__init__.py: -------------------------------------------------------------------------------- 1 | from robot_api.api.ansible import Ansible 2 | from robot_api.api.dockerize import Docker 3 | from robot_api.api.aggregation import Aggregation 4 | from robot_api.api.upload import Forum, Mattermost, Slack 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | docker~=4.0 2 | requests~=2.0 3 | netaddr~=0.7.19 4 | mattermostdriver~=6.0 5 | shodan~=1.0 6 | certifi~=2019.9 7 | beautifulsoup4~=4.0 8 | argparse 9 | tqdm~=4.0 10 | dicttoxml~=1.0 11 | slackclient~=2.0 12 | idna-ssl~=1.1 13 | -------------------------------------------------------------------------------- /docs/source/tests.rst: -------------------------------------------------------------------------------- 1 | tests package 2 | ============= 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | tests.unit 10 | 11 | Module contents 12 | --------------- 13 | 14 | .. automodule:: tests 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | minio: 4 | image: minio/minio:RELEASE.2018-03-30T00-38-44Z 5 | volumes: 6 | - ~/.drrobot/output/:/export 7 | ports: 8 | - "9000:9000" 9 | environment: 10 | MINIO_ACCESS_KEY: YOURACCESSKEY 11 | MINIO_SECRET_KEY: YOURSECRETKEY 12 | command: server /export 13 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | ## Pipenv 2 | This tool has been made with Pipenv in mind. That being said you should easily be able to convert it to a virtualenv if you hate pipenv. 3 | 4 | ``` 5 | cd INSTALL_DIRECTORY 6 | pipenv install && pipenv shell 7 | ``` 8 | Possible integration with pip in near future as cited in Pipenv docs: https://github.com/pypa/pipfile 9 | 10 | ## Virtualenv 11 | 12 | ``` 13 | cd INSTALL_DIRECTORY 14 | virtualenv python3 env 15 | . env/bin/activate 16 | pip install --requirements Pipfile 17 | ``` 18 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Dr.ROBOT documentation master file, created by 2 | sphinx-quickstart on Wed Nov 7 15:50:56 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Dr.ROBOT's documentation! 7 | ==================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | drrobot.rst 13 | src.rst 14 | modules.rst 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | dr-robot = {editable = true,path = "."} 8 | idna-ssl = "*" 9 | 10 | [dev-packages] 11 | docker = "~=4.0" 12 | requests = "~=2.0" 13 | netaddr = "~=0.7.19" 14 | mattermostdriver= "~=6.0" 15 | shodan = "~=1.0" 16 | certifi = "~=2019.9" 17 | beautifulsoup4 = "~=4.0" 18 | tqdm = "~=4.0" 19 | dicttoxml = "~=1.0" 20 | slackclient = "~=2.0" 21 | argparse= "*" 22 | "flake8" = "*" 23 | pylint = "*" 24 | "autopep8" = "*" 25 | sphinx = "*" 26 | sphinx-rtd-theme = "*" 27 | 28 | [requires] 29 | python_version = "3" 30 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Aquatone.tmp: -------------------------------------------------------------------------------- 1 | FROM andrius/alpine-ruby 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | 6 | RUN if [ -n $dns ];\ 7 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 8 | fi; \ 9 | apk update && apk upgrade && apk add --no-cache git build-base ruby ruby-bundler ruby-dev gcc libffi ruby-json libc-dev make linux-headers libffi-dev 10 | 11 | RUN if [ -n $dns ]; \ 12 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 13 | fi; \ 14 | if [ -n "$$http_proxy" ]; \ 15 | then gem install --http-proxy=$http_proxy aquatone;\ 16 | else gem install aquatone; \ 17 | fi; 18 | 19 | RUN mkdir -p $output 20 | 21 | ENTRYPOINT aquatone-discover --thread 40 --ignore-private --domain "$target" && mv /root/aquatone/$target/hosts.txt $output/aquatone.txt 22 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Subfinder.tmp: -------------------------------------------------------------------------------- 1 | FROM golang:1.11.4-alpine3.7 AS build-env 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | 6 | RUN if [ -n $dns ]; \ 7 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 8 | fi;\ 9 | apk add --no-cache --upgrade git openssh-client ca-certificates 10 | 11 | ADD certs/ /usr/local/share/ca-certificates/ 12 | RUN update-ca-certificates 13 | 14 | RUN if [ -n $dns ]; \ 15 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 16 | fi;\ 17 | go get github.com/golang/dep/cmd/dep 18 | 19 | WORKDIR /go/src/app 20 | 21 | # Install 22 | RUN if [ -n $dns ]; \ 23 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 24 | fi;\ 25 | go get -u github.com/subfinder/subfinder 26 | 27 | RUN mkdir -p $output 28 | 29 | ENTRYPOINT subfinder -d "$target" -o $output/subfinder.json -oJ 30 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Aquatone.Flyover.tmp: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | 6 | RUN if [ -n $dns ];\ 7 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 8 | fi; \ 9 | apk update && apk upgrade && apk add --no-cache wget chromium ca-certificates 10 | 11 | ADD certs/ /usr/local/share/ca-certificates/ 12 | RUN update-ca-certificates 13 | 14 | RUN if [ -n $dns ];\ 15 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 16 | fi; \ 17 | wget https://github.com/michenriksen/aquatone/releases/download/v1.7.0/aquatone_linux_amd64_1.7.0.zip -O aquatone.zip && \ 18 | unzip aquatone.zip && \ 19 | mv aquatone /usr/bin/ 20 | 21 | RUN mkdir -p $output/aquatone-flyover 22 | 23 | WORKDIR $output/aquatone-flyover 24 | ENTRYPOINT cat $infile | aquatone -chrome-path /usr/bin/chromium-browser 25 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Subbrute.tmp: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | WORKDIR /home 4 | 5 | ENV http_proxy $proxy 6 | ENV https_proxy $proxy 7 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 8 | 9 | RUN mkdir -p $output 10 | 11 | RUN if [ -n $dns ];\ 12 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 13 | fi;\ 14 | apt install -y git ca-certificates;\ 15 | pip install --trusted-host pypi.org dnspython; 16 | 17 | ADD certs/ /usr/local/share/ca-certificates/ 18 | ADD certs/ /etc/ssl/certs/ 19 | RUN update-ca-certificates 20 | 21 | RUN if [ -n $dns ];\ 22 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 23 | fi;\ 24 | git clone https://github.com/TheRook/subbrute.git /home/subbrute 25 | 26 | RUN mkdir -p $output 27 | 28 | WORKDIR /home/subbrute 29 | 30 | ENTRYPOINT ./subbrute.py -o $output/subbrute.txt -r resolvers.txt -p -v $target 31 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.PDList.tmp: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | 3 | WORKDIR /root 4 | ENV http_proxy $proxy 5 | ENV https_proxy $proxy 6 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 7 | 8 | RUN mkdir -p $output 9 | 10 | RUN if [ -n $dns ]; \ 11 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 12 | fi;\ 13 | apt-get install git ca-certificates 14 | 15 | ADD certs/ /usr/local/share/ca-certificates/ 16 | ADD certs/ /etc/ssl/certs/ 17 | RUN update-ca-certificates 18 | 19 | RUN if [ -n $dns ];\ 20 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 21 | fi;\ 22 | git clone https://github.com/gnebbia/pdlist /root/pdlist 23 | 24 | WORKDIR /root/pdlist 25 | 26 | RUN if [ -n $dns ]; \ 27 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 28 | fi;\ 29 | pip3 install -r requirements.txt 30 | 31 | ENTRYPOINT pdlist $target > $output/sublist3r.txt 32 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Massdns.tmp: -------------------------------------------------------------------------------- 1 | FROM ubuntu:16.04 2 | 3 | WORKDIR /home 4 | ENV http_proxy $proxy 5 | ENV https_proxy $proxy 6 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 7 | 8 | RUN mkdir -p $output 9 | 10 | RUN if [ -n $dns ]; \ 11 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 12 | fi;\ 13 | apt-get update &&\ 14 | apt-get install -y libldns-dev git build-essential python wget 15 | 16 | ADD certs/ /usr/local/share/ca-certificates/ 17 | ADD certs/ /etc/ssl/certs/ 18 | RUN update-ca-certificates 19 | 20 | RUN if [ -n $dns ];\ 21 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 22 | fi;\ 23 | git clone https://github.com/blechschmidt/massdns.git /home/massdns 24 | 25 | WORKDIR /home/massdns 26 | 27 | RUN make 28 | RUN echo $target > target.txt 29 | 30 | 31 | ENTRYPOINT /bin/massdns -r /home/massdns/lists/resolvers.txt -w $output/massdns.txt -t AAAA target.txt 32 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Sublist3r.tmp: -------------------------------------------------------------------------------- 1 | FROM python:3.4 2 | 3 | WORKDIR /home 4 | ENV http_proxy $proxy 5 | ENV https_proxy $proxy 6 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 7 | 8 | RUN mkdir -p $output 9 | 10 | RUN if [ -n $dns ]; \ 11 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 12 | fi;\ 13 | apt-get install git ca-certificates 14 | 15 | ADD certs/ /usr/local/share/ca-certificates/ 16 | ADD certs/ /etc/ssl/certs/ 17 | RUN update-ca-certificates 18 | 19 | RUN if [ -n $dns ];\ 20 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 21 | fi;\ 22 | git clone https://github.com/aboul3la/Sublist3r.git /home/sublist 23 | 24 | WORKDIR /home/sublist 25 | 26 | RUN if [ -n $dns ]; \ 27 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 28 | fi;\ 29 | pip3 install -r requirements.txt 30 | 31 | ENTRYPOINT python3 sublist3r.py --domain $target -o $output/sublist3r.txt 32 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Nmap.tmp: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | ENV TARGET $target 6 | ENV OUTPUT /tmp/output/eyewitness 7 | ENV INFILE $infile 8 | ARG user=eyewitness 9 | 10 | RUN if [ -n $dns ]; \ 11 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 12 | apt update; \ 13 | apt install -y nmap git wget ca-certificates 14 | 15 | ADD certs/ /usr/local/share/ca-certificates/ 16 | RUN update-ca-certificates 17 | 18 | RUN if [ -n $dns ]; \ 19 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 20 | git clone https://github.com/SpiderLabs/Nmap-Tools.git \ 21 | && cp Nmap-Tools/NSE/http-screenshot.nse /usr/share/nmap/scripts/http-screenshot.nse \ 22 | && nmap --script-updatedb \ 23 | && wget -O wkhtml.deb https://downloads.wkhtmltopdf.org/0.12/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ 24 | && dpkg -i wkhtml.deb 25 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Turbolist3r.tmp: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | 3 | WORKDIR /root 4 | ENV http_proxy $proxy 5 | ENV https_proxy $proxy 6 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 7 | 8 | RUN mkdir -p $output 9 | 10 | RUN if [ -n $dns ]; \ 11 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 12 | fi;\ 13 | apt-get install git ca-certificates 14 | 15 | ADD certs/ /usr/local/share/ca-certificates/ 16 | ADD certs/ /etc/ssl/certs/ 17 | RUN update-ca-certificates 18 | 19 | RUN if [ -n $dns ];\ 20 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 21 | fi;\ 22 | git clone https://github.com/fleetcaptain/Turbolist3r.git /root/turbo 23 | 24 | WORKDIR /root/turbo 25 | 26 | RUN if [ -n $dns ]; \ 27 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 28 | fi;\ 29 | pip3 install -r requirements.txt 30 | 31 | ENTRYPOINT python3 turbolist3r.py --domain $target -o $output/sublist3r.txt 32 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Gowitness.Screenshot.tmp: -------------------------------------------------------------------------------- 1 | FROM golang:1.13.1-buster as build 2 | 3 | RUN if [ -n $dns ]; \ 4 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 5 | fi;\ 6 | apt update && \ 7 | apt install -y git ca-certificates chromium; 8 | 9 | ADD certs/ /usr/local/share/ca-certificates/ 10 | RUN update-ca-certificates 11 | 12 | RUN if [ -n $dns ]; \ 13 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 14 | fi;\ 15 | go get -u github.com/sensepost/gowitness 16 | 17 | ENV GO111MODULE on 18 | 19 | WORKDIR /go/src/github.com/sensepost/gowitness 20 | 21 | RUN if [ -n $dns ]; \ 22 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 23 | fi;\ 24 | go build && \ 25 | cp gowitness /usr/bin/ 26 | 27 | ENV http_proxy $proxy 28 | ENV https_proxy $proxy 29 | ENV HOME / 30 | 31 | 32 | ENTRYPOINT mkdir -p $output/gowitness && cd $output/gowitness && gowitness file -s $infile --threads 20 33 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.CT.tmp: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 6 | 7 | RUN if [ -n $dns ]; \ 8 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 9 | apt-get update && apt-get install -y python-dev git ca-certificates 10 | 11 | ADD certs/ /usr/local/share/ca-certificates/ 12 | ADD certs/ /etc/ssl/certs/ 13 | RUN update-ca-certificates 14 | 15 | RUN if [ -n $dns ]; \ 16 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 17 | git clone https://github.com/chris408/ct-exposer.git ctexposure 18 | 19 | 20 | WORKDIR /ctexposure 21 | 22 | RUN if [ -n $dns ]; \ 23 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 24 | pip3 install --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements.txt 25 | 26 | ENV INFILE $infile 27 | 28 | RUN mkdir -p $output 29 | 30 | ENTRYPOINT python ct-exposer.py -d "$target" -m -u > $output/ctexposure.txt 31 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.CTFR.tmp: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 6 | 7 | RUN if [ -n $dns ]; \ 8 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 9 | apt-get update && apt-get install -y --no-install-recommends \ 10 | python-dev \ 11 | git \ 12 | ca-certificates 13 | 14 | ADD certs/ /usr/local/share/ca-certificates/ 15 | ADD certs/ /etc/ssl/certs/ 16 | RUN update-ca-certificates 17 | 18 | RUN if [ -n $dns ]; \ 19 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 20 | git clone https://github.com/UnaPibaGeek/ctfr.git ctfr 21 | 22 | 23 | WORKDIR /ctfr 24 | 25 | RUN if [ -n $dns ]; \ 26 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 27 | pip install --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host files.pythonhosted.org -r requirements.txt 28 | 29 | RUN mkdir -p $output 30 | 31 | ENTRYPOINT python ctfr.py -d $target -o $output/ctfr.txt 32 | -------------------------------------------------------------------------------- /docs/source/robot_api.rst: -------------------------------------------------------------------------------- 1 | robot\_api package 2 | ================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | robot_api.api 10 | 11 | Submodules 12 | ---------- 13 | 14 | robot\_api.cli module 15 | --------------------- 16 | 17 | .. automodule:: robot_api.cli 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | robot\_api.config module 23 | ------------------------ 24 | 25 | .. automodule:: robot_api.config 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | 30 | robot\_api.parse module 31 | ----------------------- 32 | 33 | .. automodule:: robot_api.parse 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | robot\_api.robot module 39 | ----------------------- 40 | 41 | .. automodule:: robot_api.robot 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | 46 | 47 | Module contents 48 | --------------- 49 | 50 | .. automodule:: robot_api 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | -------------------------------------------------------------------------------- /docs/source/src.rst: -------------------------------------------------------------------------------- 1 | src package 2 | =========== 3 | 4 | Submodules 5 | ---------- 6 | 7 | src.ansible module 8 | ------------------ 9 | 10 | .. automodule:: src.ansible 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | src.dockerize module 16 | -------------------- 17 | 18 | .. automodule:: src.dockerize 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | src.robot module 24 | ---------------- 25 | 26 | .. automodule:: src.robot 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | src.upload module 32 | ----------------- 33 | 34 | .. automodule:: src.upload 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | src.web\_resources module 40 | ------------------------- 41 | 42 | .. automodule:: src.web_resources 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: src 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Altdns.tmp: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | ENV DNS $dns 6 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 7 | 8 | 9 | RUN if [ -n $dns ]; \ 10 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 11 | fi;\ 12 | apt-get install git ca-certificates 13 | 14 | ADD certs/ /usr/local/share/ca-certificates/ 15 | ADD certs/ /etc/ssl/certs/ 16 | RUN update-ca-certificates 17 | 18 | RUN if [ -n $dns ]; \ 19 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 20 | fi;\ 21 | cat /etc/resolv.conf ;\ 22 | git clone https://github.com/infosec-au/altdns.git altdns 23 | 24 | WORKDIR altdns 25 | 26 | RUN if [ -n $dns ]; \ 27 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 28 | fi;\ 29 | python setup.py install 30 | 31 | RUN cp $infile /tmp/infile.txt; \ 32 | echo $target >> /tmp/infile.txt 33 | 34 | RUN mkdir -p $output 35 | 36 | WORKDIR $output 37 | 38 | ENTRYPOINT altdns -i /tmp/infile.txt -w /altdns/words.txt -o altered.txt -s altnds.txt -r 39 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Spyse.tmp: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | WORKDIR /home 4 | ENV http_proxy $proxy 5 | ENV https_proxy $proxy 6 | 7 | RUN if [ -n $dns ]; \ 8 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 9 | fi;\ 10 | apt-get update && \ 11 | apt-get install -y git python-dnspython ca-certificates && \ 12 | rm -rf /var/lib/apt/lists/* 13 | 14 | ADD certs/ /usr/local/share/ca-certificates/ 15 | RUN update-ca-certificates 16 | 17 | RUN if [ -n $dns ]; \ 18 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 19 | fi;\ 20 | git clone https://github.com/guelfoweb/knock.git /home/knock 21 | 22 | WORKDIR /home/knock 23 | 24 | RUN if [ -n $dns ]; \ 25 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 26 | fi;\ 27 | python setup.py install 28 | 29 | RUN if [ -n "$VT_KEY" ]; \ 30 | then sed -i ".orig" 's/""/"$VT_KEY"/g' knockpy/config.json;\ 31 | fi 32 | 33 | ENV TARGET $target 34 | ENV OUTPUT $output 35 | ENV VT_KEY $vt_key 36 | 37 | RUN mkdir -p $output 38 | WORKDIR $output 39 | 40 | ENTRYPOINT knockpy -c $target 41 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Anubis.tmp: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 6 | 7 | RUN if [ -n $dns ]; \ 8 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 9 | apt-get update && apt-get install -y --no-install-recommends \ 10 | build-essential \ 11 | libssl-dev \ 12 | libffi-dev \ 13 | python-dev \ 14 | ca-certificates \ 15 | git \ 16 | && rm -rf /var/lib/apt/lists/* 17 | 18 | ADD certs/ /usr/local/share/ca-certificates/ 19 | ADD certs/ /etc/ssl/certs/ 20 | RUN update-ca-certificates 21 | 22 | RUN if [ -n $dns ]; \ 23 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 24 | git clone https://github.com/jonluca/Anubis.git anubis 25 | 26 | 27 | WORKDIR /anubis 28 | 29 | RUN if [ -n $dns ]; \ 30 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 31 | pip3 install -r requirements.txt; \ 32 | pip3 install . 33 | 34 | RUN mkdir -p $output 35 | 36 | ENTRYPOINT anubis -t "$target" -o $output/anubis.txt -i 37 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Webscreenshot.Screenshot.tmp: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | 6 | RUN mkdir -p $output/webscreenshot 7 | 8 | RUN if [ -n $dns ]; \ 9 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 10 | fi;\ 11 | apt-get update \ 12 | && apt-get install -y wget git python python-pip chromium-browser xvfb firefox phantomjs ca-certificates 13 | 14 | ADD certs/ /usr/local/share/ca-certificates/ 15 | RUN update-ca-certificates 16 | 17 | 18 | RUN if [ -n $dns ]; \ 19 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 20 | fi;\ 21 | git clone https://github.com/maaaaz/webscreenshot.git webscreenshot 22 | 23 | WORKDIR webscreenshot 24 | 25 | RUN if [ -n $dns ]; \ 26 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 27 | fi;\ 28 | pip install -r requirements.txt 29 | 30 | RUN if [ -n $dns ]; \ 31 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 32 | fi;\ 33 | pip install -r requirements.txt 34 | 35 | 36 | ENTRYPOINT python webscreenshot.py -i $infile -o $output/webscreenshot -r $tool --no-xserver 37 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Knock.tmp: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | WORKDIR /home 4 | ENV http_proxy $proxy 5 | ENV https_proxy $proxy 6 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 7 | 8 | RUN if [ -n $dns ]; \ 9 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 10 | fi;\ 11 | apt-get update && \ 12 | apt-get install -y git python-dnspython ca-certificates && \ 13 | rm -rf /var/lib/apt/lists/* 14 | 15 | ADD certs/ /usr/local/share/ca-certificates/ 16 | ADD certs/ /etc/ssl/certs/ 17 | RUN update-ca-certificates 18 | 19 | RUN if [ -n $dns ]; \ 20 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 21 | fi;\ 22 | git clone https://github.com/guelfoweb/knock.git /home/knock 23 | 24 | WORKDIR /home/knock 25 | 26 | RUN if [ -n $dns ]; \ 27 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 28 | fi;\ 29 | python setup.py install 30 | 31 | RUN if [ -n "$VT_KEY" ]; \ 32 | then sed -i ".orig" 's/""/"$VT_KEY"/g' knockpy/config.json;\ 33 | fi 34 | 35 | ENV TARGET $target 36 | ENV OUTPUT $output 37 | ENV VT_KEY $vt_key 38 | 39 | RUN mkdir -p $output 40 | WORKDIR $output 41 | 42 | ENTRYPOINT knockpy -c $target 43 | -------------------------------------------------------------------------------- /LICENSES/LICENSE.pipenv.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright 2017 Kenneth Reitz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Amass.tmp: -------------------------------------------------------------------------------- 1 | FROM golang:1.13.0-alpine3.10 as build 2 | 3 | RUN if [ -n $dns ]; \ 4 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 5 | fi;\ 6 | apk --no-cache add git ca-certificates; 7 | 8 | ADD certs/ /usr/local/share/ca-certificates/ 9 | RUN update-ca-certificates 10 | 11 | RUN if [ -n $dns ]; \ 12 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 13 | fi;\ 14 | go get github.com/OWASP/Amass; exit 0 15 | 16 | ENV GO111MODULE on 17 | 18 | WORKDIR /go/src/github.com/OWASP/Amass 19 | 20 | RUN if [ -n $dns ]; \ 21 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 22 | fi;\ 23 | go install ./... 24 | 25 | FROM alpine:latest 26 | 27 | RUN if [ -n $dns ]; \ 28 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 29 | fi;\ 30 | apk --no-cache add ca-certificates 31 | 32 | COPY --from=build /go/bin/amass /bin/amass 33 | COPY --from=build /go/src/github.com/OWASP/Amass/examples/wordlists/ /wordlists/ 34 | 35 | ENV http_proxy $proxy 36 | ENV https_proxy $proxy 37 | ENV HOME / 38 | 39 | RUN mkdir -p $output 40 | 41 | ENTRYPOINT /bin/amass enum --passive -d "$target" -o $output/amass.txt 42 | -------------------------------------------------------------------------------- /LICENSES/LICENSE.aquatone.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Michael Henriksen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Nmap.Screenshot.tmp: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | ENV DNS $dns 6 | 7 | RUN if [ -n $dns ]; \ 8 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 9 | apt update; \ 10 | apt install -y nmap git wget libjpeg-turbo8 fontconfig libfreetype6 xfonts-base libxrender1 xfonts-75dpi chromium-browser firefox ca-certificates 11 | 12 | ADD certs/ /usr/local/share/ca-certificates/ 13 | RUN update-ca-certificates 14 | 15 | RUN if [ -n $dns ]; \ 16 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 17 | git clone https://github.com/CrimsonK1ng/nmap-screenshot.git \ 18 | && cp nmap-screenshot/http-screenshot.nse /usr/share/nmap/scripts/http-screenshot.nse \ 19 | && nmap --script-updatedb 20 | 21 | RUN if [ -n $dns ]; \ 22 | then echo "nameserver $dns" >> /etc/resolv.conf; fi;\ 23 | wget -O wkhtml.deb https://downloads.wkhtmltopdf.org/0.12/0.12.5/wkhtmltox_0.12.5-1.bionic_amd64.deb \ 24 | && dpkg -i wkhtml.deb 25 | 26 | WORKDIR $output 27 | 28 | ENTRYPOINT mkdir -p $output/nmapscreen && cd $output/nmapscreen && nmap --script http-screenshot --script-args tool=$tool -iL $infile -p "80,8080,443,8888" 29 | -------------------------------------------------------------------------------- /docs/source/robot_api.api.rst: -------------------------------------------------------------------------------- 1 | robot\_api.api package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | robot\_api.api.aggregation module 8 | --------------------------------- 9 | 10 | .. automodule:: robot_api.api.aggregation 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | robot\_api.api.ansible module 16 | ----------------------------- 17 | 18 | .. automodule:: robot_api.api.ansible 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | robot\_api.api.dockerize module 24 | ------------------------------- 25 | 26 | .. automodule:: robot_api.api.dockerize 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | robot\_api.api.upload module 32 | ---------------------------- 33 | 34 | .. automodule:: robot_api.api.upload 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | robot\_api.api.web\_resources module 40 | ------------------------------------ 41 | 42 | .. automodule:: robot_api.api.web_resources 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: robot_api.api 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/source/tests.unit.rst: -------------------------------------------------------------------------------- 1 | tests.unit package 2 | ================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | tests.unit.NO\_test\_upload module 8 | ---------------------------------- 9 | 10 | .. automodule:: tests.unit.NO_test_upload 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | tests.unit.test\_ansible module 16 | ------------------------------- 17 | 18 | .. automodule:: tests.unit.test_ansible 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | tests.unit.test\_dockerize module 24 | --------------------------------- 25 | 26 | .. automodule:: tests.unit.test_dockerize 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | tests.unit.test\_drrobot module 32 | ------------------------------- 33 | 34 | .. automodule:: tests.unit.test_drrobot 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | tests.unit.test\_robot module 40 | ----------------------------- 41 | 42 | .. automodule:: tests.unit.test_robot 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: tests.unit 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import codecs 4 | 5 | from setuptools import setup, find_packages 6 | 7 | def get_requirements(): 8 | with open('requirements.txt', 'r') as fp: 9 | reqs = [req.strip() for req in fp.readlines()] 10 | return reqs 11 | 12 | 13 | setup( 14 | name="drrobot", 15 | version="1.1.2", 16 | package_dir={"": "src"}, 17 | packages=find_packages(where="src"), 18 | url='https://github.com/sandialabs/dr_robot', 19 | author='Aleksandar Straumann', 20 | author_email='astraum@sandia.gov', 21 | description='This tool can be used to enumerate the subdomains associated' 22 | + 'with a company by aggregating the results of multiple OSINT' 23 | + '(Open Source Intelligence) tools.', 24 | keywords=[ 25 | 'environment variables', 26 | 'settings', 27 | 'env', 28 | 'encryption', 29 | 'dotenv', 30 | 'configurations', 31 | 'python' 32 | ], 33 | long_description=codecs.open('README.md', encoding="utf8").read(), 34 | include_package_data=True, 35 | entry_points={ 36 | 'console_scripts': ['drrobot=robot_api.cli:run'] 37 | }, 38 | install_requires=[ 39 | ], 40 | setup_requires=[ 41 | 'pytest-runner' 42 | ], 43 | tests_require=[ 44 | 'pytest', 45 | ] 46 | ) 47 | -------------------------------------------------------------------------------- /LICENSES/LICENSE.shodanpython.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014- John Matherly 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | Except as contained in this notice, the name(s) of the above 16 | copyright holders shall not be used in advertising or otherwise 17 | to promote the sale, use or other dealings in this Software 18 | without prior written authorization. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 21 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 22 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 23 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 24 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 25 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 26 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 27 | OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Eyewitness.tmp: -------------------------------------------------------------------------------- 1 | FROM phusion/baseimage 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | 6 | ARG user=eyewitness 7 | 8 | RUN if [ -n $dns ]; \ 9 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 10 | fi;\ 11 | apt-get update && \ 12 | apt-get install -y git wget ca-certificates && \ 13 | rm -rf /var/lib/apt/lists/* 14 | 15 | 16 | ADD certs/ /usr/local/share/ca-certificates/ 17 | RUN update-ca-certificates 18 | 19 | RUN if [ -n $dns ]; \ 20 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 21 | fi;\ 22 | export uid=1000 gid=1000 && \ 23 | mkdir -p /home/$$user && \ 24 | echo "$$user:x:$${uid}:$${gid}:$$user,,,:/home/$$user:/bin/bash" >> /etc/passwd && \ 25 | echo "$$user:x:$${uid}:" >> /etc/group && \ 26 | chown $${uid}:$${gid} -R /home/$$user 27 | 28 | WORKDIR /home/$$user 29 | 30 | RUN if [ -n $dns ]; \ 31 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 32 | fi;\ 33 | git clone https://github.com/ChrisTruncer/EyeWitness.git && \ 34 | cd EyeWitness 35 | 36 | WORKDIR /home/$$user/EyeWitness 37 | 38 | RUN cd setup && \ 39 | ./setup.sh && \ 40 | cd .. && \ 41 | chown -R $$user:$$user /home/$$user/EyeWitness && \ 42 | mkdir -p /tmp/EyeWitness 43 | 44 | ENTRYPOINT mkdir -p $output/Eyewitness && python EyeWitness.py -d $output/EyeWitness -f $infile --no-prompt --web --threads 40 --max-retries 1 45 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.HTTPScreenshot.tmp: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | 6 | RUN if [ -n $dns ]; \ 7 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 8 | fi;\ 9 | apt-get update \ 10 | && apt-get install -y wget git python-dev python-pip libfontconfig unzip firefox ca-certificates 11 | 12 | 13 | ADD certs/ /usr/local/share/ca-certificates/ 14 | RUN update-ca-certificates 15 | 16 | RUN if [ -n $dns ]; \ 17 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 18 | fi;\ 19 | wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \ 20 | && dpkg -i google-chrome-stable_current_amd64.deb || true \ 21 | && apt-get -y -f install 22 | 23 | RUN if [ -n $dns ]; \ 24 | then echo "nameserver $dns" >> /etc/resolv.conf;\ 25 | fi;\ 26 | git clone https://github.com/CrimsonK1ng/httpscreenshot.git 27 | 28 | RUN cd httpscreenshot \ 29 | && ./install-dependencies.sh \ 30 | && chmod +x httpscreenshot.py \ 31 | && ln -s /httpscreenshot/httpscreenshot.py /usr/bin/httpscreenshot \ 32 | && wget https://chromedriver.storage.googleapis.com/2.44/chromedriver_linux64.zip \ 33 | && unzip -o chromedriver_linux64.zip \ 34 | && ln -s /httpscreenshot/chromedriver /usr/bin/chromedriver 35 | 36 | ENTRYPOINT mkdir -p $output/httpscreenshot && cd $output/httpscreenshot && httpscreenshot -i $infile -b chrome -p -w 40 -a -vH 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # custom 107 | .idea/* 108 | src/robot_api/configs/user_config.json 109 | src/robot_api/docker_buildfiles/*Dockerfile* 110 | !src/robot_api/docker_buildfiles/*tmp 111 | src/robot_api/ansible_plays/*.retry 112 | logs/* 113 | output/* 114 | dbs/* 115 | serve_api/Dockerfile.Django 116 | -------------------------------------------------------------------------------- /src/robot_api/ansible_plays/eyewitness_play.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: "{{ variable_host|quote }}" 3 | remote_user: "{{ variable_user|quote }}" 4 | 5 | tasks: 6 | - name: Apt install git 7 | become: true 8 | apt: 9 | name: git 10 | force: yes 11 | 12 | - name: Apt install python 13 | become: true 14 | apt: 15 | name: python 16 | force: yes 17 | 18 | - name: Git install EyeWitness repo 19 | git: 20 | repo: https://github.com/FortyNorthSecurity/EyeWitness.git 21 | update: yes 22 | dest: /tmp/EyeWitness 23 | 24 | - name: Pip install EyeWitness 25 | become: true 26 | command: 27 | chdir=/tmp/EyeWitness/setup 28 | ./setup.sh 29 | 30 | - name: Copy target hosts file to remote 31 | become: true 32 | copy: 33 | src: "{{ infile|quote }}" #This is how you access a passed in variable --extra-vars "version=1.23.45 other_variable=foo" 34 | dest: /tmp/EyeWitness/hosts_targets.txt 35 | owner: root 36 | group: root 37 | force: true 38 | mode: u+rw,g+rw,o+rw 39 | 40 | - name: Run Scan 41 | become: true 42 | command: 43 | chdir=/tmp/EyeWitness 44 | python3 EyeWitness.py -f hosts_targets.txt -d /tmp/EyeWitness/reportout --no-prompt --web --threads 40 --max-retries 1 45 | async: 100000 46 | poll: 60 47 | 48 | - name: Permissions 49 | become: true 50 | file: 51 | group: "{{ variable_user|quote }}" 52 | owner: "{{ variable_user|quote }}" 53 | recurse: yes 54 | path: /tmp/EyeWitness 55 | 56 | - name: Create tar of all files 57 | archive: 58 | path: /tmp/EyeWitness/reportout 59 | dest: /tmp/EyeWitness.tar 60 | 61 | - name: Fetch tar file from server 62 | fetch: 63 | src: /tmp/EyeWitness.tar 64 | dest: "{{ outfile|quote }}" 65 | flat: true 66 | 67 | - name: Killall phantomJS processes 68 | become: true 69 | command: 70 | killall phantomjs 71 | ignore_errors: yes 72 | 73 | - hosts: localhost 74 | 75 | tasks: 76 | - name: Create output directory 77 | file: 78 | path: "{{ outfolder|quote }}" 79 | state: directory 80 | mode: 0755 81 | 82 | - name: Unpack tar file 83 | unarchive: 84 | src: "{{ outfile|quote }}" 85 | dest: "{{ outfolder|quote }}" 86 | ... 87 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights in this software 2 | 3 | NOTICE: 4 | 5 | For five (5) years from 9/4/2018 the United States Government is granted for itself and others acting on its behalf a paid-up, nonexclusive, irrevocable worldwide license in this data to reproduce, prepare derivative works, and perform publicly and display publicly, by or on behalf of the Government. There is provision for the possible extension of the term of this license. Subsequent to that period or any extension granted, the United States Government is granted for itself and 6 | others acting on its behalf a paid-up, nonexclusive, irrevocable worldwide license in this data to reproduce, prepare derivative works, distribute copies to the public, perform publicly and display publicly, and to permit others to do so. The specific term of the license can be identified by inquiry made to National Technology and Engineering Solutions of Sandia, LLC or DOE. 7 | 8 | NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR NATIONAL TECHNOLOGY AND ENGINEERING SOLUTIONS OF SANDIA, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. 9 | 10 | MIT License 11 | 12 | Copyright (c) 2018 Sandia National Laboratories 13 | 14 | Permission is hereby granted, free of charge, to any person obtaining a copy 15 | of this software and associated documentation files (the "Software"), to deal 16 | in the Software without restriction, including without limitation the rights 17 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 18 | copies of the Software, and to permit persons to whom the Software is 19 | furnished to do so, subject to the following conditions: 20 | 21 | The above copyright notice and this permission notice shall be included in all 22 | copies or substantial portions of the Software. 23 | 24 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 26 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 27 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 28 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 29 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | SOFTWARE. 31 | -------------------------------------------------------------------------------- /src/robot_api/ansible_plays/webscreenshot_play.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: "{{ variable_host|quote }}" 3 | remote_user: "{{ variable_user|quote }}" 4 | 5 | tasks: 6 | - name: Apt install git 7 | become: true 8 | apt: 9 | name: git 10 | force: yes 11 | 12 | - name: Apt install python 13 | become: true 14 | apt: 15 | name: python 16 | force: yes 17 | 18 | - name: Apt install phantomjs 19 | become: true 20 | apt: 21 | name: phantomjs 22 | force: yes 23 | 24 | - name: Git install Webscreenshot repo 25 | git: 26 | repo: https://github.com/maaaaz/webscreenshot.git 27 | update: yes 28 | dest: /tmp/Webscreenshot 29 | 30 | - name: Pip install Webscreenshot 31 | become: true 32 | command: 33 | chdir=/tmp/Webscreenshot 34 | pip3 install -r requirements.txt 35 | 36 | - name: Copy target hosts file to remote 37 | become: true 38 | copy: 39 | src: "{{ infile|quote }}" #This is how you access a passed in variable --extra-vars "version=1.23.45 other_variable=foo" 40 | dest: /tmp/Webscreenshot/hosts_targets.txt 41 | owner: root 42 | group: root 43 | force: true 44 | mode: u+rw,g+rw,o+rw 45 | 46 | - name: Run Scan 47 | become: true 48 | command: 49 | chdir=/tmp/Webscreenshot 50 | python3 webscreenshot.py -i hosts_targets.txt -r phantomjs -o /tmp/Webscreenshot/results --no-xserver 51 | async: 100000 52 | poll: 60 53 | 54 | - name: Permissions 55 | become: true 56 | file: 57 | group: "{{ variable_user|quote }}" 58 | owner: "{{ variable_user|quote }}" 59 | recurse: yes 60 | path: /tmp/Webscreenshot 61 | 62 | - name: Create tar of all files 63 | archive: 64 | path: /tmp/Webscreenshot/results/ 65 | dest: /tmp/Webscreenshot.tar 66 | 67 | - name: Fetch tar file from server 68 | fetch: 69 | src: /tmp/Webscreenshot.tar 70 | dest: "{{ outfile|quote }}" 71 | flat: true 72 | 73 | - name: Killall phantomJS processes 74 | become: true 75 | command: 76 | killall phantomjs 77 | ignore_errors: yes 78 | 79 | - hosts: localhost 80 | 81 | tasks: 82 | - name: Create output directory 83 | file: 84 | path: "{{ outfolder|quote }}" 85 | state: directory 86 | mode: 0755 87 | 88 | - name: Unpack tar file 89 | unarchive: 90 | src: "{{ outfile|quote }}" 91 | dest: "{{ outfolder|quote }}" 92 | ... 93 | -------------------------------------------------------------------------------- /src/robot_api/docker_buildfiles/Dockerfile.Reconng.tmp: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | 3 | ENV http_proxy $proxy 4 | ENV https_proxy $proxy 5 | ENV dns $dns 6 | ENV GIT_SSL_NO_VERIFY 1 7 | ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ 8 | 9 | RUN mkdir -p /home/ 10 | 11 | RUN if [ -n $dns ]; \ 12 | then echo "nameserver " >> /etc/resolv.conf;\ 13 | fi;\ 14 | apt-get install -y git ca-certificates 15 | 16 | ADD certs/ /usr/local/share/ca-certificates/ 17 | ADD certs/ /etc/ssl/certs/ 18 | RUN update-ca-certificates 19 | 20 | RUN if [ -n $dns ]; \ 21 | then echo "nameserver " >> /etc/resolv.conf;\ 22 | fi;\ 23 | git clone https://github.com/lanmaster53/recon-ng.git /home/recon-ng 24 | 25 | WORKDIR /home/recon-ng 26 | 27 | RUN if [ -n $dns ]; \ 28 | then echo "nameserver " >> /etc/resolv.conf;\ 29 | fi;\ 30 | pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org -r REQUIREMENTS; 31 | 32 | RUN if [ -n $dns ]; \ 33 | then echo "nameserver " >> /etc/resolv.conf;\ 34 | fi;\ 35 | echo -e "marketplace install recon/domains-hosts/\nexit" > /tmp/loadmodules || true && \ 36 | ./recon-ng -r /tmp/loadmodules 37 | 38 | RUN echo './recon-cli -m recon/domains-hosts/threatcrowd -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/threatcrowd.txt' > run.sh 39 | 40 | RUN echo './recon-cli -m recon/domains-hosts/bing_domain_web -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/bing_domain_web.txt' >> run.sh 41 | 42 | RUN echo './recon-cli -m recon/domains-hosts/bing_domain_api -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/bing_domain_api.txt' >> run.sh 43 | 44 | RUN echo './recon-cli -m recon/domains-hosts/findsubdomains -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/findsubdomains.txt' >> run.sh 45 | 46 | RUN echo './recon-cli -m recon/domains-hosts/google_site_web -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/google_site_web.txt' >> run.sh 47 | 48 | RUN echo './recon-cli -m recon/domains-hosts/netcraft -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/netcraft.txt' >> run.sh 49 | 50 | RUN echo './recon-cli -m recon/domains-hosts/threatminer -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/threatminer.txt' >> run.sh 51 | 52 | RUN echo './recon-cli -m recon/domains-hosts/ssl_san -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/ssl_san.txt' >> run.sh 53 | 54 | RUN echo './recon-cli -m recon/domains-hosts/binaryedge -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/threatcrowd.txt' > run.sh 55 | 56 | RUN echo './recon-cli -m recon/domains-hosts/ssl_san -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/threatcrowd.txt' > run.sh 57 | 58 | RUN echo './recon-cli -m recon/domains-hosts/mx_spf_ip -o SOURCE="$target" -o PROXY="$http_proxy" -o NAMESERVER="$dns" -x | grep host > $output/threatcrowd.txt' > run.sh 59 | 60 | RUN chmod +x run.sh 61 | 62 | RUN mkdir -p $output 63 | 64 | ENTRYPOINT ./run.sh 65 | -------------------------------------------------------------------------------- /src/robot_api/ansible_plays/httpscreenshot_play.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: "{{ variable_host|quote }}" 3 | remote_user: "{{ variable_user|quote }}" 4 | 5 | tasks: 6 | - name: Apt install swig 7 | become: true 8 | apt: 9 | name: swig 10 | force: yes 11 | 12 | - name: Apt install swig2 13 | become: true 14 | apt: 15 | name: swig2.0 16 | force: yes 17 | 18 | - name: Apt install libssl-dev 19 | become: true 20 | apt: 21 | name: libssl-dev 22 | force: yes 23 | 24 | - name: Apt install python-dev 25 | become: true 26 | apt: 27 | name: swig 28 | force: yes 29 | 30 | - name: Apt install python-pip 31 | become: true 32 | apt: 33 | name: swig 34 | force: yes 35 | 36 | - name: Apt install git 37 | become: true 38 | apt: 39 | name: git 40 | force: yes 41 | 42 | - name: Git install httpscreen repo 43 | git: 44 | repo: https://github.com/breenmachine/httpscreenshot.git 45 | update: no 46 | dest: /tmp/httpscreenshot 47 | 48 | - name: Pip install httpscreenshot 49 | command: 50 | chdir=/tmp/httpscreenshot 51 | python -m pip install -r requirements.txt 52 | 53 | - name: Copy target hosts file to remote 54 | copy: 55 | src: "{{ infile|quote }}" #This is how you access a passed in variable --extra-vars "version=1.23.45 other_variable=foo" 56 | dest: /tmp/httpscreenshot/hosts_targets.txt 57 | owner: admin 58 | group: admin 59 | mode: u+rw,g+rw,o+rw 60 | 61 | - name: Create temp file location for screenshots 62 | file: 63 | path: /tmp/httpscreenshot/screenshot 64 | state: directory 65 | mode: 0755 66 | 67 | - name: Run Scan 68 | command: 69 | chdir=/tmp/httpscreenshot/screenshot 70 | python ../httpscreenshot.py -l ../hosts_targets.txt -p -w 40 -vH -a 71 | async: 10000 72 | poll: 60 73 | 74 | - name: Permissions 75 | become: true 76 | file: 77 | group: "{{ variable_user|quote }}" 78 | owner: "{{ variable_user|quote }}" 79 | recurse: yes 80 | path: /tmp/httpscreenshot 81 | 82 | - name: Create tar of all files 83 | archive: 84 | path: /tmp/httpscreenshot/screenshot 85 | dest: /tmp/httpscreenshot.tar 86 | 87 | - name: Fetch tar file from server 88 | fetch: 89 | src: /tmp/httpscreenshot.tar 90 | dest: "{{ outfile|quote }}" 91 | flat: true 92 | 93 | - hosts: localhost 94 | tasks: 95 | - name: Create output directory 96 | file: 97 | path: "{{ outfolder|quote }}" 98 | state: directory 99 | mode: 0755 100 | 101 | - name: Unpack tar file 102 | unarchive: 103 | src: "{{ outfile|quote }}" 104 | dest: "{{ outfolder|quote }}" 105 | ... 106 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: bionic 2 | language: python 3 | python: 4 | - '3.7' 5 | 6 | services: 7 | - docker 8 | 9 | install: 10 | - pip install -r requirements.txt 11 | - pip install -e . 12 | 13 | jobs: 14 | include: 15 | - stage: help 16 | script: 17 | - drrobot --help 18 | - stage: start scans 19 | name: Sublist3r 20 | script: 21 | - drrobot gather -sub localhost 22 | - script: 23 | - drrobot gather -knock localhost 24 | name: Knock 25 | - script: 26 | - drrobot gather -amass localhost 27 | name: Amass 28 | - script: 29 | - drrobot gather -turbo localhost 30 | name: Turbolist3r 31 | - script: 32 | - drrobot gather -sfinder localhost 33 | name: Subfinder 34 | - script: 35 | - drrobot gather -recon localhost 36 | name: ReconNG 37 | - script: 38 | - drrobot gather -altdns localhost 39 | name: AltDNS 40 | - script: 41 | - drrobot gather -anubis localhost 42 | name: Anubis 43 | - script: 44 | - drrobot gather -ctexpo localhost 45 | name: CTExpose 46 | - script: 47 | - drrobot gather -ctfr localhost 48 | name: CTFR 49 | - script: 50 | - drrobot gather -pdlist localhost 51 | name: PDList 52 | - stage: Test screenshots 53 | name: Webscreenshot 54 | script: 55 | - drrobot -h 56 | - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated 57 | - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt 58 | - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt -webscreen singlewebsitetest 59 | - script: 60 | - drrobot -h 61 | - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated 62 | - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt 63 | - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt -nmapscreen singlewebsitetest 64 | name: NMAP 65 | - script: 66 | - drrobot -h 67 | - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated 68 | - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt 69 | - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt -eye singlewebsitetest 70 | name: Eyewitness 71 | - script: 72 | - drrobot -h 73 | - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated 74 | - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt 75 | - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt -http singlewebsitetest 76 | name: HTTPScreenshot 77 | - script: 78 | - drrobot -h 79 | - mkdir -p $HOME/.drrobot/output/singlewebsitetest/aggregated 80 | - echo "http://localhost" > $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt 81 | - drrobot inspect --file $HOME/.drrobot/output/singlewebsitetest/aggregated/aggregated_protocol_hostnames.txt -gowitness singlewebsitetest 82 | name: GOWitness 83 | -------------------------------------------------------------------------------- /readmes/usage.md: -------------------------------------------------------------------------------- 1 | # Usage 2 | 3 | ## Gather 4 | 5 | To gather data simply run the following command: 6 | 7 | ``` 8 | drrobot gather --help 9 | usage: drrobot gather [-h] [-aqua] [-sub] [-turbo] [-brute] [-sfinder] 10 | [-knock] [-amass] [-recon] [-altdns] [-anubis] [-ctexpo] 11 | [-ctfr] [-pdlist] [-shodan] [-arin] [-hack] [-dump] 12 | [-virus] [--ignore IGNORE] [--headers] 13 | domain 14 | 15 | drrobot gather 16 | ... 17 | 18 | ``` 19 | 20 | 21 | ## Inspection 22 | 23 | The inspection process will generate noise due to the headless automated browsing, which is easily detectable. If your list is large enough this could cause your IP to be blocked. 24 | 25 | ``` 26 | drrobot inspect --help 27 | usage: drrobot inspect [-h] [-http] [-eye] [-nmapscreen] [-webscreen] domain 28 | 29 | positional arguments: 30 | domain Domain to run scan against 31 | 32 | optional arguments: 33 | -h, --help show this help message and exit 34 | -http, --HTTPScreenshot 35 | Post enumeration tool for screen grabbing websites. 36 | All images will be downloaded to outfile: 37 | httpscreenshot.tar and unpacked httpscreenshots 38 | -eye, --Eyewitness Post enumeration tool for screen grabbing websites. 39 | All images will be downloaded to outfile: 40 | Eyewitness.tar and unpacked in Eyewitness 41 | -nmapscreen, --Nmap Post enumeration tool for screen grabbing websites. 42 | (Chrome is not installed in the dockerfile due. 43 | Options are chromium-browser/firefox/wkhtmltoimage) 44 | -webscreen, --Webscreenshot 45 | Post enumeration tool for screen grabbing websites. 46 | (Chrome is not installed in the dockerfile due. 47 | Options are chromium-browser/firefox/wkhtmltoimage) 48 | 49 | --file FILE File to use for inspection 50 | ``` 51 | 52 | ## Upload 53 | 54 | Upload to Slack/Mattermost 55 | 56 | ``` 57 | drrobot upload --help 58 | usage: drrobot upload [-h] [-matter] [-slack] filepath 59 | 60 | positional arguments: 61 | filepath Filepath to the folder containing imagesto upload. 62 | This is relative to the domain specified. By default 63 | this will be the path to the output folder 64 | 65 | optional arguments: 66 | -h, --help show this help message and exit 67 | -matter, --Mattermost 68 | Mattermost server 69 | -slack, --Slack Slack server 70 | ``` 71 | 72 | ## Rebuild 73 | 74 | Rebuild will use the config file to determine what files to use when aggregating data. This will update the database with new information. This does not recreate the aggregated files 75 | 76 | ``` 77 | drrobot rebuild --help 78 | usage: drrobot rebuild [-h] [-f [FILES [FILES ...]]] [--headers] domain 79 | 80 | positional arguments: 81 | domain Domain to dump output of 82 | 83 | optional arguments: 84 | -h, --help show this help message and exit 85 | -f [FILES [FILES ...]], --files [FILES [FILES ...]] 86 | Additional files to supply outside of the config file 87 | --headers Rebuild with headers 88 | ``` 89 | 90 | ## Dumpdb 91 | Dump the database to aggregated files and the header files as well 92 | 93 | ``` 94 | drrobot dumpdb --help 95 | usage: drrobot dumpdb [-h] domain 96 | 97 | positional arguments: 98 | domain Domain to show data for 99 | 100 | optional arguments: 101 | -h, --help show this help message and exit 102 | ``` 103 | 104 | ## Output 105 | Generate output file as JSON or XML 106 | 107 | ``` 108 | drrobot output --help 109 | usage: drrobot output [-h] [--output OUTPUT] {json,xml} domain 110 | 111 | positional arguments: 112 | {json,xml} Generate json file under outputs folder (format) 113 | domain Domain to dump output of 114 | 115 | optional arguments: 116 | -h, --help show this help message and exit 117 | --output OUTPUT Alternative location to create output file 118 | ``` 119 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). 4 | 5 | 6 | ## [1.3.0] - August 2019 7 | 8 | Updates 9 | 10 | ### Added 11 | 12 | * Threading for docker containers 13 | * CI/CD with travis to test building of containers 14 | * MassDNS 15 | * Certificate usage for all containers 16 | * Minio docker-compose to serve up data locally for easy viewing 17 | * Config files in $HOME/.drrobot!!! 18 | 19 | ### Removed 20 | 21 | * Server for Dr.ROBOT 22 | * Test directory. Most testing involves the network. Will eventually add unit tests again. 23 | 24 | ### Changed 25 | 26 | * File layout 27 | * Broke up drrobot into multiple pieces: cli.py robot.py config.py 28 | * Moved robot specific classes into api folder: dockerize.py ansible.py aggregation.py upload.py web_resources.py 29 | * Template files are not longer in same directory as filled in files 30 | * Config and templates are stored in user $HOME/.drrobot directories for convenience 31 | * Users can now use virtualenv or pipenv 32 | * drrobot can now be installed locally for usage as a module 33 | 34 | ### Fixed 35 | 36 | * Aggregation not parsing correctly 37 | * Host/IP resolution error 38 | * Redundant license 39 | * Weird output with multiprocessing 40 | * Logging 41 | 42 | ## [1.2.0] - August 2019 43 | 44 | Updates 45 | 46 | ### Added 47 | 48 | * **Django** docker container to server domain data 49 | * Added dbfile option for specification of personal database 50 | * **Webscreenshot** (Currently phantomjs receives the best results in Docker) 51 | * **Altdns** 52 | * **Slack** configuration. Requires you to set up a channel and make a robot with correct permissions similar to other Plugins. 53 | 54 | ### Removed 55 | 56 | * Removed global domain option for all subparsers 57 | * Removed domain requirement for upload 58 | 59 | ### Changed 60 | 61 | * **SQLlite** refactor 62 | * Use two tables 1 for list of domains, 1 for all data gather. 63 | * Threading for reading files. Sometimes this threading would cause unknown hangs. Working on creating a more improved version of the file parsing. 64 | * Minor packages updates for urllib3 and Jinja 65 | 66 | ### Fixed 67 | 68 | * ValueError exception was not caught when building web tools. If you were missing a field it would crash the program 69 | * Amass Dockerfile 70 | * Changes Knock to output CSV rather than Json 71 | * KeyboardInterrupt when running docker containers caused error message. Now cancels gracefully. 72 | * Slack client was updated. Changed code to reflect new client changes. 73 | 74 | ## [1.1.0] - Feb 2019 75 | 76 | Current Release 77 | 78 | ### Added 79 | 80 | * Changelog :) 81 | 82 | * **Nmap-screenshot** docker container which allows one to use the nmap screenshot NSE script 83 | * **Knockpy** docker container added 84 | * **Eyewitness** docker container added 85 | * **Amass** docker container added 86 | * ```---verbosity``` flag added as well as some useful log files that should contain any exceptions and debugging information respectively. 87 | 88 | ### Removed 89 | 90 | * ```--domain``` option in favor of required domain. (Can't run tool without domain) 91 | 92 | ### Changed 93 | 94 | * Changed default configuration options 95 | * ```network_mode``` allows us to pass host network to the docker daemon for ease of use 96 | * ```DOCKER``` or ```ANSIBLE ``` mode for Scanners and Inspection tools. Change for what mode you would like them to utilize 97 | * Ansible configuration optoins 98 | * ```"flags": "-e '$extra' -i configs/ansible_inventory",``` This option allows you to have a **dr_robot** specific ansible inventory. 99 | * ```variable_user``` option to specify what user Ansible should use to log in 100 | * Changed folders for docker containers to *docker_buildfiles* 101 | * Tests to utilize Travis CI 102 | 103 | ### Fixed 104 | 105 | * Updated Eyewitness to use a specific commit hash. ```--headless``` flag was removed in favor of ```--web``` 106 | * Added user created and generated files to ```.gitignore``` 107 | * Fixed Duplication of IP and Hostnames in the aggregated section 108 | * Duplicated docker containers showing up. 109 | * Docker containers running pass **Keyboard Interrupt** 110 | 111 | ## [1.0.0] - Nov 2018 112 | 113 | Initial Release 114 | -------------------------------------------------------------------------------- /src/robot_api/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """ Config Module 3 | 4 | Utility methods for loading configuration files, 5 | Generating template and config files into HOME directory 6 | Checking if tools exist 7 | """ 8 | import json 9 | from subprocess import Popen 10 | from os import devnull, environ, path, makedirs 11 | import pkg_resources 12 | try: 13 | import errno 14 | except Exception: 15 | from os import errno 16 | 17 | 18 | def generate_configs(): 19 | """Loads configuration files and templates into users home directory. 20 | 21 | Returns: 22 | None 23 | """ 24 | 25 | CONFIG_DIR = path.join(environ.get("HOME","."),".drrobot") 26 | if not path.exists(CONFIG_DIR): 27 | makedirs(CONFIG_DIR) 28 | 29 | if not path.exists(path.join(CONFIG_DIR, "docker_buildfiles")): 30 | makedirs(path.join(CONFIG_DIR, "docker_buildfiles")) 31 | 32 | if not path.exists(path.join(CONFIG_DIR, "docker_active")): 33 | makedirs(path.join(CONFIG_DIR, "docker_active")) 34 | 35 | if not path.exists(path.join(CONFIG_DIR, "tarfiles")): 36 | makedirs(path.join(CONFIG_DIR, "tarfiles")) 37 | 38 | if not path.exists(path.join(CONFIG_DIR, "certs")): 39 | makedirs(path.join(CONFIG_DIR, "certs")) 40 | 41 | if not path.isfile(path.join(CONFIG_DIR, "config.json")): 42 | with open(path.join(CONFIG_DIR, "config.json"), 'wb') as _file: 43 | config = pkg_resources.resource_string(__name__, 44 | 'configs/default_config.json') 45 | _file.write(config) 46 | 47 | if not path.isfile(path.join(CONFIG_DIR, "ansible_inventory")): 48 | with open(path.join(CONFIG_DIR, "ansible_inventory"), 'wb') as _file: 49 | config = pkg_resources.resource_string(__name__, 50 | 'configs/ansible_inventory') 51 | _file.write(config) 52 | 53 | 54 | for _file in pkg_resources.resource_listdir(__name__, "docker_buildfiles"): 55 | if not path.isfile(path.join(CONFIG_DIR, "docker_buildfiles", _file)): 56 | fpath = path.join("docker_buildfiles", _file) 57 | contents = pkg_resources.resource_string(__name__, fpath) 58 | with open(path.join(CONFIG_DIR, "docker_buildfiles", _file), 59 | 'wb') as _file: 60 | _file.write(contents) 61 | 62 | if not path.exists(path.join(CONFIG_DIR, "ansible_plays")): 63 | makedirs(path.join(CONFIG_DIR, "ansible_plays")) 64 | 65 | for _file in pkg_resources.resource_listdir(__name__, "ansible_plays"): 66 | if not path.isfile(path.join(CONFIG_DIR, "ansible_plays", _file)): 67 | fpath = path.join("ansible_plays", _file) 68 | contents = pkg_resources.resource_string(__name__, fpath) 69 | with open(path.join(CONFIG_DIR, "ansible_plays", _file), 70 | 'wb') as _file: 71 | _file.write(contents) 72 | 73 | 74 | def tool_check(): 75 | """Checks if Docker or Ansible exists 76 | """ 77 | 78 | TOOLS = ["ansible", "docker"] 79 | for name in TOOLS: 80 | try: 81 | dnull = open(devnull) 82 | Popen([name], stdout=dnull, stderr=dnull).communicate() 83 | except OSError as error: 84 | if error.errno == errno.ENOENT: 85 | print(f"[!!] Tool {name} not found in path. " + 86 | "If we error out it is your fault.") 87 | 88 | 89 | def load_config(config): 90 | """Load all data from the json file in the USERS home directory 91 | 92 | Returns: 93 | A dict of the USER defined tools and their options for running them : 94 | 95 | { "scanners" : {...}, 96 | "webtools" : {...}, 97 | "enumeration" : {...}, 98 | "upload_dest" : {...}, 99 | """ 100 | 101 | with open(config, 'r') as f: 102 | config = json.load(f) 103 | 104 | scanners = config.get('Scanners', {}) 105 | webtools = config.get('WebTools', {}) 106 | enumeration = config.get('Enumeration', {}) 107 | upload_dest = config.get('Upload', {}) 108 | return {"scanners": scanners, 109 | "webtools": webtools, 110 | "enumeration": enumeration, 111 | "upload_dest": upload_dest} 112 | 113 | 114 | def get_config(): 115 | """Utility to fetch the path to the configuration file. 116 | 117 | If HOME is not defined just check the current directory. 118 | 119 | Returns: 120 | A string path that points to the config.json file. 121 | """ 122 | return path.join(environ.get("HOME", "."), ".drrobot", "config.json") 123 | -------------------------------------------------------------------------------- /src/robot_api/api/ansible.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """ Ansible module 3 | 4 | Module for running ansible playbooks 5 | 6 | Attributes: 7 | ansible_arguments (dict): Contains config, flags, and extra_flags options 8 | ansible_file_location (str): location to playbook 9 | domain (str): target domain 10 | ansible_base (str): Base command string 11 | output_dir (str): location to dump output of playbook 12 | infile (str): Infile for playbook job 13 | verbose (bool): More output Yes/No 14 | final_command (str): Final command to run 15 | """ 16 | import logging 17 | import subprocess 18 | from string import Template 19 | from robot_api.parse import join_abs 20 | 21 | LOG = logging.getLogger(__name__) 22 | 23 | 24 | class Ansible: 25 | def __init__(self, **kwargs): 26 | """ 27 | Build Ansible object 28 | 29 | Args: 30 | 31 | **kwargs: { 32 | "ansible_arguments" : { 33 | "config" : "$config/httpscreenshot_play.yml", 34 | "flags": "-e '$extra' -i configs/ansible_inventory", 35 | "extra_flags":{ 36 | "1" : "variable_host=localhost", 37 | "2" : "variable_user=user", 38 | "3" : "infile=$infile", 39 | "4" : "outfile=$outfile/httpscreenshots.tar", 40 | "5" : "outfolder=$outfile/httpscreenshots" 41 | } 42 | }, 43 | "ansible_file_location" : "location", 44 | "verbose" : True, 45 | "domain" : "target.domain" 46 | } 47 | 48 | Returns: 49 | 50 | """ 51 | self.ansible_base = "ansible-playbook $flags $config" 52 | self.ansible_arguments = kwargs.get('ansible_arguments') 53 | self.ansible_file = kwargs.get('ansible_file_location', None) 54 | if not self.ansible_file: 55 | raise TypeError( 56 | "argument ansible_file must be of type string, not 'NoneType'") 57 | 58 | self.domain = kwargs.get('domain', None) 59 | if not self.domain: 60 | raise TypeError( 61 | "argument domain must be of type string, not 'NoneType'") 62 | self.output_dir = kwargs.get('output_dir') 63 | self.infile = kwargs.get('infile', None) 64 | if not self.infile: 65 | self.infile = join_abs(self.output_dir, "aggregated", "aggregated_protocol_hostnames.txt") 66 | 67 | self.verbose = kwargs.get('verbose', False) 68 | self.final_command = None 69 | 70 | def _print(self, msg): 71 | """Utility for logging 72 | """ 73 | if self.verbose: 74 | print("[D] " + msg) 75 | LOG.debug(msg) 76 | 77 | def build(self): 78 | """Build the final command for the ansible process. 79 | Uses the arguments provided in the ansible_arguments 80 | 81 | Args: 82 | 83 | Returns: 84 | 85 | """ 86 | try: 87 | system_replacements = { 88 | "infile": self.infile, 89 | "outdir": self.output_dir, 90 | "config": self.ansible_file 91 | } 92 | 93 | extra_flags = self.ansible_arguments.get('extra_flags', None) 94 | extra_replace_string = "" 95 | 96 | self._print(f"building with extra flags {extra_flags}") 97 | 98 | if extra_flags: 99 | for _, val in extra_flags.items(): 100 | cur_str = Template(val) 101 | extra_replace_string += cur_str.safe_substitute( 102 | system_replacements) + " " 103 | 104 | flags = Template(self.ansible_arguments.get("flags")) 105 | config = Template(self.ansible_arguments.get("config")) 106 | 107 | """ 108 | This may seem redundant but this prepends the 109 | path of the ansible location to the ansible file 110 | specified in the config. 111 | All the flags are added to the flags 112 | argument to be input into the final command 113 | """ 114 | substitutes = { 115 | "flags": flags.safe_substitute(extra=extra_replace_string), 116 | "config": config.safe_substitute(system_replacements) 117 | } 118 | 119 | _temp = Template(self.ansible_base) 120 | self.final_command = _temp.safe_substitute(substitutes) 121 | 122 | self._print(f"Final ansible command {self.final_command}") 123 | except BaseException: 124 | raise TypeError("NoneType object supplied in Dict build") 125 | 126 | def run(self): 127 | """Run the final command built at runtime 128 | 129 | Args: 130 | 131 | Returns: 132 | 133 | """ 134 | try: 135 | self.build() 136 | _ = subprocess.check_output( 137 | self.final_command, 138 | shell=True, 139 | stderr=subprocess.STDOUT, 140 | universal_newlines=True) 141 | except subprocess.CalledProcessError: 142 | print(f"[!] CallProcessError check logs") 143 | LOG.exception("Called Process Error Ansible") 144 | except OSError: 145 | print(f"[!] OSError check logs") 146 | LOG.exception("OSError in Ansible") 147 | except subprocess.SubprocessError: 148 | print(f"[!] SubprocessError check logs") 149 | LOG.exception("Subprocess Error in Ansible") 150 | except TypeError: 151 | print(f"[!] TypeError check logs") 152 | LOG.exception("Type Error in Ansible") 153 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | 18 | BASE_DIR=os.path.abspath("../../") 19 | print(BASE_DIR) 20 | 21 | sys.path.insert(0, os.path.join(BASE_DIR)) 22 | 23 | 24 | # -- Project information ----------------------------------------------------- 25 | 26 | project = 'Dr.ROBOT' 27 | copyright = '2018, Aleksandar Straumann' 28 | author = 'Aleksandar Straumann' 29 | 30 | # The short X.Y version 31 | version = '2.0' 32 | # The full version, including alpha/beta/rc tags 33 | release = '2.0.0' 34 | 35 | 36 | # -- General configuration --------------------------------------------------- 37 | 38 | # If your documentation needs a minimal Sphinx version, state it here. 39 | # 40 | # needs_sphinx = '1.0' 41 | 42 | # Add any Sphinx extension module names here, as strings. They can be 43 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 44 | # ones. 45 | extensions = [ 46 | 'sphinx.ext.autodoc', 47 | 'sphinx.ext.doctest', 48 | 'sphinx.ext.todo', 49 | 'sphinx.ext.coverage', 50 | 'sphinx.ext.viewcode', 51 | 'sphinx.ext.napoleon', 52 | ] 53 | 54 | # Add any paths that contain templates here, relative to this directory. 55 | templates_path = ['_templates'] 56 | 57 | # The suffix(es) of source filenames. 58 | # You can specify multiple suffix as a list of string: 59 | # 60 | # source_suffix = ['.rst', '.md'] 61 | source_suffix = '.rst' 62 | 63 | # The master toctree document. 64 | master_doc = 'index' 65 | 66 | # The language for content autogenerated by Sphinx. Refer to documentation 67 | # for a list of supported languages. 68 | # 69 | # This is also used if you do content translation via gettext catalogs. 70 | # Usually you set "language" from the command line for these cases. 71 | language = None 72 | 73 | # List of patterns, relative to source directory, that match files and 74 | # directories to ignore when looking for source files. 75 | # This pattern also affects html_static_path and html_extra_path. 76 | exclude_patterns = ["*tests*", "setup*"] 77 | 78 | # The name of the Pygments (syntax highlighting) style to use. 79 | pygments_style = None 80 | 81 | 82 | # -- Options for HTML output ------------------------------------------------- 83 | 84 | # The theme to use for HTML and HTML Help pages. See the documentation for 85 | # a list of builtin themes. 86 | # 87 | html_theme = 'sphinx_rtd_theme' 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | # 93 | # html_theme_options = {} 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | # Custom sidebar templates, must be a dictionary that maps document names 101 | # to template names. 102 | # 103 | # The default sidebars (for documents that don't match any pattern) are 104 | # defined by theme itself. Builtin themes are using these templates by 105 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 106 | # 'searchbox.html']``. 107 | # 108 | # html_sidebars = {} 109 | 110 | 111 | # -- Options for HTMLHelp output --------------------------------------------- 112 | 113 | # Output file base name for HTML help builder. 114 | htmlhelp_basename = 'DrROBOTdoc' 115 | 116 | 117 | # -- Options for LaTeX output ------------------------------------------------ 118 | 119 | latex_elements = { 120 | # The paper size ('letterpaper' or 'a4paper'). 121 | # 122 | # 'papersize': 'letterpaper', 123 | 124 | # The font size ('10pt', '11pt' or '12pt'). 125 | # 126 | # 'pointsize': '10pt', 127 | 128 | # Additional stuff for the LaTeX preamble. 129 | # 130 | # 'preamble': '', 131 | 132 | # Latex figure (float) alignment 133 | # 134 | # 'figure_align': 'htbp', 135 | } 136 | 137 | # Grouping the document tree into LaTeX files. List of tuples 138 | # (source start file, target name, title, 139 | # author, documentclass [howto, manual, or own class]). 140 | latex_documents = [ 141 | (master_doc, 'DrROBOT.tex', 'Dr.ROBOT Documentation', 142 | 'Aleksandar Straumann', 'manual'), 143 | ] 144 | 145 | 146 | # -- Options for manual page output ------------------------------------------ 147 | 148 | # One entry per manual page. List of tuples 149 | # (source start file, name, description, authors, manual section). 150 | man_pages = [ 151 | (master_doc, 'drrobot', 'Dr.ROBOT Documentation', 152 | [author], 1) 153 | ] 154 | 155 | 156 | # -- Options for Texinfo output ---------------------------------------------- 157 | 158 | # Grouping the document tree into Texinfo files. List of tuples 159 | # (source start file, target name, title, author, 160 | # dir menu entry, description, category) 161 | texinfo_documents = [ 162 | (master_doc, 'DrROBOT', 'Dr.ROBOT Documentation', 163 | author, 'DrROBOT', 'One line description of project.', 164 | 'Miscellaneous'), 165 | ] 166 | 167 | 168 | # -- Options for Epub output ------------------------------------------------- 169 | 170 | # Bibliographic Dublin Core info. 171 | epub_title = project 172 | 173 | # The unique identifier of the text. This can be a ISBN number 174 | # or the project homepage. 175 | # 176 | # epub_identifier = '' 177 | 178 | # A unique identification for the text. 179 | # 180 | # epub_uid = '' 181 | 182 | # A list of files that should not be packed into the epub file. 183 | epub_exclude_files = ['search.html'] 184 | 185 | 186 | # -- Extension configuration ------------------------------------------------- 187 | 188 | # -- Options for todo extension ---------------------------------------------- 189 | 190 | # If true, `todo` and `todoList` produce output, else they produce nothing. 191 | todo_include_todos = True 192 | -------------------------------------------------------------------------------- /readmes/config.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | Dr.ROBOT is built in a modular fashion, making it easy to add new tools. You have three options for adding a new tool to Dr.ROBOT: 4 | 5 | #### Important: To make sure no issues come from adding your own tool, make sure the key used to identify a json item, the name, and docker_name are all unique. 6 | 7 | ## 1. Docker container 8 | Use the config file found at `$HOME/.drrobot/config.json`. Use the existing modules as a template. For this example I am using the Amass tool: 9 | ``` 10 | "Amass": { 11 | "name": "Amass", 12 | "default" : false, 13 | "mode" : "DOCKER", 14 | "network_mode": "host", 15 | "docker_name" : "amass", 16 | "default_conf": "docker_buildfiles/Dockerfile.Amass.tmp", 17 | "active_conf": "docker_buildfiles/Dockerfile.Amass", 18 | "description": "The OWASP Amass tool suite obtains subdomain names by scraping data sources, recursive brute forcing, crawling web archives, permuting/altering names and reverse DNS sweeping.", 19 | "src": "https://github.com/OWASP/Amass", 20 | "output": "/root/amass", 21 | "output_folder": "amass" 22 | }, 23 | ``` 24 | 25 | To add your own tool you can simply replace the above data as necessary with your tools information. 26 | 27 | Now add a Dockerfile to `$HOME/.drrobot/docker_buildfiles`. Simply follow the naming scheme for default and active conf when creating your file. 28 | ``` 29 | FROM golang:1.12.6-alpine3.10 as build 30 | RUN apk --no-cache add git 31 | RUN go get github.com/OWASP/Amass; exit 0 32 | 33 | ENV GO111MODULE on 34 | 35 | WORKDIR /go/src/github.com/OWASP/Amass 36 | 37 | RUN go install ./... 38 | 39 | FROM alpine:latest 40 | COPY --from=build /go/bin/amass /bin/amass 41 | COPY --from=build /go/src/github.com/OWASP/Amass/wordlists/ /wordlists/ 42 | 43 | ENV http_proxy $proxy 44 | ENV https_proxy $proxy 45 | ENV DNS $dns 46 | ENV HOME / 47 | 48 | RUN mkdir -p $output 49 | 50 | ENV TARGET $target 51 | ENV OUTPUT $output/amass.txt 52 | ENV INFILE $infile 53 | 54 | ENTRYPOINT amass enum --passive -d "$target" -o $$OUTPUT 55 | ``` 56 | As you can see there are some ENV variables that are passed in when running our tool. If you have any specific ones that you would like to pass into the docker container, you can add them to the above JSON using a name which you will then reference in the Dockerfile. For example you will notice `$output` is used. `$output` comes from the above json blob and is then replaced during the runtime of Dr.ROBOT. 57 | 58 | 59 | ## 2. Ansible Playbook 60 | Similar to adding a Docker container we first add our tool to the configuration file. 61 | ``` 62 | "HTTPScreenshot": { 63 | "name" : "HTTPScreenshot", 64 | "short_name" : "http", 65 | "mode" : "ANSIBLE", 66 | "ansible_arguments" : { 67 | "config" : "$config/httpscreenshot_play.yml", 68 | "flags": "-e '$extra' -i configs/ansible_inventory", 69 | "extra_flags":{ 70 | "1" : "variable_host=localhost", 71 | "2" : "variable_user=user", 72 | "3" : "infile=$infile", 73 | "4" : "outfile=$outdir/httpscreenshots.tar", 74 | "5" : "outfolder=$outdir/httpscreenshots" 75 | }, 76 | "run_as_thread": false, 77 | "infile" : "aggregated/aggregated_protocol_hostnames.txt" 78 | }, 79 | "description" : "Post enumeration tool for screen grabbing websites. All images will be downloaded to outfile: httpscreenshot.tar and unpacked httpscreenshots", 80 | "output" : "/tmp/output", 81 | "infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt", 82 | }, 83 | ``` 84 | 85 | Take special note of the `ansible_arguments`. The two required items are `config` and `flags`. These two items tell Dr.ROBOT what file to use and what inventory and flags will be used. `extra flags` is a nested JSON block where you can specify any parameters that need to go into the ansible playbook. 86 | 87 | Note: 88 | 89 | * `$infile` comes from the ansible_arguments **infile**, so that it is consistent for both docker and ansible. You can use a full path to a file for input if you desire. 90 | * `$outdir` comes from Dr.ROBOT. It will generate a path that points to ```$HOME/.drrobot/output// ``` Again, you can specify a custom path if you like. ` 91 | * run_as_thread will allow you to run the playbook similar to docker containers. This requires that you have ansible configured to handle complete automation. By default we set this to False 92 | 93 | #### The Playbook 94 | This will simply be a standard playbook with a few changes so that Dr.ROBOT can use the parameters we fed it. To make sure a parameter that we specified in the "extra_flags" JSON blob is available, use Ansible syntax for variables: ```"{{ variable_name|quote }}"``` (Note the *quote* helps prevent issues with variable names) 95 | 96 | ``` 97 | --- 98 | - hosts: "{{ variable_host|quote }}" 99 | remote_user: "{{ variable_user|quote }}" 100 | 101 | tasks: 102 | - name: Apt install git 103 | become: true 104 | ... 105 | ``` 106 | 107 | ## 3. Web Module 108 | Again we start with the `config.json` file. For web modules, you will be writing Python code that Dr.ROBOT can leverage for domain enumeration. As before, the names must be unique, however, for Web Modules the **class_name** must exactly match the class name inside web_resource.py. 109 | 110 | ``` 111 | "Dumpster" : 112 | { 113 | "short_name" : "dump", 114 | "class_name" : "Dumpster", 115 | "default" : false, 116 | "description" : "Use the limited response of DNSDumpster. Requires API access for better results.", 117 | "output_file" : "dumpster.txt" 118 | }, 119 | ``` 120 | 121 | #### The Module 122 | 123 | Dr.ROBOT will use the JSON input to load classes at runtime which allows us to run your custom code! To add your custom code to the web_resource.py file there are some caveats: 124 | 125 | 1. It must extend the WebTool abstract base class. This allos DrROBOT to treat all imported classes as the same and run the only method we require: **do_query**. 126 | 2. If you want your output to be written to the correct folder you will store your results under the **self.results** list and call **_write_results** which will write to the output_file in your config.json. 127 | -------------------------------------------------------------------------------- /src/robot_api/parse.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import pkg_resources 4 | from os import devnull, environ, path, makedirs 5 | 6 | 7 | def join_abs(*args): 8 | """Utility method to call abspath and join 9 | without having to write it out every single time 10 | 11 | Args: 12 | *args: Multiple string like parameters to be joined 13 | 14 | Returns: 15 | string to the absolute path pointed to by *args 16 | 17 | """ 18 | return path.abspath(path.join(*args)) 19 | 20 | def parse_args( 21 | scanners=None, 22 | enumeration=None, 23 | webtools=None, 24 | upload_dest=None, 25 | root_dir="."): 26 | """Generate the argparse options given the configuration file. 27 | 28 | Args: 29 | scanners: dict of scanners defined in config.json 30 | enumeration: dict of enumeration tools defined in config.json 31 | webtools: dict of webtools defined in config.json 32 | upload_dest: List of upload locations defined in config.json 33 | root_dir: Root directory of the config.json 34 | 35 | Returns: 36 | parser given all available options provided at cli 37 | """ 38 | 39 | if scanners is None: 40 | scanners = {} 41 | 42 | if enumeration is None: 43 | enumeration = {} 44 | 45 | if webtools is None: 46 | webtools = {} 47 | 48 | if upload_dest is None: 49 | upload_dest = {} 50 | 51 | parser = argparse.ArgumentParser(description="Docker DNS recon tool") 52 | 53 | parser.add_argument('--proxy', 54 | default=None, 55 | type=str, 56 | help="Proxy server URL to set DOCKER http_proxy too") 57 | 58 | parser.add_argument( 59 | '--dns', 60 | default=None, 61 | type=str, 62 | help="DNS server to add to resolv.conf of DOCKER containers") 63 | 64 | parser.add_argument('--verbose', 65 | default=False, 66 | action="store_true", 67 | help="Display verbose statements") 68 | 69 | parser.add_argument('--dbfile', 70 | default=join_abs(root_dir, "dbs", "drrobot.db"), 71 | type=str, 72 | help="Specify what db file to use for saving data too") 73 | 74 | subparser = parser.add_subparsers(dest='actions') 75 | ########################## 76 | # GATHER 77 | ########################## 78 | 79 | parser_gather = subparser.add_parser( 80 | 'gather', 81 | help="Runs gather phase with tools under the webtools/scanners") 82 | 83 | for k in scanners.keys(): 84 | parser_gather.add_argument( 85 | f"-{scanners[k].get('docker_name', None)}", 86 | f'--{k}', 87 | action='store_true', 88 | help=scanners[k].get( 89 | 'description', 90 | "No description provided"), 91 | default=False) 92 | 93 | for k in webtools.keys(): 94 | parser_gather.add_argument( 95 | f"-{webtools[k].get('short_name', None)}", 96 | f'--{k}', 97 | action='store_true', 98 | help=webtools[k].get( 99 | 'description', 100 | "No description provided"), 101 | default=False) 102 | 103 | parser_gather.add_argument( 104 | '--ignore', 105 | default=None, 106 | type=str, 107 | action='append', 108 | help="Space seperated list of subnets to ignore") 109 | 110 | parser_gather.add_argument( 111 | '--headers', 112 | default=False, 113 | action='store_true', 114 | help="If headers should be scraped from ip addresses gathered") 115 | 116 | parser_gather.add_argument("domain", 117 | type=str, 118 | help="Domain to run scan against") 119 | 120 | ########################## 121 | # INSPECT 122 | ########################## 123 | 124 | parser_inspect = subparser.add_parser( 125 | 'inspect', 126 | help="Run inspection phase against aggregated data " 127 | "Note: you must either supply a file containing a list of IP/Hostnames" 128 | " or the targeted domain must have a db under the dbs folder") 129 | 130 | for k in enumeration.keys(): 131 | parser_inspect.add_argument( 132 | f"-{enumeration[k].get('short_name')}", 133 | f"--{k}", 134 | action='store_true', 135 | help=enumeration[k].get( 136 | 'description', 137 | "No description provided"), 138 | default=False) 139 | 140 | parser_inspect.add_argument("--file", 141 | type=str, 142 | help="File to use for inspection") 143 | 144 | parser_inspect.add_argument("domain", 145 | type=str, 146 | help="Domain to run scan against") 147 | ########################## 148 | # UPLOAD 149 | ########################## 150 | 151 | parser_upload = subparser.add_parser( 152 | 'upload', help="Upload recon data to Mattermost/Slack") 153 | 154 | for k in upload_dest.keys(): 155 | parser_upload.add_argument( 156 | f"-{upload_dest[k].get('short_name')}", 157 | f"--{k}", 158 | action='store_true', 159 | help=upload_dest[k].get( 160 | 'description', 161 | "No description provided")) 162 | 163 | parser_upload.add_argument( 164 | f"filepath", type=str, help="Filepath to the folder containing images" 165 | "to upload. This is relative to the domain " 166 | "specified. By default this will be the path to the output folder") 167 | ########################## 168 | # REBUILD 169 | ########################## 170 | 171 | parser_rebuild = subparser.add_parser( 172 | 'rebuild', 173 | help="Rebuild the database with additional files from previous runs") 174 | 175 | parser_rebuild.add_argument("domain", 176 | type=str, 177 | help="Domain to dump output of") 178 | 179 | parser_rebuild.add_argument( 180 | "-f", 181 | "--files", 182 | nargs="*", 183 | help="Additional files to supply outside of the config file") 184 | 185 | parser_rebuild.add_argument( 186 | "--headers", 187 | action="store_true", 188 | default=False, 189 | help="Rebuild with headers") 190 | 191 | ########################## 192 | # DUMPDB 193 | ########################## 194 | 195 | parser_dumpdb = subparser.add_parser( 196 | "dumpdb", 197 | help="Dump contents of database (ip,hostname,banners) to a text file") 198 | 199 | parser_dumpdb.add_argument("domain", 200 | type=str, 201 | help="Domain to show data for") 202 | ########################## 203 | # OUTPUT 204 | ########################## 205 | parser_output = subparser.add_parser( 206 | "output", 207 | help="Generate output in specified format. Contains all " 208 | "information from scans (images, headers, hostnames, ips)") 209 | 210 | parser_output.add_argument( 211 | "format", 212 | choices=[ 213 | "json", 214 | "xml"], 215 | default="json", 216 | help="Generate json file under outputs folder (format)") 217 | 218 | parser_output.add_argument( 219 | "--output", 220 | default=None, 221 | help="Alternative location to create output file") 222 | 223 | parser_output.add_argument("domain", 224 | type=str, 225 | help="Domain to dump output of") 226 | 227 | return parser 228 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ``` 4 | .______ .______ .______ ._______ ._______ ._______ _____._ 5 | :_ _ \ : __ \ : __ \ : .___ \ : __ / : .___ \ \__ _:| 6 | | | || \____|| \____|| : | || |> \ | : | | | :| 7 | | . | || : \ | : \ | : || |> \| : | | | 8 | |. ____/ | |___\| |___\ \_. ___/ |_______/ \_. ___/ | | 9 | :/ |___| |___| :/ :/ |___| 10 | : : : 11 | ``` 12 | 13 | [![Dc27Badge](https://img.shields.io/badge/DEF%20CON-27-green)](https://defcon.org/html/defcon-27/dc-27-demolabs.html#Dr.%20ROBOT)[![License](http://img.shields.io/:license-mit-blue.svg)](https://github.com/sandialabs/dr_robot/blob/master/LICENSE)[![Build Status](https://travis-ci.org/sandialabs/dr_robot.svg?branch=master)](https://travis-ci.org/sandialabs/dr_robot)[![GitHub release (latest by date)](https://img.shields.io/github/v/release/sandialabs/dr_robot)](https://github.com/sandialabs/dr_robot/blob/master/CHANGELOG.md) 14 | 15 | Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights in this software. 16 | 17 | ## Introduction 18 | 19 | Dr.ROBOT is a tool for **Domain Reconnaissance and Enumeration**. By utilizing containers to reduce the overhead of dealing with dependencies, inconsistencies across operating systems, and different languages, Dr.ROBOT is built to be highly portable and configurable. 20 | 21 | **Use Case**: Gather as many public facing servers that an organization possesses. Querying DNS resources enables us to quickly develop a large list of possible targets that you can run further analysis on. 22 | 23 | **Note**: Dr.ROBOT is not just a one trick pony. You can easily customize the tools that are used to gather information, so that you can enjoy the benefits of using latest and greatest along with your battle tested favorites. 24 | 25 | ### List of current tools 26 | * Altdns 27 | * Amass 28 | * Anubis 29 | * Aquatone (Discover portion, when aquatone had multiple parts) 30 | * CT-Exposer 31 | * CTFR 32 | * Eyewitness 33 | * HTTPScreenshot 34 | * Knock 35 | * NMap Screenshot 36 | * NMap 37 | * Reconng 38 | * Subbrute 39 | * Subfinder 40 | * Sublist3r 41 | * Webscreenshot 42 | * GoWitness 43 | 44 | ## Config Files 45 | 46 | Dr.ROBOT adds config files, templates, logs, output files, and db files to your ```$HOME``` directory under ```.drrobot``` 47 | 48 | The directory structure will look like this: 49 | ``` 50 | -rw-r--r-- 1 0 Sep 16 12:15 ansible_inventory 51 | drwxr-xr-x 5 160 Sep 16 12:18 ansible_plays 52 | -rw-r--r-- 1 13576 Sep 16 12:41 config.json 53 | drwxr-xr-x 4 128 Sep 17 10:48 dbs 54 | drwxr-xr-x 21 672 Sep 16 13:51 docker_buildfiles 55 | drwxr-xr-x 4 128 Sep 16 15:38 logs 56 | drwxr-xr-x 3 96 Sep 16 12:46 output 57 | ``` 58 | If you ever break your config beyond saving, you can delete the config.json file in your ```$HOME``` directory and rerun Dr.ROBOT, which will generate a new config file for you. 59 | 60 | ## Installation (with pip) 61 | 62 | ``` 63 | git clone 64 | cd gitrepo 65 | pip install -r requirements.txt 66 | pip install -e . 67 | drrobot --help 68 | 69 | 70 | usage: drrobot [-h] [--proxy PROXY] [--dns DNS] [--verbose] [--dbfile DBFILE] 71 | {gather,inspect,upload,rebuild,dumpdb,output,serve} ... 72 | 73 | Docker DNS recon tool 74 | 75 | positional arguments: 76 | {gather,inspect,upload,rebuild,dumpdb,output,serve} 77 | gather Runs initial scanning phase where tools under the 78 | webtools/scannerscategory will run and gather 79 | information used in the following phases 80 | inspect Run further tools against domain information gathered 81 | from previous step.Note: you must either supply a file 82 | which contains a list of IP/Hostnames orThe targeted 83 | domain must have a db under the dbs folder 84 | upload Upload recon data to Mattermost/Slack 85 | rebuild Rebuild the database with additional files/all files 86 | from previous runtime 87 | dumpdb Dump contents of database (ip,hostname,banners) to a 88 | text file with hostname for filename 89 | output Generate output in specified format. Contains all 90 | information from scans (images, headers, hostnames, 91 | ips) 92 | serve Serve database file in docker container using django 93 | 94 | optional arguments: 95 | -h, --help show this help message and exit 96 | --proxy PROXY Proxy server URL to set DOCKER http_proxy too 97 | --dns DNS DNS server to add to resolv.conf of DOCKER containers 98 | --verbose Display verbose statements 99 | --dbfile DBFILE Specify what db file to use for saving data too 100 | 101 | ``` 102 | 103 | ## Installation (pipenv) 104 | 105 | ``` 106 | git clone 107 | cd gitrepo 108 | pipenv sync 109 | pipenv shell 110 | drrobot --help 111 | 112 | 113 | usage: drrobot [-h] [--proxy PROXY] [--dns DNS] [--verbose] [--dbfile DBFILE] 114 | {gather,inspect,upload,rebuild,dumpdb,output,serve} ... 115 | 116 | Docker DNS recon tool 117 | 118 | positional arguments: 119 | {gather,inspect,upload,rebuild,dumpdb,output,serve} 120 | gather Runs initial scanning phase where tools under the 121 | webtools/scannerscategory will run and gather 122 | information used in the following phases 123 | inspect Run further tools against domain information gathered 124 | from previous step.Note: you must either supply a file 125 | which contains a list of IP/Hostnames orThe targeted 126 | domain must have a db under the dbs folder 127 | upload Upload recon data to Mattermost/Slack 128 | rebuild Rebuild the database with additional files/all files 129 | from previous runtime 130 | dumpdb Dump contents of database (ip,hostname,banners) to a 131 | text file with hostname for filename 132 | output Generate output in specified format. Contains all 133 | information from scans (images, headers, hostnames, 134 | ips) 135 | serve Serve database file in docker container using django 136 | 137 | optional arguments: 138 | -h, --help show this help message and exit 139 | --proxy PROXY Proxy server URL to set DOCKER http_proxy too 140 | --dns DNS DNS server to add to resolv.conf of DOCKER containers 141 | --verbose Display verbose statements 142 | --dbfile DBFILE Specify what db file to use for saving data too 143 | ``` 144 | 145 | ## Certs 146 | 147 | Running this behind a proxy was a pain. To make this less painful we create a certs directory under the ```$HOME/.drrobot/*``` where you can add your crt files. As part of the dockerfile build process we now generate tarfiles with the certificates so that applications, such as Amass, can run. 148 | 149 | ## Minio 150 | 151 | Included with Dr.ROBOT is a docker-compose.yml file. This file contains a simple compose file to serve up Minio and the files gathered during runtime. 152 | 153 | To use: 154 | ``` 155 | cd /path/to/drrobot/ 156 | docker-compose up 157 | ``` 158 | 159 | ## Docker 160 | 161 | This tool relies heavily on Docker. 162 | 163 | See installation instructions here: 164 | * [Docker Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) 165 | * [Docker MacOS](https://docs.docker.com/docker-for-mac/install/) 166 | * [Docker Windows](https://docs.docker.com/docker-for-windows/install/) 167 | 168 | 169 | ## Ansible 170 | 171 | You can make any module support Ansible. 172 | 173 | See [Installation](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#intro-installation-guide) guide for instructions. 174 | 175 | * If using a mac you will need to install gnu-tar for Ansible to unpack compressed files: ```brew install gnu-tar``` 176 | * If you have an encrypted ssh key that requires a password to use and would not like to enter their password for every command ran remotely look into using an **ssh-agent** 177 | ``` 178 | eval $(ssh-agent) 179 | ssh-add /path/to/keyfile 180 | ```` 181 | 182 | 183 | ## Documentation 184 | 185 | To add your own tool see the [Configuration](readmes/config.md) to get started. 186 | 187 | For usage see [Usage](readmes/usage.md) to get started. 188 | -------------------------------------------------------------------------------- /src/robot_api/api/upload.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """ Upload module 3 | 4 | This module handles the uploading of files and images to a 5 | supplied destination. This module contains one base class 6 | from which all modules are derived so Dr.ROBOT may call and treat 7 | all extended classes equally 8 | """ 9 | from abc import ABC, abstractmethod 10 | import logging 11 | import datetime 12 | from os import walk, stat 13 | from os.path import exists, abspath, isdir, isfile, basename 14 | from mattermostdriver import Driver, exceptions 15 | import slack 16 | 17 | from robot_api.parse import join_abs 18 | 19 | LOG = logging.getLogger(__name__) 20 | 21 | 22 | class Forum(ABC): 23 | def __init__(self, **kwargs): 24 | """ABC class 25 | Args: 26 | **kwargs: 27 | api_key (str): api key for service 28 | username (str): username for service 29 | password (str): password for service 30 | url (str): url for service 31 | domain (str): domain for service 32 | 33 | """ 34 | self.api_key = kwargs.get('api_key', None) 35 | self.username = kwargs.get('username', None) 36 | self.password = kwargs.get('password', None) 37 | self.url = kwargs.get('url', None) 38 | self.domain = kwargs.get('domain', None) 39 | 40 | @abstractmethod 41 | def upload(self, **kwargs): 42 | """Abstractmethod 43 | 44 | Method to be used in order to keep instantiation 45 | and execution simple in calling function 46 | Args: 47 | **kwargs: Any extra values needed for execution. 48 | 49 | Returns: 50 | 51 | """ 52 | pass 53 | 54 | 55 | class Mattermost(Forum): 56 | def __init__(self, **kwargs): 57 | """Initialize Mattermost class which extends Chat ABC 58 | Args: 59 | **kwargs (dict): args to pass to Chat ABC 60 | team_channel (str): required for posting to channel 61 | channel_name (str): required for posting to channel 62 | """ 63 | super().__init__(**kwargs) 64 | self.inst = Driver({ 65 | 'url': self.url, 66 | 'verify': False, 67 | 'token': self.api_key, 68 | 'username': self.username, 69 | 'port': kwargs.get('port', 8065) 70 | }) 71 | self.team_name = kwargs.get("team_name", None) 72 | self.channel_name = kwargs.get("channel_name", None) 73 | self.filepath = kwargs.get("filepath", None) 74 | 75 | def _upload_files(self, file_location, channel_id): 76 | file_ids = [] 77 | for root, dirs, files in walk(file_location): 78 | for filename in files: 79 | # TODO add optional parameters for adjusting size. Implement 80 | # file splitting 81 | print(f"[...] Uploading {filename}") 82 | if stat(join_abs(root, filename)).st_size / 1024 ** 2 > 49: 83 | print(f"[!]\tFile {filename} is to big, ignoring for now") 84 | continue 85 | else: 86 | file_ids += [ 87 | self.inst.files.upload_file( 88 | channel_id=channel_id, 89 | files={ 90 | 'files': ( 91 | filename, 92 | open( 93 | join_abs( 94 | root, 95 | filename), 96 | 'rb'))})['file_infos'][0]['id']] 97 | if len(file_ids) >= 5: 98 | self.inst.posts.create_post(options={ 99 | 'channel_id': channel_id, 100 | 'message': f"Recon Data {datetime.datetime.now()}", 101 | 'file_ids': file_ids 102 | }) 103 | file_ids = [] 104 | if len(file_ids) > 0: 105 | self.inst.posts.create_post(options={ 106 | 'channel_id': channel_id, 107 | 'message': f"Recon Data {datetime.datetime.now()}", 108 | 'file_ids': file_ids 109 | }) 110 | 111 | def upload(self, **kwargs): 112 | """File upload 113 | 114 | Args: 115 | filepath (str): optional filepath to check for files to upload 116 | Returns: 117 | 118 | """ 119 | print("[*] doing post message") 120 | try: 121 | self.inst.login() 122 | 123 | team_id = self.inst.teams.get_team_by_name(self.team_name)['id'] 124 | channel_id = self.inst.channels.get_channel_by_name( 125 | channel_name=self.channel_name, team_id=team_id)['id'] 126 | except exceptions.NoAccessTokenProvided as er: 127 | print(f"[!] NoAccessTokenProvided {er}") 128 | LOG.exception() 129 | except exceptions.InvalidOrMissingParameters as er: 130 | print(f"[!] InvalidOrMissingParameters {er}") 131 | LOG.exception() 132 | 133 | try: 134 | if isfile(self.filepath): 135 | file_ids = [ 136 | self.inst.files.upload_file( 137 | channel_id=channel_id, files={ 138 | 'files': ( 139 | basename( 140 | self.filepath), open( 141 | join_abs( 142 | self.filepath), 'rb'))})['file_infos'][0]['id']] 143 | 144 | self.inst.posts.create_post(options={ 145 | 'channel_id': channel_id, 146 | 'message': f"Recon Data {datetime.datetime.now()}", 147 | 'file_ids': file_ids 148 | }) 149 | 150 | elif isdir(self.filepath): 151 | file_location = abspath(self.filepath) 152 | 153 | self._upload_files(file_location, channel_id) 154 | 155 | except exceptions.ContentTooLarge: 156 | print(f"[!] ContentTooLarge in upload") 157 | LOG.exception() 158 | except exceptions.ResourceNotFound: 159 | print(f"[!] ResourceNotFound in upload") 160 | LOG.exception() 161 | except OSError: 162 | print(f"[!] File not found in upload") 163 | LOG.exception() 164 | 165 | 166 | class Slack(Forum): 167 | def __init__(self, **kwargs): 168 | """Initialize Slack Client 169 | """ 170 | super().__init__(**kwargs) 171 | 172 | self.channel_name = kwargs.get("channel_name", None) 173 | self.filepath = kwargs.get("filepath", None) 174 | 175 | def _get_files(self, file_location, max_filesize=50): 176 | """Generator to grab individual files for upload 177 | 178 | Args: 179 | file_location (str): Location of file(s) to upload 180 | max_filesize (int): Max allowed file size, in megabytes 181 | 182 | Returns: 183 | (Tuple) (filename, path to file) 184 | """ 185 | if not exists(file_location): 186 | return None 187 | 188 | for root, _, files in walk(file_location): 189 | for filename in files: 190 | # TODO add optional parameters for adjusting size. Implement 191 | # file splitting 192 | print(f"[...] Uploading {filename}") 193 | if stat(join_abs(root, filename)).st_size / \ 194 | 1024 ** 2 > max_filesize: 195 | print(f"[!]\tFile {filename} is to big, ignoring for now") 196 | continue 197 | else: 198 | yield (filename, join_abs(root, filename)) 199 | 200 | def upload(self, **kwargs): 201 | slack_client = slack.WebClient(self.api_key) 202 | 203 | # try: 204 | if isfile(self.filepath): 205 | slack_client.files_upload( 206 | channels=self.channel_name, 207 | file=self.filepath) 208 | 209 | elif isdir(self.filepath): 210 | print(self.filepath) 211 | if self.filepath and exists(abspath(self.filepath)): 212 | file_location = abspath(self.filepath) 213 | 214 | for filename, filepath in self._get_files(file_location): 215 | print(filename, filepath) 216 | slack_client.files_upload( 217 | channels=self.channel_name, 218 | file=filepath) 219 | # except Exception as ex: 220 | # print(str(ex)) 221 | # LOG.exception(ex) 222 | -------------------------------------------------------------------------------- /src/robot_api/api/dockerize.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """ Docker wrapper 3 | 4 | Build docker images and containers using templates for Dr.ROBOT 5 | 6 | Attributes: 7 | _docker_options (dict): docker options for template provided from config.json 8 | _default_config_path (str): location of template file 9 | _active_config_path (str): location of active Dockerfile 10 | verbose (bool): More output Yes/No 11 | container (docker.Container): container object when running 12 | status (str): If running or not 13 | name (str): name of docker image 14 | """ 15 | from os.path import isfile, basename 16 | from string import Template 17 | import logging 18 | import time 19 | import tarfile 20 | import json 21 | import multiprocessing 22 | from tqdm import tqdm 23 | import docker 24 | from docker.errors import APIError, BuildError, ContainerError, ImageNotFound, NotFound 25 | 26 | from robot_api.parse import join_abs 27 | 28 | LOG = logging.getLogger(__name__) 29 | 30 | 31 | class Docker: 32 | def __init__(self, **kwargs): 33 | """Constructor 34 | 35 | Args: 36 | **kwargs: 37 | docker_options : (Dict) of arguments to build docker with (name, descriptions, output) 38 | active_config_path: (String) filepath of config to use for "active" configuration 39 | default_config_path: (String) filepath of config to use as default template. 40 | output_dir: (String) output directory to mount on docker 41 | 42 | Returns: 43 | 44 | """ 45 | self._docker_options = kwargs.get('docker_options', None) 46 | 47 | self._default_config_path = kwargs.get('default_config_path', None) 48 | self._active_config_path = kwargs.get('active_config_path', None) 49 | self.name = self._docker_options['name'] 50 | self.network_mode = self._docker_options.get('network_mode', 'host') 51 | 52 | self.verbose = kwargs.get('verbose', False) 53 | self.client = docker.from_env() 54 | self.image = None 55 | self.container = None 56 | self.status = None 57 | self.error = False 58 | self.done_building = False 59 | self.OUTPUT_DIR = kwargs.get('output_dir', None) 60 | 61 | def _print(self, msg): 62 | if self.verbose: 63 | print("[D] " + msg) 64 | LOG.debug(msg) 65 | 66 | def kill(self): 67 | """Kills container 68 | 69 | Tries to kill container. 70 | 71 | """ 72 | try: 73 | self.container.kill() 74 | except docker.errors.APIError: 75 | self._print( 76 | "Error when trying to send kill signal to docker container.") 77 | LOG.exception("Killing container") 78 | 79 | def gen_config(self): 80 | """Creates active configuration from template 81 | 82 | Raises: 83 | OSError 84 | """ 85 | if isfile(self._default_config_path): 86 | LOG.info("Creating Dockerfile for %s", self._docker_options['name']) 87 | elif not isfile(self._default_config_path): 88 | raise OSError( 89 | 'Default configuration file is not found, please fix') 90 | 91 | self._print(f"Making config with args:{json.dumps(self._docker_options, indent=4)}") 92 | 93 | with open(self._default_config_path, 'r') as cfg: 94 | t = Template(cfg.read()) 95 | with open(self._active_config_path, 'w') as out: 96 | out.write(t.safe_substitute({k: v if 97 | v else '\'\'' 98 | for k, v 99 | in self._docker_options.items() 100 | })) 101 | def gen_tarfile(self): 102 | tarname = join_abs(self._docker_options['tarfiles'], basename(self._active_config_path) + ".tar.gz") 103 | with tarfile.open(name=tarname, mode="w:gz") as tar: 104 | tar.add(self._active_config_path, "Dockerfile") 105 | tar.add(self._docker_options['certs'], 'certs') 106 | return tarname 107 | 108 | 109 | def build(self): 110 | """ 111 | Generates docker image from active_config 112 | 113 | Args: 114 | 115 | Returns: 116 | 117 | """ 118 | try: 119 | self.gen_config() 120 | tarfile = self.gen_tarfile() 121 | self._print(f"""Built with options: 122 | -f {self._active_config_path} 123 | -t {self._docker_options['docker_name']}:{self._docker_options['docker_name']} 124 | --rm 125 | --network {self.network_mode} 126 | """) 127 | with open(tarfile, 'rb') as _file: 128 | self.image = self.client.images.build(fileobj=_file, 129 | tag=f"{self._docker_options['docker_name']}:{self._docker_options['docker_name']}", 130 | custom_context=True, 131 | encoding='gzip', 132 | rm=True, 133 | network_mode=self.network_mode, 134 | use_config_proxy=True) 135 | 136 | self.done_building = True 137 | except BuildError as error: 138 | # print(f"[!] Build Error {self.name}. Check logs") 139 | LOG.exception("[!] BuildError: %s", self.name) 140 | self.error = True 141 | # if "net/http" in str(error): 142 | # print("[!] This could be a proxy issue, see " + 143 | # "https://docs.docker.com/config/daemon/systemd/#httphttps-proxy for help") 144 | except APIError: 145 | # print(f"[!] APIError: {self.name}") 146 | LOG.exception("[!] APIError: %s", self.name) 147 | self.error = True 148 | except KeyError: 149 | # print(f"[!] KeyError Output or Docker Name " + 150 | # "is not defined!!: {scanner.name}") 151 | LOG.exception("[!] KeyError Output or Docker Name " + 152 | "is not defined!!: %s", self.name) 153 | self.error = True 154 | except OSError: 155 | LOG.exception("[!] Output directory could not be created, " + 156 | "please verify permissions") 157 | self.error = True 158 | 159 | def run(self): 160 | """ 161 | Builds and runs docker container. 162 | 163 | Args: 164 | 165 | Returns: 166 | 167 | """ 168 | 169 | try: 170 | volumes = self._docker_options.get("volumes", None) 171 | if volumes is None: 172 | volumes = { 173 | self.OUTPUT_DIR: { 174 | 'bind': self._docker_options['output'], 175 | 'mode': 'rw' 176 | } 177 | } 178 | self.container = self.client.containers.run( 179 | image=f"{self._docker_options['docker_name']}:{self._docker_options['docker_name']}", 180 | # dns=[self._docker_options.get('dns')] if self._docker_options.get('dns', None) else None, # REMOVED in latest due to issues :/ 181 | auto_remove=True, 182 | tty=True, 183 | detach=True, 184 | network_mode=self.network_mode, 185 | command=self._docker_options.get( 186 | 'command', 187 | None), 188 | ports=self._docker_options.get( 189 | 'ports', 190 | None), 191 | volumes=volumes) 192 | 193 | self.status = self.container.status 194 | 195 | self._print(f"mount point specified here: {volumes}") 196 | 197 | except ImageNotFound: 198 | self.error = True 199 | # print(f"[!] ImageNotFound: {self.name}") 200 | LOG.exception("[!] ImageNotFound: %s", self.name) 201 | except ContainerError: 202 | self.error = True 203 | # print(f"[!] Container Error: {self.name}") 204 | LOG.exception("[!] Container Error: %s", self.name) 205 | except APIError: 206 | self.error = True 207 | # print(f"[!] APIError: {self.name}") 208 | LOG.exception("[!] APIError: %s", self.name) 209 | except KeyError: 210 | self.error = True 211 | # print(f"[!] KeyError Output or Docker Name " + 212 | # "is not defined!!: {scanner.name}") 213 | LOG.exception("[!] KeyError Output or Docker Name " + 214 | "is not defined!!: %s", self.name) 215 | except OSError: 216 | self.error = True 217 | LOG.exception("[!] Output directory could not be created, " + 218 | "please verify permissions") 219 | 220 | def monitor_build(self): 221 | """Monitor build status of docker 222 | 223 | """ 224 | with tqdm() as pbar: 225 | pbar.set_description(f"[#] Docker image {self.name}, building...") 226 | while self.image is None and not self.error and not self.done_building: 227 | time.sleep(2) 228 | pbar.update(1) 229 | 230 | def update_status(self): 231 | """ 232 | Thread watcher to update status of container 233 | 234 | Args: 235 | 236 | Returns: 237 | 238 | """ 239 | if self.error: 240 | return 241 | try: 242 | with tqdm() as pbar: 243 | pbar.set_description( 244 | f"[#] Docker container {self.name}, running...") 245 | while True: 246 | self.container.reload() 247 | time.sleep(2) 248 | self.status = self.container.status 249 | pbar.update(1) 250 | 251 | except NotFound: 252 | self._print(f"[*] Docker container {self._docker_options['docker_name']} exited") 253 | self.status = 'exited' 254 | except AttributeError: 255 | LOG.exception("AttributeError in update_status. Check logs") 256 | except KeyboardInterrupt: 257 | LOG.exception("KeyboardInterrupt in update_status. Check logs") 258 | self.kill() 259 | -------------------------------------------------------------------------------- /LICENSES/LICENSE.dockerpy.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2016 Docker, Inc. 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. -------------------------------------------------------------------------------- /LICENSES/LICENSE.docker.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | https://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2013-2018 Docker, Inc. 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | https://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | -------------------------------------------------------------------------------- /src/robot_api/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Cli module for Dr.ROBOT 3 | 4 | This module sets up the logger and configuration files for cli usage. 5 | """ 6 | from os import makedirs, path, environ 7 | import logging 8 | import json 9 | import sys 10 | from enum import Enum 11 | from sqlite3 import DatabaseError 12 | 13 | from robot_api.robot import Robot 14 | from robot_api.parse import parse_args, join_abs 15 | from robot_api.config import load_config, generate_configs, tool_check, get_config 16 | 17 | 18 | ROOT_DIR = path.join(environ.get("HOME","."),".drrobot") 19 | 20 | generate_configs() 21 | 22 | 23 | class Mode(Enum): 24 | """Class for simple option handling when loading the config file. 25 | 26 | Leave for extending upon later 27 | 28 | """ 29 | DOCKER = 1 30 | ANSIBLE = 2 31 | 32 | 33 | def setup_logger(): 34 | """Setup our logging instance. 35 | Returns: 36 | A logger for writing to two seperate files depending on error or debug messages 37 | """ 38 | logger = logging.getLogger() 39 | logger.setLevel(logging.NOTSET) 40 | formatter = logging.Formatter( 41 | '%(asctime)s {%(pathname)s:%(lineno)d}: \t%(message)s') 42 | 43 | handler = logging.FileHandler( 44 | filename=join_abs( 45 | ROOT_DIR, 46 | "logs", 47 | "drrobot.err")) 48 | handler.setLevel(logging.ERROR) 49 | handler.setFormatter(formatter) 50 | logger.addHandler(handler) 51 | 52 | handler = logging.FileHandler( 53 | filename=join_abs( 54 | ROOT_DIR, 55 | "logs", 56 | "drrobot.dbg")) 57 | handler.setLevel(logging.DEBUG) 58 | handler.setFormatter(formatter) 59 | logger.addHandler(handler) 60 | 61 | logging.getLogger("urllib3.connectionpool").disabled = True 62 | 63 | return logger 64 | 65 | 66 | def create_dirs(parser): 67 | """Create directories for domain under output folder 68 | """ 69 | args = parser.parse_args() 70 | if getattr(args, 'domain', None): 71 | if not path.exists( 72 | join_abs(ROOT_DIR, "output", getattr(args, 'domain'))): 73 | makedirs( 74 | join_abs(ROOT_DIR, "output", getattr(args, 'domain'))) 75 | 76 | if not path.exists( 77 | join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "aggregated")): 78 | makedirs( 79 | join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "aggregated")) 80 | 81 | 82 | def start_gather(drrobot, tools, parser): 83 | """Gather data using OSINT tools 84 | 85 | This method starts off the gather phase of Dr.ROBOT. 86 | Meant to be passive, HOWEVER, pay attention to the tools 87 | you are using. Not all tools are passive and as such it is 88 | important to double check what you are running. 89 | 90 | 91 | TODO: 92 | * Track passive tools vs none passive 93 | """ 94 | args = parser.parse_args() 95 | print("Beginning gather") 96 | verbose = getattr(args, "verbose", False) 97 | webtools = { 98 | k: v for k, v in tools.get("webtools").items() if getattr(args, k) 99 | } 100 | if verbose: 101 | print(f"Webtools:\n{webtools}") 102 | 103 | scanners_dockers = {k: v for k, v 104 | in tools.get("scanners").items() 105 | if getattr(args, k) 106 | is True and Mode[v["mode"]] == Mode.DOCKER} 107 | if verbose: 108 | print(f"Scanners as Dockers: \ 109 | {json.dumps(scanners_dockers, indent=4)}") 110 | 111 | scanners_ansible = {k: v for k, v 112 | in tools.get("scanners").items() 113 | if getattr(args, k) 114 | is True 115 | and Mode[v["mode"]] == Mode.ANSIBLE} 116 | if verbose: 117 | print(f"Scanners as Ansible Play: \ 118 | {json.dumps(scanners_ansible, indent=4)}") 119 | 120 | if not webtools and \ 121 | not scanners_ansible and \ 122 | not scanners_dockers: 123 | print("[*] No scanners/webtools provided, exiting...") 124 | parser.print_help() 125 | sys.exit(0) 126 | 127 | drrobot.gather( 128 | webtools=webtools, 129 | scanners_dockers=scanners_dockers, 130 | scanners_ansible=scanners_ansible, 131 | headers=getattr( 132 | args, 133 | "headers", 134 | False)) 135 | 136 | 137 | def start_inspect(drrobot, tools, parser): 138 | """Begin inspection of domain using aggregated data. 139 | 140 | This module starts off the "inspection" phase of Dr.ROBOT 141 | Inspection is a none-passive portion of this tool and is meant 142 | to generate more interesting output using the aggregated data 143 | """ 144 | print("Beginning inspection") 145 | args = parser.parse_args() 146 | verbose = getattr(args, "verbose", False) 147 | post_enum_dockers = {k: v for k, v 148 | in tools.get("enumeration").items() 149 | if getattr(args, k) 150 | is True 151 | and Mode[v["mode"]] == Mode.DOCKER} 152 | if verbose: 153 | print(f"Inspection dockers \ 154 | {json.dumps(post_enum_dockers, indent=4)}") 155 | 156 | post_enum_ansible = {k: v for k, v 157 | in tools.get("enumeration").items() 158 | if getattr(args, k) 159 | is True 160 | and Mode[v["mode"]] == Mode.ANSIBLE} 161 | if verbose: 162 | print(f"Inspection ansible \ 163 | {json.dumps(post_enum_ansible, indent=4)}") 164 | 165 | if not post_enum_ansible and not post_enum_dockers: 166 | print("[*] No scanners/webtools provided, exiting...") 167 | parser.print_help() 168 | sys.exit(0) 169 | 170 | _file = getattr(args, 'file', None) 171 | drrobot.inspection( 172 | post_enum_ansible=post_enum_ansible, 173 | post_enum_dockers=post_enum_dockers, 174 | file=_file) 175 | 176 | 177 | def start_upload(drrobot, tools, parser): 178 | """Uploads files/images to destination 179 | 180 | TODO: 181 | * Other upload destinations 182 | """ 183 | args = parser.parse_args() 184 | filepath = getattr(args, "filepath") 185 | if filepath is None: 186 | print("No filepath provided, exiting...") 187 | sys.exit(0) 188 | elif not path.exists(filepath): 189 | print("Filepath does not exists, exiting...") 190 | sys.exit(0) 191 | 192 | print(f"Beginning upload with file path: {filepath}") 193 | 194 | upload_dest = {k: v for k, v in tools.get("upload_dest").items() if 195 | getattr(args, k) is True} 196 | print( 197 | f"Upload tools: {json.dumps(upload_dest, indent=4)}") 198 | 199 | drrobot.upload(filepath=filepath, upload_dest=upload_dest) 200 | 201 | 202 | def start_rebuild(drrobot, tools, parser): 203 | """Rebuild database with given files/directory 204 | 205 | Parsers the config file and files argument and loads all relevant files to 206 | throw through the aggregation module 207 | """ 208 | args = parser.parse_args() 209 | files = getattr(args, "files", None) 210 | headers = getattr(args, "headers", False) 211 | if files is None: 212 | files = [] 213 | 214 | def gen_dict_extract(key, var): 215 | if hasattr(var, 'items'): 216 | for _key2, _val2 in var.items(): 217 | if _key2 == key: 218 | yield _val2 219 | if isinstance(_val2, dict): 220 | for result in gen_dict_extract(key, _val2): 221 | yield result 222 | elif isinstance(_val2, list): 223 | for _dict in _val2: 224 | for result in gen_dict_extract(key, _dict): 225 | yield result 226 | 227 | for output in gen_dict_extract("output_file", tools): 228 | files += [output] 229 | for folder in gen_dict_extract("output_folder", tools): 230 | files += [folder] 231 | 232 | drrobot.rebuild(files=files, headers=headers) 233 | 234 | 235 | def start_dumpdb(drrobot, parser): 236 | """Dump database to output folder for given domain 237 | 238 | Generates all header text files and aggregated files under 239 | $HOME/.drrobot/output//aggregated 240 | """ 241 | args = parser.parse_args() 242 | dbpath = getattr(args, "dbfile") 243 | 244 | if path.exists(dbpath): 245 | if not path.exists( 246 | join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "headers")): 247 | makedirs(join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "headers")) 248 | drrobot.dumpdb() 249 | else: 250 | print("[!] DB file does not exists, try running gather first") 251 | 252 | 253 | def start_output(drrobot, parser): 254 | """Generate output 255 | 256 | Dump database and generate json/xml file 257 | 258 | TODO: 259 | * Other output forms 260 | """ 261 | args = parser.parse_args() 262 | _format = getattr(args, "format") 263 | output = getattr(args, "output") 264 | drrobot.generate_output(_format, output) 265 | 266 | def run(): 267 | """Main method for running Dr.ROBOT. 268 | 269 | Returns: 270 | Nothing. 271 | """ 272 | try: 273 | if not path.exists(join_abs(ROOT_DIR, "logs")): 274 | makedirs(join_abs(ROOT_DIR, "logs")) 275 | 276 | log = setup_logger() 277 | 278 | tools = load_config(get_config()) 279 | 280 | parser = parse_args(**tools, root_dir=ROOT_DIR) 281 | 282 | if len(sys.argv) <= 1: 283 | parser.print_help() 284 | sys.exit(1) 285 | 286 | args = parser.parse_args() 287 | 288 | log.debug(args) 289 | 290 | tool_check() 291 | 292 | drrobot = Robot(root_dir=ROOT_DIR, 293 | user_config=get_config(), 294 | **tools, 295 | dns=getattr(args, 'dns', None), 296 | proxy=getattr(args, 'proxy', None), 297 | domain=getattr(args, 'domain', None), 298 | verbose=getattr(args, 'verbose'), 299 | dbfile=getattr(args, 'dbfile'), 300 | verify=getattr(args, 'verify', None)) 301 | 302 | if not path.exists(join_abs(ROOT_DIR, "dbs")): 303 | makedirs(join_abs(ROOT_DIR, "dbs")) 304 | 305 | create_dirs(parser) 306 | log.debug(f"Dumping tools for run :{tools}") 307 | 308 | if args.actions in "gather": 309 | start_gather(drrobot, tools, parser) 310 | 311 | if args.actions in 'inspect': 312 | start_inspect(drrobot, tools, parser) 313 | 314 | if args.actions in "upload": 315 | start_upload(drrobot, tools, parser) 316 | 317 | if args.actions in "rebuild": 318 | start_rebuild(drrobot, tools, parser) 319 | 320 | if args.actions in "output": 321 | start_output(drrobot, parser) 322 | 323 | if args.actions in "dumpdb": 324 | start_dumpdb(drrobot, parser) 325 | 326 | except json.JSONDecodeError as error: 327 | print(f"[!] JSON load error, configuration file is bad.\n {error}") 328 | log.exception(error) 329 | except DatabaseError as error: 330 | print(f"[!] Something went wrong with SQLite\n {error}") 331 | log.exception(error) 332 | except KeyboardInterrupt: 333 | print("[!] KeyboardInterrup, exiting...") 334 | except OSError as error: 335 | log.exception(error) 336 | print(f"[!] OSError {error}") 337 | except TypeError as error: 338 | log.exception(error) 339 | print(f"[!] {error}") 340 | 341 | 342 | if __name__ == "__main__": 343 | run() 344 | -------------------------------------------------------------------------------- /src/robot_api/api/web_resources.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf8 -*- 2 | """WebTool module 3 | 4 | Contains all web based modules for Dr.ROBOT. All modules 5 | implement the Abstract Base Class WebTool to make running 6 | any user created modules easier. 7 | 8 | """ 9 | import requests 10 | import netaddr 11 | import socket 12 | import logging 13 | import shodan 14 | import json 15 | import re 16 | from bs4 import BeautifulSoup 17 | from abc import ABC, abstractmethod 18 | 19 | LOG = logging.getLogger(__name__) 20 | 21 | 22 | class WebTool(ABC): 23 | def __init__(self, **kwargs): 24 | """ 25 | ABC class 26 | Args: 27 | **kwargs: 28 | api_key (str): api key for service 29 | user (str): ) user for service 30 | password (str): password for service 31 | proxies (str): proxies for service 32 | domain (str): domain for service 33 | output (str): output for service 34 | 35 | """ 36 | self.proxies = kwargs.get('proxies', None) 37 | self.domain = kwargs.get('domain', None) 38 | self.output = kwargs.get('output_file', None) 39 | self.verbose = kwargs.get('verbose', False) 40 | self.results = [] 41 | 42 | @abstractmethod 43 | def do_query(self): 44 | """Abstract method for query 45 | 46 | All modules implement this method. This method 47 | is called by Dr.ROBOT so that we can dynamically 48 | import and call modules without having to do 49 | anything special 50 | """ 51 | pass 52 | 53 | def _print(self, msg): 54 | if self.verbose: 55 | print("[D] " + msg) 56 | LOG.debug(msg) 57 | 58 | def _write_results(self): 59 | """ 60 | Write output of queries to file 61 | Returns: 62 | 63 | """ 64 | with open(self.output, 'w') as f: 65 | for item in self.results: 66 | f.write(f"{item}\n") 67 | 68 | 69 | class Arin(WebTool): 70 | """Arin web module 71 | 72 | Reaches out to ARIN api to grab IP ranges. Generates A TON 73 | of IPS. Best not used unless you have time and are loud 74 | 75 | """ 76 | def __init__(self, **kwargs): 77 | super().__init__(**kwargs) 78 | self.api_key = kwargs.get('api_key', None) 79 | if not self.api_key: 80 | raise ValueError("API Key cannot be empty") 81 | 82 | def __getattr__(self, item): 83 | return self 84 | 85 | def _lookup_org(self): 86 | """ 87 | Queries ARIN for the organization handle given the domain. 88 | 89 | Returns: 90 | (String) Handle for Organization 91 | 92 | """ 93 | url = "https://whois.arin.net/ui/query.do" 94 | headers = {'Content-Type': 'application/x-www-form-urlencoded', 95 | 'Accept': 'application/json'} 96 | ipv4 = socket.gethostbyname(self.domain) 97 | data = {'xslt': 'https://localhost:8080/whoisrws-servlet/arin.xsl', 98 | 'flushCache': False, 99 | 'queryinput': ipv4, 100 | 'whoisSubmitButton': '+'} 101 | res = requests.post( 102 | url, 103 | headers=headers, 104 | data=data, 105 | verify=False, 106 | proxies=self.proxies, 107 | timeout=10) 108 | return res.json().get('ns4:pft').get('org').get('handle').get('$') 109 | 110 | def _generate_ips(self, all_cidrs): 111 | """ 112 | Convert cidr ranges to ip addresses 113 | 114 | Returns: 115 | (List) ip address' 116 | """ 117 | ips = [] 118 | for cidr in all_cidrs: 119 | if not cidr.is_private() and cidr.prefixlen >= 16: 120 | ips += [ip for ip in cidr] 121 | return ips 122 | 123 | def do_query(self): 124 | """ 125 | Queries ARIN CIDR ranges for the domain passed in on instantiation 126 | 127 | Returns: 128 | 129 | """ 130 | print("[*] Beginning Arin Query") 131 | LOG.info('Starting ARIN Query for ' + self.domain) 132 | try: 133 | org_name = self._lookup_org() 134 | if not org_name: 135 | raise ValueError( 136 | "[!] Org name was not found ARIN query cannot continue") 137 | if not self.api_key: 138 | raise ValueError("[!] Arin API Key not found") 139 | url = 'http://whois.arin.net/rest/org/' + \ 140 | org_name + '/nets?apikey=' + self.api_key 141 | self._print(f"Making request to url {url}") 142 | 143 | headers = {'Accept': 'application/json'} 144 | LOG.info('Getting ' + org_name + ' ARIN Query from url: ' + url) 145 | result = requests.get( 146 | url, 147 | headers=headers, 148 | proxies=self.proxies, 149 | verify=False) 150 | self._print( 151 | f"Result content and status_code {result.content}, {result.status_code}") 152 | if result.status_code == 200: 153 | all_cidrs = [] 154 | result_json = result.json() 155 | 156 | self._print(f"{json.dumps(result_json, indent=4)}") 157 | 158 | if isinstance(result_json['nets']['netRef'], list): 159 | for x in result_json['nets']['netRef']: 160 | result_cidr_list = netaddr.iprange_to_cidrs( 161 | x['@startAddress'], x['@endAddress']) 162 | all_cidrs.append(result_cidr_list[0]) 163 | else: 164 | result_cidr = netaddr.iprange_to_cidrs( 165 | result_json['nets']['netRef']['@startAddress'], 166 | result_json['nets']['netRef']['@endAddress']) 167 | all_cidrs.append(result_cidr[0]) 168 | 169 | LOG.info('CIDR of ' + org_name + ' is: %s', all_cidrs) 170 | self.results = self._generate_ips(all_cidrs) 171 | self._write_results() 172 | else: 173 | self._print('Failed to get data for: ' + self.org_name) 174 | 175 | print("[*] Finished ARIN Query") 176 | 177 | except requests.exceptions.HTTPError: 178 | LOG.exception("HTTPError in Arin Scan") 179 | except requests.exceptions.ConnectionError: 180 | LOG.exception("Connection Error in Arin Scan") 181 | except requests.exceptions.RequestException: 182 | LOG.exception("RequestException Err in Arin Scan") 183 | except ValueError: 184 | LOG.exception("ValueError in Arin Scan") 185 | 186 | 187 | class Shodan(WebTool): 188 | def __init__(self, **kwargs): 189 | super().__init__(**kwargs) 190 | self.api_key = kwargs.get('api_key', None) 191 | if not self.api_key: 192 | raise ValueError("API Key cannot be None") 193 | 194 | def do_query(self): 195 | """ 196 | Queries Shodan for the domain passed in on instantiation 197 | 198 | Returns: 199 | 200 | """ 201 | print("[*] Beginning Shodan Query") 202 | 203 | try: 204 | shod = shodan.Shodan(self.api_key) 205 | shod._session.proxies = self.proxies 206 | shod._session.verify = False 207 | res = shod.search(self.domain) 208 | for item in res['matches']: 209 | if item['hostnames']: 210 | self.results += item['hostnames'] 211 | self._print( 212 | "Host: {} \n" 213 | "\t Product: {} \n" 214 | "\t PORT : {} \n" 215 | "\t Country Code : {} \n" 216 | "\t timestamp: {} \n" .format( 217 | " ".join(item.get("hostnames", "")), 218 | item.get("product", ""), 219 | item.get("port", ""), 220 | item.get("location", ""), 221 | item.get("timestamp", ""))) 222 | self._write_results() 223 | print("[*] Finished Shodan Query") 224 | except shodan.APIError: 225 | print("[!] Shodan Error. See log for more details") 226 | LOG.exception() 227 | 228 | 229 | class Dumpster(WebTool): 230 | def __init__(self, **kwargs): 231 | super().__init__(**kwargs) 232 | self.ENDPOINT = "https://dnsdumpster.com" 233 | self.hostnames = [] 234 | 235 | def _grab_csrf(self): 236 | """ 237 | Gather CSRF token from response so that we can continue making requests. 238 | 239 | Returns: 240 | (String) CSRF token 241 | """ 242 | response = requests.get( 243 | self.ENDPOINT, 244 | verify=False, 245 | proxies=self.proxies) 246 | reg = re.compile(r"csrftoken=([0-9A-Za-z]*)") 247 | if response.status_code == 200: 248 | match = reg.search(response.headers.get("Set-Cookie")) 249 | return match.group(1) 250 | return None 251 | 252 | def do_query(self): 253 | """ 254 | Queries DNSDumpster.com for domain information. 255 | 256 | Returns: 257 | 258 | """ 259 | print("[*] Beginning Dumpster Query") 260 | try: 261 | csrf = self._grab_csrf() 262 | cookies = { 263 | "csrftoken": csrf 264 | } 265 | headers = { 266 | "referer": self.ENDPOINT 267 | } 268 | data = { 269 | "csrfmiddlewaretoken": csrf, 270 | "targetip": self.domain 271 | } 272 | res = requests.post( 273 | self.ENDPOINT, 274 | data=data, 275 | headers=headers, 276 | cookies=cookies, 277 | proxies=self.proxies, 278 | verify=False, 279 | ) 280 | self._print(f"Dumpster query at url {self.ENDPOINT}" + 281 | f"\nwith data {data}\n" + 282 | f"and headers{headers}\n" + 283 | f"proxies {self.proxies}\n") 284 | soup = BeautifulSoup(res.content, 'html.parser') 285 | tds = soup.findAll('td', {'class': 'col-md-4'}) 286 | for td in tds: 287 | if td.text: 288 | self.results += [td.text.strip()] 289 | 290 | self._write_results() 291 | except requests.exceptions.ConnectionError: 292 | LOG.exception("[!] Connection Error check network configuration") 293 | except requests.exceptions.RequestException: 294 | LOG.exception(f"[!] Request failed SHODAN") 295 | except IndexError: 296 | LOG.exception(f"[!] No CSRF in response SHODAN") 297 | self._print("[*] End Dumpster Query") 298 | 299 | 300 | class HackerTarget(WebTool): 301 | """Module for HackerTarget.com 302 | """ 303 | def __init__(self, **kwargs): 304 | super().__init__(**kwargs) 305 | self.ENDPOINT = "https://api.hackertarget.com/hostsearch/?q={}".format( 306 | self.domain) 307 | 308 | def do_query(self): 309 | """ 310 | Queries HackerTarget.com for domain information 311 | 312 | Returns: 313 | 314 | """ 315 | print("[*] Beginning HackerTarget Query") 316 | try: 317 | res = requests.get( 318 | self.ENDPOINT, 319 | verify=False, 320 | proxies=self.proxies) 321 | self._print(f"Making request to url {self.ENDPOINT}" + 322 | f"with proxies {self.proxies}") 323 | lines = res.content.splitlines() 324 | if len(lines) < 2: 325 | print("Domain not found on hackertarget") 326 | return 327 | for line in res.content.split(): 328 | unused_hostname, ip = str(line, 'utf-8').split(',') 329 | self.results += [ip.strip()] 330 | self._write_results() 331 | except requests.exceptions.ConnectionError as er: 332 | LOG.exception("[!] Connection Error check network configuration") 333 | except requests.exceptions.RequestException as er: 334 | LOG.exception(f"[!] Request failed HackerTarget") 335 | except OSError as er: 336 | LOG.exception("OSError in HackerTarget") 337 | self._print("[*] End HackerTarget Query") 338 | 339 | 340 | class VirusTotal(WebTool): 341 | """Module for VirusTotal.com 342 | """ 343 | def __init__(self, **kwargs): 344 | super().__init__(**kwargs) 345 | self.ENDPOINT = "https://www.virustotal.com/ui/domains/{}/subdomains?limit=40".format( 346 | self.domain) 347 | 348 | def do_query(self): 349 | """ 350 | Queries VirusTotal for domain inforamtion 351 | 352 | Returns: 353 | 354 | """ 355 | headers = { 356 | "Content-Type": "application/json" 357 | } 358 | print("[*] Begin VirusTotal Query") 359 | try: 360 | res = requests.get( 361 | self.ENDPOINT, 362 | proxies=self.proxies, 363 | verify=False, 364 | headers=headers) 365 | self._print(f"Making request to url {self.ENDPOINT}" + 366 | f"with proxies {self.proxies}" + 367 | f"with headers {headers}") 368 | 369 | next_group = res.json().get('links', None).get('next', None) 370 | 371 | while next_group: 372 | for subdomain in res.json().get('data', None): 373 | self.results += [subdomain.get('id').strip()] 374 | 375 | next_group = res.json().get('links', None).get('next', None) 376 | if next_group: 377 | res = requests.get( 378 | str(next_group), 379 | proxies=self.proxies, 380 | verify=False, 381 | headers=headers) 382 | 383 | self._write_results() 384 | 385 | except requests.ConnectionError: 386 | LOG.exception("[!] Connection Error check network configuration") 387 | except requests.exceptions.RequestException: 388 | LOG.exception("[!] Request failed Virus") 389 | except OSError: 390 | LOG.exception("OSError in Virus") 391 | self._print("[*] End VirtusTotal Query") 392 | -------------------------------------------------------------------------------- /src/robot_api/configs/default_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "WebTools": 3 | { 4 | "Shodan" : 5 | { 6 | "short_name": "shodan", 7 | "class_name": "Shodan", 8 | "description" : "Query SHODAN for publicly facing sites of given domain", 9 | "output_file" : "shodan.txt", 10 | "api_key" : null, 11 | "endpoint" : null, 12 | "username" : null, 13 | "password" : null 14 | }, 15 | "Arin" : 16 | { 17 | "short_name" : "arin", 18 | "class_name": "Arin", 19 | "description" : "Query ARIN for public CIDR ranges. This is better as a brute force option as the ranges", 20 | "output_file" : "arin.txt", 21 | "api_key" : null, 22 | "username" : null, 23 | "password" : null 24 | }, 25 | "HackerTarget" : 26 | { 27 | "short_name" : "hack", 28 | "class_name" : "HackerTarget", 29 | "description" : "This query will display the forward DNS records discovered using the data sets outlined above.", 30 | "api_call_unused" : "https://api.hackertarget.com/hostsearch/?q=example.com", 31 | "output_file" : "hacker.txt" 32 | }, 33 | "Dumpster" : 34 | { 35 | "short_name" : "dump", 36 | "class_name" : "Dumpster", 37 | "description" : "Use the limited response of DNSDumpster. Requires API access for better results.", 38 | "output_file" : "dumpster.txt" 39 | }, 40 | "VirusTotal" : 41 | { 42 | "short_name" : "virus", 43 | "class_name" : "VirusTotal", 44 | "description" : "Utilize VirusTotal's Observer Subdomain Search", 45 | "output_file" : "virustotal.txt" 46 | } 47 | }, 48 | "Upload" : { 49 | "Mattermost": 50 | { 51 | "class_name" : "Mattermost", 52 | "short_name" : "matter", 53 | "api_key": "", 54 | "token_id" : "", 55 | "port" : null, 56 | "username" : "", 57 | "url" : "", 58 | "team_name" : "", 59 | "channel_name" : "", 60 | "description" : "Mattermost server" 61 | }, 62 | "Slack" : 63 | { 64 | "class_name" : "Slack", 65 | "short_name" : "slack", 66 | "api_key": "", 67 | "channel_name" : "", 68 | "description" : "Slack server" 69 | 70 | } 71 | }, 72 | "Scanners" : 73 | { 74 | "Aquatone" : { 75 | "name": "Aquatone", 76 | "mode" : "DOCKER", 77 | "network_mode": "host", 78 | "docker_name": "aqua", 79 | "default_conf": "docker_buildfiles/Dockerfile.Aquatone.tmp", 80 | "active_conf": "docker_active/Dockerfile.Aquatone", 81 | "description": "AQUATONE is a set of tools for performing reconnaissance on domain names", 82 | "src": "https://github.com/michenriksen/aquatone", 83 | "output": "/aqua", 84 | "output_folder": "aquatone" 85 | }, 86 | "Sublist3r": { 87 | "name": "Sublist3r", 88 | "mode" : "DOCKER", 89 | "network_mode": "host", 90 | "docker_name": "sub", 91 | "default_conf": "docker_buildfiles/Dockerfile.Sublist3r.tmp", 92 | "active_conf": "docker_active/Dockerfile.Sublist3r", 93 | "description": "Sublist3r is a python tool designed to enumerate subdomains of websites using OSINT", 94 | "src": "https://github.com/aboul3la/Sublist3r", 95 | "output": "/root/sublist3r", 96 | "output_folder": "sublist3r" 97 | }, 98 | "Turbolist3r": { 99 | "name": "Turbolist3r", 100 | "mode" : "DOCKER", 101 | "network_mode": "host", 102 | "docker_name": "turbo", 103 | "default_conf": "docker_buildfiles/Dockerfile.Turbolist3r.tmp", 104 | "active_conf": "docker_active/Dockerfile.Turbolist3r", 105 | "description": "Turbolist3r is a fork of the sublist3r subdomain discovery tool", 106 | "src": "https://github.com/fleetcaptain/Turbolist3r.git", 107 | "output": "/root/turbooutput", 108 | "output_folder": "turbolist3r" 109 | }, 110 | "Subbrute": { 111 | "name": "Subbrute", 112 | "mode" : "DOCKER", 113 | "network_mode": "host", 114 | "docker_name" : "brute", 115 | "default_conf": "docker_buildfiles/Dockerfile.Subbrute.tmp", 116 | "active_conf": "docker_active/Dockerfile.Subbrute", 117 | "description": "SubBrute is a community driven project with the goal of creating the fastest, and most accurate subdomain enumeration tool.", 118 | "src": "https://github.com/TheRook/subbrute.git", 119 | "output": "/root/brute", 120 | "output_folder": "subbrute" 121 | }, 122 | "Subfinder": { 123 | "name": "Subfinder", 124 | "mode" : "DOCKER", 125 | "docker_name" : "sfinder", 126 | "network_mode": "host", 127 | "default_conf": "docker_buildfiles/Dockerfile.Subfinder.tmp", 128 | "active_conf": "docker_active/Dockerfile.Subfinder", 129 | "description": "SubFinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources", 130 | "src": "https://github.com/subfinder/subfinder", 131 | "output": "/root/subfinder", 132 | "output_folder": "subfinder" 133 | }, 134 | "Knock": { 135 | "name": "Knock", 136 | "mode" : "DOCKER", 137 | "network_mode": "host", 138 | "docker_name" : "knock", 139 | "default_conf": "docker_buildfiles/Dockerfile.Knock.tmp", 140 | "active_conf": "docker_active/Dockerfile.Knock", 141 | "description": "Knockpy is a python tool designed to enumerate subdomains on a target domain through a wordlist", 142 | "src": "https://github.com/guelfoweb/knock", 143 | "output": "/root/knock", 144 | "output_folder": "knock", 145 | "vt_key": "" 146 | }, 147 | "Amass": { 148 | "name": "Amass", 149 | "mode" : "DOCKER", 150 | "network_mode": "host", 151 | "docker_name" : "amass", 152 | "default_conf": "docker_buildfiles/Dockerfile.Amass.tmp", 153 | "active_conf": "docker_active/Dockerfile.Amass", 154 | "description": "The OWASP Amass tool suite obtains subdomain names by scraping data sources, recursive brute forcing, crawling web archives, permuting/altering names and reverse DNS sweeping.", 155 | "src": "https://github.com/OWASP/Amass", 156 | "resolvers": "", 157 | "output": "/root/amass", 158 | "output_folder": "amass" 159 | }, 160 | "Reconng": { 161 | "name": "Reconng", 162 | "mode" : "DOCKER", 163 | "network_mode": "host", 164 | "docker_name" : "recon", 165 | "default_conf": "docker_buildfiles/Dockerfile.Reconng.tmp", 166 | "active_conf": "docker_active/Dockerfile.Reconng", 167 | "description": "Recon-ng is a full-featured Web Reconnaissance framework written in Python. DrRobot utilizes several of the recon/hosts-domain modules in this framework.", 168 | "src": "https://bitbucket.org/LaNMaSteR53/recon-ng", 169 | "output": "/tmp/output", 170 | "output_folder": "reconng" 171 | }, 172 | "Altdns": { 173 | "name": "Altdns", 174 | "mode" : "DOCKER", 175 | "network_mode": "host", 176 | "docker_name" : "altdns", 177 | "default_conf": "docker_buildfiles/Dockerfile.Altdns.tmp", 178 | "active_conf": "docker_active/Dockerfile.Altdns", 179 | "description": "Generates permutations, alterations and mutations of subdomains and then resolves them", 180 | "src": "https://github.com/infosec-au/altdns", 181 | "infile" : "/root/altdns/aggregated/aggregated_hostnames.txt", 182 | "output": "/root/altdns", 183 | "output_folder": "altdns" 184 | }, 185 | "Anubis": { 186 | "name": "Anubis", 187 | "mode" : "DOCKER", 188 | "network_mode": "host", 189 | "docker_name" : "anubis", 190 | "default_conf": "docker_buildfiles/Dockerfile.Anubis.tmp", 191 | "active_conf": "docker_active/Dockerfile.Anubis", 192 | "description": "Anubis is a subdomain enumeration and information gathering tool.", 193 | "src": "https://github.com/jonluca/Anubis", 194 | "output": "/root/anubis", 195 | "output_folder": "anubis" 196 | } , 197 | "CTExposer": { 198 | "name": "CTExpose", 199 | "mode" : "DOCKER", 200 | "network_mode": "host", 201 | "docker_name" : "ctexpo", 202 | "default_conf": "docker_buildfiles/Dockerfile.CT.tmp", 203 | "active_conf": "docker_active/Dockerfile.CT", 204 | "description": "An OSINT tool that discovers sub-domains by searching Certificate Transparency ", 205 | "src": "https://github.com/chris408/ct-exposer", 206 | "output": "/root/ct", 207 | "output_folder": "ctexpose" 208 | }, 209 | "CTFR": { 210 | "name": "CTFR", 211 | "mode" : "DOCKER", 212 | "network_mode": "host", 213 | "docker_name" : "ctfr", 214 | "default_conf": "docker_buildfiles/Dockerfile.CTFR.tmp", 215 | "active_conf": "docker_active/Dockerfile.CTFR", 216 | "description": "Abusing Certificate Transparency logs for getting HTTPS websites subdomains.", 217 | "src": "https://github.com/UnaPibaGeek/ctfr", 218 | "output": "/root/ctfr", 219 | "output_folder": "ctfr" 220 | },"PDList": { 221 | "name": "PDList", 222 | "mode" : "DOCKER", 223 | "network_mode": "host", 224 | "docker_name" : "pdlist", 225 | "default_conf": "docker_buildfiles/Dockerfile.PDList.tmp", 226 | "active_conf": "docker_active/Dockerfile.PDList", 227 | "description": "pdlist is a passive subdomain finder written in python3", 228 | "src": "https://github.com/gnebbia/pdlist", 229 | "output": "/root/pdlistoutput", 230 | "output_folder": "pdlist" 231 | },"MassDNS": { 232 | "name": "MassDNS", 233 | "mode" : "DOCKER", 234 | "network_mode": "host", 235 | "docker_name" : "mass", 236 | "default_conf": "docker_buildfiles/Dockerfile.Massdns.tmp", 237 | "active_conf": "docker_active/Dockerfile.Massdns", 238 | "description": "MassDNS high performance DNS stub resolver", 239 | "src": "https://github.com/blechschmidt/massdns", 240 | "output": "/root/massdns", 241 | "output_folder": "massdns" 242 | } 243 | }, 244 | "Enumeration" : 245 | { 246 | "Aquatone" : { 247 | "name" : "Aquatone", 248 | "short_name" : "aqua", 249 | "mode" : "DOCKER", 250 | "network_mode": "host", 251 | "docker_name" : "aquatone", 252 | "default_conf" : "docker_buildfiles/Dockerfile.Aquatone.Flyover.tmp", 253 | "active_conf" : "docker_active/Dockerfile.Aquatone.Flyover", 254 | "description" : "Tool for domain flyovers", 255 | "output" : "/tmp/output", 256 | "infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt" 257 | }, 258 | "HTTPScreenshot": { 259 | "name" : "HTTPScreenshot", 260 | "short_name" : "http", 261 | "mode" : "DOCKER", 262 | "network_mode": "host", 263 | "docker_name" : "httpscreen", 264 | "default_conf" : "docker_buildfiles/Dockerfile.HTTPScreenshot.tmp", 265 | "active_conf" : "docker_active/Dockerfile.HTTPScreenshot", 266 | "ansible_arguments" : { 267 | "config" : "$config/httpscreenshot_play.yml", 268 | "flags": "-e '$extra' -i ~/.drrobot/ansible_inventory", 269 | "extra_flags":{ 270 | "1" : "variable_host=localhost", 271 | "2" : "variable_user=user", 272 | "3" : "infile=$infile", 273 | "4" : "outfile=$outdir/httpscreenshots.tar", 274 | "5" : "outfolder=$outdir/httpscreenshots" 275 | }, 276 | "run_as_thread": false, 277 | "infile" : "aggregated/aggregated_protocol_hostnames.txt" 278 | }, 279 | "description" : "Post enumeration tool for screen grabbing websites. All images will be downloaded to outfile: httpscreenshot.tar and unpacked httpscreenshots", 280 | "output" : "/tmp/output", 281 | "infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt" 282 | }, 283 | "Eyewitness": { 284 | "name" : "Eyewitness", 285 | "short_name" : "eye", 286 | "docker_name" : "eye", 287 | "mode" : "DOCKER", 288 | "network_mode": "host", 289 | "default_conf" : "docker_buildfiles/Dockerfile.Eyewitness.tmp", 290 | "active_conf" : "docker_active/Dockerfile.Eyewitness", 291 | "ansible_arguments" : { 292 | "config" : "$config/eyewitness_play.yml", 293 | "flags": "-e '$extra' -i ~/.drrobot/ansible_inventory", 294 | "extra_flags":{ 295 | "1" : "variable_host=localhost", 296 | "2" : "variable_user=user", 297 | "3" : "infile=$infile", 298 | "4" : "outfile=$outdir/Eyewitness.tar", 299 | "5" : "outfolder=$outdir/Eyewitness" 300 | }, 301 | "run_as_thread": false, 302 | "infile" : "aggregated/aggregated_protocol_hostnames.txt" 303 | }, 304 | "description" : "Post enumeration tool for screen grabbing websites. All images will be downloaded to outfile: Eyewitness.tar and unpacked in Eyewitness", 305 | "output" : "/tmp/output", 306 | "infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt" 307 | }, 308 | "Nmap" : { 309 | "name" : "Nmap Screenshot", 310 | "short_name" : "nmapscreen", 311 | "docker_name" : "nmapscreen", 312 | "mode" : "DOCKER", 313 | "network_mode": "host", 314 | "default_conf" : "docker_buildfiles/Dockerfile.Nmap.Screenshot.tmp", 315 | "active_conf" : "docker_active/Dockerfile.Nmap.Screenshot", 316 | "description" : "Post enumeration tool for screen grabbing websites. (Chrome is not installed in the dockerfile due. Options are chromium-browser/firefox/wkhtmltoimage)", 317 | "output" : "/tmp/output", 318 | "infile" : "/tmp/output/aggregated/aggregated_hostnames.txt", 319 | "tool" : "chromium-browser" 320 | }, 321 | "Gowitness" : { 322 | "name" : "Gowitness", 323 | "short_name" : "gowitness", 324 | "docker_name" : "gowitness", 325 | "mode" : "DOCKER", 326 | "network_mode": "host", 327 | "default_conf" : "docker_buildfiles/Dockerfile.Gowitness.Screenshot.tmp", 328 | "active_conf" : "docker_active/Dockerfile.Gowitness.Screenshot", 329 | "description" : "gowitness is a website screenshot utility written in Golang", 330 | "output" : "/tmp/output", 331 | "infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt" 332 | }, 333 | "Webscreenshot" : { 334 | "name" : "WebScreenshot", 335 | "short_name" : "webscreen", 336 | "docker_name" : "webscreen", 337 | "mode" : "DOCKER", 338 | "network_mode": "host", 339 | "default_conf" : "docker_buildfiles/Dockerfile.Webscreenshot.Screenshot.tmp", 340 | "active_conf" : "docker_active/Dockerfile.Webscreenshot.Screenshot", 341 | "ansible_arguments" : { 342 | "config" : "$config/webscreenshot_play.yml", 343 | "flags": "-e '$extra' -i ~/.drrobot/ansible_inventory", 344 | "extra_flags":{ 345 | "1" : "variable_host=localhost", 346 | "2" : "variable_user=user", 347 | "3" : "infile=$infile", 348 | "4" : "outfile=$outdir/Webscreenshot.tar", 349 | "5" : "outfolder=$outdir/Webscreenshot" 350 | }, 351 | "run_as_thread": false, 352 | "infile" : "aggregated/aggregated_protocol_hostnames.txt" 353 | }, 354 | "description" : "Post enumeration tool for screen grabbing websites. (Chrome is not installed in the dockerfile due. Options are chromium-browser/firefox/wkhtmltoimage)", 355 | "output" : "/tmp/output", 356 | "infile" : "/tmp/output/aggregated/aggregated_protocol_hostnames.txt", 357 | "tool" : "phantomjs" 358 | } 359 | } 360 | } 361 | -------------------------------------------------------------------------------- /src/robot_api/api/aggregation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Aggregation Module 3 | 4 | This module handles all database interaction when aggregating data source. 5 | 6 | Attributes: 7 | dbfile (str): location of database file to load 8 | domain (str): target domain 9 | output_dir (str): Where to dump files 10 | logger (Logger): Module based logger 11 | 12 | """ 13 | import json 14 | import socket 15 | import sqlite3 16 | from os import path, getcwd, walk 17 | import glob 18 | import re 19 | import logging 20 | import multiprocessing 21 | from functools import partial 22 | from tqdm import tqdm 23 | import requests 24 | 25 | from robot_api.parse import join_abs 26 | 27 | def reverse_ip_lookup(domain, queue, filename): 28 | """Read in filesnames and use regex to extract all ips and hostnames. 29 | 30 | Args: 31 | filename: string to filename to parse 32 | 33 | Returns: 34 | A list of tuples containing the extracted host and ip 35 | """ 36 | ip_reg = re.compile( 37 | r"(?:(?:1\d\d|2[0-5][0-5]|2[0-4]\d|0?[1-9]\d|0?0?\d)\.){3}(?:1\d\d|2[0-5][0-5]|2[0-4]\d|0?[1-9]\d|0?0?\d)") 38 | hostname_reg = re.compile( 39 | r"([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*?\." 40 | + domain 41 | + r"(\:?[0-9]{1,5})?") 42 | results = [] 43 | try: 44 | with open(filename, "r", encoding='utf-8') as _file: 45 | for line in tqdm(_file.readlines(), desc=f"{filename} parsing..."): 46 | _host = hostname_reg.search(line) 47 | if _host is not None: 48 | _host = _host.group(0) 49 | _ip = ip_reg.search(line) 50 | if _ip is not None: 51 | _ip = _ip.group(0) 52 | try: 53 | if _host is not None and _ip is None: 54 | _ip = socket.gethostbyname(_host) 55 | if _ip is not None and _host is None: 56 | _host = socket.gethostbyaddr(_ip) 57 | except Exception: 58 | pass 59 | if _host or _ip: 60 | queue.put((_host, _ip)) 61 | except Exception: 62 | pass 63 | 64 | return results 65 | 66 | def get_headers(queue, target): 67 | """Static method for request to scrape header information from ip 68 | 69 | Args: 70 | target: string to make request to 71 | 72 | Returns: 73 | ip/hostname and tuple containing headers 74 | """ 75 | http = None 76 | https = None 77 | # May add option later to set UserAgent 78 | headers = { 79 | "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0" 80 | } 81 | try: 82 | http = requests.get( 83 | f"http://{target}", 84 | headers=headers, 85 | timeout=1, 86 | verify=False).headers 87 | http = str(http) 88 | except requests.ConnectionError: 89 | pass 90 | except OSError: 91 | pass 92 | try: 93 | https = requests.get( 94 | f"https://{target}", 95 | headers=headers, 96 | timeout=1, 97 | verify=False).headers 98 | https = str(https) 99 | except requests.ConnectionError: 100 | pass 101 | except OSError: 102 | pass 103 | queue.put([target, (http, https)]) 104 | 105 | class Aggregation: 106 | """Aggregation module 107 | """ 108 | def __init__(self, db_filename, domain, output_dir): 109 | """Initialize aggregation object 110 | 111 | Args: 112 | dbfile (str): location of database file to load 113 | domain (str): target domain 114 | output_dir (str): Where to dump files 115 | logger (Logger): Module based logger 116 | 117 | 118 | Returns: 119 | 120 | """ 121 | self.dbfile = db_filename 122 | self.domain = domain 123 | self.output_dir = output_dir 124 | self.logger = logging.getLogger(__name__) 125 | 126 | def dump_to_file( 127 | self, 128 | dump_ips=True, 129 | dump_hostnames=True, 130 | dump_headers=False): 131 | """Dump database to file 132 | 133 | Dumps contents of database to three seperate files: 134 | 1. File with protocol (http/https) 135 | 2. File with hostname only 136 | 3. File with ip only 137 | 138 | Args: 139 | dump_ips (bool): dump ips 140 | dump_hostnames (bool): dump hostnames to file 141 | dump_headers (bool): dump headers to output firectory under headers 142 | 143 | 144 | Returns: 145 | """ 146 | dbconn = sqlite3.connect(self.dbfile) 147 | try: 148 | dbcurs = dbconn.cursor() 149 | 150 | ips = dbcurs.execute( 151 | f"""SELECT DISTINCT ip 152 | FROM data 153 | WHERE domain='{self.domain.replace('.', '_')}' 154 | AND ip IS NOT NULL""").fetchall() 155 | hostnames = dbcurs.execute( 156 | f"""SELECT DISTINCT hostname 157 | FROM data 158 | WHERE domain='{self.domain.replace('.', '_')}' 159 | AND hostname IS NOT NULL""").fetchall() 160 | 161 | """ 162 | Header options require there to have been a scan otherwise 163 | there will be no output but that should be expected. 164 | Might change db to a dataframe later... possible 165 | """ 166 | headers = dbcurs.execute( 167 | f"""SELECT DISTINCT ip, hostname, http_headers, https_headers 168 | FROM data 169 | WHERE domain='{self.domain.replace('.', '_')}' 170 | AND (http_headers IS NOT NULL 171 | AND https_headers IS NOT NULL)""").fetchall() 172 | 173 | if dump_ips: 174 | with open(join_abs(self.output_dir, 175 | 'aggregated', 176 | 'aggregated_ips.txt'), 177 | 'w') as _file: 178 | _file.writelines("\n".join(list(ip[0] for ip in ips))) 179 | 180 | if dump_hostnames: 181 | with open(join_abs(self.output_dir, 182 | 'aggregated', 183 | 'aggregated_hostnames.txt'), 184 | 'w') as _file: 185 | _file.writelines( 186 | "\n".join( 187 | list( 188 | f"{host[0]}" for host in hostnames))) 189 | 190 | with open(join_abs(self.output_dir, 191 | 'aggregated', 192 | 'aggregated_protocol_hostnames.txt'), 193 | 'w') as _file: 194 | _file.writelines( 195 | "\n".join( 196 | list(f"https://{host[0]}\nhttp://{host[0]}" 197 | for host in hostnames))) 198 | 199 | if dump_headers: 200 | keys = ["Ip", "Hostname", "Http", "Https"] 201 | for row in headers: 202 | _rows = dict(zip(keys, row)) 203 | with open(join_abs(self.output_dir, 204 | "headers", 205 | f"{_rows['Hostname']}_headers.txt"), 206 | 'w') as _file: 207 | _file.write(json.dumps(_rows, indent=2)) 208 | except sqlite3.Error: 209 | print("Failed to write to files in aggregated directory, exiting") 210 | self.logger.exception("Error in dump to file") 211 | return 212 | except OSError: 213 | print("Failed to write to files in aggregated directory, exiting") 214 | self.logger.exception("Error in dump to file") 215 | return 216 | finally: 217 | dbconn.close() 218 | 219 | def _build_db(self, queue, cursor): 220 | """Takes in ip/hostname data and inserts them into the database 221 | 222 | Args: 223 | queue (multiprocessing.Queue): list of tupes (host, ip) 224 | cursor (sqlite3.cursor): database cursor object 225 | 226 | Returns: 227 | """ 228 | cursor.execute('BEGIN TRANSACTION') 229 | domain = self.domain.replace(".", "_") 230 | while not queue.empty(): 231 | host, ipv4 = queue.get() 232 | if host is not None and type(host) is not str: 233 | host = host[0] 234 | try: 235 | cursor.execute("""INSERT OR IGNORE INTO data 236 | (ip, hostname, http_headers, https_headers, domain) 237 | VALUES (?,?, NULL, NULL, ?);""", (ipv4, host, domain)) 238 | except sqlite3.Error: 239 | print(f"Issue with the following data: {ipv4} {host} {domain}") 240 | self.logger.exception("Error in _build_db") 241 | 242 | cursor.execute('COMMIT') 243 | 244 | def aggregate(self, output_files=[], output_folders=[]): 245 | """Aggregates all output from scanners into the database 246 | 247 | Args: 248 | output_files: list of output files referenced in config.json 249 | output_folders: list of folders to for aggregation 250 | 251 | Returns: 252 | """ 253 | try: 254 | 255 | dbconn = sqlite3.connect(self.dbfile) 256 | dbcurs = dbconn.cursor() 257 | # Enable foreign key support 258 | dbcurs.execute("PRAGMA foreign_keys=1") 259 | # Simple database that contains list of domains to run against 260 | dbcurs.execute(""" 261 | CREATE TABLE IF NOT EXISTS domains ( 262 | domain VARCHAR PRIMARY KEY, 263 | UNIQUE(domain) 264 | ) 265 | """) 266 | # Setup database to keep all data from all targets. This allows us 267 | # to use a single model for hosting with Django 268 | dbcurs.execute(""" 269 | CREATE TABLE IF NOT EXISTS data ( 270 | domainid INTEGER PRIMARY KEY, 271 | ip VARCHAR, 272 | hostname VARCHAR, 273 | http_headers TEXT, 274 | https_headers TEXT, 275 | domain VARCHAR, 276 | found TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, 277 | FOREIGN KEY(domain) REFERENCES domains(domain), 278 | UNIQUE(hostname) 279 | ) 280 | """) 281 | # Quickly create entry in domains table. 282 | dbcurs.execute(f"INSERT OR IGNORE INTO domains(domain) VALUES ('{self.domain.replace('.', '_')}')") 283 | dbconn.commit() 284 | 285 | all_files = [] 286 | for name in output_files: 287 | if path.isfile(join_abs(self.output_dir, name)): 288 | all_files += [join_abs(self.output_dir, name)] 289 | elif path.isfile(name): 290 | all_files += [name] 291 | else: 292 | print( 293 | f"[!] File {name} does not exist, verify scan results") 294 | 295 | for folder in output_folders: 296 | for root, _, files in walk( 297 | join_abs(self.output_dir, folder)): 298 | for _file in files: 299 | if path.isfile(join_abs(root, _file)): 300 | all_files += [join_abs(root, _file)] 301 | # multi_queue = multiprocessing.Queue() 302 | qu_manager = multiprocessing.Manager() 303 | pool = multiprocessing.Pool(5) 304 | queue = qu_manager.Queue() 305 | reverse_partial = partial(reverse_ip_lookup, self.domain, queue) 306 | pool.map(reverse_partial, all_files) 307 | pool.close() 308 | self._build_db(queue, dbcurs) 309 | dbconn.commit() 310 | except sqlite3.Error: 311 | self.logger.exception("Error in aggregation") 312 | finally: 313 | dbconn.close() 314 | 315 | 316 | 317 | def headers(self): 318 | """Attempts to grab header data for all ips/hostnames 319 | 320 | Returns: 321 | 322 | """ 323 | dbconn = sqlite3.connect(self.dbfile) 324 | dbcurs = dbconn.cursor() 325 | 326 | print("[*] Grabbing headers from ips and hostnames") 327 | ips = dbcurs.execute(f"""SELECT ip 328 | FROM data 329 | WHERE ip IS NOT NULL 330 | AND domain='{self.domain.replace('.','_')}'""" 331 | ).fetchall() 332 | ips = [item[0] for item in ips] 333 | # Threading is done against the staticmethod. 334 | # Feel free to change the max_workers if your system allows. 335 | # May add option to specify threaded workers. 336 | pool = multiprocessing.Pool(40) 337 | 338 | qu_manager = multiprocessing.Manager() 339 | queue = qu_manager.Queue() 340 | get_headers_partial = partial(get_headers, queue) 341 | _ = list(tqdm(pool.imap_unordered(get_headers_partial, ips), total=len(ips), desc="Getting headers for ip...")) 342 | pool.close() 343 | pool.join() 344 | 345 | print("Updating database with ip headers") 346 | dbcurs.execute('BEGIN TRANSACTION') 347 | domain_rep = self.domain.replace(".", "_") 348 | while not queue.empty(): 349 | ipv4, (http, https) = queue.get() 350 | dbcurs.execute(f"""UPDATE data 351 | SET http_headers=?, https_headers=? 352 | WHERE ip = ? 353 | AND domain= ? 354 | """, 355 | (http, https, ipv4, domain_rep)) 356 | dbcurs.execute("COMMIT") 357 | 358 | hostnames = dbcurs.execute(f"""SELECT hostname 359 | FROM data 360 | WHERE ip IS NOT NULL 361 | AND domain='{self.domain.replace('.','_')}'""" 362 | ).fetchall() 363 | hostnames = [item[0] for item in hostnames] 364 | 365 | pool = multiprocessing.Pool(40) 366 | queue = qu_manager.Queue() 367 | get_headers_partial = partial(get_headers, queue) 368 | _ = list(tqdm(pool.map(get_headers_partial, hostnames), total=len(hostnames), desc="Getting headers for host...")) 369 | 370 | pool.close() 371 | pool.join() 372 | 373 | print("Updating database with hostname headers") 374 | dbcurs.execute('BEGIN TRANSACTION') 375 | domain_rep = self.domain.replace(".", "_") 376 | while not queue.empty(): 377 | hostname, (http, https) = queue.get() 378 | dbcurs.execute(f"""UPDATE data 379 | SET http_headers=?, https_headers=? 380 | WHERE hostname = ? 381 | AND domain= ?;""", 382 | (http, https, hostname, domain_rep)) 383 | dbcurs.execute("COMMIT") 384 | 385 | dbconn.close() 386 | 387 | def gen_output(self): 388 | """Generate dictionary containing all data from the database 389 | 390 | Returns: 391 | A dictionary containing all data from database 392 | 393 | """ 394 | if not path.exists(self.dbfile): 395 | print("No database file found. Exiting") 396 | return None 397 | 398 | dbconn = sqlite3.connect(self.dbfile) 399 | dbcurs = dbconn.cursor() 400 | 401 | db_headers = dbcurs.execute(f"""SELECT * 402 | FROM data 403 | WHERE domain='{self.domain.replace('.','_')}' 404 | AND (http_headers IS NOT NULL OR https_headers IS NOT NULL)""" 405 | ).fetchall() 406 | db_ips = dbcurs.execute(f"""SELECT DISTINCT ip, hostname 407 | FROM data 408 | WHERE domain='{self.domain.replace('.', '_')}'""" 409 | ).fetchall() 410 | 411 | """ 412 | (IP, HOSTNAME, HTTP, HTTPS) 413 | """ 414 | file_index = {} 415 | """ 416 | Need to be smarter about this: 417 | 418 | Multiple different sql queries 419 | 1. Grabs all those with headers: 420 | most likely that if they have headers 421 | they have a screenshot 422 | glob can run and take it's time. 423 | 2. Grab all unique ips 424 | 2a. Grab all unique hostnames 425 | 426 | 3. Update json with all documents 427 | """ 428 | for _, ipv4, hostname, http, https, _ in db_headers: 429 | ip_screenshots = glob.glob("**/*{}*".format(ipv4), recursive=True) 430 | hostname_screeshots = glob.glob( 431 | "**/*{}*".format(hostname), recursive=True) 432 | 433 | image_files = [] 434 | for _file in ip_screenshots: 435 | image_files += [join_abs(getcwd(), _file)] 436 | for _file in hostname_screeshots: 437 | image_files += [join_abs(getcwd(), _file)] 438 | file_index[ipv4] = { 439 | "hostnames": [hostname], 440 | "http_header": http, 441 | "https_header": https, 442 | "images": image_files 443 | } 444 | 445 | for ipv4, hostname in db_ips: 446 | if ipv4 not in file_index: 447 | file_index[ipv4] = { 448 | "hostnames": [hostname], 449 | "http_header": "", 450 | "https_header": "", 451 | "images": [] 452 | } 453 | elif hostname not in file_index[ipv4]['hostnames']: 454 | file_index[ipv4]['hostnames'] += [hostname] 455 | return file_index 456 | --------------------------------------------------------------------------------