├── .gitignore ├── CODEOWNERS ├── LICENSE.txt ├── README.md ├── docs ├── Makefile ├── make.bat └── source │ ├── commands.rst │ ├── conf.py │ ├── development.rst │ ├── index.rst │ └── install.rst ├── requirements.txt ├── setup.py ├── sphinx_req.txt ├── tests ├── __init__.py ├── test_esdocs.py └── test_threatshell.py ├── threatshell.py └── threatshell ├── .DS_Store ├── __init__.py ├── commands ├── __init__.py ├── conf_manager.py ├── cymru.py ├── geoip.py ├── infoblox.py ├── novetta.py ├── opendns.py ├── passivetotal.py ├── q.py ├── riskiq.py ├── shadow_server.py ├── threat_exchange.py ├── threat_q.py └── umbrella.py ├── common ├── __init__.py ├── abstracts.py ├── colors.py ├── config.py ├── constants.py ├── log.py └── logo.py ├── conf └── defaults.ini ├── core ├── __init__.py ├── config.py └── elastic.py ├── doctypes ├── __init__.py ├── cymru.py ├── generic.py ├── geoip.py ├── opendns.py ├── passivetotal.py └── shadow_server.py └── utils ├── __init__.py ├── argparse_utils.py └── keygen.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.sh 2 | *.pyc 3 | *.pyo 4 | *_pycache_* 5 | *.DS_Store 6 | .coverage 7 | .eggs 8 | threatshell.egg-info 9 | docs/build 10 | build 11 | test_config.ini 12 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Comment line immediately above ownership line is reserved for related gus information. Please be careful while editing. 2 | #ECCN:Open Source 3 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Salesforce.com, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | * Neither the name of Salesforce.com nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 22 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Threatshell 2 | =========== 3 | 4 | Threatshell is a python-based command line shell aimed at providing security 5 | researchers with a single, integrated environment for gathering information 6 | from various intelligence APIs and analysis scripts, and storing all of the 7 | obtained information into one or more elasticsearch instances. The goal of 8 | keeping the results in elasticsearch being to provide a historical search 9 | mechanism for all of the gathered information, and to start building a clever 10 | event analyzer to assist in hunting and analysis activities. 11 | 12 | 13 | Notes 14 | ======= 15 | 16 | Please see the docs for installation help and for threatshell usage details. 17 | 18 | I recommend using python virtual environments (virtualenvs) if you don't 19 | already. If you'd like to use a virtualenv, I detail (roughly) how to set one 20 | up in [**the docs**](http://threatshell.readthedocs.io/en/latest/install.html), 21 | and there are plenty of awesome tutorials for setting up and using virtualenvs 22 | out there already if you still have questions. 23 | 24 | 25 | Documentation 26 | ================= 27 | 28 | Threatshell's documentation can be found 29 | [**here**](http://threatshell.readthedocs.io/en/latest/index.html) 30 | 31 | 32 | Quick Start 33 | ============== 34 | 35 | You can get up and running with threatshell with the following few steps - 36 | 37 | First, you'll need the GeoIP library for geocoding IP addresses. 38 | 39 | On ubuntu: 40 | 41 | sudo apt-get install libgeoip-dev 42 | 43 | On OSX: 44 | 45 | brew install GeoIP 46 | 47 | Then activate your virtualenv if you're using one for the next commands 48 | 49 | # make sure pip is up to date if you want 50 | pip install --upgrade pip 51 | 52 | # you can install everything with 53 | # pip install -r requirements.txt 54 | # or 55 | # python setup.py install 56 | 57 | pip install -r requirements.txt 58 | 59 | Now you can start up threatshell with 60 | 61 | python threatshell.py 62 | 63 | If it's your first time running threatshell, it will create a config 64 | directory, `$HOME/.threatshell`, and prompt you for a password for the 65 | crypto key it generates to keep all of your config's secrets safe. Then, 66 | once your key is generated, it asks for your API keys and other settings. 67 | You can just enter through the prompts and set the keys later with the 68 | [**config management commands**](http://threatshell.readthedocs.io/en/latest/commands.html#config-management-commands) 69 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " applehelp to make an Apple Help Book" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | @echo " coverage to run coverage check of the documentation (if enabled)" 49 | 50 | .PHONY: clean 51 | clean: 52 | rm -rf $(BUILDDIR) 53 | 54 | .PHONY: html 55 | html: 56 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 57 | @echo 58 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 59 | 60 | .PHONY: dirhtml 61 | dirhtml: 62 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 63 | @echo 64 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 65 | 66 | .PHONY: singlehtml 67 | singlehtml: 68 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 69 | @echo 70 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 71 | 72 | .PHONY: pickle 73 | pickle: 74 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 75 | @echo 76 | @echo "Build finished; now you can process the pickle files." 77 | 78 | .PHONY: json 79 | json: 80 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 81 | @echo 82 | @echo "Build finished; now you can process the JSON files." 83 | 84 | .PHONY: htmlhelp 85 | htmlhelp: 86 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 87 | @echo 88 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 89 | ".hhp project file in $(BUILDDIR)/htmlhelp." 90 | 91 | .PHONY: qthelp 92 | qthelp: 93 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 94 | @echo 95 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 96 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 97 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ThreatShell.qhcp" 98 | @echo "To view the help file:" 99 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ThreatShell.qhc" 100 | 101 | .PHONY: applehelp 102 | applehelp: 103 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 104 | @echo 105 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 106 | @echo "N.B. You won't be able to view it unless you put it in" \ 107 | "~/Library/Documentation/Help or install it in your application" \ 108 | "bundle." 109 | 110 | .PHONY: devhelp 111 | devhelp: 112 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 113 | @echo 114 | @echo "Build finished." 115 | @echo "To view the help file:" 116 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ThreatShell" 117 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ThreatShell" 118 | @echo "# devhelp" 119 | 120 | .PHONY: epub 121 | epub: 122 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 123 | @echo 124 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 125 | 126 | .PHONY: latex 127 | latex: 128 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 129 | @echo 130 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 131 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 132 | "(use \`make latexpdf' here to do that automatically)." 133 | 134 | .PHONY: latexpdf 135 | latexpdf: 136 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 137 | @echo "Running LaTeX files through pdflatex..." 138 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 139 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 140 | 141 | .PHONY: latexpdfja 142 | latexpdfja: 143 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 144 | @echo "Running LaTeX files through platex and dvipdfmx..." 145 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 146 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 147 | 148 | .PHONY: text 149 | text: 150 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 151 | @echo 152 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 153 | 154 | .PHONY: man 155 | man: 156 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 157 | @echo 158 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 159 | 160 | .PHONY: texinfo 161 | texinfo: 162 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 163 | @echo 164 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 165 | @echo "Run \`make' in that directory to run these through makeinfo" \ 166 | "(use \`make info' here to do that automatically)." 167 | 168 | .PHONY: info 169 | info: 170 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 171 | @echo "Running Texinfo files through makeinfo..." 172 | make -C $(BUILDDIR)/texinfo info 173 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 174 | 175 | .PHONY: gettext 176 | gettext: 177 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 178 | @echo 179 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 180 | 181 | .PHONY: changes 182 | changes: 183 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 184 | @echo 185 | @echo "The overview file is in $(BUILDDIR)/changes." 186 | 187 | .PHONY: linkcheck 188 | linkcheck: 189 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 190 | @echo 191 | @echo "Link check complete; look for any errors in the above output " \ 192 | "or in $(BUILDDIR)/linkcheck/output.txt." 193 | 194 | .PHONY: doctest 195 | doctest: 196 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 197 | @echo "Testing of doctests in the sources finished, look at the " \ 198 | "results in $(BUILDDIR)/doctest/output.txt." 199 | 200 | .PHONY: coverage 201 | coverage: 202 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 203 | @echo "Testing of coverage in the sources finished, look at the " \ 204 | "results in $(BUILDDIR)/coverage/python.txt." 205 | 206 | .PHONY: xml 207 | xml: 208 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 209 | @echo 210 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 211 | 212 | .PHONY: pseudoxml 213 | pseudoxml: 214 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 215 | @echo 216 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 217 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 1>NUL 2>NUL 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\ThreatShell.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\ThreatShell.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # ThreatShell documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Mar 14 13:16:23 2016. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | import sys 15 | import os 16 | 17 | # If extensions (or modules to document with autodoc) are in another directory, 18 | # add these directories to sys.path here. If the directory is relative to the 19 | # documentation root, use os.path.abspath to make it absolute, like shown here. 20 | #sys.path.insert(0, os.path.abspath('.')) 21 | 22 | # -- General configuration ------------------------------------------------ 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = [ 31 | 'sphinx.ext.autodoc', 32 | 'sphinx.ext.todo', 33 | 'sphinx.ext.coverage', 34 | 'sphinx.ext.viewcode', 35 | ] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['ntemplates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The encoding of source files. 46 | #source_encoding = 'utf-8-sig' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = u'ThreatShell' 53 | copyright = u'2016, Tommy Stallings' 54 | author = u'Tommy Stallings' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = u'0.0.1' 62 | # The full version, including alpha/beta/rc tags. 63 | release = u'0.0.1' 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | # 68 | # This is also used if you do content translation via gettext catalogs. 69 | # Usually you set "language" from the command line for these cases. 70 | language = None 71 | 72 | # There are two options for replacing |today|: either, you set today to some 73 | # non-false value, then it is used: 74 | #today = '' 75 | # Else, today_fmt is used as the format for a strftime call. 76 | #today_fmt = '%B %d, %Y' 77 | 78 | # List of patterns, relative to source directory, that match files and 79 | # directories to ignore when looking for source files. 80 | exclude_patterns = [] 81 | 82 | # The reST default role (used for this markup: `text`) to use for all 83 | # documents. 84 | #default_role = None 85 | 86 | # If true, '()' will be appended to :func: etc. cross-reference text. 87 | #add_function_parentheses = True 88 | 89 | # If true, the current module name will be prepended to all description 90 | # unit titles (such as .. function::). 91 | #add_module_names = True 92 | 93 | # If true, sectionauthor and moduleauthor directives will be shown in the 94 | # output. They are ignored by default. 95 | #show_authors = False 96 | 97 | # The name of the Pygments (syntax highlighting) style to use. 98 | pygments_style = 'sphinx' 99 | 100 | # A list of ignored prefixes for module index sorting. 101 | #modindex_common_prefix = [] 102 | 103 | # If true, keep warnings as "system message" paragraphs in the built documents. 104 | #keep_warnings = False 105 | 106 | # If true, `todo` and `todoList` produce output, else they produce nothing. 107 | todo_include_todos = True 108 | 109 | 110 | # -- Options for HTML output ---------------------------------------------- 111 | 112 | # The theme to use for HTML and HTML Help pages. See the documentation for 113 | # a list of builtin themes. 114 | html_theme = 'alabaster' 115 | 116 | # Theme options are theme-specific and customize the look and feel of a theme 117 | # further. For a list of options available for each theme, see the 118 | # documentation. 119 | #html_theme_options = {} 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | #html_theme_path = [] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as html_title. 129 | #html_short_title = None 130 | 131 | # The name of an image file (relative to this directory) to place at the top 132 | # of the sidebar. 133 | #html_logo = None 134 | 135 | # The name of an image file (relative to this directory) to use as a favicon of 136 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 137 | # pixels large. 138 | #html_favicon = None 139 | 140 | # Add any paths that contain custom static files (such as style sheets) here, 141 | # relative to this directory. They are copied after the builtin static files, 142 | # so a file named "default.css" will overwrite the builtin "default.css". 143 | html_static_path = ['nstatic'] 144 | 145 | # Add any extra paths that contain custom files (such as robots.txt or 146 | # .htaccess) here, relative to this directory. These files are copied 147 | # directly to the root of the documentation. 148 | #html_extra_path = [] 149 | 150 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 151 | # using the given strftime format. 152 | #html_last_updated_fmt = '%b %d, %Y' 153 | 154 | # If true, SmartyPants will be used to convert quotes and dashes to 155 | # typographically correct entities. 156 | #html_use_smartypants = True 157 | 158 | # Custom sidebar templates, maps document names to template names. 159 | #html_sidebars = {} 160 | 161 | # Additional templates that should be rendered to pages, maps page names to 162 | # template names. 163 | #html_additional_pages = {} 164 | 165 | # If false, no module index is generated. 166 | #html_domain_indices = True 167 | 168 | # If false, no index is generated. 169 | #html_use_index = True 170 | 171 | # If true, the index is split into individual pages for each letter. 172 | #html_split_index = False 173 | 174 | # If true, links to the reST sources are added to the pages. 175 | #html_show_sourcelink = True 176 | 177 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 178 | #html_show_sphinx = True 179 | 180 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 181 | #html_show_copyright = True 182 | 183 | # If true, an OpenSearch description file will be output, and all pages will 184 | # contain a tag referring to it. The value of this option must be the 185 | # base URL from which the finished HTML is served. 186 | #html_use_opensearch = '' 187 | 188 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 189 | #html_file_suffix = None 190 | 191 | # Language to be used for generating the HTML full-text search index. 192 | # Sphinx supports the following languages: 193 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 194 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 195 | #html_search_language = 'en' 196 | 197 | # A dictionary with options for the search language support, empty by default. 198 | # Now only 'ja' uses this config value 199 | #html_search_options = {'type': 'default'} 200 | 201 | # The name of a javascript file (relative to the configuration directory) that 202 | # implements a search results scorer. If empty, the default will be used. 203 | #html_search_scorer = 'scorer.js' 204 | 205 | # Output file base name for HTML help builder. 206 | htmlhelp_basename = 'ThreatShelldoc' 207 | 208 | # -- Options for LaTeX output --------------------------------------------- 209 | 210 | latex_elements = { 211 | # The paper size ('letterpaper' or 'a4paper'). 212 | #'papersize': 'letterpaper', 213 | 214 | # The font size ('10pt', '11pt' or '12pt'). 215 | #'pointsize': '10pt', 216 | 217 | # Additional stuff for the LaTeX preamble. 218 | #'preamble': '', 219 | 220 | # Latex figure (float) alignment 221 | #'figure_align': 'htbp', 222 | } 223 | 224 | # Grouping the document tree into LaTeX files. List of tuples 225 | # (source start file, target name, title, 226 | # author, documentclass [howto, manual, or own class]). 227 | latex_documents = [ 228 | (master_doc, 'ThreatShell.tex', u'ThreatShell Documentation', 229 | u'Tommy Stallings', 'manual'), 230 | ] 231 | 232 | # The name of an image file (relative to this directory) to place at the top of 233 | # the title page. 234 | #latex_logo = None 235 | 236 | # For "manual" documents, if this is true, then toplevel headings are parts, 237 | # not chapters. 238 | #latex_use_parts = False 239 | 240 | # If true, show page references after internal links. 241 | #latex_show_pagerefs = False 242 | 243 | # If true, show URL addresses after external links. 244 | #latex_show_urls = False 245 | 246 | # Documents to append as an appendix to all manuals. 247 | #latex_appendices = [] 248 | 249 | # If false, no module index is generated. 250 | #latex_domain_indices = True 251 | 252 | 253 | # -- Options for manual page output --------------------------------------- 254 | 255 | # One entry per manual page. List of tuples 256 | # (source start file, name, description, authors, manual section). 257 | man_pages = [ 258 | (master_doc, 'threatshell', u'ThreatShell Documentation', 259 | [author], 1) 260 | ] 261 | 262 | # If true, show URL addresses after external links. 263 | #man_show_urls = False 264 | 265 | 266 | # -- Options for Texinfo output ------------------------------------------- 267 | 268 | # Grouping the document tree into Texinfo files. List of tuples 269 | # (source start file, target name, title, author, 270 | # dir menu entry, description, category) 271 | texinfo_documents = [ 272 | (master_doc, 'ThreatShell', u'ThreatShell Documentation', 273 | author, 'ThreatShell', 'One line description of project.', 274 | 'Miscellaneous'), 275 | ] 276 | 277 | # Documents to append as an appendix to all manuals. 278 | #texinfo_appendices = [] 279 | 280 | # If false, no module index is generated. 281 | #texinfo_domain_indices = True 282 | 283 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 284 | #texinfo_show_urls = 'footnote' 285 | 286 | # If true, do not generate a @detailmenu in the "Top" node's menu. 287 | #texinfo_no_detailmenu = False 288 | -------------------------------------------------------------------------------- /docs/source/development.rst: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | Foreword 5 | -------- 6 | 7 | Being able to easily develop against Threatshell is my ultimate goal. Feel 8 | free to open issues or email me if something isn't clear or isn't working, or 9 | if python isn't really your thing but you'd like to see something cool added to 10 | Threatshell. 11 | 12 | Threatshell is built on top of the python cmd module's `Cmd class`_. It will be 13 | just as useful of a source as anything I'd write to explain how things integrate into the 14 | main command loop, like adding your `def do_` which you'll see 15 | later. 16 | 17 | For now, all you need to do to integrate your favorite analysis scripts into 18 | Threatshell is follow a few easy steps (which will change in the near future). 19 | Before jumping into the steps, there are a couple of prerequisites: 20 | 21 | Prerequisites 22 | ------------- 23 | 24 | * make your script a module containing a class that has all the functions you want to put in Threatshell 25 | 26 | * decide if you want to store your data in elasticsearch or not 27 | - This will be expanded later to include additional output sinks 28 | 29 | 30 | Integration 31 | ----------- 32 | 33 | Threatshell commands are currently split up into two parts -- the functions that 34 | go in the command module, and the functions that use the functions that go in 35 | the command module. Sounds confusing, I know, but let's look at an example. 36 | Let's pretend we have a script named `example.py` and we want to make it a 37 | Threatshell module with a corresponding elasitcsearch document definition, 38 | and for extra awesome sauce, we want our example function to run with the 39 | magic "q" function. 40 | 41 | `threatshell.commands.example` 42 | 43 | .. code-block:: python 44 | :linenos: 45 | 46 | # here is the example.py module that goes in threatshell.commands 47 | from threatshell.commands.q import AutoQuery # for q function 48 | from threatshell.doctypes import example as example_docs # we'll get to this 49 | 50 | import logging 51 | import requests 52 | 53 | log = logging.getLogger(__name__) 54 | 55 | 56 | class Example: 57 | 58 | # If you have sensitive information like API keys, user/pass, etc. 59 | # they will go into the Threatshell config which can then be passed 60 | # to your module's init function. No need to worry about the security 61 | # of your keys - that's already been taken care of 62 | def __init__(self, config): 63 | self.url = "http://some.api.com" 64 | self.user = config.get("Example", "user") 65 | self.passwd = config.get("Example", "pass") 66 | 67 | 68 | # This will set our example.request_intel() function to be used on 69 | # q --domain dom1 dom2 domN 70 | # 71 | # If the function works for multiple types of indicators, simply add 72 | # them to the array passed to the use_on decorator 73 | # (e.g. ["domain", "ip", "url"]) 74 | # 75 | # Reference the threatshell.commands.q module for the list of supported 76 | # auto-query types, or add your own type to it 77 | # 78 | @AutoQuery.use_on(["domain"]) 79 | def request_intel(self, domains): 80 | 81 | # This part is important - if you want this funtion to use q 82 | # correctly, it will need to be able to iterate over a list of 83 | # queries or it will need to be able to process the list all at 84 | # once (like with a bulk API). Plus it's nice to be able to do 85 | # multiple lookups in one command 86 | if not isinstance(domains, list): 87 | domains = [domains] 88 | 89 | docs = [] 90 | for domain in domains: 91 | url = "%s/%s" % (self.url, domain) 92 | resp = requests.get(url, auth=(self.user, self.passwd)) 93 | 94 | # assuming things went well with the request 95 | 96 | doc = example_docs.ExampleIntelDoc(resp.json()) 97 | docs.append(doc) 98 | 99 | return docs 100 | 101 | 102 | For the elasticsearch document definition, all you have to do is create a simple 103 | class containing a mapping of what each key--value pair is and how it should be 104 | handled by elasticsearch. So, let's say our magic intel API we use in 105 | `threatshell.commands.example` returns a json document that looks like this 106 | :: 107 | 108 | { 109 | "domain": "somedomain.com", 110 | "malicious": false, 111 | "contact": "someone@somedomain.com", 112 | "timestamp": "2016-04-20 12:00:00" 113 | } 114 | 115 | A simple, short, sweet json document. To turn that document into a Threatshell 116 | elasticsearch document you would create a class that looks like the following 117 | 118 | `threatshell.doctypes.example` 119 | 120 | .. code-block:: python 121 | :linenos: 122 | 123 | # 124 | # GenericDoc is a base class I created to help with managing serialization, 125 | # Additional Threatshell fields, and other stuff coming in the future. 126 | # All Threatshell elasticsearch doctypes should extend this class. 127 | # 128 | # There are also some analyzers, filters, etc. in this module which you 129 | # may find helpful but the other really key thing to include is the 130 | # ThreatshellIndex decorator. You'll see why shortly. 131 | # 132 | from threatshell.doctypes.generic import ( 133 | GenericDoc, 134 | email_analyzer, 135 | ThreatshellIndex 136 | ) 137 | from elasticsearch_dsl import ( 138 | String, 139 | Boolean 140 | ) 141 | 142 | # 143 | # Here's that magical decorator. It's important because of how the 144 | # elasticsearch_dsl module works. When Threatshell starts, all of the 145 | # document decorators are collected and their respective mappings and 146 | # settings are sent to whatever elasticsearch servers are in your config 147 | # 148 | # In short, no decorator == no ES mapping. This doesn't guarantee a failure, 149 | # oh no, it's much more sneaky than that. You'll get partial docs generated 150 | # by the dynamic mapping and fields could end up being improperly configured 151 | # or behave unexpectedly when searching and whatnot. 152 | # 153 | @ThreatshellIndex.doc_type 154 | class ExampleIntelDoc(GenericDoc): 155 | 156 | # 157 | # This is to set the name of this document type in 158 | # elasticsearch 159 | # 160 | # (e.g. http://localhost:9200/threatshell/example_intel_doc/{doc_id}) 161 | # 162 | class Meta: 163 | doc_type = "example_intel_doc" 164 | 165 | # Notice that these are defined just like in the json. That's because 166 | # of how the elasticsearch_dsl DocType class works with how it manages 167 | # attributes. 168 | 169 | domain = String() 170 | malicious = Boolean() 171 | contact = String(analyzer=email_analyzer) 172 | timestamp = Date() 173 | 174 | def __init__(self, jdata={}): 175 | GenericDoc.__init__(self) 176 | for k, v in jdata.items(): 177 | 178 | # elasticsearch_dsl non-sense 179 | if v is None: 180 | v = {} 181 | 182 | setattr(self, k, v) 183 | 184 | # Notice how we directly bind the json doc to this 185 | # class instance with setattr. You can achieve the 186 | # same effect with self. = or with 187 | # ExampleIntelDoc(). = 188 | 189 | 190 | That's about all you need to get started with elasticsearch_dsl documents. It 191 | can get a bit challenging to define some of these docs. The better you define 192 | them though, the better ES can do at indexing and searching, which means you can get 193 | better correlation tips and stuff when the web interface side of Threatshell 194 | gets built out more. So do try to do a good job, and, as always, feel free to 195 | ask for help. You can also look at other doctypes I have made to see how more 196 | complicated things were achieved. The `elasticsearch_dsl ReadTheDocs`_ is also a 197 | good place for learning tricks to defining documents 198 | 199 | 200 | That is almost all you need to know to get a Threatshell module working. All 201 | that needs to be done now is to open up threatshell.py and add a few lines of 202 | code use the command module. To do so, follow these easy steps: 203 | 204 | * open `threatshell.py` and add the import statement for your module 205 | 206 | .. code-block:: python 207 | 208 | from threatshell.commands.example import Example 209 | 210 | * add the instantiation of your module to the MyPrompt.__init__ 211 | - Don't forget to pass it the config if you need it 212 | 213 | .. code-block:: python 214 | 215 | class MyPrompt(Cmd): 216 | 217 | def __init__(self, args): 218 | 219 | ... 220 | 221 | self.example_api = Example() 222 | 223 | * add a method that follows the name schema of "do_" which uses the function in your example module 224 | 225 | .. code-block:: python 226 | 227 | def do_exmpl_request_intel(self, cmd_args): 228 | """ 229 | This is a doc string which becomes the help string 230 | for this function in the CLI. So now you can type 231 | "help exmpl_request_intel" and see this doc string 232 | when you are using the shell 233 | """ 234 | 235 | # do argument parsing config 236 | 237 | split_args = shlex.split(cmd_args) 238 | args = parser.parse_args(args=split_args) 239 | docs = self.example_api.request_intel(args.domains) 240 | 241 | # Currently a function to handle output and ES document saving 242 | # but will be changed later to incorporate connectors to 243 | # other things and output formatting modules 244 | self._handle_response(docs) 245 | 246 | .. _elasticsearch_dsl ReadTheDocs: https://elasticsearch-dsl.readthedocs.org/en/latest/ 247 | .. _Cmd class: https://docs.python.org/2/library/cmd.html 248 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to ThreatShell's documentation! 2 | ======================================= 3 | 4 | Contents: 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | 9 | install 10 | 11 | commands 12 | 13 | development 14 | -------------------------------------------------------------------------------- /docs/source/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | The installation process for most operating systems follows 4 | the same set of steps for the most part. I prefer to keep 5 | things maintained in a python virtual environment. These install 6 | steps will guide you through setting up a virtual environment for 7 | each OS which you may skip if you don't want. 8 | 9 | OS X 10 | ----- 11 | 12 | 13 | Virtualenv Setup 14 | ^^^^^^^^^^^^^^^^ 15 | 16 | Before You Begin 17 | ++++++++++++++++ 18 | 19 | First up, you'll need Xcode. You can find it in the app store, 20 | but it's huge (like 4GB) so it'll take a while to download. Once 21 | you have it, install the command line tools with 22 | :: 23 | 24 | xcode-select --install 25 | 26 | Once Xcode is all taken care of, get homebrew 27 | :: 28 | 29 | ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" 30 | 31 | You should already have python, but if you'd like a different 32 | version, you may install one from brew. 33 | 34 | Setting up the Virtualenv 35 | +++++++++++++++++++++++++ 36 | 37 | :: 38 | 39 | pip install virtualenv 40 | pip install virtualenvwrapper 41 | echo 'export WORKON_HOME="$HOME/.virtualenvs"' >> ~/.bash_profile 42 | echo 'source $(which virtualenvwrapper.sh)' >> ~/.bash_profile 43 | source ~/.bash_profile 44 | 45 | Now you will have a bunch of bash helper functions for 46 | using/managing virtual environments. The one's you will use most 47 | are: 48 | :: 49 | 50 | mkvirtualenv # creates a new virtual env 51 | rmvirtualenv # deletes a virtual env 52 | mktmpenv # creates a virtual env that is deleted when `deactivate` is run 53 | workon # sets the current virtual env 54 | deactivate # unsets the current virtual env 55 | 56 | Setting up ThreatShell 57 | ++++++++++++++++++++++ 58 | 59 | Installing the requirements for ThreatShell is as simple as 60 | :: 61 | 62 | python setup.py install 63 | 64 | You can run tests with 65 | :: 66 | 67 | python setup.py test 68 | 69 | # or with coverage included like so 70 | python setup.py test --pytest-args="--cov=threatshell" 71 | 72 | To begin using it 73 | :: 74 | 75 | python threatshell.py 76 | 77 | Running ThreatShell for the first time will trigger some 78 | setup/configuration prompts. Just follow along and you'll be 79 | hunting in no time. 80 | 81 | Readline Caveat! 82 | ++++++++++++++++ 83 | 84 | OS X doesn't ship with libreadline. Instead, you get libedit which kinda sucks. 85 | The main issue with libedit is that it doesn't honor the libreadline escape 86 | characters, therefore, it will fail to properly read the shell line with its 87 | color sequences, and scrolling through history causes buggy command display 88 | issues. 89 | 90 | This can be solved by simply using `easy_install` to install readline, or by 91 | using brew to install the GNU readline and force linking it in. I recommend 92 | just using the `easy_install readline`. You will know when you have the right 93 | readline library working because Threatshell won't complain about libedit being 94 | detected ;) 95 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | alabaster==0.7.7 2 | Babel==2.2.0 3 | cffi==1.6.0 4 | coverage==4.0.3 5 | cryptography==0.9 6 | docutils==0.12 7 | elasticsearch==2.3.0 8 | elasticsearch-dsl==2.0.0 9 | enum34==1.0.4 10 | GeoIP==1.3.2 11 | idna==2.0 12 | ipaddr==2.1.11 13 | ipaddress==1.0.7 14 | Jinja2==2.8 15 | MarkupSafe==0.23 16 | ndg-httpsclient==0.4.0 17 | netaddr==0.7.15 18 | nose==1.3.6 19 | py==1.4.27 20 | pyasn1==0.1.7 21 | pycparser==2.13 22 | pycrypto==2.6.1 23 | pyflakes==0.8.1 24 | Pygments==2.1.3 25 | pyOpenSSL==0.15.1 26 | pytest==2.7.1 27 | pytest-cov==2.2.1 28 | python-dateutil==2.5.1 29 | pytz==2016.1 30 | readline==6.2.4.1 31 | requests==2.7.0 32 | six==1.10.0 33 | snowballstemmer==1.2.1 34 | Sphinx==1.3.6 35 | sphinx-rtd-theme==0.1.9 36 | urllib3==1.14 37 | wheel==0.24.0 38 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from __future__ import print_function 34 | from setuptools import setup, find_packages 35 | from setuptools.command.test import test as TestCommand 36 | import io 37 | import os 38 | import sys 39 | 40 | from threatshell.common.constants import TS_VERSION 41 | import threatshell 42 | 43 | here = os.path.abspath(os.path.dirname(__file__)) 44 | 45 | 46 | def read(*filenames, **kwargs): 47 | encoding = kwargs.get('encoding', 'utf-8') 48 | sep = kwargs.get('sep', '\n') 49 | buf = [] 50 | for filename in filenames: 51 | with io.open(filename, encoding=encoding) as f: 52 | buf.append(f.read()) 53 | return sep.join(buf) 54 | 55 | long_description = read('README.md') 56 | 57 | # TODO: fix this 58 | on_rtd = os.environ.get('READTHEDOCS') == 'True' 59 | if on_rtd: 60 | requirements = read("sphinx_req.txt") 61 | else: 62 | requirements = read("requirements.txt") 63 | 64 | 65 | class PyTest(TestCommand): 66 | 67 | user_options = [ 68 | ('pytest-args=', 'a', "Additional args for py.test"), 69 | ] 70 | 71 | def initialize_options(self): 72 | TestCommand.initialize_options(self) 73 | self.pytest_args = [] 74 | 75 | def run_tests(self): 76 | import pytest 77 | errcode = pytest.main(self.pytest_args) 78 | sys.exit(errcode) 79 | 80 | setup( 81 | name='threatshell', 82 | version=TS_VERSION, 83 | url='https://www.github.com/salesforce/threatshell', 84 | license='BSD', 85 | author=threatshell.__author__, 86 | tests_require=['pytest'], 87 | install_requires=requirements.split("\n"), 88 | cmdclass={'test': PyTest}, 89 | author_email=threatshell.__email__, 90 | description='A command line utility for hunting and analyzing', 91 | long_description=long_description, 92 | packages=find_packages(), 93 | include_package_data=True, 94 | platforms='any', 95 | test_suite='tests', 96 | classifiers=[ 97 | 'Programming Language :: Python', 98 | 'Development Status :: 4 - Beta', 99 | 'Natural Language :: English', 100 | 'Environment :: Web Environment', 101 | 'Intended Audience :: Developers', 102 | 'License :: OSI Approved :: BSD License', 103 | 'Operating System :: OS Independent', 104 | 'Topic :: Software Development :: Libraries :: Python Modules', 105 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 106 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 107 | ], 108 | extras_require={ 109 | 'testing': ['pytest'], 110 | } 111 | ) 112 | -------------------------------------------------------------------------------- /sphinx_req.txt: -------------------------------------------------------------------------------- 1 | alabaster==0.7.9 2 | Babel==2.3.4 3 | docutils==0.12 4 | imagesize==0.7.1 5 | Jinja2==2.8 6 | MarkupSafe==0.23 7 | Pygments==2.1.3 8 | pytz==2016.6.1 9 | six==1.10.0 10 | snowballstemmer==1.2.1 11 | Sphinx==1.4.6 12 | wheel==0.24.0 13 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | -------------------------------------------------------------------------------- /threatshell/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/salesforce/threatshell/645024d056087d63b8e5163c08bf9c237231e028/threatshell/.DS_Store -------------------------------------------------------------------------------- /threatshell/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __author__ = 'Tommy Stallings' 4 | __email__ = 'tommy.stallings2@gmail.com' 5 | __version__ = '1.0' 6 | -------------------------------------------------------------------------------- /threatshell/commands/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | -------------------------------------------------------------------------------- /threatshell/commands/conf_manager.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.common.constants import TS_DIR 34 | from threatshell.common.colors import blue, red 35 | from threatshell.core.config import EncryptedConfigParser 36 | 37 | from ConfigParser import RawConfigParser 38 | from StringIO import StringIO 39 | 40 | import os 41 | 42 | 43 | class ConfigManager: 44 | 45 | def __init__(self, config): 46 | self.config = config 47 | self.edit_path = os.path.join(TS_DIR, "threatshell.txt") 48 | self.target_path = os.path.join(TS_DIR, "threatshell.ini") 49 | self.aes_key = os.path.join(TS_DIR, "aes_key.enc") 50 | self.rsa_key = os.path.join(TS_DIR, "conf_key.pem") 51 | 52 | def set_option(self, section, option, value): 53 | self.config.set_option(section, option, value) 54 | 55 | def dump_config(self, to_screen=False): 56 | 57 | f = None 58 | if to_screen: 59 | f = StringIO() 60 | else: 61 | f = open(self.edit_path, "w") 62 | 63 | RawConfigParser.write(self.config.config, f) 64 | if not to_screen: 65 | f.close() 66 | print blue( 67 | "Wrote decrypted config to %s" % self.edit_path, 68 | readline=True 69 | ) 70 | else: 71 | data = f.getvalue() 72 | f.close() 73 | return data 74 | 75 | def remove_section(self, section): 76 | if not self.config.config.has_section(section): 77 | print red("Section %s does not exist" % section) 78 | return 79 | 80 | self.config.remove_section(section) 81 | print blue("Successfully removed section %s" % section, readline=True) 82 | 83 | def remove_option(self, section, option): 84 | if not self.config.config.has_option(section, option): 85 | print red( 86 | "Section '%s' has no option '%s' to remove" % ( 87 | section, 88 | option 89 | ), 90 | readline=True 91 | ) 92 | return 93 | self.config.config.remove_option(section, option) 94 | print blue( 95 | "Successfully removed '%s' from '%s'" % (option, section), 96 | readline=True 97 | ) 98 | 99 | def save(self): 100 | if os.path.exists(self.edit_path): 101 | config = EncryptedConfigParser( 102 | allow_no_value=True, 103 | private_key=self.rsa_key, 104 | aes_key=self.aes_key 105 | ) 106 | read = config.read_raw(self.edit_path) 107 | if not read: 108 | raise IOError("Failed to read config '%s'" % self.edit_path) 109 | config.write(outfile=self.target_path) 110 | else: 111 | self.config.save_config() 112 | -------------------------------------------------------------------------------- /threatshell/commands/cymru.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.commands.q import AutoQuery 34 | from threatshell.doctypes import cymru as cymru_docs 35 | from elasticsearch_dsl import String 36 | 37 | import logging 38 | import re 39 | import socket 40 | 41 | log = logging.getLogger(__name__) 42 | 43 | 44 | class Cymru: 45 | 46 | def __init__(self): 47 | self.cymru = "whois.cymru.com" 48 | self.port = 43 49 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 50 | self.ip_expr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") 51 | 52 | def _connect(self): 53 | try: 54 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 55 | self.sock.connect((self.cymru, self.port)) 56 | except Exception, e: 57 | log.error( 58 | "Failed to connect to '%s' - [%s]: %s" % ( 59 | self.cymru, 60 | str(e.__class__.__name__), 61 | str(e.message) 62 | ) 63 | ) 64 | 65 | def _close(self): 66 | try: 67 | self.sock.close() 68 | except: 69 | pass 70 | 71 | def _send_to_sock(self, query): 72 | 73 | data = "" 74 | try: 75 | 76 | self._connect() 77 | self.sock.send(query) 78 | buff = self.sock.recv(4096) 79 | 80 | while buff != '': 81 | data += buff 82 | buff = self.sock.recv(4096) 83 | 84 | except Exception, e: 85 | log.error( 86 | "Error communicating on socket - [%s]: %s" % ( 87 | str(e.__class__.__name__), 88 | str(e.message) 89 | ) 90 | ) 91 | print str(e) 92 | 93 | finally: 94 | self._close() 95 | 96 | return data 97 | 98 | def _get_ip(self, dom): 99 | try: 100 | return socket.gethostbyname(dom) 101 | except Exception, e: 102 | log.error( 103 | "Failed to get address of domain '%s' - [%s]: %s" % ( 104 | dom, 105 | str(e.__class__.__name__), 106 | str(e.message) 107 | ) 108 | ) 109 | return None 110 | 111 | def asn_info(self, target): 112 | 113 | data_map = { 114 | 'asnum': None, 115 | "ip": None, 116 | 'prefix': None, 117 | 'country': None, 118 | 'registry': None, 119 | 'allocation_date': None, 120 | 'asname': None 121 | } 122 | 123 | if not self.ip_expr.match(target): 124 | dom = target 125 | target = self._get_ip(target) 126 | if not target: 127 | record = ( 128 | "Failed to resolve IP address of %s - can't get " + 129 | "ASN origin information" 130 | ) % dom 131 | log.error(record) 132 | doc = cymru_docs.CymruASInfoDoc(data_map) 133 | setattr(doc, "successful", False) 134 | setattr(doc, "term", target) 135 | return doc 136 | 137 | data = self._send_to_sock("-v -f %s\r\n\r\n" % target) 138 | if not data: 139 | record = ( 140 | "No data returned from Cymru for IP '%s' " + 141 | "for ASN information" 142 | ) % (target) 143 | log.error(record) 144 | doc = cymru_docs.CymruASInfoDoc(data_map) 145 | setattr(doc, "successful", False) 146 | setattr(doc, "term", target) 147 | return doc 148 | 149 | data = data.strip() 150 | 151 | parts = data.split("|") 152 | data_map = { 153 | 'asnum': parts[0].strip(), 154 | "ip": parts[1].strip(), 155 | 'prefix': parts[2].strip(), 156 | 'country': parts[3].strip(), 157 | 'registry': parts[4].strip(), 158 | 'allocation_date': parts[5].strip(), 159 | 'asname': parts[6].strip() 160 | } 161 | 162 | doc = cymru_docs.CymruASInfoDoc(data_map) 163 | setattr(doc, "successful", True) 164 | setattr(doc, "term", target) 165 | 166 | return doc 167 | 168 | @AutoQuery.use_on(["ip"]) 169 | def batch_asn_info(self, targets): 170 | 171 | data_map = { 172 | 'asnum': None, 173 | "ip": None, 174 | 'prefix': None, 175 | 'country': None, 176 | 'registry': None, 177 | 'allocation_date': None, 178 | 'asname': None 179 | } 180 | 181 | docs = [] 182 | filtered_targets = [] 183 | 184 | for i in xrange(0, len(targets)): 185 | 186 | if not self.ip_expr.match(targets[i]): 187 | 188 | dom = targets[i] 189 | targets[i] = self._get_ip(targets[i]) 190 | 191 | if not targets[i]: 192 | record = ( 193 | "Failed to resolve IP address of %s - can't get " + 194 | "ASN origin information" 195 | ) % dom 196 | log.error(record) 197 | doc = cymru_docs.CymruASInfoDoc(data_map) 198 | setattr(doc, "successful", False) 199 | setattr(doc, "term", targets[i]) 200 | docs.append(doc) 201 | 202 | else: 203 | filtered_targets.append(targets[i]) 204 | else: 205 | filtered_targets.append(targets[i]) 206 | 207 | targets = filtered_targets 208 | 209 | header = "begin" 210 | options = "verbose" 211 | footer = "end" 212 | 213 | targets.insert(0, header) 214 | targets.insert(1, options) 215 | targets.append(footer) 216 | targets = "\n".join(targets) 217 | 218 | data = self._send_to_sock("%s\r\n\r\n" % targets) 219 | if not data: 220 | record = ( 221 | "No data returned from Cymru for batch lookup " + 222 | "of ASN information" 223 | ) 224 | log.error(record) 225 | doc = cymru_docs.CymruASInfoDoc(data_map) 226 | setattr(doc, "successful", False) 227 | setattr(doc, "term", "batch_asn_info") 228 | return doc 229 | 230 | data = data.split("\n") 231 | data.pop(0) 232 | data.pop() 233 | for line in data: 234 | line = line.strip() 235 | parts = line.split("|") 236 | data_map = { 237 | 'asnum': parts[0].strip(), 238 | 'ip': parts[1].strip(), 239 | 'prefix': parts[2].strip(), 240 | 'country': parts[3].strip(), 241 | 'registry': parts[4].strip(), 242 | 'allocation_date': parts[5].strip(), 243 | 'asname': parts[6].strip() 244 | } 245 | doc = cymru_docs.CymruASInfoDoc(data_map) 246 | setattr(doc, "successful", True) 247 | setattr(doc, "term", parts[1].strip()) 248 | docs.append(doc) 249 | 250 | return docs 251 | 252 | def asnum_to_name(self, target): 253 | 254 | data_map = { 255 | 'asnum': None, 256 | 'country': None, 257 | 'registry': None, 258 | 'allocation_date': None, 259 | 'asname': None 260 | } 261 | 262 | data = self._send_to_sock("-v -f AS%s\r\n\r\n" % target) 263 | if not data: 264 | record = ( 265 | "No data returned from Cymru for AS name lookup " + 266 | "on target '%s'" 267 | ) % target 268 | log.error(record) 269 | doc = cymru_docs.CymruASNumInfoDoc(data_map) 270 | setattr(doc, "successful", False) 271 | setattr(doc, "term", target) 272 | return doc 273 | 274 | data = data.strip() 275 | parts = data.split("|") 276 | data_map = { 277 | 'asnum': parts[0].strip(), 278 | 'country': parts[1].strip(), 279 | 'registry': parts[2].strip(), 280 | 'allocation_date': parts[3].strip(), 281 | 'asname': parts[4].strip() 282 | } 283 | 284 | doc = cymru_docs.CymruASNumInfoDoc(data_map) 285 | setattr(doc, "successful", True) 286 | setattr(doc, "term", target) 287 | return doc 288 | 289 | @AutoQuery.use_on(["asnum"]) 290 | def batch_asnum_to_name(self, targets): 291 | 292 | data_map = { 293 | 'asnum': None, 294 | 'country': None, 295 | 'registry': None, 296 | 'allocation_date': None, 297 | 'asname': None 298 | } 299 | 300 | for i in xrange(0, len(targets)): 301 | targets[i] = "AS%s" % targets[i] 302 | 303 | header = "begin" 304 | options = "verbose" 305 | footer = "end" 306 | 307 | targets.insert(0, header) 308 | targets.insert(1, options) 309 | targets.append(footer) 310 | targets = "\n".join(targets) 311 | 312 | data = self._send_to_sock("%s\r\n\r\n" % targets) 313 | if not data: 314 | record = ( 315 | "No data returned from Cymru for batch AS name lookup" 316 | ) 317 | log.error(record) 318 | doc = cymru_docs.CymruASNumInfoDoc(data_map) 319 | setattr(doc, "successful", False) 320 | setattr(doc, "term", "batch_asname") 321 | return doc 322 | 323 | data = data.strip() 324 | lines = data.split("\n") 325 | lines.pop(0) 326 | 327 | docs = [] 328 | for line in lines: 329 | parts = line.split("|") 330 | data_map = { 331 | 'asnum': parts[0].strip(), 332 | 'country': parts[1].strip(), 333 | 'registry': parts[2].strip(), 334 | 'allocation_date': parts[3].strip(), 335 | 'asname': parts[4].strip() 336 | } 337 | doc = cymru_docs.CymruASNumInfoDoc(data_map) 338 | setattr(doc, "successful", True) 339 | setattr(doc, "term", parts[0].strip()) 340 | docs.append(doc) 341 | 342 | return docs 343 | -------------------------------------------------------------------------------- /threatshell/commands/geoip.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.commands.q import AutoQuery 34 | from threatshell.common.colors import red, bold 35 | from threatshell.common.constants import TS_DIR 36 | from threatshell.doctypes import geoip as geo_docs 37 | from hashlib import md5 38 | import argparse 39 | import GeoIP 40 | import glob 41 | import gzip 42 | import ipaddr 43 | import logging 44 | import os 45 | import re 46 | import requests 47 | import shlex 48 | 49 | log = logging.getLogger(__name__) 50 | 51 | 52 | class GeoTools: 53 | 54 | def __init__(self, config): 55 | self.config = config 56 | self.host_url = "http://updates.maxmind.com" 57 | self.fnames_url = "%s/app/update_getfilename" % self.host_url 58 | self.ipaddr_url = "%s/app/update_getipaddr" % self.host_url 59 | self.update_url = "%s/app/update_secure" % self.host_url 60 | self.db_path = "%s/geo_db" % TS_DIR 61 | self.db_hashes = {} 62 | 63 | self.v1_asnum = "%s/GeoLiteASNum.dat" % self.db_path 64 | self.v1_city = "%s/GeoLiteCity.dat" % self.db_path 65 | self.v1_country = "%s/GeoLiteCountry.dat" % self.db_path 66 | 67 | self.g1_asnum_reader = None 68 | self.g1_city_reader = None 69 | self.g1_country_reader = None 70 | 71 | self.have_db_files = True 72 | 73 | if not os.path.exists(self.db_path): 74 | value = raw_input( 75 | bold( 76 | red( 77 | "Geolocation databases not found. Would you like " + 78 | "to download them now ([yes]/no)?: " 79 | ) 80 | ) 81 | ) 82 | 83 | value = value.strip().upper() 84 | if( 85 | value == "NO" or 86 | value == "N" 87 | ): 88 | self.have_db_files = False 89 | else: 90 | os.mkdir(self.db_path) 91 | self.update() 92 | 93 | else: 94 | 95 | db_files = glob.glob("%s/*" % self.db_path) 96 | 97 | for db_file in db_files: 98 | 99 | db_data = open(db_file).read() 100 | db_hash = md5(db_data).hexdigest() 101 | 102 | db_file = db_file.split(os.path.sep)[-1] 103 | 104 | self.db_hashes[db_file] = db_hash 105 | 106 | if db_files: 107 | self._init_readers() 108 | 109 | def _init_readers(self): 110 | self.g1_asnum_reader = GeoIP.open(self.v1_asnum, GeoIP.GEOIP_STANDARD) 111 | self.g1_city_reader = GeoIP.open(self.v1_city, GeoIP.GEOIP_STANDARD) 112 | self.g1_country_reader = GeoIP.open( 113 | self.v1_country, 114 | GeoIP.GEOIP_STANDARD 115 | ) 116 | 117 | def _close_readers(self): 118 | self.g1_asnum_reader = None 119 | self.g1_city_reader = None 120 | self.g1_country_reader = None 121 | 122 | def _reset_readers(self): 123 | self._close_readers() 124 | self._init_readers() 125 | 126 | def update(self): 127 | 128 | log.debug("Requesting IP address") 129 | db_fnames = [] 130 | resp = requests.get(self.ipaddr_url) 131 | if resp.status_code != 200: 132 | log.error( 133 | "Failed to get external IP from %s - [%d]: %s" % ( 134 | self.ipaddr_url, 135 | resp.status_code, 136 | resp.content 137 | ) 138 | ) 139 | return 140 | 141 | ip_addr = resp.content.strip() 142 | log.debug("IP Address is %s" % ip_addr) 143 | ip_key_md5 = md5(self.config.get("GeoIP", "LicenseKey")) 144 | ip_key_md5.update(ip_addr) 145 | ip_key_hash = ip_key_md5.hexdigest() 146 | log.debug("IP/Key hash is %s" % ip_key_hash) 147 | 148 | product_ids = self.config.get("GeoIP", "ProductIds").split(",") 149 | product_ids = [x.strip() for x in product_ids] 150 | 151 | fnames = {} 152 | for pid in product_ids: 153 | 154 | log.debug("Requesting name for product id %s..." % pid) 155 | resp = requests.get(self.fnames_url, params={"product_id": pid}) 156 | 157 | if resp.status_code != 200: 158 | log.error( 159 | "Failed to resolve %s - [%d]: %s" % ( 160 | self.fnames_url, 161 | resp.status_code, 162 | resp.content 163 | ) 164 | ) 165 | continue 166 | 167 | fname = resp.content.strip() 168 | log.debug("Product name is %s" % fname) 169 | fnames[fname] = pid 170 | 171 | for fname in fnames.keys(): 172 | 173 | db_hash = "0" * 32 174 | if self.db_hashes.get(fname) is not None: 175 | db_hash = self.db_hashes[fname] 176 | 177 | log.debug("Requesting db file %s" % fname) 178 | params = { 179 | "db_md5": db_hash, 180 | "challenge_md5": ip_key_hash, 181 | "user_id": self.config.get("GeoIP", "UserId"), 182 | "edition_id": fnames[fname] 183 | } 184 | resp = requests.get(self.update_url, params=params) 185 | if resp.status_code != 200: 186 | log.error( 187 | "Failed to download new db file - [%d]: %s" % ( 188 | resp.status_code, 189 | resp.content 190 | ) 191 | ) 192 | else: 193 | log.debug("Downloading new db file...") 194 | chunk_size = 4096 195 | current_pattern = re.compile( 196 | ".*?No new updates.*", re.IGNORECASE) 197 | with open("%s/%s.gz" % (self.db_path, fname), 'wb') as fd: 198 | for chunk in resp.iter_content(chunk_size): 199 | fd.write(chunk) 200 | 201 | header = open("%s/%s.gz" % (self.db_path, fname)).read(1024) 202 | if not current_pattern.match(header): 203 | 204 | log.debug("Decompressing db file") 205 | gz_istream = gzip.open( 206 | "%s/%s.gz" % (self.db_path, fname), "rb") 207 | ostream = open("%s/%s" % (self.db_path, fname), "wb") 208 | buf = gz_istream.read(4096) 209 | while buf != "": 210 | ostream.write(buf) 211 | buf = gz_istream.read(4096) 212 | 213 | gz_istream.close() 214 | ostream.close() 215 | os.unlink("%s/%s.gz" % (self.db_path, fname)) 216 | db_fnames.append("%s/%s" % (self.db_path, fname)) 217 | 218 | else: 219 | 220 | log.debug("%s is up to date" % fname) 221 | db_fnames.append("%s/%s" % (self.db_path, fname)) 222 | os.unlink("%s/%s.gz" % (self.db_path, fname)) 223 | 224 | if self.g1_country_reader is None: 225 | self._init_readers() 226 | else: 227 | self._reset_readers() 228 | 229 | return db_fnames 230 | 231 | @AutoQuery.use_on(["ip", "domain"]) 232 | def city_lookup(self, indicators): 233 | 234 | if not isinstance(indicators, list): 235 | indicators = [indicators] 236 | 237 | docs = [] 238 | for i in indicators: 239 | if re.match("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", i): 240 | docs.append(self.city_by_addr(i)) 241 | else: 242 | docs.append(self.city_by_domain(i)) 243 | 244 | return docs 245 | 246 | def city_by_domain(self, domain): 247 | 248 | result = self.g1_city_reader.record_by_name(domain) 249 | 250 | if result: 251 | result["location"] = { 252 | "lat": result["latitude"], 253 | "lon": result["longitude"] 254 | } 255 | del result["latitude"] 256 | del result["longitude"] 257 | 258 | doc = geo_docs.GeoCityDoc(result) 259 | setattr(doc, "term", domain) 260 | 261 | if result: 262 | setattr(doc, "successful", True) 263 | else: 264 | setattr(doc, "successful", False) 265 | 266 | return doc 267 | 268 | def city_by_addr(self, addr): 269 | 270 | result = self.g1_city_reader.record_by_addr(addr) 271 | doc = geo_docs.GeoCityDoc(result) 272 | 273 | if result: 274 | result["location"] = { 275 | "lat": result["latitude"], 276 | "lon": result["longitude"] 277 | } 278 | del result["latitude"] 279 | del result["longitude"] 280 | 281 | setattr(doc, "term", addr) 282 | 283 | if result: 284 | setattr(doc, "successful", True) 285 | else: 286 | setattr(doc, "successful", False) 287 | 288 | return doc 289 | 290 | def country_lookup(self, cmd_args): 291 | 292 | parser = argparse.ArgumentParser(usage="geo_country") 293 | 294 | parser.add_argument( 295 | "-cc", 296 | "--country_code", 297 | action="store_true", 298 | help="Use country code instead of name", 299 | default=False, 300 | required=False 301 | ) 302 | 303 | parser.add_argument( 304 | "target", 305 | action="store", 306 | help="The target domain or IP to geolocate" 307 | ) 308 | 309 | try: 310 | args = parser.parse_args(args=shlex.split(cmd_args)) 311 | except SystemExit, e: 312 | if str(e) != "0": 313 | log.error( 314 | "Invalid arguments (use -h or --help to see command " + 315 | "options)" 316 | ) 317 | return 318 | 319 | doc = None 320 | if re.match("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", args.target): 321 | 322 | if args.country_code: 323 | 324 | result = self.country_code_by_addr(args.target) 325 | doc = geo_docs.GeoCountryCodeDoc( 326 | {"country_code" : result} 327 | ) 328 | setattr(doc, "term", args.target) 329 | if result: 330 | setattr(doc, "successful", True) 331 | else: 332 | setattr(doc, "successful", False) 333 | 334 | else: 335 | 336 | result = self.country_name_by_addr(args.target) 337 | doc = geo_docs.GeoCountryNameDoc( 338 | {"country_name": result} 339 | ) 340 | setattr(doc, "term", args.target) 341 | 342 | if result: 343 | setattr(doc, "successful", True) 344 | else: 345 | setattr(doc, "successful", False) 346 | 347 | else: 348 | 349 | if args.country_code: 350 | 351 | result = self.country_code_by_domain(args.target) 352 | doc = geo_docs.GeoCountryCodeDoc( 353 | {"country_code": result} 354 | ) 355 | setattr(doc, "term", args.target) 356 | 357 | if result: 358 | setattr(doc, "successful", True) 359 | else: 360 | setattr(doc, "successful", False) 361 | 362 | else: 363 | 364 | result = self.country_name_by_domain(args.target) 365 | doc = geo_docs.GeoCountryNameDoc( 366 | {"country_name": result} 367 | ) 368 | setattr(doc, "term", args.target) 369 | 370 | if result: 371 | setattr(doc, "successful", True) 372 | else: 373 | setattr(doc, "successful", False) 374 | 375 | return doc 376 | 377 | def _make_country_record(self, key, value): 378 | if value: 379 | return {key: value} 380 | 381 | def country_name_by_domain(self, domain): 382 | return self.g1_country_reader.country_name_by_name(domain) 383 | 384 | def country_name_by_addr(self, addr): 385 | return self.g1_country_reader.country_name_by_addr(addr) 386 | 387 | def country_code_by_domain(self, domain): 388 | return self.g1_country_reader.country_code_by_name(domain) 389 | 390 | def country_code_by_addr(self, addr): 391 | return self.g1_country_reader.country_code_by_addr(addr) 392 | 393 | @AutoQuery.use_on(["ip", "domain"]) 394 | def as_lookup(self, args): 395 | 396 | if not isinstance(args, list): 397 | args = [args] 398 | 399 | docs = [] 400 | for arg in args: 401 | 402 | as_string = None 403 | successful = True 404 | doc = None 405 | 406 | if re.match("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", arg): 407 | 408 | as_string = self.g1_asnum_reader.org_by_addr(arg) 409 | 410 | start, end = self.g1_asnum_reader.range_by_ip(arg) 411 | start = ipaddr.IPv4Address(start) 412 | end = ipaddr.IPv4Address(end) 413 | net_range = ipaddr.summarize_address_range(start, end) 414 | doc = geo_docs.GeoIpASNDoc() 415 | setattr(doc, "ip_allocation", str(net_range[0])) 416 | 417 | else: 418 | as_string = self.g1_asnum_reader.org_by_name(arg) 419 | doc = geo_docs.GeoASNDoc() 420 | 421 | if as_string: 422 | 423 | as_parts = as_string.split(" ") 424 | setattr(doc, "as_num", as_parts[0]) 425 | setattr(doc, "as_name", " ".join(as_parts[1:])) 426 | setattr(doc, "successful", True) 427 | 428 | else: 429 | setattr(doc, "as_num", 0) 430 | setattr(doc, "as_name", "") 431 | setattr(doc, "successful", False) 432 | 433 | docs.append(doc) 434 | 435 | return docs 436 | 437 | def can_geolocate(self): 438 | return self.have_db_files 439 | -------------------------------------------------------------------------------- /threatshell/commands/infoblox.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.doctypes.generic import GenericDoc 34 | 35 | from elasticsearch_dsl import String 36 | 37 | import logging 38 | import requests 39 | 40 | log = logging.getLogger(__name__) 41 | 42 | 43 | class InfobloxDoc(GenericDoc): 44 | 45 | response = String() 46 | 47 | 48 | class Infoblox: 49 | 50 | def __init__(self, config): 51 | self.url = config.get("Infoblox", "url") 52 | self.api_endpoint = "%s/wapi/v1.7.1" % self.url 53 | self.search_api = "%s/search" % self.api_endpoint 54 | 55 | self.user = config.get("Infoblox", "username") 56 | self.passwd = config.get("Infoblox", "password") 57 | 58 | def _build_error(self, term, message): 59 | record = {term: {"error": message}} 60 | return self._build_doc( 61 | term, 62 | record, 63 | False 64 | ) 65 | 66 | def _build_doc(self, term, response, successful): 67 | return InfobloxDoc( 68 | response=response, 69 | successful=successful, 70 | term=term 71 | ) 72 | 73 | def search(self, search_target): 74 | 75 | lease_params = { 76 | '_return_type': 'json-pretty', 77 | 'objtype': 'lease', 78 | 'search_string~': search_target 79 | } 80 | 81 | full_detail_params = { 82 | '_return_type': 'json-pretty', 83 | '_return_fields+': ",".join([ 84 | 'discovered_data', 85 | 'is_invalid_mac', 86 | 'hardware', 87 | 'client_hostname', 88 | 'ends', 89 | 'never_ends', 90 | 'never_starts', 91 | 'next_binding_state', 92 | 'on_commit', 93 | 'on_expiry', 94 | 'on_release', 95 | 'option', 96 | 'protocol', 97 | 'remote_id', 98 | 'served_by', 99 | 'server_host_name' 100 | ]) 101 | } 102 | 103 | r = requests.get( 104 | self.search_api, 105 | params=lease_params, 106 | auth=(self.user, self.passwd), 107 | verify=False 108 | ) 109 | 110 | if r.status_code != requests.codes.ok: 111 | message = "No leases found (Bad error code - [%s]: %s)" % ( 112 | r.status_code, 113 | r.content 114 | ) 115 | log.error(message) 116 | return self._build_error(search_target, message) 117 | 118 | refs = r.json() 119 | 120 | if len(refs) == 0: 121 | message = "No leases found" 122 | return self._build_error(search_target, message) 123 | 124 | if refs[0].get('_ref') is None: 125 | message = "No leases found" 126 | return self._build_error(search_target, message) 127 | 128 | details = [] 129 | for seq, ref_type in enumerate(refs): 130 | 131 | lease = refs[seq]['_ref'] 132 | query_lease = "%s/%s" % (self.api_endpoint, lease) 133 | r = requests.get( 134 | query_lease, 135 | params=full_detail_params, 136 | auth=(self.user, self.passwd), 137 | verify=False 138 | ) 139 | 140 | # TODO: maybe some es error feed back here 141 | if r.status_code != requests.codes.ok: 142 | log.info( 143 | "lease %s for %s not found - %s: %s" % ( 144 | lease, 145 | search_target, 146 | r.status_code, 147 | r.content 148 | ) 149 | ) 150 | continue 151 | 152 | lease_detail = r.json() 153 | # ref = lease_detail["_ref"] 154 | # del lease_detail["_ref"] 155 | # lease_detail["ref"] = ref 156 | details.append({search_target: lease_detail}) 157 | 158 | docs = [] 159 | for entry in details: 160 | docs.append( 161 | self._build_doc( 162 | search_target, 163 | entry, 164 | True 165 | ) 166 | ) 167 | return docs 168 | -------------------------------------------------------------------------------- /threatshell/commands/novetta.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.doctypes.generic import GenericDoc 34 | 35 | from elasticsearch_dsl import String 36 | 37 | import argparse 38 | import logging 39 | import requests 40 | import shlex 41 | 42 | log = logging.getLogger(__name__) 43 | 44 | 45 | class NovettaDoc(GenericDoc): 46 | 47 | response = String() 48 | 49 | 50 | class Novetta: 51 | 52 | def __init__(self, config): 53 | self.key = config.get("Novetta", "key") 54 | self.url = "https://whodat.novetta-arg.com" 55 | self.raw = "v0/raw" 56 | self.parsed = "v0/parsed" 57 | 58 | def _error(self, arg, endpoint, resp): 59 | log.error( 60 | "Failed to query whodat endpoint %s - %s: %s" % ( 61 | endpoint, 62 | resp.status_code, 63 | resp.content 64 | ) 65 | ) 66 | record = { 67 | arg: "Failed on endpoint %s with status code %s - %s" % ( 68 | endpoint, 69 | resp.status_code, 70 | resp.content 71 | ) 72 | } 73 | return self._build_doc(arg, record, False) 74 | 75 | def _query(self, endpoint, indicator): 76 | 77 | query = "%s/%s/%s" % (self.url, endpoint, indicator) 78 | auth_header = { 79 | "Authorization": "Bearer %s" % self.key 80 | } 81 | 82 | response = requests.get( 83 | query, 84 | verify=False, 85 | headers=auth_header 86 | ) 87 | 88 | if response.status_code == 202: 89 | message = "Queued for processing. Check back later" 90 | record = {indicator: message} 91 | return self._build_doc(indicator, record, False) 92 | 93 | if response.status_code == 204: 94 | record = { 95 | indicator: ( 96 | "The service was unable to parse the results. Try using " + 97 | "the --raw option" 98 | ) 99 | } 100 | return self._build_doc(indicator, record, False) 101 | 102 | if response.status_code != requests.codes.ok: 103 | return self._error(indicator, endpoint, response) 104 | 105 | record = {} 106 | if endpoint == self.raw: 107 | record = {indicator: response.content} 108 | else: 109 | record = {indicator: response.json()} 110 | 111 | return self._build_doc(indicator, record, True) 112 | 113 | def _build_doc(self, term, response, successful): 114 | return NovettaDoc( 115 | response=response, 116 | term=term, 117 | successful=successful 118 | ) 119 | 120 | def whodat(self, args): 121 | 122 | parser = argparse.ArgumentParser( 123 | usage="nvwhois" 124 | ) 125 | 126 | parser.add_argument( 127 | "indicator", 128 | action="store", 129 | help="Specify the indicator to query for" 130 | ) 131 | 132 | parser.add_argument( 133 | "--raw", 134 | action="store_true", 135 | default=False, 136 | required=False, 137 | help=( 138 | "Query for the unparsed whois data" 139 | ) 140 | ) 141 | 142 | try: 143 | parsed_args = parser.parse_args(args=shlex.split(args)) 144 | except SystemExit, e: 145 | if str(e) != "0": 146 | log.error( 147 | "Invalid argument for query (use -h or --help " + 148 | "to see command options)" 149 | ) 150 | return 151 | 152 | endpoint = self.parsed 153 | if parsed_args.raw: 154 | endpoint = self.raw 155 | 156 | return self._query(endpoint, parsed_args.indicator) 157 | -------------------------------------------------------------------------------- /threatshell/commands/q.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | import logging 34 | 35 | log = logging.getLogger(__name__) 36 | 37 | 38 | # TODO: Add an "all" option 39 | class AutoQuery: 40 | 41 | query_table = {} 42 | sup_query_types = [ 43 | "ip", 44 | "url", 45 | "domain", 46 | "asnum", 47 | "asname", 48 | "nameserver", 49 | "email", 50 | "hash", 51 | "address", 52 | "phone", 53 | "md5", 54 | "sha1" 55 | ] 56 | 57 | @classmethod 58 | def use_on(cls, query_types): 59 | if not isinstance(query_types, list): 60 | query_types = [query_types.lower()] 61 | else: 62 | query_types = [t.lower() for t in query_types] 63 | 64 | # log.debug("Query types requested: %s" % ", ".join(query_types)) 65 | for t in cls.sup_query_types: 66 | if t not in cls.sup_query_types: 67 | log.warn("%s isn't a supported indicator type yet" % t) 68 | query_types.remove(t) 69 | 70 | def decorator(method): 71 | 72 | # log.debug( 73 | # "Adding method %s to support query types of %s" % ( 74 | # method.__name__, 75 | # ", ".join(query_types) 76 | # ) 77 | # ) 78 | for ioc_type in query_types: 79 | if cls.query_table.get(ioc_type) is None: 80 | cls.query_table[ioc_type] = [method] 81 | else: 82 | cls.query_table[ioc_type].append(method) 83 | 84 | return method 85 | 86 | return decorator 87 | -------------------------------------------------------------------------------- /threatshell/commands/riskiq.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.doctypes.generic import GenericDoc 34 | 35 | from Crypto.Hash import HMAC, SHA 36 | from elasticsearch_dsl import String 37 | from requests.auth import HTTPBasicAuth 38 | 39 | import argparse 40 | import base64 41 | import logging 42 | import re 43 | import requests 44 | import shlex 45 | 46 | # To disable this annoying message 47 | # InsecureRequestWarning: Unverified HTTPS request is being made. 48 | # Adding certificate verification is strongly advised. 49 | # 50 | # Will want to maybe make this an option later 51 | try: 52 | requests.packages.urllib3.disable_warnings() 53 | except: 54 | pass 55 | 56 | log = logging.getLogger(__name__) 57 | 58 | 59 | class RiskIQDoc(GenericDoc): 60 | 61 | response = String() 62 | 63 | 64 | class RiskIQ: 65 | 66 | def __init__(self, config): 67 | self.key = config.get("RiskIQ", "key") 68 | self.token = config.get("RiskIQ", "token") 69 | self.url = "https://ws.riskiq.net" 70 | self.json_header = "application/json" 71 | self.xml_header = "text/xml" 72 | self.dns_name = "v1/dns/name" 73 | self.dns_data = "v1/dns/data" 74 | 75 | def _build_doc(self, term, response, successful): 76 | return RiskIQDoc( 77 | response=response, 78 | successful=successful, 79 | term=term 80 | ) 81 | 82 | def _get_indicator(self, params): 83 | 84 | indicator = None 85 | 86 | if params.get("ip") is not None: 87 | indicator = params["ip"] 88 | elif params.get("name") is not None: 89 | indicator = params["name"] 90 | else: 91 | indicator = params["raw"] 92 | 93 | return indicator 94 | 95 | def _error(self, arg, endpoint, resp): 96 | 97 | message = "Failed on endpoint %s with status code %s - %s" % ( 98 | endpoint, 99 | resp.status_code, 100 | resp.content 101 | ) 102 | 103 | log.error(message) 104 | record = {"error": message} 105 | 106 | return self._build_doc(arg, record, False) 107 | 108 | def _query(self, endpoint, params): 109 | 110 | # TODO: Figure this bullshit out 111 | import httplib 112 | httplib.HTTPConnection.debuglevel = 1 113 | requests_log = logging.getLogger("requests.packages.urllib3") 114 | requests_log.setLevel(logging.DEBUG) 115 | requests_log.propagate = True 116 | 117 | query_url = "%s/%s" % (self.url, endpoint) 118 | 119 | hmac = HMAC.new(self.key, digestmod=SHA) 120 | 121 | param_string = "=".join(params.items()[0]) 122 | sig_data = "GET\n/%s\n%s\n\n" % (endpoint, param_string) 123 | 124 | log.debug(sig_data) 125 | 126 | hmac.update(sig_data) 127 | signature = hmac.digest() 128 | encoded = base64.b64encode(signature) 129 | 130 | r = requests.get( 131 | query_url, 132 | params=params, 133 | verify=False, 134 | headers={ 135 | "Accept": self.json_header, 136 | "Authorization": "RiskIQ %s:%s" % (self.token, encoded) 137 | } 138 | ) 139 | if r.status_code != requests.codes.ok: 140 | return self._error( 141 | self._get_indicator(params), 142 | endpoint, 143 | r 144 | ) 145 | 146 | return r.json() 147 | 148 | def _ba_query(self, endpoint, params): 149 | 150 | query_url = "%s/%s" % (self.url, endpoint) 151 | indicator = self._get_indicator(params) 152 | 153 | r = requests.get( 154 | query_url, 155 | params=params, 156 | verify=False, 157 | auth=HTTPBasicAuth(self.token, self.key), 158 | headers={"Accept": self.json_header} 159 | ) 160 | 161 | if r.status_code == 204: 162 | record = {indicator: "No results found"} 163 | return self._build_doc(indicator, record, False) 164 | 165 | if r.status_code != requests.codes.ok: 166 | record = {indicator: "%s: %s" % (r.status_code, r.content)} 167 | return self._build_doc( 168 | indicator, 169 | record, 170 | False 171 | ) 172 | 173 | record = {indicator: r.json()} 174 | return self._build_doc(indicator, record, True) 175 | 176 | def passive_dns(self, args): 177 | 178 | parser = argparse.ArgumentParser( 179 | usage="riq_pdns", 180 | epilog=( 181 | "See docs at " + 182 | "https://sf.riskiq.net/crawlview/api/docs/controllers/" + 183 | "DnsController.html for more details" 184 | ) 185 | ) 186 | 187 | parser.add_argument( 188 | "indicator", 189 | action="store", 190 | help="Specify the indicator to query for" 191 | ) 192 | 193 | parser.add_argument( 194 | "--data", 195 | action="store_true", 196 | default=False, 197 | required=False, 198 | help=( 199 | "query for matches in the resource record data rather " + 200 | "than by resource record names" 201 | ) 202 | ) 203 | 204 | parser.add_argument( 205 | "--raw", 206 | action="store_true", 207 | default=False, 208 | required=False, 209 | help=( 210 | "Indicate that the given indicator should be used as a raw " + 211 | "string in the search criteria" 212 | ) 213 | ) 214 | 215 | parser.add_argument( 216 | "--rr_type", 217 | action="store", 218 | choices=["A", "NS", "MX", "TXT", "PTR"], 219 | default=None, 220 | required=False, 221 | help=( 222 | "Filter results by a resource record type" 223 | ) 224 | ) 225 | 226 | parser.add_argument( 227 | "--limit", 228 | action="store", 229 | default=None, 230 | required=False, 231 | help=( 232 | "Specify a maximum number of results to be returned " + 233 | "(A default of 100 is imposed by RiskIQ)" 234 | ) 235 | ) 236 | 237 | try: 238 | parsed_args = parser.parse_args(args=shlex.split(args)) 239 | except SystemExit, e: 240 | if str(e) != "0": 241 | log.error( 242 | "Invalid argument for query (use -h or --help " + 243 | "to see command options)" 244 | ) 245 | return 246 | 247 | endpoint = self.dns_name 248 | if parsed_args.data: 249 | endpoint = self.dns_data 250 | 251 | params = {} 252 | if parsed_args.rr_type is not None: 253 | params["rrType"] = parsed_args.rr_type 254 | 255 | if parsed_args.raw: 256 | params["raw"] = parsed_args.indicator.encode("hex") 257 | else: 258 | if re.match( 259 | "\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", 260 | parsed_args.indicator 261 | ): 262 | params["ip"] = parsed_args.indicator 263 | else: 264 | params["name"] = parsed_args.indicator 265 | 266 | if parsed_args.limit is not None: 267 | params["maxResults"] = parsed_args.limit 268 | 269 | return self._ba_query(endpoint, params) 270 | -------------------------------------------------------------------------------- /threatshell/commands/shadow_server.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.commands.q import AutoQuery 34 | from threatshell.doctypes import shadow_server as ss_docs 35 | from elasticsearch_dsl import String 36 | 37 | import logging 38 | import re 39 | import socket 40 | 41 | log = logging.getLogger(__name__) 42 | 43 | 44 | class ShadowServer: 45 | 46 | def __init__(self): 47 | self.ss = "asn.shadowserver.org" 48 | self.port = 43 49 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 50 | self.ip_expr = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") 51 | 52 | def _connect(self): 53 | try: 54 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 55 | self.sock.connect((self.ss, self.port)) 56 | except Exception, e: 57 | log.error( 58 | "Failed to connect to '%s' - [%s]: %s" % ( 59 | self.ss, 60 | str(e.__class__.__name__), 61 | str(e.message) 62 | ) 63 | ) 64 | 65 | def _close(self): 66 | try: 67 | self.sock.close() 68 | except: 69 | pass 70 | 71 | def _send_to_sock(self, query): 72 | 73 | data = "" 74 | try: 75 | 76 | self._connect() 77 | self.sock.send(query) 78 | buff = self.sock.recv(4096) 79 | 80 | while buff != '': 81 | data += buff 82 | buff = self.sock.recv(4096) 83 | 84 | except Exception, e: 85 | log.error( 86 | "Error communicating on socket - [%s]: %s" % ( 87 | str(e.__class__.__name__), 88 | str(e.message) 89 | ) 90 | ) 91 | print str(e) 92 | 93 | finally: 94 | self._close() 95 | 96 | return data 97 | 98 | def _get_ip(self, dom): 99 | try: 100 | return socket.gethostbyname(dom) 101 | except Exception, e: 102 | log.error( 103 | "Failed to get address of domain '%s' - [%s]: %s" % ( 104 | dom, 105 | str(e.__class__.__name__), 106 | str(e.message) 107 | ) 108 | ) 109 | return None 110 | 111 | def asn_origin(self, target): 112 | 113 | if not self.ip_expr.match(target): 114 | dom = target 115 | target = self._get_ip(target) 116 | if not target: 117 | record = ( 118 | "Failed to resolve IP address of %s - can't get " + 119 | "ASN origin information" 120 | ) % dom 121 | log.error(record) 122 | doc = geo_docs.ASOriginDoc({}) 123 | setattr(doc, "term", dom) 124 | setattr(doc, "successful", False) 125 | return doc 126 | 127 | data = self._send_to_sock("origin %s\r\n\r\n" % target) 128 | if not data: 129 | record = ( 130 | "No data returned from Shadow Server for IP '%s' " + 131 | "for ASN origin information" 132 | ) % (target) 133 | log.error(record) 134 | doc = ss_docs.ASOriginDoc({}) 135 | setattr(doc, "term", target) 136 | setattr(doc, "successful", False) 137 | return doc 138 | 139 | data = data.strip() 140 | 141 | parts = data.split("|") 142 | data_map = { 143 | 'asnum': parts[0].strip(), 144 | 'prefix': parts[1].strip(), 145 | 'asname': parts[2].strip(), 146 | 'country': parts[3].strip(), 147 | 'domain': parts[4].strip(), 148 | 'isp': parts[5].strip() 149 | } 150 | 151 | doc = ss_docs.ASOriginDoc(data_map) 152 | setattr(doc, "successful", True) 153 | setattr(doc, "term", target) 154 | return doc 155 | 156 | @AutoQuery.use_on(["ip", "domain"]) 157 | def batch_asn_origin(self, targets): 158 | 159 | docs = [] 160 | filtered_targets = [] 161 | 162 | for i in xrange(0, len(targets)): 163 | 164 | if not self.ip_expr.match(targets[i]): 165 | 166 | dom = targets[i] 167 | targets[i] = self._get_ip(targets[i]) 168 | 169 | if not targets[i]: 170 | record = ( 171 | "Failed to resolve IP address of %s - can't get " + 172 | "ASN origin information" 173 | ) % dom 174 | log.error(record) 175 | doc = ss_docs.ASOriginDoc({}) 176 | setattr(doc, "term", dom) 177 | setattr(doc, "successful", False) 178 | docs.append(doc) 179 | 180 | else: 181 | filtered_targets.append(targets[i]) 182 | else: 183 | filtered_targets.append(targets[i]) 184 | 185 | targets = filtered_targets 186 | 187 | header = "begin origin" 188 | footer = "end" 189 | 190 | targets.insert(0, header) 191 | targets.append(footer) 192 | targets = "\n".join(targets) 193 | 194 | data = self._send_to_sock("%s\r\n\r\n" % targets) 195 | if not data: 196 | record = ( 197 | "No data returned from Shadow Server for batch lookup " + 198 | "of ASN information" 199 | ) 200 | log.error(record) 201 | doc = ss_docs.ASOriginDoc({}) 202 | setattr(doc, "term", "batch_origin") 203 | setattr(doc, "successful", False) 204 | return doc 205 | 206 | data = data.split("\n") 207 | data.pop() 208 | for line in data: 209 | line = line.strip() 210 | parts = line.split("|") 211 | data_map = { 212 | 'asnum': parts[1].strip(), 213 | 'prefix': parts[2].strip(), 214 | 'asname': parts[3].strip(), 215 | 'country': parts[4].strip(), 216 | 'domain': parts[5].strip(), 217 | 'isp': parts[6].strip() 218 | } 219 | doc = ss_docs.ASOriginDoc(data_map) 220 | setattr(doc, "term", parts[0]) 221 | setattr(doc, "successful", True) 222 | docs.append(doc) 223 | 224 | return docs 225 | 226 | def asn_peers(self, target): 227 | 228 | if not self.ip_expr.match(target): 229 | dom = target 230 | target = self._get_ip(target) 231 | if not target: 232 | record = ( 233 | "Failed to resolve IP address of %s - can't get " + 234 | "ASN peer information" 235 | ) % dom 236 | log.error(record) 237 | doc = ss_docs.ASPeersDoc({}) 238 | setattr(doc, "term", dom) 239 | setattr(doc, "successful", False) 240 | return doc 241 | 242 | data = self._send_to_sock("peer %s\r\n\r\n" % target) 243 | if not data: 244 | record = ( 245 | "No data returned from Shadow Server for peer lookup " + 246 | "of ASN information on target '%s'" 247 | ) % target 248 | log.error(record) 249 | doc = ss_docs.ASPeersDoc({}) 250 | setattr(doc, "term", target) 251 | setattr(doc, "successful", False) 252 | return doc 253 | 254 | parts = data.split("|") 255 | peers = parts[0] 256 | peers = peers.split(" ") 257 | peers.pop() 258 | for i in xrange(0, len(peers)): 259 | peers[i] = peers[i].strip() 260 | 261 | data = data.strip() 262 | parts = data.split("|") 263 | parts.pop(0) 264 | 265 | data_map = { 266 | 'peers': peers, 267 | 'asnum': parts[0].strip(), 268 | 'prefix': parts[1].strip(), 269 | 'asname': parts[2].strip(), 270 | 'country': parts[3].strip(), 271 | 'domain': parts[4].strip(), 272 | 'isp': parts[5].strip() 273 | } 274 | 275 | doc = ss_docs.ASPeersDoc(data_map) 276 | setattr(doc, "term", target) 277 | setattr(doc, "successful", True) 278 | return doc 279 | 280 | @AutoQuery.use_on(["ip", "domain"]) 281 | def batch_asn_peers(self, targets): 282 | 283 | docs = [] 284 | filtered_targets = [] 285 | 286 | for i in xrange(0, len(targets)): 287 | 288 | if not self.ip_expr.match(targets[i]): 289 | 290 | dom = targets[i] 291 | targets[i] = self._get_ip(targets[i]) 292 | 293 | if not targets[i]: 294 | record = ( 295 | "Failed to resolve IP address of %s - can't get " + 296 | "ASN peer information" 297 | ) % dom 298 | log.error(record) 299 | doc = ss_docs.ASPeersDoc({}) 300 | setattr(doc, "term", dom) 301 | setattr(doc, "successful", False) 302 | docs.append(doc) 303 | else: 304 | filtered_targets.append(targets[i]) 305 | else: 306 | filtered_targets.append(targets[i]) 307 | 308 | targets = filtered_targets 309 | 310 | header = "begin peer" 311 | footer = "end" 312 | 313 | targets.insert(0, header) 314 | targets.append(footer) 315 | targets = "\n".join(targets) 316 | 317 | data = self._send_to_sock("%s\r\n\r\n" % targets) 318 | if not data: 319 | record = ( 320 | "No data returned from Shadow Server for batch lookup " + 321 | "of ASN information" 322 | ) 323 | log.error(record) 324 | doc = ss_docs.ASPeersDoc({}) 325 | setattr(doc, "term", "batch_peer") 326 | setattr(doc, "successful", False) 327 | return doc 328 | 329 | data = data.split("\n") 330 | data.pop() 331 | 332 | for line in data: 333 | 334 | parts = line.split("|") 335 | peers = parts[1].split(" ") 336 | peers.pop() 337 | peers.pop(0) 338 | for i in xrange(0, len(peers)): 339 | peers[i] = peers[i].strip() 340 | 341 | line = line.strip() 342 | line = line.split("|") 343 | line.pop(1) 344 | 345 | data_map = { 346 | 'peers': peers, 347 | 'asnum': line[1].strip(), 348 | 'prefix': line[2].strip(), 349 | 'asname': line[3].strip(), 350 | 'country': line[4].strip(), 351 | 'domain': line[5].strip(), 352 | 'isp': line[6].strip() 353 | } 354 | doc = ss_docs.ASPeersDoc(data_map) 355 | setattr(doc, "term", line[0]) 356 | setattr(doc, "successful", True) 357 | docs.append(doc) 358 | 359 | return docs 360 | 361 | @AutoQuery.use_on(["asnum"]) 362 | def asnum_to_prefix(self, target): 363 | data = self._send_to_sock("prefix %s\r\n\r\n" % target) 364 | if not data: 365 | record = ( 366 | "No data returned from Shadow Server for AS prefix lookup " + 367 | "on target '%s'" 368 | ) % target 369 | log.error(record) 370 | doc = ss_docs.ASPrefixDoc({}) 371 | setattr(doc, "term", target) 372 | setattr(doc, "successful", True) 373 | return doc 374 | 375 | data = data.split("\n") 376 | data.pop() 377 | for i in xrange(0, len(data)): 378 | data[i] = data[i].strip() 379 | 380 | doc = ss_docs.ASPrefixDoc( 381 | { 382 | "prefixes": data 383 | } 384 | ) 385 | setattr(doc, "term", target) 386 | setattr(doc, "successful", True) 387 | return doc 388 | -------------------------------------------------------------------------------- /threatshell/commands/threat_q.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | import argparse 34 | import logging 35 | import json 36 | import requests 37 | import shlex 38 | 39 | # To disable this annoying message 40 | # InsecureRequestWarning: Unverified HTTPS request is being made. 41 | # Adding certificate verification is strongly advised. 42 | # 43 | # Will want to maybe make this an option later 44 | try: 45 | requests.packages.urllib3.disable_warnings() 46 | except: 47 | pass 48 | 49 | log = logging.getLogger(__name__) 50 | 51 | 52 | # TODO - add status code checking and error handling on bad status codes 53 | class ThreatQ: 54 | 55 | def __init__(self, config): 56 | self.url = config.get("ThreatQ", "url") 57 | if self.url[-1] == "/": 58 | self.url = self.url[0:-1] 59 | self.key = config.get("ThreatQ", "key") 60 | 61 | def _get_status_map(self): 62 | statuses = self.indicator_statuses(None) 63 | status_map = {} 64 | for item in statuses: 65 | status_name = item["indicator_status"] 66 | status_id = item["indicator_status_id"] 67 | status_map[status_name] = status_id 68 | 69 | return status_map 70 | 71 | def _get_type_map(self): 72 | types = self.indicator_types(None) 73 | type_map = {} 74 | for item in types: 75 | indicator_type = item["indicator_type"] 76 | type_id = item["indicator_type_id"] 77 | type_map[indicator_type] = type_id 78 | 79 | return type_map 80 | 81 | def query(self, args): 82 | 83 | parser = argparse.ArgumentParser(usage="tq_search") 84 | parser.add_argument( 85 | "indicator", 86 | action="store", 87 | help="Specify the indicator to query for" 88 | ) 89 | 90 | try: 91 | parsed_args = parser.parse_args(args=shlex.split(args)) 92 | except SystemExit, e: 93 | if str(e) != "0": 94 | log.error( 95 | "Invalid argument for query (use -h or --help " + 96 | "to see command options)" 97 | ) 98 | return {} 99 | 100 | query_url = "%s/api/search/" % self.url 101 | params = {"api_key": self.key} 102 | data = {"indicator": parsed_args.indicator} 103 | 104 | r = requests.post(query_url, params=params, data=data, verify=False) 105 | 106 | jdata = r.json() 107 | if not jdata: 108 | jdata = {parsed_args.indicator: "Nothin'..."} 109 | return jdata 110 | 111 | def indicator_statuses(self, args): 112 | 113 | query_url = "%s/api/indicator-statuses" % self.url 114 | params = {"api_key": self.key} 115 | r = requests.get(query_url, params=params, verify=False) 116 | 117 | jdata = r.json() 118 | if not jdata: 119 | jdata = {} 120 | return jdata 121 | 122 | def indicator_types(self, args): 123 | 124 | query_url = "%s/api/indicator-types" % self.url 125 | params = {"api_key": self.key} 126 | r = requests.get(query_url, params=params, verify=False) 127 | 128 | jdata = r.json() 129 | if not jdata: 130 | jdata = {} 131 | return jdata 132 | 133 | def add_indicator(self, args): 134 | 135 | type_map = self._get_type_map() 136 | status_map = self._get_status_map() 137 | 138 | parser = argparse.ArgumentParser(usage="tq_add") 139 | parser.add_argument( 140 | "indicator", 141 | action="store", 142 | help="Specify the indicator to be added" 143 | ) 144 | 145 | parser.add_argument( 146 | "-c", 147 | "--class_type", 148 | action="store", 149 | choices=["network", "host"], 150 | metavar="TYPE", 151 | required=True, 152 | help=( 153 | "Specify indicator class. Valid choices are: %s" 154 | ) % ", ".join(["network", "host"]) 155 | ) 156 | 157 | parser.add_argument( 158 | "-t", 159 | "--type", 160 | action="store", 161 | choices=type_map.keys(), 162 | metavar="TYPE", 163 | required=True, 164 | help=( 165 | "Specify the indicator type. Valid choices are: %s" 166 | ) % (", ".join(type_map.keys())) 167 | ) 168 | 169 | parser.add_argument( 170 | "-s", 171 | "--status", 172 | action="store", 173 | choices=status_map.keys(), 174 | metavar="STATUS", 175 | required=True, 176 | help=( 177 | "Specify the indicator status. Valid choices are: %s" 178 | ) % ", ".join(status_map.keys()) 179 | ) 180 | 181 | try: 182 | parsed_args = parser.parse_args(args=shlex.split(args)) 183 | except SystemExit, e: 184 | if str(e) != "0": 185 | log.error( 186 | "Invalid argument for query (use -h or --help " + 187 | "to see command options)" 188 | ) 189 | return {} 190 | 191 | query_url = "%s/api/indicators/" % self.url 192 | params = { 193 | "api_key": self.key, 194 | "indicator": parsed_args.indicator, 195 | "indicator_class": parsed_args.class_type, 196 | "indicator_type_id": str(type_map[parsed_args.type]), 197 | "indicator_status_id": str(status_map[parsed_args.status]) 198 | } 199 | 200 | r = requests.post( 201 | query_url, 202 | params=params, 203 | verify=False 204 | ) 205 | 206 | if r.status_code != 200: 207 | log.error("[tqadd] Got back status code %s" % r.status_code) 208 | return { 209 | parsed_args.indicator: "Failed with error code %s" % ( 210 | r.status_code 211 | ) 212 | } 213 | 214 | rc = r.content 215 | if not rc: 216 | return "No reply" 217 | return rc 218 | 219 | def update_indicator_status(self, args): 220 | 221 | status_map = self._get_status_map() 222 | 223 | parser = argparse.ArgumentParser(usage="tqcs") 224 | parser.add_argument( 225 | "indicator_id", 226 | action="store", 227 | help="Specify the indicator ID to change the status of" 228 | ) 229 | 230 | parser.add_argument( 231 | "--class_type", 232 | action="store", 233 | choices=["network", "host"], 234 | required=True, 235 | help="indicator class" 236 | ) 237 | 238 | parser.add_argument( 239 | "--status", 240 | action="store", 241 | choices=status_map.keys(), 242 | required=True, 243 | help="indicator status to set indicator to" 244 | ) 245 | try: 246 | parsed_args = parser.parse_args(args=shlex.split(args)) 247 | except SystemExit, e: 248 | if str(e) != "0": 249 | log.error( 250 | "Invalid argument for query (use -h or --help " + 251 | "to see command options)" 252 | ) 253 | return {} 254 | 255 | query_url = "%s/api/indicators/%s" % ( 256 | self.url, 257 | parsed_args.indicator_id 258 | ) 259 | params = { 260 | "api_key": self.key, 261 | "indicator_class": parsed_args.class_type, 262 | "indicator_status_id": str(status_map[parsed_args.status]) 263 | } 264 | data = { 265 | "indicator_id": parsed_args.indicator_id 266 | } 267 | 268 | r = requests.put(query_url, data=data, params=params, verify=False) 269 | if r.status_code != 200: 270 | log.error( 271 | "[tqcs] Failed to update status - error code %s" % ( 272 | r.status_code 273 | ) 274 | ) 275 | return { 276 | parsed_args.indicator_id: "Failed with error code %s" % ( 277 | r.status_code 278 | ) 279 | } 280 | 281 | rc = r.content 282 | if not rc: 283 | return "No reply" 284 | return rc 285 | -------------------------------------------------------------------------------- /threatshell/commands/umbrella.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | import datetime 34 | import json 35 | import requests 36 | 37 | 38 | class Umbrella: 39 | 40 | def __init__(self, config): 41 | 42 | url = "https://s-platform.api.opendns.com/1.0" 43 | self.api_token = config.get("Umbrella", "api_token") 44 | self.block_endpoint = "%s/events" % url 45 | self.query_endpoint = "%s/domains" % url 46 | 47 | def list_blocked_domains(self): 48 | 49 | params = { 50 | "customerKey": self.api_token 51 | } 52 | 53 | res = requests.get( 54 | self.query_endpoint, 55 | params=params, 56 | headers={"Content-Type": "application/json"} 57 | ) 58 | 59 | if res.status_code != requests.codes.ok: 60 | return { 61 | "error": "[%s]: %s" % (res.status_code, res.content) 62 | } 63 | 64 | return res.json() 65 | 66 | def add_blocked_domain(self, url): 67 | 68 | if "/" in domain: 69 | domain = url.split("/")[2] 70 | 71 | params = { 72 | "customerKey": self.api_token 73 | } 74 | 75 | time_stamp = datetime.datetime.strftime( 76 | datetime.datetime.now(), 77 | '%Y-%m-%dT%H:%M:%S.0Z' 78 | ) 79 | 80 | body = { 81 | "alertTime": time_stamp, 82 | "deviceId": "cbc387aa-fffb-490e-a75a-0056f49eca11", 83 | "deviceVersion": "1.0", 84 | "dstDomain": domain, 85 | "dstUrl": url, 86 | "eventTime": time_stamp, 87 | "protocolVersion": "1.0a", 88 | "providerName": "Security Platform" 89 | } 90 | 91 | res = requests.post( 92 | self.block_endpoint, 93 | params=params, 94 | data=json.dumps(body), 95 | headers={"Content-Type": "application/json"} 96 | ) 97 | 98 | if res.status_code != requests.codes.ok: 99 | return { 100 | "error": "[%s]: %s" % (res.status_code, res.content) 101 | } 102 | 103 | def delete_blocked_domain(self, domain): 104 | 105 | params = { 106 | "customerKey": self.api_token, 107 | "where[name]": domain 108 | } 109 | 110 | res = requests.delete( 111 | self.query_endpoint, 112 | params=params, 113 | headers={"Content-Type": "application/json"} 114 | ) 115 | 116 | if res.status_code != requests.codes.ok: 117 | return { 118 | "error": "[%s]: %s" % (res.status_code, res.content) 119 | } 120 | 121 | return res.json() 122 | -------------------------------------------------------------------------------- /threatshell/common/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | -------------------------------------------------------------------------------- /threatshell/common/abstracts.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | import json 34 | import logging 35 | 36 | log = logging.getLogger(__name__) 37 | 38 | 39 | class JSONEntry(dict): 40 | 41 | def __setattr__(self, key, value): 42 | self.__dict__[key] = value 43 | dict.__setitem__(self, key, value) 44 | 45 | def __getattr__(self, key): 46 | return self.__dict__.get(key) 47 | dict.__getitem__(self, key) 48 | 49 | def __init__(self, jdata): 50 | 51 | if jdata is None: 52 | return 53 | 54 | for key in jdata.keys(): 55 | 56 | if not hasattr(self, key): 57 | log.warn( 58 | ( 59 | "API change detected! %s object has no " + 60 | "attribute '%s'" 61 | ) % (type(self).__name__, key) 62 | ) 63 | 64 | if( 65 | isinstance(getattr(self, key), list) and 66 | not isinstance(jdata[key], list) 67 | ): 68 | setattr(self, key, [jdata[key]]) 69 | else: 70 | setattr(self, key, jdata[key]) 71 | 72 | def __str__(self): 73 | return json.dumps(self, indent=4) 74 | 75 | def to_json(self): 76 | return str(self) 77 | -------------------------------------------------------------------------------- /threatshell/common/colors.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | import os 34 | import sys 35 | 36 | 37 | def color(text, color_code, readline=True): 38 | """Colorize text. 39 | @param text: text. 40 | @param color_code: color. 41 | @return: colorized text. 42 | """ 43 | # $TERM under Windows: 44 | # cmd.exe -> "" (what would you expect..?) 45 | # cygwin -> "cygwin" (should support colors, but doesn't work somehow) 46 | # mintty -> "xterm" (supports colors) 47 | if sys.platform == "win32" and os.getenv("TERM") != "xterm": 48 | return text 49 | if readline: 50 | # special readline escapes to fix colored input promps 51 | # http://bugs.python.org/issue17337 52 | return "\001\033[%dm\002%s\001\033[0m\002" % (color_code, text) 53 | return "\x1b[%dm%s\x1b[0m" % (color_code, text) 54 | 55 | 56 | def black(text, readline=True): 57 | return color(text, 30, readline) 58 | 59 | 60 | def red(text, readline=True): 61 | return color(text, 31, readline) 62 | 63 | 64 | def green(text, readline=True): 65 | return color(text, 32, readline) 66 | 67 | 68 | def yellow(text, readline=True): 69 | return color(text, 33, readline) 70 | 71 | 72 | def blue(text, readline=True): 73 | return color(text, 34, readline) 74 | 75 | 76 | def magenta(text, readline=True): 77 | return color(text, 35, readline) 78 | 79 | 80 | def cyan(text, readline=True): 81 | return color(text, 36, readline) 82 | 83 | 84 | def white(text, readline=True): 85 | return color(text, 37, readline) 86 | 87 | 88 | def bold(text, readline=True): 89 | return color(text, 1, readline) 90 | -------------------------------------------------------------------------------- /threatshell/common/config.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.common.colors import red, magenta, bold, blue 34 | from threatshell.common.constants import TS_ROOT, TS_DIR 35 | from threatshell.core.config import EncryptedConfigParser 36 | from threatshell.utils.keygen import KeyMaker 37 | 38 | from getpass import getpass 39 | 40 | import os 41 | 42 | 43 | class Config: 44 | 45 | def __init__(self, config_file="threatshell.ini"): 46 | """ 47 | @param config_file - a config file name to read or write 48 | 49 | throws IOException if reading the config file fails 50 | """ 51 | 52 | aes_key = os.path.join(TS_DIR, "aes_key.enc") 53 | rsa_key = os.path.join(TS_DIR, "conf_key.pem") 54 | 55 | self.target_path = os.path.join(TS_DIR, config_file) 56 | config_path = self.target_path 57 | edit_path = os.path.join(TS_DIR, "threatshell.txt") 58 | 59 | config = None 60 | 61 | if not os.path.exists(config_path): 62 | 63 | print red( 64 | ( 65 | "\nThis appears to be the first time you've ran " + 66 | "threatshell. Let's load the config defaults, then you " + 67 | "can setup the appropriate values. Make sure to have " + 68 | "your API keys handy ;)\n\n" 69 | ), 70 | readline=True 71 | ) 72 | config_path = os.path.join(TS_ROOT, "conf", "defaults.ini") 73 | key_maker = KeyMaker(rsa_key, "PEM", 2048) 74 | 75 | print "\n" 76 | print bold( 77 | red( 78 | "Building RSA key for encrypted config", 79 | readline=True 80 | ), 81 | readline=True 82 | ) 83 | key_maker.generate_key() 84 | 85 | while True: 86 | 87 | passphrase = getpass( 88 | bold( 89 | magenta("RSA key passphrase: ", readline=True), 90 | readline=True 91 | ) 92 | ) 93 | 94 | if passphrase: 95 | 96 | confirm = getpass( 97 | bold( 98 | magenta("RSA key passphrase: ", readline=True), 99 | readline=True 100 | ) 101 | ) 102 | 103 | if passphrase != confirm: 104 | print bold( 105 | red( 106 | "Passwords do not match! Please try again.\n", 107 | readline=True 108 | ), 109 | readline=True 110 | ) 111 | else: 112 | break 113 | else: 114 | break 115 | 116 | key_maker.save_key(passphrase) 117 | 118 | config = EncryptedConfigParser( 119 | allow_no_value=True, 120 | aes_key=aes_key, 121 | private_key=rsa_key 122 | ) 123 | 124 | read = config.read_raw(config_path) 125 | if not read: 126 | raise IOError("Failed to read config file '%s'" % config_path) 127 | self._configure(config) 128 | config.write(outfile=self.target_path) 129 | print red( 130 | "\n\nConfiguration complete - Happy hunting!", 131 | readline=True 132 | ) 133 | 134 | elif os.path.exists(edit_path): 135 | config = EncryptedConfigParser( 136 | allow_no_value=True, 137 | aes_key=aes_key, 138 | private_key=rsa_key 139 | ) 140 | print magenta( 141 | "Found decrypted config file - loading and encrypting...", 142 | readline=True 143 | ) 144 | read = config.read_raw(edit_path) 145 | if not read: 146 | raise IOError("Failed to read config file '%s'" % edit_path) 147 | config.write(outfile=self.target_path) 148 | os.unlink(edit_path) 149 | 150 | else: 151 | config = EncryptedConfigParser( 152 | allow_no_value=True, 153 | aes_key=aes_key, 154 | private_key=rsa_key 155 | ) 156 | config.read(self.target_path) 157 | 158 | self.config = config 159 | 160 | def _configured(self, config): 161 | 162 | for section in config.sections(): 163 | for key, value in config.items(section): 164 | if not value: 165 | return False 166 | 167 | return True 168 | 169 | def _configure(self, config): 170 | 171 | for section in config.sections(): 172 | print red("Configuring %s" % section, readline=True) 173 | for option, value in config.items(section): 174 | entry = raw_input( 175 | bold(magenta( 176 | "\t[%s] (default currently '%s'): " % ( 177 | option, 178 | str(value) 179 | ), 180 | readline=True 181 | )) 182 | ) 183 | if entry: 184 | value = entry 185 | if not value: 186 | value = "NOT SET" 187 | config.set(section, option, value) 188 | 189 | def get_int(self, section, option): 190 | """ 191 | Get the integer value of an existing option 192 | 193 | @param section - the config section name containing the option 194 | @param option - the option to be retrieved 195 | 196 | returns an integer value of the option 197 | throws NoOptionError if option is not in section 198 | throws ValueError if option value can't be converted to an integer 199 | """ 200 | return self.config.getint(section, option) 201 | 202 | def get_boolean(self, section, option): 203 | """ 204 | Get the boolean value of an existing option 205 | 206 | @param section - the config section name containing the option 207 | @param option - the option to be retrieved 208 | 209 | returns a boolean value of the option 210 | throws NoOptionError if option is not in section 211 | throws ValueError if option value can't be converted to a boolean 212 | """ 213 | return self.config.getboolean(section, option) 214 | 215 | def get(self, section, option): 216 | """ 217 | Get the raw string value of an option 218 | 219 | @param section - the config section name containing the option 220 | @param option - the option to be retrieved 221 | 222 | returns the raw string value of the option 223 | throws NoOptionError if option is not in section 224 | """ 225 | return self.config.get(section, option) 226 | 227 | def set_option(self, section, option, value): 228 | """ 229 | Creates or overwrites existing options 230 | 231 | @param section - the config section name that will contain the option 232 | @param option - the name of the option to create or modify 233 | @param value - the value for the given option to have 234 | """ 235 | if section not in self.config.sections(): 236 | resp = raw_input( 237 | bold( 238 | magenta( 239 | "Section %s does not exist. Create? (y/[n]): " % ( 240 | section 241 | ), 242 | readline=True 243 | ), 244 | readline=True 245 | ) 246 | ) 247 | if not resp or resp.lower() == "n": 248 | print red("Opted out of creating new section", readline=True) 249 | return 250 | else: 251 | self.config.add_section(section) 252 | self.config.set(section, option, value) 253 | print blue( 254 | "Added '%s' -> '%s' to new section '%s'" % ( 255 | option, 256 | value, 257 | section 258 | ), 259 | readline=True 260 | ) 261 | return 262 | 263 | if not self.config.has_option(section, option): 264 | resp = raw_input( 265 | bold( 266 | magenta( 267 | "%s does not have option %s. Create? (y/[n]): " % ( 268 | section, 269 | option 270 | ), 271 | readline=True 272 | ), 273 | readline=True 274 | ) 275 | ) 276 | if not resp or resp.lower() == "n": 277 | print red("Opted out of creating new option", readline=True) 278 | else: 279 | self.config.set(section, option, value) 280 | print blue( 281 | "Successfully added '%s' -> '%s'" % ( 282 | section, 283 | option 284 | ), 285 | readline=True 286 | ) 287 | 288 | else: 289 | self.config.set(section, option, value) 290 | 291 | def save_config(self): 292 | """ 293 | saves current configuration to file 294 | """ 295 | self.config.write(outfile=self.target_path) 296 | -------------------------------------------------------------------------------- /threatshell/common/constants.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | import os 34 | import subprocess 35 | 36 | __current_dir = os.path.abspath(os.path.dirname(__file__)) 37 | 38 | TS_ROOT = os.path.normpath(os.path.join(__current_dir, "..")) 39 | TS_DIR = os.path.join(os.path.expanduser("~"), ".threatshell") 40 | 41 | rev_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]) 42 | rev_count = subprocess.check_output(["git", "rev-list", "HEAD", "--count"]) 43 | 44 | TS_BUILD_VERSION = "1.0.%s-%s" % (rev_count.strip(), rev_hash.strip()) 45 | TS_VERSION = "1.0.%s" % (rev_count.strip()) 46 | 47 | if not os.path.exists(TS_DIR): 48 | os.mkdir(TS_DIR, 0o0700) 49 | -------------------------------------------------------------------------------- /threatshell/common/log.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.common.colors import red, yellow, green, cyan 34 | from threatshell.common.constants import TS_DIR 35 | 36 | from logging.handlers import RotatingFileHandler 37 | 38 | import copy 39 | import logging 40 | import os 41 | 42 | log = logging.getLogger() 43 | log.setLevel(logging.DEBUG) 44 | 45 | FORMAT = "%(asctime)s [%(levelname)s]: %(name)s - %(message)s" 46 | 47 | 48 | class ConsoleHandler(logging.StreamHandler): 49 | 50 | def emit(self, record): 51 | 52 | colored = copy.copy(record) 53 | 54 | if record.levelname == "WARNING": 55 | colored.msg = yellow(record.msg) 56 | 57 | elif record.levelname in ["ERROR", "CRITICAL", "FATAL"]: 58 | colored.msg = red(record.msg) 59 | 60 | elif record.levelname == "INFO": 61 | colored.msg = green(record.msg) 62 | 63 | else: 64 | colored.msg = cyan(record.msg) 65 | 66 | logging.StreamHandler.emit(self, colored) 67 | 68 | 69 | def init_file_logger(log_level=logging.DEBUG): 70 | 71 | formatter = logging.Formatter(FORMAT) 72 | 73 | log_path = "%s/logs/threatshell.log" % TS_DIR 74 | 75 | path_parts = [] 76 | start = 0 77 | if log_path[0] == os.path.sep: 78 | path_parts.append('') 79 | start += 1 80 | 81 | for path in log_path.split(os.path.sep)[start: -1]: 82 | 83 | path_parts.append(path) 84 | if not os.path.exists(os.path.sep.join(path_parts)): 85 | os.mkdir(os.path.sep.join(path_parts), 0o0700) 86 | 87 | file_h = RotatingFileHandler( 88 | log_path, 89 | mode="a", 90 | maxBytes=100 * 1000 * 1000, # 100MB 91 | backupCount=5 92 | ) 93 | file_h.setLevel(log_level) 94 | file_h.setFormatter(formatter) 95 | log.addHandler(file_h) 96 | 97 | 98 | def init_console_logger(log_level=logging.INFO): 99 | 100 | formatter = logging.Formatter(FORMAT) 101 | ch = ConsoleHandler() 102 | ch.setFormatter(formatter) 103 | ch.setLevel(log_level) 104 | log.addHandler(ch) 105 | 106 | 107 | def init_logging(con_level=logging.INFO, f_level=logging.DEBUG): 108 | 109 | init_file_logger(f_level) 110 | init_console_logger(con_level) 111 | -------------------------------------------------------------------------------- /threatshell/conf/defaults.ini: -------------------------------------------------------------------------------- 1 | [PassiveTotal] 2 | username = 3 | key = 4 | 5 | [Umbrella] 6 | api_token = 7 | 8 | [ThreatQ] 9 | url = 10 | key = 11 | 12 | [Infoblox] 13 | username = 14 | password = 15 | url = 16 | 17 | [GeoIP] 18 | UserId = 999999 19 | LicenseKey = 000000000000 20 | ProductIds = 506, 517, 533 21 | 22 | [OpenDNS] 23 | api_token = 24 | 25 | [Novetta] 26 | key = 27 | 28 | [ThreatExchange] 29 | app_secret = 30 | app_id = 31 | 32 | [ElasticSearch] 33 | servers = http://localhost:9200 34 | 35 | [RiskIQ] 36 | token = 37 | key = 38 | -------------------------------------------------------------------------------- /threatshell/core/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | -------------------------------------------------------------------------------- /threatshell/core/config.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.common.colors import bold, magenta 34 | 35 | from Crypto import Random 36 | from Crypto.Cipher import AES 37 | from Crypto.PublicKey import RSA 38 | from ConfigParser import RawConfigParser 39 | from getpass import getpass 40 | from StringIO import StringIO 41 | 42 | import os 43 | import struct 44 | 45 | 46 | class Error(Exception): 47 | 48 | def _get_message(self): 49 | return self.__message 50 | 51 | def _set_message(self, message): 52 | self.__message = message 53 | 54 | message = property(_get_message, _set_message) 55 | 56 | def __init__(self, msg=""): 57 | self.message = msg 58 | Exception.__init__(self, msg) 59 | 60 | def __repr__(self): 61 | return self.message 62 | 63 | __str__ = __repr__ 64 | 65 | 66 | class KeyNotFoundError(Error): 67 | 68 | def __init__(self, message="Private key not found"): 69 | Error.__init__(self, message) 70 | 71 | 72 | class ConfigDecryptionError(Error): 73 | 74 | def __init__(self): 75 | Error.__init__(self, "Config decrytption failed") 76 | 77 | 78 | class AESKeygen: 79 | 80 | def __init__(self, key=None, iv=None): 81 | 82 | if key is None: 83 | key = Random.new().read(32) 84 | 85 | if iv is None: 86 | iv = Random.new().read(AES.block_size) 87 | 88 | self.aes_key = AES.new(key, AES.MODE_CBC, IV=iv) 89 | self.key = key 90 | self.iv = iv 91 | 92 | def get_key(self): 93 | return self.key 94 | 95 | def get_iv(self): 96 | return self.iv 97 | 98 | def get_encryptor(self): 99 | return self.aes_key 100 | 101 | 102 | class EncryptedConfigParser(RawConfigParser): 103 | 104 | def __init__( 105 | self, 106 | defaults=None, 107 | dict_type=dict, 108 | allow_no_value=False, 109 | private_key="conf_key.private.pem", 110 | aes_key="aes_key.enc" 111 | ): 112 | 113 | RawConfigParser.__init__( 114 | self, 115 | defaults, 116 | dict_type, 117 | allow_no_value 118 | ) 119 | 120 | pkey_data = None 121 | try: 122 | pkey_h = open(private_key, "rb") 123 | pkey_data = pkey_h.read() 124 | pkey_h.close() 125 | except IOError: 126 | raise KeyNotFoundError() 127 | 128 | self.private_key = RSA.importKey( 129 | pkey_data, 130 | passphrase=getpass( 131 | bold( 132 | magenta("RSA key passphrase: ", readline=True), 133 | readline=True 134 | ) 135 | ) 136 | ) 137 | self.public_key = self.private_key.publickey() 138 | self.aes_key_name = aes_key 139 | 140 | def _decrypt_config(self, key, fhandle, chunksize=1024): 141 | 142 | orig_size = struct.unpack(' 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | -------------------------------------------------------------------------------- /threatshell/doctypes/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | -------------------------------------------------------------------------------- /threatshell/doctypes/cymru.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.doctypes.generic import ( 34 | convert_cidr, 35 | GenericDoc, 36 | ThreatshellIndex 37 | ) 38 | from elasticsearch_dsl import ( 39 | Date, 40 | Index, 41 | Integer, 42 | Ip, 43 | String 44 | ) 45 | 46 | 47 | @ThreatshellIndex.doc_type 48 | class CymruASInfoDoc(GenericDoc): 49 | 50 | class Meta: 51 | doc_type = "cymru_asinfo" 52 | 53 | allocation_date = Date() 54 | asnum = Integer() 55 | ip = Ip() 56 | asname = String() 57 | prefix = String() 58 | registry = String() 59 | country = String() 60 | 61 | # def __setattr__(self, key, value): 62 | # if key == "prefix": 63 | # value = convert_cidr(value) 64 | # super(CymruASNumInfoDoc, self).__setattr__(key, value) 65 | 66 | def __init__(self, jdata={}): 67 | GenericDoc.__init__(self) 68 | for k, v in jdata.items(): 69 | if v is None: 70 | v = {} 71 | setattr(self, k, v) 72 | 73 | 74 | @ThreatshellIndex.doc_type 75 | class CymruASNumInfoDoc(GenericDoc): 76 | 77 | class Meta: 78 | doc_type = "cymru_asnum_info" 79 | 80 | country = String() 81 | allocation_date = Date() 82 | asname = String() 83 | asnum = Integer() 84 | registry = String() 85 | 86 | def __init__(self, jdata={}): 87 | GenericDoc.__init__(self) 88 | for k, v in jdata.items(): 89 | if v is None: 90 | v = {} 91 | setattr(self, k, v) 92 | -------------------------------------------------------------------------------- /threatshell/doctypes/generic.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from datetime import date, datetime 34 | from dateutil import parser as date_parser 35 | from elasticsearch_dsl import ( 36 | analyzer, 37 | Boolean, 38 | Date, 39 | DocType, 40 | Index, 41 | Field, 42 | String, 43 | tokenizer, 44 | token_filter 45 | ) 46 | from netaddr import IPNetwork 47 | import json 48 | 49 | # TODO: Add some analyzers to help coerce data into the correct types 50 | # e.g. a location array where geo points are mixed between floats and longs 51 | 52 | ThreatshellIndex = Index("threatshell") 53 | 54 | email_filter = token_filter( 55 | "email_token_filter", 56 | type="pattern_capture", 57 | preserve_original=True, 58 | patterns=[ 59 | "([^@]+)", 60 | "(\\p{L}+)", 61 | "(\\d+)", 62 | "@(.+)", 63 | "([^-@]+)" 64 | ] 65 | ) 66 | 67 | email_analyzer = analyzer( 68 | "email_analyzer", 69 | tokenizer="uax_url_email", 70 | filter=[email_filter, "lowercase", "unique"], 71 | type="custom" 72 | ) 73 | 74 | hostname_analyzer = analyzer( 75 | "hostname", 76 | tokenizer=tokenizer( 77 | "hostname_tokenizer", 78 | "pattern", 79 | pattern="([^.]+)", 80 | group=0 81 | ), 82 | filter=["lowercase"], 83 | type="custom" 84 | ) 85 | 86 | 87 | def convert_cidr(cidr): 88 | network = IPNetwork(cidr) 89 | return [str(x) for x in list(network)] 90 | 91 | 92 | class GenericDoc(DocType): 93 | 94 | session_uuid = String() 95 | timestamp = Date() 96 | successful = Boolean() 97 | term = String() 98 | 99 | class Meta: 100 | index = "threatshell" 101 | 102 | def save(self, **kwargs): 103 | self.timestamp = datetime.now() 104 | return super(GenericDoc, self).save(**kwargs) 105 | 106 | def to_json(self): 107 | 108 | def convert(x): 109 | return date_parser.parse(x) 110 | 111 | def default(x): 112 | 113 | try: 114 | dt = convert(x) 115 | return str(dt) 116 | except: 117 | pass 118 | 119 | if( 120 | hasattr(x, "_d_") and 121 | getattr(x, "_d_") is not None and 122 | getattr(x, "_d_") != {} 123 | ): 124 | return x._d_ 125 | 126 | return str(x) 127 | 128 | return json.dumps(self, indent=4, default=default) 129 | 130 | def __eq__(self, other): 131 | 132 | self_json = self.to_json() 133 | other_json = other.to_json() 134 | 135 | return self_json == other_json 136 | 137 | 138 | class BetterDate(Field): 139 | 140 | name = 'date' 141 | _coerce = True 142 | 143 | def _deserialize(self, data): 144 | 145 | if not data: 146 | return None 147 | 148 | if isinstance(data, date): 149 | return data 150 | 151 | try: 152 | return date_parser.parse(data) 153 | except: 154 | pass 155 | 156 | try: 157 | return datetime.fromtimestamp(data) 158 | except: 159 | # Might be an elasticsearch formatter for it so don't 160 | # error out just yet. 161 | return data 162 | -------------------------------------------------------------------------------- /threatshell/doctypes/geoip.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.doctypes.generic import GenericDoc, ThreatshellIndex 34 | from elasticsearch_dsl import ( 35 | GeoPoint, 36 | String, 37 | Integer 38 | ) 39 | 40 | 41 | @ThreatshellIndex.doc_type 42 | class GeoCityDoc(GenericDoc): 43 | 44 | class Meta: 45 | doc_type = "maxmind_geo_city" 46 | 47 | city = String() 48 | region_name = String() 49 | region = String() 50 | area_code = Integer() 51 | time_zone = String() 52 | location = GeoPoint() 53 | metro_code = Integer() 54 | country_code = String() 55 | postal_code = Integer() 56 | dma_code = Integer() 57 | country_code3 = String() 58 | country_name = String() 59 | 60 | def __init__(self, jdata={}): 61 | GenericDoc.__init__(self) 62 | for k, v in jdata.items(): 63 | if v is None: 64 | v = {} 65 | setattr(self, k, v) 66 | 67 | 68 | @ThreatshellIndex.doc_type 69 | class GeoASNDoc(GenericDoc): 70 | 71 | class Meta: 72 | doc_type = "maxmind_geo_asn" 73 | 74 | asnum = Integer() 75 | asname = String() 76 | 77 | def __init__(self, jdata={}): 78 | GenericDoc.__init__(self) 79 | for k, v in jdata.items(): 80 | if v is None: 81 | v = {} 82 | setattr(self, k, v) 83 | 84 | 85 | @ThreatshellIndex.doc_type 86 | class GeoIpASNDoc(GeoASNDoc): 87 | 88 | class Meta: 89 | doc_type = "maxmind_geo_ip_asn" 90 | 91 | ip_allocation = String() 92 | 93 | def __init__(self, jdata={}): 94 | GeoASNDoc.__init__(self, jdata=jdata) 95 | 96 | 97 | @ThreatshellIndex.doc_type 98 | class GeoCountryCodeDoc(GenericDoc): 99 | 100 | class Meta: 101 | doc_type = "maxmind_geo_country_code" 102 | 103 | country_code = String() 104 | 105 | def __init__(self, jdata={}): 106 | GenericDoc.__init__(self) 107 | for k, v in jdata.items(): 108 | if v is None: 109 | v = {} 110 | setattr(self, k, v) 111 | 112 | 113 | @ThreatshellIndex.doc_type 114 | class GeoCountryNameDoc(GenericDoc): 115 | 116 | class Meta: 117 | doc_type = "maxmind_geo_count_name" 118 | 119 | country_name = String() 120 | 121 | def __init__(self, jdata={}): 122 | GenericDoc.__init__(self) 123 | for k, v in jdata.items(): 124 | if v is None: 125 | v = {} 126 | setattr(self, k, v) 127 | -------------------------------------------------------------------------------- /threatshell/doctypes/opendns.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.doctypes.generic import ( 34 | convert_cidr, 35 | BetterDate, 36 | email_analyzer, 37 | GenericDoc, 38 | ThreatshellIndex 39 | ) 40 | from elasticsearch_dsl import( 41 | Boolean, 42 | Date, 43 | Float, 44 | GeoPoint, 45 | Index, 46 | Integer, 47 | Ip, 48 | Nested, 49 | String, 50 | analyzer, 51 | char_filter 52 | ) 53 | import json 54 | 55 | 56 | status_filter = char_filter( 57 | "opendns_status_filter", 58 | type="mapping", 59 | mappings=[ 60 | "1 => benign", 61 | "0 => unknown", 62 | "-1 => malicious" 63 | ] 64 | ) 65 | 66 | rir_filter = char_filter( 67 | "opendns_rir_filter", 68 | type="mapping", 69 | mappings=[ 70 | "0 => Unknown", 71 | "1 => AfriNIC", 72 | "2 => APNIC", 73 | "3 => Arin", 74 | "4 => LACNIC", 75 | "5 => RIPE" 76 | ] 77 | ) 78 | 79 | status_analyzer = analyzer( 80 | "opendns_status_analyzer", 81 | tokenizer="standard", 82 | char_filter=[status_filter] 83 | ) 84 | 85 | rir_analyzer = analyzer( 86 | "opendns_rir_analyzer", 87 | tokenizer="standard", 88 | char_filter=[rir_filter], 89 | filter=["lowercase"] 90 | ) 91 | 92 | 93 | @ThreatshellIndex.doc_type 94 | class DomainCategorizationDoc(GenericDoc): 95 | 96 | class Meta: 97 | doc_type = "opendns_domain_catigorization" 98 | 99 | status = String(analyzer=status_analyzer) 100 | security_categories = String(analyzer=status_analyzer) 101 | content_categories = String() 102 | 103 | def __init__(self, jdata={}): 104 | GenericDoc.__init__(self) 105 | for k, v in jdata.items(): 106 | if v is None: 107 | v = {} 108 | setattr(self, k, v) 109 | 110 | 111 | @ThreatshellIndex.doc_type 112 | class DomainScoreDoc(GenericDoc): 113 | 114 | class Meta: 115 | doc_type = "opendns_domain_score" 116 | 117 | domain = String() 118 | status = String(analyzer=status_analyzer) 119 | 120 | def __init__(self, jdata={}): 121 | GenericDoc.__init__(self) 122 | for k, v in jdata.items(): 123 | if v is None: 124 | v = {} 125 | 126 | setattr(self, k, v) 127 | 128 | 129 | @ThreatshellIndex.doc_type 130 | class PFS2Doc(GenericDoc): 131 | 132 | class Meta: 133 | doc_type = "opendns_co_occur_pfs2" 134 | 135 | domain = String() 136 | score = Float() 137 | 138 | 139 | @ThreatshellIndex.doc_type 140 | class CoOccurrencesDoc(GenericDoc): 141 | 142 | class Meta: 143 | doc_type = "opendns_co_occurrences" 144 | 145 | pfs2 = Nested( 146 | doc_class=PFS2Doc 147 | ) 148 | 149 | found = Boolean() 150 | 151 | def __init__(self, jdata={}): 152 | GenericDoc.__init__(self) 153 | for k, v in jdata.items(): 154 | if v is None: 155 | v = {} 156 | 157 | setattr(self, k, v) 158 | 159 | 160 | @ThreatshellIndex.doc_type 161 | class TB1Doc(GenericDoc): 162 | 163 | class Meta: 164 | doc_type = "opendns_related_domains_tb1" 165 | 166 | domain = String() 167 | score = Float() 168 | 169 | 170 | @ThreatshellIndex.doc_type 171 | class RelatedDomainsDoc(GenericDoc): 172 | 173 | class Meta: 174 | doc_type = "opendns_related_domains" 175 | 176 | tb1 = Nested( 177 | doc_class=TB1Doc 178 | ) 179 | found = Boolean() 180 | 181 | def __init__(self, jdata={}): 182 | GenericDoc.__init__(self) 183 | for k, v in jdata.items(): 184 | if v is None: 185 | v = {} 186 | 187 | setattr(self, k, v) 188 | 189 | 190 | @ThreatshellIndex.doc_type 191 | class GeoDiversityDoc(GenericDoc): 192 | 193 | class Meta: 194 | doc_type = "opendns_security_info_geodiversity" 195 | 196 | country_code = String() 197 | score = Float() 198 | 199 | 200 | @ThreatshellIndex.doc_type 201 | class DomainSecurityInfoDoc(GenericDoc): 202 | 203 | class Meta: 204 | doc_type = "opendns_security_info" 205 | 206 | dga_score = Float() 207 | perplexity = Float() 208 | entropy = Float() 209 | securerank2 = Float() 210 | pagerank = Float() 211 | asn_score = Float() 212 | prefix_score = Float() 213 | rip_score = Float() 214 | fastflux = Boolean() 215 | popularity = Float() 216 | geodiversity = Nested( 217 | doc_class=GeoDiversityDoc 218 | ) 219 | geodiversity_normalized = Nested( 220 | doc_class=GeoDiversityDoc 221 | ) 222 | tld_geodiversity = Nested( 223 | doc_class=GeoDiversityDoc 224 | ) 225 | geoscore = Float() 226 | ks_test = Float() 227 | attack = String() 228 | threat_type = String() 229 | found = Boolean() 230 | 231 | def __init__(self, jdata={}): 232 | GenericDoc.__init__(self) 233 | for k, v in jdata.items(): 234 | if v is None: 235 | v = {} 236 | setattr(self, k, v) 237 | 238 | 239 | @ThreatshellIndex.doc_type 240 | class ResourceRecordDoc(GenericDoc): 241 | 242 | class Meta: 243 | doc_type = "opendns_resource_record" 244 | 245 | name = String() 246 | ttl = Integer() 247 | class_type = String() 248 | query_type = String() 249 | rr = Ip() 250 | 251 | 252 | @ThreatshellIndex.doc_type 253 | class DomainResourceRecordDoc(GenericDoc): 254 | 255 | class Meta: 256 | doc_type = "opendns_domain_resource_record" 257 | 258 | first_seen = Date() 259 | last_seen = Date() 260 | rrs = Nested( 261 | doc_class=ResourceRecordDoc 262 | ) 263 | 264 | 265 | @ThreatshellIndex.doc_type 266 | class DomainFeaturesDoc(GenericDoc): 267 | 268 | class Meta: 269 | doc_type = "opendns_domain_features" 270 | 271 | age = Integer() 272 | ttls_min = Integer() 273 | ttls_max = Integer() 274 | ttls_mean = Float() 275 | ttls_median = Float() 276 | ttls_stddev = Float() 277 | country_codes = String() 278 | country_count = Integer() 279 | asns = Integer() 280 | asns_count = Integer() 281 | prefixes = String() 282 | prefix_count = Integer() 283 | rips = Integer() 284 | div_rips = Float() 285 | locations = GeoPoint() 286 | locations_count = Integer() 287 | geo_distance_sum = Float() 288 | geo_distance_mean = Float() 289 | non_routable = Boolean() 290 | mail_exchanger = Boolean() 291 | cname = Boolean() 292 | ff_candidate = Boolean() 293 | rips_stability = Float() 294 | base_domain = String() 295 | is_subdomain = Boolean() 296 | 297 | 298 | @ThreatshellIndex.doc_type 299 | class DomainResourceRecordHistoryDoc(GenericDoc): 300 | 301 | class Meta: 302 | doc_type = "opendns_domain_resource_record_history" 303 | 304 | rrs_tf = Nested( 305 | doc_class=DomainResourceRecordDoc 306 | ) 307 | features = Nested( 308 | doc_class=DomainFeaturesDoc 309 | ) 310 | 311 | def __init__(self, jdata={}): 312 | GenericDoc.__init__(self) 313 | for k, v in jdata.items(): 314 | if v is None: 315 | v = {} 316 | setattr(self, k, v) 317 | 318 | 319 | @ThreatshellIndex.doc_type 320 | class IPFeaturesDoc(GenericDoc): 321 | 322 | class Meta: 323 | doc_type = "opendns_ip_features" 324 | 325 | rr_count = Integer() 326 | ld2_count = Integer() 327 | ld3_count = Integer() 328 | ld2_1_count = Integer() 329 | ld2_2_count = Integer() 330 | div_ld2 = Float() 331 | div_ld3 = Float() 332 | div_ld2_1 = Float() 333 | div_ld2_2 = Float() 334 | 335 | 336 | @ThreatshellIndex.doc_type 337 | class IPResourceRecordHistoryDoc(GenericDoc): 338 | 339 | class Meta: 340 | doc_type = "opendns_ip_resource_record_history" 341 | 342 | rrs = Nested( 343 | doc_class=ResourceRecordDoc 344 | ) 345 | features = Nested( 346 | doc_class=IPFeaturesDoc 347 | ) 348 | 349 | def __init__(self, jdata={}): 350 | GenericDoc.__init__(self) 351 | for k, v in jdata.items(): 352 | if v is None: 353 | v = {} 354 | 355 | setattr(self, k, v) 356 | 357 | 358 | @ThreatshellIndex.doc_type 359 | class ASInformationDoc(GenericDoc): 360 | 361 | class Meta: 362 | doc_type = "opendns_asn_information" 363 | 364 | creation_date = Date() 365 | ir = String(analyzer=rir_analyzer) 366 | description = String() 367 | asn = Integer() 368 | cidr = String() 369 | 370 | def __init__(self, jdata={}): 371 | GenericDoc.__init__(self) 372 | for k, v in jdata.items(): 373 | if v is None: 374 | v = {} 375 | setattr(self, k, v) 376 | 377 | 378 | @ThreatshellIndex.doc_type 379 | class ASPrefixInformationDoc(GenericDoc): 380 | 381 | class Meta: 382 | doc_type = "opendns_asn_prefix_information" 383 | 384 | cidr = String() 385 | geo = Nested( 386 | properties={ 387 | "country_name": String(), 388 | "country_code": Integer() 389 | } 390 | ) 391 | 392 | def __init__(self, jdata={}): 393 | GenericDoc.__init__(self) 394 | for k, v in jdata.items(): 395 | if v is None: 396 | v = {} 397 | setattr(self, k, v) 398 | 399 | 400 | @ThreatshellIndex.doc_type 401 | class WhoisEmailToDomainDoc(GenericDoc): 402 | 403 | class Meta: 404 | doc_type = "opendns_whois_email_to_domain" 405 | 406 | domain = String() 407 | email = String(analyzer=email_analyzer) 408 | current = Boolean() 409 | 410 | def __init__(self, jdata={}): 411 | GenericDoc.__init__(self) 412 | for k, v in jdata.items(): 413 | if v is None: 414 | v = {} 415 | setattr(self, k, v) 416 | 417 | 418 | @ThreatshellIndex.doc_type 419 | class WhoisNameServerToDomainDoc(GenericDoc): 420 | 421 | class Meta: 422 | doc_type = "opendns_whois_nameserver_to_domain" 423 | 424 | nameserver = String() 425 | domain = String() 426 | current = Boolean() 427 | 428 | def __init__(self, jdata={}): 429 | GenericDoc.__init__(self) 430 | for k, v in jdata.items(): 431 | if v is None: 432 | v = {} 433 | setattr(self, k, v) 434 | 435 | 436 | @ThreatshellIndex.doc_type 437 | class WhoisDomainRecordDoc(GenericDoc): 438 | 439 | class Meta: 440 | doc_type = "opendns_whois_record" 441 | 442 | addresses = String() 443 | administrativeContactCity = String() 444 | administrativeContactCountry = String() 445 | administrativeContactEmail = String(analyzer=email_analyzer) 446 | administrativeContactFax = String() 447 | administrativeContactFaxExt = String() 448 | administrativeContactName = String() 449 | administrativeContactOrganization = String() 450 | administrativeContactPostalCode = String() 451 | administrativeContactState = String() 452 | administrativeContactStreet = String() 453 | administrativeContactTelephone = String() 454 | administrativeContactTelephoneExt = String() 455 | auditUpdatedDate = Date() 456 | billingContactCity = String() 457 | billingContactCountry = String() 458 | billingContactEmail = String(analyzer=email_analyzer) 459 | billingContactFax = String() 460 | billingContactFaxExt = String() 461 | billingContactName = String() 462 | billingContactOrganization = String() 463 | billingContactPostalCode = String() 464 | billingContactState = String() 465 | billingContactStreet = String() 466 | billingContactTelephone = String() 467 | billingContactTelephoneExt = String() 468 | created = Date() 469 | domainName = String() 470 | emails = String(analyzer=email_analyzer) 471 | expires = Date() 472 | hasRawText = Boolean() 473 | nameServers = String() 474 | recordExpired = Boolean() 475 | registrantCity = String() 476 | registrantCountry = String() 477 | registrantEmail = String(analyzer=email_analyzer) 478 | registrantFax = String() 479 | registrantFaxExt = String() 480 | registrantName = String() 481 | registrantOrganization = String() 482 | registrantPostalCode = String() 483 | registrantState = String() 484 | registrantStreet = String() 485 | registrantTelephone = String() 486 | registrantTelephoneExt = String() 487 | registrarIANAID = Integer() 488 | registrarName = String() 489 | record_status = String() 490 | technicalContactCity = String() 491 | technicalContactCountry = String() 492 | technicalContactEmail = String(analyzer=email_analyzer) 493 | technicalContactFax = String() 494 | technicalContactFaxExt = String() 495 | technicalContactName = String() 496 | technicalContactOrganization = String() 497 | technicalContactPostalCode = String() 498 | technicalContactState = String() 499 | technicalContactStreet = String() 500 | technicalContactTelephone = String() 501 | technicalContactTelephoneExt = String() 502 | timeOfLatestRealtimeCheck = BetterDate(format="epoch_millis") 503 | timestamp = Date() 504 | updated = Date() 505 | whoisServers = String() 506 | zoneContactCity = String() 507 | zoneContactCountry = String() 508 | zoneContactEmail = String(analyzer=email_analyzer) 509 | zoneContactFax = String() 510 | zoneContactFaxExt = String() 511 | zoneContactName = String() 512 | zoneContactOrganization = String() 513 | zoneContactPostalCode = String() 514 | zoneContactState = String() 515 | zoneContactStreet = String() 516 | zoneContactTelephone = String() 517 | zoneContactTelephoneExt = String() 518 | 519 | def __init__(self, jdata={}): 520 | GenericDoc.__init__(self) 521 | for k, v in jdata.items(): 522 | if v is None: 523 | v = {} 524 | setattr(self, k, v) 525 | 526 | 527 | @ThreatshellIndex.doc_type 528 | class LatestMaliciousDomsDoc(GenericDoc): 529 | 530 | class Meta: 531 | doc_type = "opendns_latest_malicious_domains" 532 | 533 | domain_id = Integer() 534 | name = String() 535 | 536 | def __init__(self, jdata={}): 537 | GenericDoc.__init__(self) 538 | for k, v in jdata.items(): 539 | setattr(self, k, v) 540 | -------------------------------------------------------------------------------- /threatshell/doctypes/shadow_server.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.doctypes.generic import GenericDoc, ThreatshellIndex 34 | from elasticsearch_dsl import ( 35 | Ip, 36 | Integer, 37 | Index, 38 | String 39 | ) 40 | 41 | 42 | @ThreatshellIndex.doc_type 43 | class ASOriginDoc(GenericDoc): 44 | 45 | class Meta: 46 | doc_type = "shadowserver_as_origin" 47 | 48 | domain = String() 49 | asnum = Integer() 50 | country = String() 51 | isp = String() 52 | prefix = String() 53 | asname = String() 54 | 55 | # def __setattr__(self, key, value): 56 | # if key == "prefix": 57 | # value = convert_cidr(value) 58 | # super(ASOriginDoc, self).__setattr__(key, value) 59 | 60 | def __init__(self, jdata={}): 61 | GenericDoc.__init__(self) 62 | for k, v in jdata.items(): 63 | if v is None: 64 | v = {} 65 | setattr(self, k, v) 66 | 67 | 68 | @ThreatshellIndex.doc_type 69 | class ASPeersDoc(GenericDoc): 70 | 71 | class Meta: 72 | doc_type = "shadowserver_as_peers" 73 | 74 | peers = Integer() 75 | asnum = Integer() 76 | country = String() 77 | isp = String() 78 | domain = String() 79 | prefix = String() 80 | asname = String() 81 | 82 | # def __setattr__(self, key, value): 83 | # if key == "prefix": 84 | # value = convert_cidr(value) 85 | 86 | def __init__(self, jdata={}): 87 | GenericDoc.__init__(self) 88 | for k, v in jdata.items(): 89 | if v is None: 90 | v = {} 91 | setattr(self, k, v) 92 | 93 | 94 | @ThreatshellIndex.doc_type 95 | class ASPrefixDoc(GenericDoc): 96 | 97 | class Meta: 98 | doc_type = "shadowserver_asnum_prefix" 99 | 100 | prefixes = String() 101 | 102 | # def __setattr__(self, key, value): 103 | # 104 | # if key == "prefixes": 105 | # 106 | # if not isinstance(value, list): 107 | # value = [value] 108 | # 109 | # ips = [] 110 | # for cidr in value: 111 | # ips.extend(convert_cidr(cidr)) 112 | 113 | def __init__(self, jdata={}): 114 | GenericDoc.__init__(self) 115 | for k, v in jdata.items(): 116 | if v is None: 117 | v = {} 118 | setattr(self, k, v) 119 | -------------------------------------------------------------------------------- /threatshell/utils/__init__.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | -------------------------------------------------------------------------------- /threatshell/utils/argparse_utils.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | import argparse 34 | import time 35 | 36 | 37 | def validate_datetime(dt): 38 | 39 | try: 40 | time.strptime(dt, "%m-%d-%Y %H:%M:%S") 41 | except Exception, e: 42 | raise argparse.ArgumentTypeError( 43 | ( 44 | "%s does not match the format of MM-DD-YYYY HH:MM:SS" + 45 | " - Exception info: [%s] - %s" 46 | ) % (dt, e.__class__.__name__, e.message) 47 | ) 48 | 49 | return dt 50 | 51 | 52 | class ConvertDateTimeAction(argparse.Action): 53 | 54 | def __init__( 55 | self, 56 | option_strings, 57 | dest, 58 | nargs=None, 59 | const=None, 60 | default=None, 61 | type=None, 62 | choices=None, 63 | required=False, 64 | help=None, 65 | metavar=None 66 | ): 67 | 68 | default = self._make_time_epoch(default) 69 | argparse.Action.__init__( 70 | self, 71 | option_strings=option_strings, 72 | dest=dest, 73 | nargs=nargs, 74 | const=const, 75 | default=default, 76 | type=type, 77 | choices=choices, 78 | required=required, 79 | help=help, 80 | metavar=metavar, 81 | ) 82 | 83 | def _make_time_epoch(self, dt): 84 | try: 85 | return int( 86 | time.mktime( 87 | time.strptime( 88 | dt, 89 | "%m-%d-%Y %H:%M:%S" 90 | ) 91 | ) 92 | ) 93 | except: 94 | return None 95 | 96 | def __call__(self, parser, namespace, values, option_string=None): 97 | 98 | datetime_stamp = None 99 | if isinstance(values, list): 100 | datetime_stamp = values[0] 101 | else: 102 | datetime_stamp = values 103 | 104 | time_epoch = self._make_time_epoch(datetime_stamp) 105 | 106 | setattr(namespace, self.dest, time_epoch) 107 | -------------------------------------------------------------------------------- /threatshell/utils/keygen.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | # Author: Tommy Stallings 3 | # Copyright (c) 2016, Salesforce.com, Inc. 4 | # All rights reserved. 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, 10 | # this list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright 13 | # notice, this list of conditions and the following disclaimer in the 14 | # documentation and/or other materials provided with the distribution. 15 | # 16 | # * Neither the name of Salesforce.com nor the names of its contributors 17 | # may be used to endorse or promote products derived from this software 18 | # without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 24 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 | # POSSIBILITY OF SUCH DAMAGE. 31 | ############################################################################## 32 | 33 | from threatshell.common.colors import magenta, red, bold 34 | 35 | from Crypto.PublicKey import RSA 36 | from getpass import getpass 37 | import argparse 38 | 39 | 40 | class KeyMaker: 41 | 42 | def __init__( 43 | self, 44 | keyname, 45 | export_format, 46 | key_size=2048 47 | ): 48 | self.keyname = keyname 49 | self.format = export_format 50 | self.key_size = key_size 51 | 52 | def generate_key(self): 53 | self.key = RSA.generate(int(self.key_size)) 54 | 55 | def save_key(self, passphrase): 56 | f = open(self.keyname, "wb") 57 | 58 | if passphrase is None or not passphrase: 59 | response = raw_input( 60 | red( 61 | "No password provided - Enter a password? ([y]/n): ", 62 | readline=True 63 | ) 64 | ).strip().lower() 65 | 66 | if response == "y": 67 | 68 | while True: 69 | 70 | passphrase = getpass( 71 | bold( 72 | magenta( 73 | "Password: ", 74 | readline=True 75 | ), 76 | readline=True 77 | ) 78 | ) 79 | confirm = getpass( 80 | bold( 81 | magenta( 82 | "Confirm password: ", 83 | readline=True 84 | ), 85 | readline=True 86 | ) 87 | ) 88 | 89 | if passphrase != confirm: 90 | print bold( 91 | red( 92 | "Passwords don't match! Please try again.\n", 93 | readline=True 94 | ) 95 | ) 96 | else: 97 | break 98 | 99 | f.write( 100 | self.key.exportKey( 101 | format=self.format, 102 | passphrase=passphrase, 103 | ) 104 | ) 105 | 106 | f.close() 107 | 108 | 109 | def main(args): 110 | 111 | keyname = "%s.%s.prv" % (args.user.lower(), args.export_format.lower()) 112 | key_maker = KeyMaker(keyname, args.export_format, args.key_size) 113 | key_maker.generate_key() 114 | key_maker.save_key(args.passphrase) 115 | 116 | print "done!" 117 | 118 | if __name__ == "__main__": 119 | 120 | parser = argparse.ArgumentParser( 121 | description="Generate an RSA keypair for use with Alerter" 122 | ) 123 | 124 | parser.add_argument( 125 | "--user", 126 | action="store", 127 | required=True, 128 | help="The user name of the recipient of the keys" 129 | ) 130 | 131 | parser.add_argument( 132 | "--passphrase", 133 | action="store", 134 | required=False, 135 | help="The pass phrase to encrypt the private key with" 136 | ) 137 | 138 | parser.add_argument( 139 | "--key-size", 140 | action="store", 141 | required=False, 142 | default=2048, 143 | help="The key strength in bits" 144 | ) 145 | 146 | parser.add_argument( 147 | "--export-format", 148 | action="store", 149 | choices=["DER", "PEM", "OpenSSH"], 150 | required=False, 151 | default="PEM", 152 | help="The format to export the keys in" 153 | ) 154 | 155 | args = parser.parse_args() 156 | main(args) 157 | --------------------------------------------------------------------------------