├── favicon.ico ├── app ├── core │ ├── __init__.py │ ├── blueprints │ │ ├── ajax.py │ │ ├── admin.py │ │ ├── index.py │ │ ├── ttp.py │ │ └── user.py │ └── decorators │ │ └── authentication.py ├── utils │ ├── __init__.py │ ├── functions.py │ └── elasticsearch.py ├── static │ ├── fonts │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ └── glyphicons-halflings-regular.woff2 │ └── css │ │ └── shop-item.css ├── templates │ ├── contact.html │ ├── error_403.html │ ├── admin_choices.html │ ├── issues.html │ ├── account_reverify.html │ ├── about.html │ ├── login.html │ ├── password_reset.html │ ├── register.html │ ├── view_all.html │ ├── admin.html │ ├── index.html │ ├── layouts │ │ └── base.html │ └── ttp.html ├── config │ └── settings.py └── __init__.py ├── robots.txt ├── .gitignore ├── actor.wsgi ├── license.txt ├── README.md ├── setup ├── schema.sql ├── setup_steps.sh └── load_data.py └── updates └── 2016_05_24 └── update_mappings.py /favicon.ico: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /cgi-bin/ 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] -------------------------------------------------------------------------------- /app/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jalewis/actortrackr/HEAD/app/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /app/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jalewis/actortrackr/HEAD/app/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /app/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jalewis/actortrackr/HEAD/app/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /app/static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jalewis/actortrackr/HEAD/app/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /actor.wsgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4 2 | import os 3 | import sys 4 | 5 | # set PATH so imports are correct 6 | TOP_DIR = os.path.dirname(os.path.realpath(__file__)) 7 | APP_PATH = TOP_DIR+"/app" 8 | 9 | sys.path.insert(0, TOP_DIR) 10 | sys.path.insert(0, APP_PATH) 11 | 12 | # Fire up our application 13 | from app import app as application -------------------------------------------------------------------------------- /app/templates/contact.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | {% if error %} 5 |
6 | {{ error }} 7 |
8 | {% endif %} 9 | 10 |
11 |
12 | Questions, Comments, Issues? Contact ctig@lookingglasscyber.com 13 |
14 |
15 | {% endblock %} -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Copyright 2016 Lookingglass Cyber Solutions 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /app/templates/error_403.html: -------------------------------------------------------------------------------- 1 | {% set page_title="Forbidden" %} 2 | {% extends "layouts/base.html" %} 3 | {% block container %} 4 | 5 | {% if error %} 6 |
7 | {{ error }} 8 |
9 | {% endif %} 10 | 11 |
12 |
13 | {% if session['logged_in'] %} 14 | Your account does not have permission to perform this action. If you feel you should, send an email to ctig@lgscout.com 15 | {% else %} 16 | You must log in and have an account with permission to perform this action 17 | {% endif %} 18 |
19 |
20 | {% endblock %} -------------------------------------------------------------------------------- /app/static/css/shop-item.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Start Bootstrap - Shop Item HTML Template (http://startbootstrap.com) 3 | * Code licensed under the Apache License v2.0. 4 | * For details, see http://www.apache.org/licenses/LICENSE-2.0. 5 | */ 6 | 7 | body { 8 | padding-top: 70px; /* Required padding for .navbar-fixed-top. Remove if using .navbar-static-top. Change if height of navigation changes. */ 9 | } 10 | 11 | .thumbnail img { 12 | width: 100%; 13 | } 14 | 15 | .ratings { 16 | padding-right: 10px; 17 | padding-left: 10px; 18 | color: #d17581; 19 | } 20 | 21 | .thumbnail { 22 | padding: 0; 23 | } 24 | 25 | .thumbnail .caption-full { 26 | padding: 9px; 27 | color: #333; 28 | } 29 | 30 | footer { 31 | margin: 50px 0; 32 | } -------------------------------------------------------------------------------- /app/templates/admin_choices.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 |
5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | {% for data in simple_choices %} 16 | 17 | 18 | 19 | 20 | 21 | 22 | {% endfor %} 23 | 24 |
TypeValueReferenced CountActions
{{ data['type'] }}{{ data['value'] }}{{ data['count'] }}-
25 |
26 | 27 | {% endblock %} -------------------------------------------------------------------------------- /app/templates/issues.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | {% if error %} 5 |
6 | {{ error }} 7 |
8 | {% endif %} 9 | 10 |
11 |
12 |

Forgot Password?

13 | To reset your password click here 14 |
15 |
16 |

Account not verified

17 | To resend account verification email click here 18 |
19 |
20 |
21 |
22 |

Account not approved

23 | If your account is not approved yet, send us an email at ctig@lookingglasscyber.com 24 |
25 |
26 |

Other issues

27 | Email us at ctig@lookingglasscyber.com 28 |
29 |
30 | 31 | {% endblock %} -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ActorTrackr 2 | ActorTrackr is an open source web application for storing/searching/linking Actor related data. The primary sources are from users and various public repositories. Examples of useful repos: APTNotes and "APT Groups and Operations" spreadsheet. Without the hard work of others, this application wouldn't be possible. We hope to continue the usefulness by improving the search and linking of data stored in the system. 3 | 4 | We encourage users to contribute publicly available information here to facilitate sharing and create their own ActorTrackr internally for sensitive data. If you extend the system and find your changes useful for others, please submit a pull request. 5 | 6 | # Installation 7 | See setup/setup_steps.sh for notes on how to install. Better instruction in work 8 | 9 | # License Info 10 | Copyright 2017 Lookingglass Cyber Solutions 11 | 12 | Licensed under the Apache License, Version 2.0 (the "License"); 13 | you may not use this file except in compliance with the License. 14 | You may obtain a copy of the License at 15 | 16 | http://www.apache.org/licenses/LICENSE-2.0 17 | 18 | Unless required by applicable law or agreed to in writing, software 19 | distributed under the License is distributed on an "AS IS" BASIS, 20 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | See the License for the specific language governing permissions and 22 | limitations under the License. 23 | -------------------------------------------------------------------------------- /app/core/blueprints/ajax.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import Blueprint 4 | from flask import flash 5 | from flask import request 6 | from flask import redirect 7 | from flask import jsonify 8 | from flask import Markup 9 | from flask import g 10 | 11 | import functools 12 | import json 13 | import hashlib 14 | import logging 15 | import os 16 | import sys 17 | import time 18 | import uuid 19 | from datetime import datetime 20 | from operator import itemgetter 21 | from urllib.parse import quote_plus 22 | 23 | from config.settings import * 24 | from core.forms import forms 25 | from app import log, get_es, get_mysql 26 | from utils.elasticsearch import * 27 | from utils.functions import * 28 | 29 | #blue print def 30 | ajax_blueprint = Blueprint('ajax', __name__, url_prefix="/ajax") 31 | logger_prefix = "ajax.py:" 32 | 33 | #dynamic select populator 34 | @ajax_blueprint.route("/fetch/<_type>/", methods = ['GET']) 35 | def fetch(_type, value): 36 | logging_prefix = logger_prefix + "fetch({},{}) - ".format(_type,value) 37 | log.info(logging_prefix + "Starting") 38 | 39 | r = fetch_child_data(_type,value) 40 | return jsonify(r), 200 41 | 42 | 43 | #dynamic select populator 44 | @ajax_blueprint.route("/related/<_type>", methods = ['GET']) 45 | @ajax_blueprint.route("/related/<_type>/", methods = ['GET']) 46 | def populate_related_elements(_type): 47 | logging_prefix = logger_prefix + "populate_related_elements({}) - ".format(_type) 48 | log.info(logging_prefix + "Starting") 49 | 50 | r = fetch_related_elements(_type) 51 | return jsonify(r), 200 -------------------------------------------------------------------------------- /app/templates/account_reverify.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | {% if error %} 5 |
6 | {{ error }} 7 |
8 | {% endif %} 9 | 10 | 11 |
12 |
13 |
14 | {{ form.hidden_tag() }} 15 | {% if form.errors %} 16 |
17 |
18 | There were errors submitting the form! 19 | {% if 'csrf_token' in form.errors %} 20 | Invalid CSRF Token, try submitting the form again 21 | {% endif %} 22 |
23 | 26 |
27 | {% endif %} 28 | 29 |
30 | 31 | {{ form.user_email(class="form-control") }} 32 | {% if form.user_email.errors %} 33 |
{% for error in form.user_email.errors %}{{ error }}
{% endfor %}
34 | {% endif %} 35 |
36 | 37 |
38 | 39 |
40 | 41 |
42 |
43 |
44 | 45 | {% endblock %} -------------------------------------------------------------------------------- /app/templates/about.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | {% if error %} 5 |
6 | {{ error }} 7 |
8 | {% endif %} 9 | 10 |
11 |
12 |

13 | ActorTrackr is an open source web application for storing/searching/linking Actor related data. The primary sources are from users and various public repositories. Examples of useful repos: APTNotes and "APT Groups and Operations" spreadsheet. Without the hard work of others, this application wouldn't be possible. We hope to continue the usefulness by improving the search and linking of data stored in the system. 14 |

15 | 16 |

17 | We encourage users to contribute publicly available information here to facilitate sharing and create their own ActorTrackr internally for sensitive data. If you extend the system and find your changes useful for others, please submit a pull request. 18 |

19 | 20 |

21 | ActorTrackr is maintained by LookingGlass CTIG.
22 | Visit our blog at http://deaddrop.threatpool.com/
23 | Follow us on Twitter: https://twitter.com/LG_CTIG
24 |

25 |
26 |
27 |

License Info

28 |
29 |
30 |

31 | Copyright 2017 Lookingglass Cyber Solutions 32 |

33 | Licensed under the Apache License, Version 2.0 (the "License"); 34 | you may not use this file except in compliance with the License. 35 | You may obtain a copy of the License at 36 |

37 | http://www.apache.org/licenses/LICENSE-2.0 38 |

39 | Unless required by applicable law or agreed to in writing, software 40 | distributed under the License is distributed on an "AS IS" BASIS, 41 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 42 | See the License for the specific language governing permissions and 43 | limitations under the License. 44 |

45 |
46 |
47 | {% endblock %} 48 | -------------------------------------------------------------------------------- /setup/schema.sql: -------------------------------------------------------------------------------- 1 | -- phpMyAdmin SQL Dump 2 | -- version 4.0.10deb1 3 | -- http://www.phpmyadmin.net 4 | -- 5 | -- Host: localhost 6 | -- Generation Time: May 25, 2016 at 07:20 PM 7 | -- Server version: 10.1.13-MariaDB-1~trusty 8 | -- PHP Version: 5.5.9-1ubuntu4.17 9 | 10 | SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; 11 | SET time_zone = "+00:00"; 12 | 13 | 14 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 15 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 16 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 17 | /*!40101 SET NAMES utf8 */; 18 | 19 | -- 20 | -- Database: `threat_actors` 21 | -- 22 | CREATE DATABASE IF NOT EXISTS `threat_actors` DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci; 23 | USE `threat_actors`; 24 | 25 | -- -------------------------------------------------------- 26 | 27 | -- 28 | -- Table structure for table `password_reset_hashes` 29 | -- 30 | 31 | CREATE TABLE `password_reset_hashes` ( 32 | `id` int(11) NOT NULL AUTO_INCREMENT, 33 | `user_id` int(11) NOT NULL, 34 | `hash` varchar(64) NOT NULL, 35 | PRIMARY KEY (`id`), 36 | KEY `user_id` (`user_id`,`hash`) 37 | ) ENGINE=MyISAM DEFAULT CHARSET=latin1; 38 | 39 | -- -------------------------------------------------------- 40 | 41 | -- 42 | -- Table structure for table `users` 43 | -- 44 | 45 | CREATE TABLE `users` ( 46 | `id` int(11) NOT NULL AUTO_INCREMENT, 47 | `email` varchar(256) NOT NULL, 48 | `password` varchar(64) NOT NULL, 49 | `name` varchar(256) NOT NULL, 50 | `company` varchar(256) DEFAULT NULL, 51 | `justification` varchar(2048) NOT NULL DEFAULT 'Grandfathered in', 52 | `email_verified` int(1) NOT NULL DEFAULT '0', 53 | `verification_hash` varchar(64) NOT NULL, 54 | `approved` int(1) NOT NULL DEFAULT '0', 55 | `write_permission` int(1) NOT NULL DEFAULT '0', 56 | `delete_permission` int(1) NOT NULL DEFAULT '0', 57 | `admin` int(1) NOT NULL DEFAULT '0', 58 | `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 59 | `last_login` timestamp NULL DEFAULT NULL, 60 | PRIMARY KEY (`id`), 61 | KEY `verification_hash` (`verification_hash`) 62 | ) ENGINE=MyISAM DEFAULT CHARSET=latin1; 63 | 64 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 65 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 66 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 67 | -------------------------------------------------------------------------------- /app/templates/login.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | 5 |
6 |
7 |
8 | {{ form.hidden_tag() }} 9 | {% if form.errors %} 10 |
11 |
12 | There were errors submitting the form! 13 | {% if 'csrf_token' in form.errors %} 14 | Invalid CSRF Token, try submitting the form again 15 | {% endif %} 16 |
17 | 20 |
21 | {% endif %} 22 | 23 |
24 | 25 | {{ form.user_email(class="form-control") }} 26 | {% if form.user_email.errors %} 27 |
{% for error in form.user_email.errors %}{{ error }}
{% endfor %}
28 | {% endif %} 29 |
30 | 31 |
32 | 33 | {{ form.user_password(class="form-control") }} 34 | {% if form.user_password.errors %} 35 |
{% for error in form.user_password.errors %}{{ error }}
{% endfor %}
36 | {% endif %} 37 |
38 | 39 |
40 | 41 |
42 | 43 |
44 |
45 |
46 | Not registered?
47 |
48 |
49 | Having trouble logging in? 50 |
51 |
52 |
53 |
54 |
55 |
56 | 57 | {% endblock %} 58 | 59 | {% block javascript %} 60 | 63 | {% endblock %} 64 | -------------------------------------------------------------------------------- /app/templates/password_reset.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | {% if error %} 5 |
6 | {{ error }} 7 |
8 | {% endif %} 9 | 10 | 11 |
12 |
13 |
14 | {{ form.hidden_tag() }} 15 | {% if form.errors %} 16 |
17 |
18 | There were errors submitting the form! 19 | {% if 'csrf_token' in form.errors %} 20 | Invalid CSRF Token, try submitting the form again 21 | {% endif %} 22 |
23 | 26 |
27 | {% endif %} 28 | 29 | {% if page_type=="REQUEST" %} 30 |
31 | 32 | {{ form.user_email(class="form-control") }} 33 | {% if form.user_email.errors %} 34 |
{% for error in form.user_email.errors %}{{ error }}
{% endfor %}
35 | {% endif %} 36 |
37 | 38 | {% elif page_type=="PERFORM" %} 39 |
40 | 41 | {{ form.user_password(class="form-control") }} 42 | Note: Your password must contain at least 1 uppercase letter, 1 lowercase letter, 1 number and 1 symbol 43 | {% if form.user_password.errors %} 44 |
{% for error in form.user_password.errors %}{{ error }}
{% endfor %}
45 | {% endif %} 46 |
47 |
48 | 49 | {{ form.user_password2(class="form-control") }} 50 | {% if form.user_password2.errors %} 51 |
{% for error in form.user_password2.errors %}{{ error }}
{% endfor %}
52 | {% endif %} 53 |
54 | {% endif %} 55 | 56 |
57 | 58 |
59 | 60 |
61 |
62 |
63 | 64 | {% endblock %} -------------------------------------------------------------------------------- /app/config/settings.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Elasticsearch Settings 3 | ''' 4 | 5 | ES_PREFIX = "tp-" 6 | ES_HOSTS = ['http://localhost:9200',] 7 | 8 | ''' 9 | MySQL Settings 10 | ''' 11 | 12 | MYSQL_USER = '' 13 | MYSQL_PASSWD = '' 14 | MYSQL_DB = 'threat_actors' 15 | 16 | ''' 17 | Log Settings 18 | ''' 19 | 20 | LOG_FILE = "/var/log/actortrackr/actortrackr.log" 21 | LOG_TO_CONSOLE = True 22 | LOG_LEVEL = "DEBUG" #NOSET, DEBUG, INFO, WARNING, ERROR, CRITICAL 23 | 24 | ''' 25 | Email Settings 26 | ''' 27 | 28 | #the email address of the sender 29 | EMAIL_SENDER = "Your email here" 30 | 31 | #alerts go to these addresses 32 | EMAIL_ADDRESSES = [ "Your email here", ] 33 | 34 | ''' 35 | Application Settings 36 | ''' 37 | 38 | MAINTENANCE_MODE = False 39 | 40 | APPLICATION_DOMAIN = "http://actortrackr.com/" 41 | APPLICATION_ORG = "Lookingglass" 42 | APPLICATION_NAME = "ActorTrackr" 43 | 44 | TLPS = [ 45 | ("0", "White"), 46 | ("1", "Green"), 47 | ("2", "Amber"), 48 | ("3", "Red"), 49 | ("4", "Black") 50 | ] 51 | 52 | SOURCE_RELIABILITY = [ 53 | ("A", "A. Reliable - No doubt about the source's authenticity, trustworthiness, or competency. History of complete reliability."), 54 | ("B", "B. Usually Reliable - Minor doubts. History of mostly valid information."), 55 | ("C", "C. Fairly Reliable - Doubts. Provided valid information in the past."), 56 | ("D", "D. Not Usually Reliable - Significant doubts. Provided valid information in the past."), 57 | ("E", "E. Unreliable - Lacks authenticity, trustworthiness, and competency. History of invalid information."), 58 | ("F", "F. Can’t Be Judged - Insufficient information to evaluate reliability. May or may not be reliable.") 59 | ] 60 | 61 | INFORMATION_RELIABILITY = [ 62 | ("1", "1. Confirmed - Logical, consistent with other relevant information, confirmed by independent sources."), 63 | ("2", "2. Probably True - Logical, consistent with other relevant information, not confirmed by independent sources."), 64 | ("3", "3. Possibly True - Reasonably logical, agrees with some relevant information, not confirmed."), 65 | ("4", "4. Doubtfully True - Not logical but possible, no other information on the subject, not confirmed."), 66 | ("5", "5. Improbable - Not logical, contradicted by other relevant information."), 67 | ("6", "6. Can’t Be Judged - The validity of the information can not be determined.") 68 | ] 69 | 70 | SALTS = { 71 | "actor" : "salt", 72 | "report" : "salt", 73 | "ttp" : "salt", 74 | "user" : "salt", 75 | "email_verification" : "salt" 76 | } 77 | 78 | SESSION_EXPIRE = -1 # in seconds, -1 to disable 79 | 80 | ''' 81 | Recaptcha Settings 82 | ''' 83 | 84 | RECAPTCHA_ENABLED = True 85 | RECAPTCHA_PUBLIC_KEY = '' 86 | RECAPTCHA_PRIVATE_KEY = '' 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /setup/setup_steps.sh: -------------------------------------------------------------------------------- 1 | #Actor DB Web Server setup 2 | 3 | ################################################################### 4 | # Apache 5 | ################################################################### 6 | 7 | apt-get -y install apache2 8 | apt-get -y install libapache2-mod-wsgi-py3 9 | 10 | ################################################################### 11 | # MariaDB 12 | ################################################################### 13 | 14 | #Important note root password when going thru this, its needed for phpmyadmin and the application 15 | 16 | sudo apt-get install software-properties-common 17 | sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xcbcb082a1bb943db 18 | sudo add-apt-repository 'deb [arch=amd64,i386] http://mirror.jmu.edu/pub/mariadb/repo/10.1/ubuntu trusty main' 19 | sudo apt-get update 20 | sudo apt-get install mariadb-server 21 | 22 | #command line client, also need for python lib 23 | sudo apt-get install mysql-client 24 | 25 | ################################################################### 26 | # phpmyadmin (optional), make sure its not on port 80 27 | ################################################################### 28 | sudo apt-get install php5 libapache2-mod-php5 php5-mcrypt 29 | 30 | vim /etc/apache2/mods-enabled/dir.conf 31 | 32 | #move index.php to front of list 33 | 34 | DirectoryIndex index.php index.html index.cgi index.pl index.xhtml index.htm 35 | 36 | 37 | sudo service apache2 restart 38 | 39 | sudo apt-get update 40 | sudo apt-get install phpmyadmin 41 | 42 | sudo php5enmod mcrypt 43 | sudo service apache2 restart 44 | 45 | ################################################################### 46 | # Install pip 47 | ################################################################### 48 | 49 | #for python 3.4 50 | cd ~ 51 | wget https://bootstrap.pypa.io/get-pip.py 52 | python3.4 get-pip.py 53 | rm get-pip.py 54 | 55 | ################################################################### 56 | # Python modules 57 | ################################################################### 58 | 59 | python3.4 -m pip install flask #flask 60 | python3.4 -m pip install Flask-WTF #flask forms 61 | python3.4 -m pip install flask-compress #flask gzip compression extension 62 | python3.4 -m pip install requests #requests 63 | python3.4 -m pip install elasticsearch 64 | python3.4 -m pip install PyMySQL 65 | 66 | ################################################################### 67 | # Configuration 68 | ################################################################### 69 | 70 | ########################################### 71 | #/etc/apache2/sites-enabled/actortrackr.com.conf 72 | 73 | 74 | ServerAdmin ctig@lookingglasscyber.com 75 | DocumentRoot /var/www/actortrackr.com 76 | ServerName actortrackr.com 77 | ServerAlias www.actortrackr.com 78 | RewriteEngine On 79 | RewriteOptions inherit 80 | CustomLog /var/log/apache2/actortrackr.com.log combined 81 | Options -Indexes 82 | 83 | WSGIScriptAlias / /var/www/actortrackr.com/actor.wsgi 84 | 85 | 86 | Require all granted 87 | WSGIScriptReloading On 88 | 89 | 90 | 91 | LimitRequestBody 52428800 92 | 93 | 94 | 95 | 96 | ########################################### 97 | #/etc/apache2/ports.conf 98 | 99 | #where phpmyadmin is running 100 | Listen 8080 101 | 102 | 103 | ################################################################### 104 | # Start Server, hit some pages, and check for errors 105 | ################################################################### 106 | clear; service apache2 restart; tail -f /var/log/apache2/error.log 107 | 108 | -------------------------------------------------------------------------------- /app/templates/register.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | 5 |
6 |
7 |
8 | {{ form.hidden_tag() }} 9 | {% if form.errors %} 10 |
11 |
12 | There were errors submitting the form! 13 | {% if 'csrf_token' in form.errors %} 14 | Invalid CSRF Token, try submitting the form again 15 | {% endif %} 16 |
17 | 20 |
21 | {% endif %} 22 | 23 |
24 | 25 | {{ form.user_name(class="form-control") }} 26 | {% if form.user_name.errors %} 27 |
{% for error in form.user_name.errors %}{{ error }}
{% endfor %}
28 | {% endif %} 29 |
30 | 31 |
32 | 33 | {{ form.user_email(class="form-control") }} 34 | {% if form.user_email.errors %} 35 |
{% for error in form.user_email.errors %}{{ error }}
{% endfor %}
36 | {% endif %} 37 |
38 | 39 |
40 | 41 | {{ form.user_password(class="form-control") }} 42 | Note: Your password must contain at least 1 uppercase letter, 1 lowercase letter, 1 number and 1 symbol 43 | {% if form.user_password.errors %} 44 |
{% for error in form.user_password.errors %}{{ error }}
{% endfor %}
45 | {% endif %} 46 |
47 | 48 |
49 | 50 | {{ form.user_password2(class="form-control") }} 51 | {% if form.user_password2.errors %} 52 |
{% for error in form.user_password2.errors %}{{ error }}
{% endfor %}
53 | {% endif %} 54 |
55 | 56 |
57 | 58 | {{ form.user_company(class="form-control") }} 59 | {% if form.user_company.errors %} 60 |
{% for error in form.user_company.errors %}{{ error }}
{% endfor %}
61 | {% endif %} 62 |
63 | 64 |
65 | 66 | {{ form.user_reason(class="form-control") }} 67 | {% if form.user_reason.errors %} 68 |
{% for error in form.user_reason.errors %}{{ error }}
{% endfor %}
69 | {% endif %} 70 |
71 | 72 | {% if recaptcha_enabled %} 73 |
74 | {% endif %} 75 | 76 |
77 | 78 |
79 |
80 |
81 |
82 | 83 | {% endblock %} 84 | 85 | {% block javascript %} 86 | 87 | 90 | {% endblock %} 91 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4 2 | if __name__ == "__main__": 3 | import os 4 | import sys 5 | 6 | TOP_DIR = os.path.dirname(os.path.realpath(__file__)) 7 | APP_PATH = TOP_DIR+"/app" 8 | 9 | sys.path.insert(0, TOP_DIR) 10 | sys.path.insert(0, APP_PATH) 11 | 12 | from config.settings import * 13 | 14 | import logging 15 | from logging.handlers import TimedRotatingFileHandler 16 | from logging import StreamHandler 17 | log = logging.getLogger(__name__) 18 | 19 | log.setLevel(logging.getLevelName(LOG_LEVEL)) 20 | 21 | #log formatter 22 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 23 | 24 | # add a rotating handler 25 | handler = TimedRotatingFileHandler(LOG_FILE, when='d', interval=1, backupCount=5) #creates daily logs for 5 days 26 | handler.setFormatter(formatter) 27 | log.addHandler(handler) 28 | 29 | # add a console hander 30 | if LOG_TO_CONSOLE: 31 | consoleHandler = StreamHandler() 32 | consoleHandler.setFormatter(formatter) 33 | log.addHandler(consoleHandler) 34 | 35 | try: 36 | from flask import Flask 37 | from flask import render_template 38 | from flask import flash 39 | from flask import request 40 | from flask import redirect 41 | from flask import jsonify 42 | from flask import Markup 43 | from flask import g 44 | except Exception as e: 45 | print("Error: {}\nFlask is not installed, try 'pip install flask'".format(e)) 46 | exit(1) 47 | 48 | try: 49 | from flask.ext.compress import Compress 50 | except Exception as e: 51 | print("Error: {}\Flask compress library is not installed, try 'pip install flask-compress'".format(e)) 52 | exit(1) 53 | try: 54 | from elasticsearch import Elasticsearch 55 | from elasticsearch import exceptions 56 | except Exception as e: 57 | print("Error: {}\nElasticsearch library is not installed, try 'pip install elasticsearch'".format(e)) 58 | exit(1) 59 | 60 | 61 | try: 62 | import pymysql 63 | from pymysql.cursors import DictCursor 64 | except Exception as e: 65 | print("Error: {}\PyMySQL library is not installed, try 'pip install PyMySQL'".format(e)) 66 | exit(1) 67 | 68 | #if you want a lot of elastic logs uncomment this section 69 | ''' 70 | es_logger = logging.getLogger('elasticsearch') 71 | es_logger.propagate = False 72 | es_logger.setLevel(logging.DEBUG) 73 | es_logger_handler=logging.StreamHandler() 74 | es_logger.addHandler(es_logger_handler) 75 | 76 | es_tracer = logging.getLogger('elasticsearch.trace') 77 | es_tracer.propagate = False 78 | es_tracer.setLevel(logging.INFO) 79 | es_tracer_handler=logging.StreamHandler() 80 | es_tracer.addHandler(es_tracer_handler) 81 | ''' 82 | 83 | app = Flask(__name__) 84 | app.secret_key = "Fgtqweds5ywDJsQW87uQnL" 85 | 86 | #configure gzip compression 87 | app.config['COMPRESS_LEVEL'] = 9 88 | app.config['COMPRESS_MIN_SIZE'] = 1 89 | Compress(app) 90 | 91 | def get_es(): 92 | try: 93 | db = getattr(g, 'es', None) 94 | if db is None: 95 | db = g.es = Elasticsearch(ES_HOSTS) 96 | except RuntimeError as rte: 97 | db = Elasticsearch(ES_HOSTS) 98 | 99 | return db 100 | 101 | def get_mysql(): 102 | try: 103 | db = getattr(g, 'mysql', None) 104 | if db is None: 105 | db = g.mysql = pymysql.connect(user=MYSQL_USER,passwd=MYSQL_PASSWD,db=MYSQL_DB, cursorclass=DictCursor) 106 | except RuntimeError as rte: 107 | db = g.mysql = pymysql.connect(user=MYSQL_USER,passwd=MYSQL_PASSWD,db=MYSQL_DB, cursorclass=DictCursor) 108 | 109 | return db 110 | 111 | @app.before_request 112 | def before_request(): 113 | if MAINTENANCE_MODE: 114 | # Or alternatively, dont redirect 115 | return 'Sorry, off for maintenance! Be back in 5', 503 116 | 117 | g.es = get_es() 118 | g.mysql = get_mysql() 119 | 120 | @app.teardown_request 121 | def teardown_request(exception): 122 | get_mysql().close() 123 | pass 124 | 125 | 126 | from core.blueprints.actor import actor_blueprint 127 | from core.blueprints.admin import admin_blueprint 128 | from core.blueprints.ajax import ajax_blueprint 129 | from core.blueprints.index import index_blueprint 130 | from core.blueprints.report import report_blueprint 131 | from core.blueprints.ttp import ttp_blueprint 132 | from core.blueprints.user import user_blueprint 133 | 134 | app.register_blueprint(actor_blueprint) 135 | app.register_blueprint(admin_blueprint) 136 | app.register_blueprint(ajax_blueprint) 137 | app.register_blueprint(index_blueprint) 138 | app.register_blueprint(report_blueprint) 139 | app.register_blueprint(ttp_blueprint) 140 | app.register_blueprint(user_blueprint) 141 | 142 | 143 | 144 | if __name__ == "__main__": 145 | 146 | #start flask 147 | app.run( 148 | host = '0.0.0.0', 149 | port = 8888, 150 | threaded=True, 151 | debug=True 152 | ) 153 | -------------------------------------------------------------------------------- /app/templates/view_all.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 |
5 |
6 |

Search

7 |
8 |
9 |
10 |
11 |
12 | 13 | {{ form.hidden_tag() }} 14 | 15 |
16 |
17 | {{ form.query(class="form-control", style="width:100%") }} 18 | {% if form.query.errors %} 19 |
{% for error in form.query.errors %}{{ error }}
{% endfor %}
20 | {% endif %} 21 |
22 |
23 | 24 |
25 |
26 | 27 |
28 |
29 |
30 | 31 |
32 |
33 |

{{ data_header }} {{results_text}}

34 |
35 | 36 | {% if session['write'] %} 37 |
38 | 39 | 42 | 43 |
44 | {% endif %} 45 |
46 | 47 | {% if data['hits']['hits'] %} 48 |
49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | {% for d in data['hits']['hits'] %} 58 | 59 | 60 | 83 | 84 | {% endfor %} 85 | 86 |
{{ field_header }}Actions
{{ d['_source']['name']|e }} 61 | 62 | 65 | 66 | 67 | {% if session['write'] %} 68 | 69 | 72 | 73 | {% endif %} 74 | 75 | {% if session['delete'] %} 76 | 77 | 80 | 81 | {% endif %} 82 |
87 |
88 | 89 |
90 |
91 | {% if prev_url %} 92 | 93 | 97 | 98 | {% else %} 99 | 103 | {% endif %} 104 |
105 |
106 | {% if next_url %} 107 | 108 | 112 | 113 | {% else %} 114 | 118 | {% endif %} 119 |
120 |
121 | {% else %} 122 |
123 |
124 | No results found 125 |
126 |
127 | {% endif %} 128 | 129 | {% endblock %} -------------------------------------------------------------------------------- /app/core/decorators/authentication.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import Blueprint 4 | from flask import flash 5 | from flask import request 6 | from flask import redirect 7 | from flask import jsonify 8 | from flask import Markup 9 | from flask import session 10 | from flask import abort 11 | from flask import g 12 | 13 | import functools 14 | import json 15 | import hashlib 16 | import logging 17 | import math 18 | import os 19 | import sys 20 | import time 21 | import uuid 22 | from datetime import datetime 23 | from functools import wraps 24 | from pymysql.cursors import DictCursor 25 | from operator import itemgetter 26 | from urllib.parse import quote_plus 27 | 28 | from config.settings import * 29 | from core.forms import forms 30 | from app import log, get_es, get_mysql 31 | from utils.elasticsearch import * 32 | from utils.functions import * 33 | 34 | logger_prefix = "authentication.py:" 35 | 36 | PUBLIC = 0 37 | WRITE = 1 38 | DELETE = 2 39 | ADMIN = 3 40 | 41 | def access(access_level=PUBLIC): 42 | logging_prefix = logger_prefix + "access() - " 43 | 44 | def decorated(f): 45 | @wraps(f) 46 | 47 | def wrapped(*args, **kwargs): 48 | 49 | r = quote_plus(request.url) 50 | try: 51 | if access_level != PUBLIC: 52 | if 'logged_in' not in session: 53 | flash("You must log in to continue", 'danger') 54 | return redirect("/user/login/?r={}".format(r), code=307) 55 | elif not session['logged_in']: 56 | flash("You must log in to continue", 'danger') 57 | return redirect("/user/login/?r={}".format(r), code=307) 58 | elif session['expires'] < math.ceil(time.time()) and SESSION_EXPIRE != -1: 59 | flash("Your session has expired, log in below to continue", 'danger') 60 | return redirect("/user/logout/?r={}".format(r), code=307) 61 | 62 | 63 | #since the user is logged in requery the database for their current permissons 64 | if 'id' not in session: 65 | log.error(logging_prefix - "ID not in session: {}".format(session)) 66 | flash("Theres an issue with your session, please log in again. Error 001.", 'danger') 67 | return redirect("/user/login/?r={}".format(r), code=307) 68 | 69 | conn = get_mysql().cursor(DictCursor) 70 | conn.execute("SELECT email_verified, approved, write_permission, delete_permission, admin FROM users WHERE id = %s", (session['id'],)) 71 | user = conn.fetchone() 72 | conn.close() 73 | 74 | if not user: 75 | log.error(logging_prefix - "User not found: {}".format(session)) 76 | flash("Theres an issue with your session, please log in again. Error 002.", 'danger') 77 | return redirect("/user/logout/?r={}".format(r), code=307) 78 | 79 | if user['email_verified'] != 1: 80 | flash("Your email address has not been verified yet", 'danger') 81 | return redirect("/user/logout/?r={}".format(r), code=307) 82 | 83 | session['approved'] = (user['approved'] == 1) 84 | session['write'] = (user['write_permission'] == 1) 85 | session['delete'] = (user['delete_permission'] == 1) 86 | session['admin'] = (user['admin'] == 1) 87 | 88 | #now that we know this user is logged in set the expiration to much later 89 | session['expires'] = math.ceil(time.time()) + SESSION_EXPIRE 90 | 91 | #compare the users current access to the access level needed for this page 92 | if not session['approved']: 93 | #if read is not set, the users account has been disabled 94 | flash("Your account has been disabled", 'danger') 95 | return redirect("/user/logout/?r={}".format(r), code=307) 96 | 97 | 98 | elif access_level == WRITE and not session['write']: 99 | return render_template("error_403.html") 100 | elif access_level == DELETE and not session['delete']: 101 | return render_template("error_403.html") 102 | elif access_level == ADMIN and not session['admin']: 103 | return render_template("error_403.html") 104 | 105 | except Exception as e: 106 | log.exception("Error performing user authentication") 107 | flash("Your account could not be verified, please log in again", 'danger') 108 | return redirect("/user/logout/?r={}".format(r), code=307) 109 | 110 | return f(*args, **kwargs) 111 | 112 | return wrapped 113 | return decorated -------------------------------------------------------------------------------- /updates/2016_05_24/update_mappings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4 2 | # update_mappings.py 3 | # 4 | # adds new mappings to each index to track which users modified an item 5 | # this is a multi step script 6 | # step 1. export all of the data from each index to a temp file 7 | # step 2. delete the index/template for each index, recreate the template 8 | # step 3. upload the dumped data 9 | 10 | #import ES 11 | try: 12 | from elasticsearch import Elasticsearch 13 | from elasticsearch import exceptions 14 | from elasticsearch.helpers import scan 15 | except Exception as e: 16 | print("Error: {}\nElasticsearch library is not installed, try 'pip install elasticsearch'".format(e)) 17 | exit(1) 18 | 19 | # set up the path so this can be run and use the settings.py file 20 | import json 21 | import os 22 | import requests 23 | import sys 24 | import time 25 | 26 | # set PATH so imports are correct 27 | TOP_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) 28 | 29 | APP_PATH = TOP_DIR+"/app" 30 | 31 | sys.path.insert(0, TOP_DIR) 32 | sys.path.insert(0, APP_PATH) 33 | 34 | # Import our settings 35 | from config.settings import * 36 | 37 | #import the new mappings 38 | import mappings_file 39 | 40 | PERFORM_DELETE = True 41 | NOOP = False 42 | 43 | def getIndicesMatchingPattern(name): 44 | if name == '*' or name=='_all': 45 | raise Exception("Invalid Argument") 46 | 47 | print(name) 48 | url = "{}/{}".format(ES_HOSTS[0],name) 49 | print(url) 50 | r = requests.get(url) 51 | 52 | response = json.loads(r.content.decode('utf-8')) 53 | 54 | indices = [] 55 | for k in response: 56 | indices.append(k) 57 | 58 | return indices 59 | 60 | def delete_index_pattern(es, index): 61 | if not PERFORM_DELETE: 62 | print("NOT PERFORM INDEX DELETE DUE TO SETTINGS") 63 | return 64 | 65 | #Delete the Index 66 | try: 67 | print("deleting indexes...") 68 | indices = getIndicesMatchingPattern(index) 69 | 70 | print(indices) 71 | time.sleep(3) 72 | 73 | for i in indices: 74 | if NOOP: 75 | print("NOOP - Deleting Index {}".format(i)) 76 | else: 77 | print("[DELETE INDEX] Deleting Index {}".format(i)) 78 | print(es.indices.delete(index=i)) 79 | print("done") 80 | except exceptions.NotFoundError as e: 81 | print("Index not found") 82 | 83 | def delete_template(es, template): 84 | if not PERFORM_DELETE: 85 | print("NOT PERFORM TEMPLATE DELETE DUE TO SETTINGS") 86 | return 87 | #Delete the Template 88 | try: 89 | if NOOP: 90 | print("NOOP - Deleting Template {}".format(template)) 91 | else: 92 | print("[DELETE TEMPLATE] Deleting Template {}".format(template)) 93 | print(es.indices.delete_template(name=template)) 94 | except exceptions.NotFoundError as e: 95 | print("Template not found") 96 | 97 | def create_template(es,name,body): 98 | if NOOP: 99 | print("NOOP - Creating Template {}".format(name)) 100 | else: 101 | print("[CREATE TEMPLATE] Creating Template {}".format(name)) 102 | print(es.indices.put_template(name=name, body=body)) 103 | 104 | 105 | def dump_to_file(es, index, doc_type): 106 | data = [] 107 | 108 | query = { 109 | "query" : { 110 | "match_all" : {} 111 | } 112 | } 113 | 114 | results = scan(es,query=query,index=index,doc_type=doc_type) 115 | 116 | for i in results: 117 | data.append(i) 118 | 119 | with open(index + "__" + doc_type + ".dat", 'w') as f: 120 | f.write(json.dumps(data)) 121 | 122 | def delete_recreate(es, index, mapping): 123 | delete_index_pattern(es, index + "*") 124 | delete_template(es,index) 125 | 126 | create_template(es,name=index, body=mapping) 127 | 128 | def import_from_file(es, index, doc_type): 129 | with open(index + "__" + doc_type + ".dat", 'r') as f: 130 | json_string = "" 131 | for line in f: 132 | json_string += line 133 | 134 | data = json.loads(json_string) 135 | 136 | count = 0 137 | for i in data: 138 | source = i['_source'] 139 | 140 | source['editor'] = [] 141 | count += 1 142 | print(es.index(index=index,doc_type=doc_type, body=source, id=i['_id'])) 143 | 144 | print("{}.{} = {}".format(index,doc_type,count)) 145 | 146 | 147 | 148 | es = Elasticsearch(ES_HOSTS) 149 | 150 | try: 151 | STEP = sys.argv[1] 152 | except: 153 | STEP = None 154 | 155 | if not STEP: 156 | exit("Usage {} \nstep values\n\t1 - Export\n\t2 - Delete\n\t3 - Import".format(sys.argv[0])) 157 | 158 | if STEP == "1": 159 | dump_to_file(es, ES_PREFIX + "threat_actors","actor") 160 | dump_to_file(es, ES_PREFIX + "threat_reports","report") 161 | dump_to_file(es, ES_PREFIX + "threat_ttps","ttp") 162 | 163 | elif STEP == "2": 164 | delete_recreate(es, ES_PREFIX + "threat_actors", mappings_file.get_actor_mapping(ES_PREFIX)) 165 | delete_recreate(es, ES_PREFIX + "threat_reports",mappings_file.get_report_mapping(ES_PREFIX)) 166 | delete_recreate(es, ES_PREFIX + "threat_ttps", mappings_file.get_ttp_mapping(ES_PREFIX)) 167 | 168 | elif STEP == "3": 169 | import_from_file(es, ES_PREFIX + "threat_actors","actor") 170 | import_from_file(es, ES_PREFIX + "threat_reports","report") 171 | import_from_file(es, ES_PREFIX + "threat_ttps","ttp") 172 | 173 | else: 174 | print("Invalid step '{}'".format(STEP)) 175 | 176 | 177 | -------------------------------------------------------------------------------- /app/utils/functions.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import json 3 | import hashlib 4 | import logging 5 | import os 6 | import sys 7 | import time 8 | import uuid 9 | from datetime import datetime 10 | from operator import itemgetter 11 | from pymysql.cursors import DictCursor 12 | from urllib.parse import quote_plus 13 | 14 | #email imports 15 | import smtplib 16 | from email.mime.multipart import MIMEMultipart 17 | from email.mime.text import MIMEText 18 | 19 | from config.settings import * 20 | from core.forms import forms 21 | from app import log 22 | from utils.elasticsearch import * 23 | from utils.functions import * 24 | 25 | logger_prefix = "functions.py:" 26 | 27 | def sha256(salt, s): 28 | s = salt + s 29 | hash_object = hashlib.sha256(s.encode('utf-8')) 30 | return hash_object.hexdigest() 31 | 32 | def print_tpx(tpx): 33 | print(json.dumps(tpx, sort_keys=True, indent=4, separators=(',', ': '))) 34 | 35 | def cmp(a, b): 36 | return (a > b) - (a < b) 37 | 38 | def multikeysort(items, columns): 39 | from operator import itemgetter 40 | comparers = [ ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns] 41 | def comparer(left, right): 42 | for fn, mult in comparers: 43 | result = cmp(fn(left), fn(right)) 44 | if result: 45 | return mult * result 46 | else: 47 | return 0 48 | return sorted(items, key=functools.cmp_to_key(comparer)) 49 | 50 | ''' 51 | Edit Tracking Helpers 52 | ''' 53 | 54 | def get_editor_names(mysql, es_editors): 55 | user_ids = [] 56 | editors = [] 57 | 58 | #get a uniq list of user ids 59 | for editor in es_editors: 60 | user_id = editor.get('user_id') 61 | if user_id: 62 | if user_id not in user_ids: 63 | user_ids.append(user_id) 64 | 65 | #with the unique list query mysql 66 | if user_ids: 67 | query = "SELECT id, name FROM users WHERE id IN ({})".format(','.join(str(x) for x in user_ids)) 68 | conn = mysql.cursor(DictCursor) 69 | conn.execute(query) 70 | results = conn.fetchall() 71 | conn.close() 72 | 73 | user_dict = {} #maps id to name 74 | for user in results: 75 | user_dict[user['id']] = user['name'] 76 | 77 | for editor in es_editors: 78 | editors.append({ 79 | "user_name" : user_dict[editor['user_id']], 80 | "ts" : editor['ts'] 81 | }) 82 | 83 | 84 | editors.reverse() 85 | return editors 86 | 87 | 88 | def get_editor_list(es, index, doc_type, item_id, user_id): 89 | #get the current list of editors 90 | try: 91 | #if this is the "Add" function, the id wont exist, so catch the exception 92 | current_data = es.get(index=index, doc_type=doc_type, id=item_id) 93 | editors = current_data['_source']['editor'] 94 | except: 95 | editors = [] 96 | 97 | if user_id: 98 | editors.append({ 99 | 'user_id' : user_id, 100 | 'ts' : datetime.now().strftime("%Y-%m-%dT%H:%M:%S") 101 | }) 102 | else: 103 | raise Exception("Unable to locate User ID for session") 104 | 105 | return editors 106 | 107 | ''' 108 | Email functions 109 | ''' 110 | 111 | def sendAccountVerificationEmail(email, verification_hash): 112 | to = [ email, ] 113 | sender = EMAIL_SENDER 114 | subject = "{}: Account Activation".format(APPLICATION_NAME) 115 | body = 'Thank you for registering for the {} {}. To verify your email address, click here.

If you were not expecting this email, please send an email to ctig@lookingglasscyber.com.'.format(APPLICATION_ORG, APPLICATION_NAME, APPLICATION_DOMAIN, quote_plus(quote_plus(email)), verification_hash) 116 | 117 | sendHTMLEmail(to, sender, subject, body) 118 | 119 | def sendNewUserToApproveEmail(user_id, user_email, user_name, user_company, user_justification): 120 | to = EMAIL_ADDRESSES 121 | sender = EMAIL_SENDER 122 | subject = "{}: New Verified User".format(APPLICATION_NAME) 123 | body = 'A new user has signed up and verified their email. They can be approved on the Admin page.

Name: {}
Email: {}
Company: {}
Justification: {}'.format(user_name, user_email, user_company, user_justification) 124 | 125 | sendHTMLEmail(to, sender, subject, body) 126 | 127 | 128 | def sendAccountApprovedEmail(user_email): 129 | to = [ user_email, ] 130 | sender = EMAIL_SENDER 131 | subject = "{}: Account Approved".format(APPLICATION_NAME) 132 | body = 'Your account for the {} {} has been approved

Click here to log in.'.format(APPLICATION_ORG, APPLICATION_NAME, APPLICATION_DOMAIN) 133 | 134 | sendHTMLEmail(to, sender, subject, body) 135 | 136 | def sendAccountDisapprovedEmail(user_email): 137 | to = [ user_email, ] 138 | sender = EMAIL_SENDER 139 | subject = "{}: Account Access Removed".format(APPLICATION_NAME) 140 | body = 'Your access to the {} {} has been removed

If you feel this has been done in error, contact ctig@lookingglasscyber.com'.format(APPLICATION_ORG, APPLICATION_NAME) 141 | 142 | sendHTMLEmail(to, sender, subject, body) 143 | 144 | def sendPasswordResetEmail(user_email, reset_link): 145 | to = [ user_email, ] 146 | sender = EMAIL_SENDER 147 | subject = "{}: Password Reset".format(APPLICATION_NAME) 148 | body = 'To reset your password, click here'.format(APPLICATION_DOMAIN, reset_link) 149 | 150 | sendHTMLEmail(to, sender, subject, body) 151 | 152 | 153 | def sendCustomEmail(user_email, subject, body): 154 | to = [ user_email, ] 155 | sender = EMAIL_SENDER 156 | 157 | sendPlainTextEmail(to, sender, subject, body) 158 | 159 | ''' 160 | Email Senders 161 | ''' 162 | 163 | def sendPlainTextEmail(to, sender, subject, body): 164 | msg = MIMEText(body) 165 | msg['Subject'] = subject 166 | msg['From'] = sender 167 | msg['To'] = ", ".join(to) 168 | s = smtplib.SMTP('localhost') 169 | s.sendmail(sender, to, msg.as_string()) 170 | s.quit() 171 | 172 | print("Email sent") 173 | 174 | def sendHTMLEmail(to, sender, subject, body): 175 | msg = MIMEMultipart('alternative') 176 | msg['Subject'] = subject 177 | msg['From'] = sender 178 | msg['To'] = ", ".join(to) 179 | 180 | msgbody = MIMEText(body, 'html') 181 | msg.attach(msgbody) 182 | 183 | s = smtplib.SMTP('localhost') 184 | s.sendmail(sender, to, msg.as_string()) 185 | s.quit() 186 | 187 | print("Email sent") -------------------------------------------------------------------------------- /app/templates/admin.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | {% if users %} 5 |
6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | {% for user in users %} 24 | 25 | 26 | 32 | 33 | 34 | 37 | 40 | 43 | 46 | 47 | 48 | 61 | 62 | {% endfor %} 63 | 64 |
NameEmailCompanyJustificationApprovedWriteDeleteAdminCreatedLast LoginActions
{{ user['name']|e }} 27 | {{ user['email']|e }} 28 | {% if user['email_verified'] == 0 %} 29 | (Not Verified) 30 | {% endif %} 31 | {{ user['company']|e }}{{ user['justification']|e }} 35 | 36 | 38 | 39 | 41 | 42 | 44 | 45 | {{ user['created']|e }}{{ user['last_login']|e }} 49 | 50 | 53 | 54 | 55 | 60 |
65 |
66 | 67 | 86 | {% else %} 87 |
88 |
89 | No results found 90 |
91 |
92 | {% endif %} 93 | 94 | {% endblock %} 95 | 96 | {% block javascript %} 97 | 180 | {% endblock %} -------------------------------------------------------------------------------- /app/utils/elasticsearch.py: -------------------------------------------------------------------------------- 1 | from flask import flash, g 2 | from app import log, get_es 3 | from config.settings import * 4 | 5 | from elasticsearch import TransportError 6 | from elasticsearch.helpers import scan 7 | 8 | logger_prefix = "elasticsearch.py:" 9 | 10 | SIMPLE_CHOICES = {} 11 | 12 | def escape(v): 13 | ''' 14 | Function for escaping data before it goes into ES 15 | 16 | Parameters: 17 | v - the value to be escaped 18 | Returns: 19 | v - the escaped value, or the original value if escaping is not possible (int) 20 | ''' 21 | 22 | #TODO: escape this before returning it, maybe? 23 | 24 | #trim the string 25 | try: 26 | v = v.strip() 27 | except AttributeError as a: 28 | #if its an integer strip doesnt apply 29 | pass 30 | return v 31 | 32 | 33 | def populate_simple_choices(): 34 | ''' 35 | Populates the SIMPLE_CHOICES dictionary with options 36 | ''' 37 | 38 | global SIMPLE_CHOICES 39 | 40 | SIMPLE_CHOICES = {} 41 | try: 42 | body = { 43 | "query" : { 44 | "match_all" : {} 45 | }, 46 | "size" : 1000 47 | } 48 | results = get_es().search(ES_PREFIX + 'threat_actor_simple', 'data', body) 49 | 50 | for r in results['hits']['hits']: 51 | c_type = r['_source']['type'] 52 | c_value = r['_source']['value'] 53 | 54 | if c_type not in SIMPLE_CHOICES: 55 | SIMPLE_CHOICES[c_type] = [] 56 | 57 | SIMPLE_CHOICES[c_type].append(c_value) 58 | 59 | for k,v in SIMPLE_CHOICES.items(): 60 | SIMPLE_CHOICES[k] = sorted(v) 61 | 62 | #print(SIMPLE_CHOICES) 63 | 64 | except Exception as e: 65 | error = "There was an error fetching choices. Details: {}".format(t, e) 66 | flash(error,'danger') 67 | log.exception(logging_prefix + error) 68 | 69 | def fetch_data(c_type, add_new=False, add_unknown=False): 70 | d = [] 71 | 72 | if not SIMPLE_CHOICES: 73 | populate_simple_choices() 74 | 75 | for value in sorted(SIMPLE_CHOICES[c_type]): 76 | d.append((value,value)) 77 | 78 | if add_new: 79 | d.append(("_NEW_","Other")) 80 | 81 | if add_unknown: 82 | d.insert(0, ("Unknown", "Unknown")) 83 | 84 | #return the data 85 | return d 86 | 87 | def fetch_parent_data(_type, add_new=False, add_unknown=False): 88 | d = [] 89 | 90 | #query 91 | body = { 92 | "query" : { 93 | "term" : { 94 | "type" : _type 95 | } 96 | }, 97 | "sort": { 98 | "value": { 99 | "order": "asc" 100 | } 101 | }, 102 | "size" : 1000 103 | } 104 | 105 | #get the data! 106 | results = get_es().search(ES_PREFIX + 'threat_actor_pc', 'parent', body) 107 | for r in results['hits']['hits']: 108 | d.append((r['_source']['value'],r['_source']['value'])) 109 | 110 | if add_new: 111 | d.append(("_NEW_","Other")) 112 | 113 | if add_unknown: 114 | d.insert(0, ("Unknown", "Unknown")) 115 | 116 | #return the data 117 | return d 118 | 119 | def fetch_child_data(_type, parent, add_new=False, add_unknown=False): 120 | d = [] 121 | 122 | #query 123 | body = { 124 | "query" : { 125 | "term" : { 126 | "_parent" : _type + " " + parent 127 | } 128 | }, 129 | "sort": { 130 | "value": { 131 | "order": "asc" 132 | } 133 | }, 134 | "size" : 1000 135 | } 136 | 137 | #get the data! 138 | results = get_es().search(ES_PREFIX + 'threat_actor_pc', 'child', body) 139 | for r in results['hits']['hits']: 140 | d.append((r['_source']['value'],r['_source']['value'])) 141 | 142 | if add_new: 143 | d.append(("_NEW_","Other")) 144 | 145 | if add_unknown: 146 | d.insert(0, ("Unknown", "Unknown")) 147 | 148 | #return the data 149 | return d 150 | 151 | def fetch_related_choices(t): 152 | logging_prefix = logger_prefix + "fetch_related_choices({}) - ".format(t) 153 | 154 | choices = [('_NONE_', 'n/a')] 155 | 156 | if t == 'actor': 157 | index = ES_PREFIX + "threat_actors" 158 | doc_type = "actor" 159 | elif t == 'report': 160 | index = ES_PREFIX + "threat_reports" 161 | doc_type = "report" 162 | elif t == 'ttp': 163 | index = ES_PREFIX + "threat_ttps" 164 | doc_type = "ttp" 165 | else: 166 | raise Exception("Invalid type '{}'. Expected 'actor', 'ttp' or 'report'") 167 | 168 | es_query = { 169 | "query": { 170 | "match_all": {} 171 | }, 172 | "size": 1000, 173 | "fields" : ["name"], 174 | "sort": { 175 | "name": { 176 | "order": "asc" 177 | } 178 | } 179 | } 180 | 181 | try: 182 | results = get_es().search(index, doc_type, es_query) 183 | for r in results['hits']['hits']: 184 | choices.append((r['_id'] + ":::" + r['fields']['name'][0],r['fields']['name'][0])) 185 | 186 | except TransportError as te: 187 | 188 | #if the index was not found, this is most likely becuase theres no data there 189 | if te.status_code == 404: 190 | log.warning("Index '{}' was not found".format(index)) 191 | else: 192 | error = "There was an error fetching related {}s. Details: {}".format(t, te) 193 | flash(error,'danger') 194 | log.exception(logging_prefix + error) 195 | 196 | except Exception as e: 197 | error = "There was an error fetching related {}s. Details: {}".format(t, e) 198 | flash(error,'danger') 199 | log.exception(logging_prefix + error) 200 | 201 | return choices 202 | 203 | def fetch_related_elements(t): 204 | if t == 'Actor': 205 | index = "tp-threat_actors" 206 | doc_type = "actor" 207 | elif t == 'Report': 208 | index = "tp-threat_reports" 209 | doc_type = "report" 210 | elif t == 'TTP': 211 | index = "tp-threat_ttps" 212 | doc_type = "ttp" 213 | else: 214 | return [] 215 | 216 | query = { 217 | "query" : { 218 | "match_all" : {} 219 | }, 220 | "sort" : { 221 | "name" : { 222 | "order" : "asc" 223 | } 224 | }, 225 | "fields" : ["name"] 226 | } 227 | 228 | results = scan(get_es(),query=query,index=index,doc_type=doc_type,preserve_order=True) 229 | 230 | choices = [] 231 | for r in results: 232 | _id = r["_id"] 233 | name = r["fields"]["name"][0] 234 | choices.append((_id + ":::" + name, name)) 235 | 236 | return choices 237 | 238 | def add_to_store(t, v, add_to_local=False): 239 | body = { 240 | "value" : escape(v), 241 | "type" : escape(t) 242 | } 243 | doc_id = t + " " + v 244 | get_es().index(ES_PREFIX + "threat_actor_simple", "data", body, doc_id) 245 | 246 | if add_to_local: 247 | global SIMPLE_CHOICES 248 | SIMPLE_CHOICES[t].append(v) 249 | 250 | print("Added {} {} to ES".format(v,t)) 251 | 252 | def get_score(classification_family, classification_id): 253 | #query 254 | body = { 255 | "query" : { 256 | "query_string" : { 257 | "query" : '+_parent:"tpx_classification {}" +value:"{}"'.format(escape(classification_family),escape(classification_id)) 258 | } 259 | }, 260 | "sort": { 261 | "value": { 262 | "order": "asc" 263 | } 264 | }, 265 | "size" : 1 266 | } 267 | 268 | #get the data! 269 | results = get_es().search(ES_PREFIX + 'threat_actor_pc', 'child', body) 270 | if results['hits']['total'] != 1: 271 | raise Exception("Unable to perform score look up for {} - {}".format(classification_family,classification_id)) 272 | 273 | return results['hits']['hits'][0]['_source']['score'] 274 | -------------------------------------------------------------------------------- /app/core/blueprints/admin.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import Blueprint 4 | from flask import flash 5 | from flask import request 6 | from flask import redirect 7 | from flask import jsonify 8 | from flask import Markup 9 | from flask import g 10 | 11 | import functools 12 | import json 13 | import hashlib 14 | import logging 15 | import os 16 | import requests 17 | import sys 18 | import time 19 | import uuid 20 | from datetime import datetime 21 | from pymysql.cursors import DictCursor 22 | from operator import itemgetter 23 | from urllib.parse import unquote_plus, quote_plus 24 | 25 | from config.settings import * 26 | from core.decorators import authentication 27 | from core.forms import forms 28 | from app import log, get_es, get_mysql 29 | from utils.elasticsearch import * 30 | from utils.functions import * 31 | 32 | #blue print def 33 | admin_blueprint = Blueprint('admin', __name__, url_prefix="/admin") 34 | logger_prefix = "admin.py:" 35 | 36 | @admin_blueprint.route("/user////", methods = ['GET']) 37 | @admin_blueprint.route("/user/////", methods = ['GET']) 38 | @authentication.access(authentication.ADMIN) 39 | def edit_user_permissions(action,value,user_id,user_c): 40 | logging_prefix = logger_prefix + "edit_user_permissions({},{},{},{}) - ".format(action,value,user_id,user_c) 41 | log.info(logging_prefix + "Starting") 42 | 43 | action_column_map = {} 44 | action_column_map['approve'] = "approved" 45 | action_column_map['write_perm'] = "write_permission" 46 | action_column_map['delete_perm'] = "delete_permission" 47 | action_column_map['admin_perm'] = "admin" 48 | 49 | success = 1 50 | try: 51 | #make sure the value is valid 52 | if value not in ["0","1"]: 53 | raise Exception("Invald value: {}".format(value)) 54 | 55 | #make sure the action is valid 56 | try: 57 | column = action_column_map[action] 58 | except Exception as f: 59 | log.warning(logging_prefix + "Action '{}' not found in action_column_map".format(action)) 60 | raise f 61 | 62 | #check the hash 63 | if user_c == sha256(SALTS['user'], str(user_id)): 64 | 65 | #if action is approve, emails need to be sent 66 | if action == "approve": 67 | conn = get_mysql().cursor(DictCursor) 68 | conn.execute("SELECT name, email FROM users WHERE id = %s", (user_id,)) 69 | user = conn.fetchone() 70 | conn.close() 71 | 72 | if value == "1": 73 | log.info(logging_prefix + "Setting approved=1 for user {}".format(user_id)) 74 | sendAccountApprovedEmail(user['email']) 75 | else: 76 | log.info(logging_prefix + "Setting approved=0 for user {}".format(user_id)) 77 | sendAccountDisapprovedEmail(user['email']) 78 | 79 | #now update the desired setting 80 | conn = get_mysql().cursor() 81 | conn.execute("UPDATE users SET "+column+" = %s WHERE id = %s", (value,user_id)) 82 | get_mysql().commit() 83 | conn.close() 84 | log.info(logging_prefix + "Successfully update {} to {} for user id {}".format(column,value,user_id)) 85 | 86 | else: 87 | log.warning(logging_prefix + "Hash mismatch {} {}".format(user_id, user_c)) 88 | except Exception as e: 89 | success = 0 90 | error = "There was an error completing your request. Details: {}".format(e) 91 | log.exception(logging_prefix + error) 92 | 93 | return jsonify({ "success" : success, "new_value" : value }) 94 | 95 | @admin_blueprint.route("/user/delete//", methods = ['GET']) 96 | @admin_blueprint.route("/user/delete///", methods = ['GET']) 97 | @authentication.access(authentication.ADMIN) 98 | def user_delete(user_id, user_id_hash): 99 | logging_prefix = logger_prefix + "user_delete({},{}) - ".format(user_id, user_id_hash) 100 | log.info(logging_prefix + "Starting") 101 | 102 | redirect_url = "/admin/" 103 | try: 104 | redirect_url = request.args.get("_r") 105 | if not redirect_url: 106 | log.warning(logging_prefix + "redirect_url not set, using default") 107 | redirect_url = "/admin/" 108 | 109 | #check user_id against user_id_hash and perform delete if match 110 | if user_id_hash == sha256( SALTS['user'], user_id ): 111 | #now delete the user 112 | conn=get_mysql().cursor(DictCursor) 113 | conn.execute("DELETE FROM users WHERE id=%s", (user_id,)) 114 | conn.close() 115 | flash("The user has been deleted", "success") 116 | else: 117 | flash("Unable to delete user", "danger") 118 | 119 | except Exception as e: 120 | error = "There was an error completing your request. Details: {}".format(e) 121 | flash(error,'danger') 122 | log.exception(logging_prefix + error) 123 | 124 | return redirect(redirect_url) 125 | 126 | 127 | @admin_blueprint.route("/email", methods = ['POST']) 128 | @admin_blueprint.route("/email/", methods = ['POST']) 129 | @authentication.access(authentication.ADMIN) 130 | def email_user(): 131 | 132 | logging_prefix = logger_prefix + "email_user() - " 133 | log.info(logging_prefix + "Starting") 134 | 135 | try: 136 | email = request.form['email'] 137 | subject = request.form['subject'] 138 | body = request.form['body'] 139 | 140 | log.debug("Email: {}, Subject: {}, Body: {}".format(email, subject, body)) 141 | sendCustomEmail(email, subject, body) 142 | except Exception as e: 143 | error = "There was an error completing your request. Details: {}".format(e) 144 | log.exception(logging_prefix + error) 145 | 146 | return jsonify({ 'success' : False, 'error' : str(e) }) 147 | 148 | return jsonify({ 'success' : True }) 149 | 150 | ''' 151 | Admin Pages 152 | ''' 153 | 154 | @admin_blueprint.route("/", methods = ['GET','POST']) 155 | @admin_blueprint.route("", methods = ['GET','POST']) 156 | @authentication.access(authentication.ADMIN) 157 | def main(): 158 | logging_prefix = logger_prefix + "main() - " 159 | log.info(logging_prefix + "Starting") 160 | 161 | try: 162 | conn=get_mysql().cursor(DictCursor) 163 | conn.execute("SELECT id, name, email, company, justification, email_verified, approved, write_permission, delete_permission, admin, created, last_login FROM users ORDER BY created DESC") 164 | 165 | users = conn.fetchall() 166 | for user in users: 167 | user['id_hash'] = sha256( SALTS['user'], str(user['id']) ) 168 | 169 | conn.close() 170 | 171 | email_user_form = forms.sendUserEmailForm() 172 | 173 | except Exception as e: 174 | error = "There was an error completing your request. Details: {}".format(e) 175 | flash(error,'danger') 176 | log.exception(logging_prefix + error) 177 | 178 | 179 | return render_template("admin.html", 180 | page_title="Admin", 181 | url = "/admin/", 182 | users=users, 183 | email_user_form = email_user_form 184 | ) 185 | @admin_blueprint.route("/choices", methods = ['GET','POST']) 186 | @admin_blueprint.route("/choices/", methods = ['GET','POST']) 187 | @authentication.access(authentication.ADMIN) 188 | def choices(): 189 | logging_prefix = logger_prefix + "choices() - " 190 | log.info(logging_prefix + "Starting") 191 | 192 | simple_choices = None 193 | try: 194 | 195 | body = { 196 | "query" : { 197 | "match_all" : {} 198 | }, 199 | "size" : 1000 200 | } 201 | 202 | results = get_es().search(ES_PREFIX + 'threat_actor_simple', 'data', body) 203 | 204 | parsed_results = [] 205 | for r in results['hits']['hits']: 206 | d = {} 207 | d['type'] = r['_source']['type'] 208 | d['value'] = r['_source']['value'] 209 | d['id'] = r['_id'] 210 | 211 | #determine how many actor profiles use this value 212 | 213 | query_string = None 214 | if d['type'] == "classification": 215 | query_string = "type:\"" + escape(d['value']) + "\"" 216 | elif d['type'] == "communication": 217 | query_string = "communication_address.type:\"" + escape(d['value']) + "\"" 218 | elif d['type'] == "country": 219 | query_string = "country_affiliation:\"" + escape(d['value']) + "\" origin:\"" + escape(d['value']) + "\"" 220 | 221 | if query_string: 222 | body = { 223 | "query" : { 224 | "query_string" : { 225 | "query" : query_string 226 | } 227 | }, 228 | "size" : 0 229 | } 230 | 231 | count = get_es().search(ES_PREFIX + 'threat_actors', 'actor', body) 232 | 233 | d['count'] = count['hits']['total'] 234 | d['q'] = quote_plus(query_string) 235 | else: 236 | d['count'] = "-" 237 | d['q'] = "" 238 | 239 | parsed_results.append(d) 240 | 241 | simple_choices = multikeysort(parsed_results, ['type', 'value']) 242 | 243 | except Exception as e: 244 | error = "There was an error completing your request. Details: {}".format(e) 245 | flash(error,'danger') 246 | log.exception(logging_prefix + error) 247 | 248 | return render_template("admin_choices.html", 249 | page_title="Admin", 250 | simple_choices = simple_choices 251 | ) 252 | 253 | 254 | -------------------------------------------------------------------------------- /app/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 |
5 |
6 |

Search

7 |
8 |
9 |
10 |
11 |
12 | 13 | {{ form.hidden_tag() }} 14 | 15 |
16 |
17 | {{ form.query(class="form-control", style="width:100%") }} 18 | {% if form.query.errors %} 19 |
{% for error in form.query.errors %}{{ error }}
{% endfor %}
20 | {% endif %} 21 |
22 |
23 | 24 |
25 |
26 | 27 |
28 |
29 |
30 | 31 |
32 |
33 |

Recent Actors View all

34 |
35 | 36 | {% if session['write'] %} 37 |
38 | 39 | 42 | 43 |
44 | {% endif %} 45 |
46 | 47 | {% set action_width = "70" %} 48 | {% if session['write'] and session['delete'] %} 49 | {% set action_width = "100" %} 50 | {% endif %} 51 | 52 | 53 | 54 | {% if actors['hits']['hits'] %} 55 |
56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | {% for actor in actors['hits']['hits'] %} 65 | 66 | 67 | 102 | 103 | {% endfor %} 104 | 105 |
Actor NameActions
{{ actor['_source']['name']|e }} 68 | 69 | 72 | 73 | 74 | {% if session['write'] %} 75 | 76 | 79 | 80 | {% endif %} 81 | 82 | 93 | 94 | {% if session['delete'] %} 95 | 96 | 99 | 100 | {% endif %} 101 |
106 |
107 | {% else %} 108 |
109 |
110 | No results found 111 |
112 |
113 | {% endif %} 114 | 115 |
116 |
117 |

Recent Reports View all

118 |
119 | 120 | {% if session['write'] %} 121 |
122 | 123 | 126 | 127 |
128 | {% endif %} 129 |
130 | 131 | {% if reports['hits']['hits'] %} 132 |
133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | {% for report in reports['hits']['hits'] %} 142 | 143 | 144 | 167 | 168 | {% endfor %} 169 | 170 |
Report NameActions
{{ report['_source']['name']|e }} 145 | 146 | 149 | 150 | 151 | {% if session['write'] %} 152 | 153 | 156 | 157 | {% endif %} 158 | 159 | {% if session['delete'] %} 160 | 161 | 164 | 165 | {% endif %} 166 |
171 |
172 | {% else %} 173 |
174 |
175 | No results found 176 |
177 |
178 | {% endif %} 179 | 180 |
181 |
182 |

Recent TTPs View all

183 |
184 | 185 | {% if session['write'] %} 186 |
187 | 188 | 191 | 192 |
193 | {% endif %} 194 |
195 | 196 | {% if ttps['hits']['hits'] %} 197 |
198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | {% for ttp in ttps['hits']['hits'] %} 207 | 208 | 209 | 232 | 233 | {% endfor %} 234 | 235 |
TTP NameActions
{{ ttp['_source']['name']|e }} 210 | 211 | 214 | 215 | 216 | {% if session['write'] %} 217 | 218 | 221 | 222 | {% endif %} 223 | 224 | {% if session['delete'] %} 225 | 226 | 229 | 230 | {% endif %} 231 |
236 |
237 | {% else %} 238 |
239 |
240 | No results found 241 |
242 |
243 | {% endif %} 244 | 245 | {% endblock %} -------------------------------------------------------------------------------- /app/templates/layouts/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | {% if page_title %} 13 | {{ page_title }} 14 | {% else %} 15 | ActorTrackr 16 | {% endif %} 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 30 | 31 | 32 | 33 | 62 | 63 | 64 | 65 | 66 | 67 | 131 | 132 | 133 |
134 | 135 | {% with messages = get_flashed_messages(with_categories=true) %} 136 | {% if messages %} 137 | {% for category, message in messages %} 138 |
139 | {{message}} 140 |
141 | {% endfor %} 142 | {% endif %} 143 | {% endwith %} 144 | 145 |

{{ page_title }}

146 | 147 | {% block container %}{% endblock %} 148 | 149 | {% if session['admin'] %} 150 | {% if editors %} 151 |

Edit History

152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | {% for editor in editors %} 162 | 163 | 164 | 165 | 166 | {% endfor %} 167 | 168 |
NameTimestamp
{{ editor['user_name'] }}{{ editor['ts'] }}
169 | {% endif %} 170 | {% endif %} 171 | 172 | 173 | 174 |
175 |
176 | 177 | 178 | 192 |
193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 431 | 432 | 433 | {% block javascript %}{% endblock %} 434 | 435 | 436 | 437 | -------------------------------------------------------------------------------- /app/core/blueprints/index.py: -------------------------------------------------------------------------------- 1 | 2 | from flask import Flask 3 | from flask import render_template 4 | from flask import Blueprint 5 | from flask import flash 6 | from flask import request 7 | from flask import redirect 8 | from flask import make_response 9 | from flask import jsonify 10 | from flask import Markup 11 | from flask import g 12 | 13 | from elasticsearch import TransportError 14 | from elasticsearch.helpers import scan 15 | 16 | import functools 17 | import json 18 | import hashlib 19 | import logging 20 | import os 21 | import sys 22 | import time 23 | import uuid 24 | from datetime import datetime 25 | from operator import itemgetter 26 | from urllib.parse import quote_plus 27 | 28 | from config.settings import * 29 | from core.decorators import authentication 30 | from core.forms import forms 31 | from app import log, get_es, get_mysql 32 | 33 | 34 | #blue print def 35 | index_blueprint = Blueprint('index', __name__, url_prefix="") 36 | logger_prefix = "index.py:" 37 | 38 | 39 | @index_blueprint.route("/about", methods = ['GET']) 40 | @authentication.access(authentication.PUBLIC) 41 | def about(): 42 | search_form = forms.searchForm() 43 | 44 | return render_template("about.html", 45 | page_title="About", 46 | search_form = search_form 47 | ) 48 | 49 | @index_blueprint.route("/contact", methods = ['GET']) 50 | @authentication.access(authentication.PUBLIC) 51 | def contact(): 52 | search_form = forms.searchForm() 53 | 54 | return render_template("contact.html", 55 | page_title="Contact", 56 | search_form = search_form 57 | ) 58 | 59 | @index_blueprint.route("/", methods = ['GET','POST']) 60 | @authentication.access(authentication.PUBLIC) 61 | def index(): 62 | logging_prefix = logger_prefix + "index() - " 63 | log.info(logging_prefix + "Loading home page") 64 | 65 | form = forms.searchForm(request.form) 66 | error = None 67 | url = "/" 68 | query_string_url = "" 69 | try: 70 | #this is the default query for actors in ES, i'd imagine this will be recently added/modified actors 71 | es_query = { 72 | "query": { 73 | "match_all": {} 74 | }, 75 | "size": 10, 76 | "sort": { 77 | "last_updated_s": { 78 | "order": "desc" 79 | } 80 | } 81 | } 82 | 83 | #pull the query out of the url 84 | query_string = request.args.get("q") 85 | 86 | #someone is searching for something 87 | if request.method == 'POST' and not query_string: 88 | if form.validate(): 89 | 90 | #get the value 91 | value = form.query.data 92 | 93 | #redirect to this same page, but setting the query value in the url 94 | return redirect("/?q={}".format(quote_plus(value)), code=307) 95 | 96 | else: 97 | #if there was an error print the error dictionary to the console 98 | # temporary help, these should also appear under the form field 99 | print(form.errors) 100 | 101 | elif query_string: 102 | #now that the query_string is provided as ?q=, perform the search 103 | print("VALID SEARCH OPERATION DETECTED") 104 | 105 | #do some searching... 106 | es_query = { 107 | "query": { 108 | "query_string": { 109 | "query" : query_string 110 | } 111 | }, 112 | "size": 10 113 | } 114 | 115 | url += "?q=" + query_string 116 | query_string_url = "?q=" + query_string 117 | 118 | #set the form query value to what the user is searching for 119 | form.query.data = query_string 120 | 121 | ''' 122 | Fetch the data from ES 123 | ''' 124 | 125 | actors = {} 126 | actors['hits'] = {} 127 | actors['hits']['hits'] = [] 128 | 129 | reports = dict(actors) 130 | 131 | ttps = dict(actors) 132 | 133 | try: 134 | actors = get_es().search(ES_PREFIX + 'threat_actors', 'actor', es_query) 135 | except TransportError as te: 136 | #if the index was not found, this is most likely becuase theres no data there 137 | if te.status_code == 404: 138 | log.warning("Index 'threat_actors' was not found") 139 | else: 140 | error = "There was an error fetching actors. Details: {}".format(te) 141 | flash(error,'danger') 142 | log.exception(logging_prefix + error) 143 | except Exception as e: 144 | error = "The was an error fetching Actors. Error: {}".format(e) 145 | log.exception(error) 146 | flash(error, "danger") 147 | 148 | try: 149 | reports = get_es().search(ES_PREFIX + 'threat_reports', 'report', es_query) 150 | except TransportError as te: 151 | #if the index was not found, this is most likely becuase theres no data there 152 | if te.status_code == 404: 153 | log.warning("Index 'threat_reports' was not found") 154 | else: 155 | error = "There was an error fetching reports. Details: {}".format(te) 156 | flash(error,'danger') 157 | log.exception(logging_prefix + error) 158 | except Exception as e: 159 | error = "The was an error fetching Reports. Error: {}".format(e) 160 | log.exception(error) 161 | flash(error, "danger") 162 | 163 | try: 164 | ttps = get_es().search(ES_PREFIX + 'threat_ttps', 'ttp', es_query) 165 | except TransportError as te: 166 | #if the index was not found, this is most likely becuase theres no data there 167 | if te.status_code == 404: 168 | log.warning("Index 'threat_ttps' was not found") 169 | else: 170 | error = "There was an error ttps. Details: {}".format(te) 171 | flash(error,'danger') 172 | log.exception(logging_prefix + error) 173 | except Exception as e: 174 | error = "The was an error fetching TTPs. Error: {}".format(e) 175 | log.exception(error) 176 | flash(error, "danger") 177 | 178 | ''' 179 | Modify the data as needed 180 | ''' 181 | 182 | for actor in actors['hits']['hits']: 183 | s = SALTS['actor'] + actor['_id'] 184 | hash_object = hashlib.sha256(s.encode('utf-8')) 185 | hex_dig = hash_object.hexdigest() 186 | actor["_source"]['id_hash'] = hex_dig 187 | 188 | for report in reports['hits']['hits']: 189 | s = SALTS['report'] + report['_id'] 190 | hash_object = hashlib.sha256(s.encode('utf-8')) 191 | hex_dig = hash_object.hexdigest() 192 | report["_source"]['id_hash'] = hex_dig 193 | 194 | for ttp in ttps['hits']['hits']: 195 | s = SALTS['ttp'] + ttp['_id'] 196 | hash_object = hashlib.sha256(s.encode('utf-8')) 197 | hex_dig = hash_object.hexdigest() 198 | ttp["_source"]['id_hash'] = hex_dig 199 | 200 | except Exception as e: 201 | error = "There was an error completing your request. Details: {}".format(e) 202 | log.exception(error) 203 | flash(error, "danger") 204 | 205 | #render the template, passing the variables we need 206 | # templates live in the templates folder 207 | return render_template("index.html", 208 | page_title="ActorTrackr", 209 | form=form, 210 | query_string_url=query_string_url, 211 | actors=actors, 212 | reports=reports, 213 | ttps=ttps, 214 | url = quote_plus(url) 215 | ) 216 | 217 | @index_blueprint.route("/export", methods = ['GET']) 218 | @index_blueprint.route("/export/", methods = ['GET']) 219 | @authentication.access(authentication.PUBLIC) 220 | def export_all_the_data(): 221 | logging_prefix = logger_prefix + "export_all_the_data() - " 222 | log.info(logging_prefix + "Exporting") 223 | 224 | try: 225 | dump = {} 226 | dump['actors'] = [] 227 | dump['reports'] = [] 228 | dump['ttps'] = [] 229 | dump['choices'] = {} 230 | 231 | query = { 232 | "query" : { 233 | "match_all" : {} 234 | } 235 | } 236 | 237 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actors",doc_type="actor") 238 | 239 | 240 | for i in results: 241 | dump['actors'].append(i) 242 | 243 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_reports",doc_type="report") 244 | 245 | for i in results: 246 | dump['reports'].append(i) 247 | 248 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_ttps",doc_type="ttp") 249 | 250 | for i in results: 251 | dump['ttps'].append(i) 252 | 253 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actor_pc",doc_type="parent") 254 | 255 | dump['choices']['parents'] = [] 256 | for i in results: 257 | dump['choices']['parents'].append(i) 258 | 259 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actor_pc",doc_type="child") 260 | 261 | dump['choices']['children'] = [] 262 | for i in results: 263 | dump['choices']['children'].append(i) 264 | 265 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actor_simple",doc_type="data") 266 | 267 | dump['choices']['simple'] = [] 268 | for i in results: 269 | dump['choices']['simple'].append(i) 270 | 271 | # We need to modify the response, so the first thing we 272 | # need to do is create a response out of the Dictionary 273 | response = make_response(json.dumps(dump)) 274 | 275 | # This is the key: Set the right header for the response 276 | # to be downloaded, instead of just printed on the browser 277 | response.headers["Content-Disposition"] = "attachment; filename=export.json" 278 | response.headers["Content-Type"] = "text/json; charset=utf-8" 279 | 280 | return response 281 | 282 | except Exception as e: 283 | error = "There was an error completing your request. Details: {}".format(e) 284 | log.exception(error) 285 | flash(error, "danger") 286 | return redirect("/") 287 | 288 | @index_blueprint.route("/", methods = ['GET','POST']) 289 | @index_blueprint.route("//", methods = ['GET','POST']) 290 | @index_blueprint.route("//", methods = ['GET','POST']) 291 | @index_blueprint.route("///", methods = ['GET','POST']) 292 | def view_all(t,page=1): 293 | if t == 'favicon.ico': 294 | return jsonify({}),404 295 | 296 | page = int(page) 297 | 298 | logging_prefix = logger_prefix + "view_all() - " 299 | log.info(logging_prefix + "Loading view all page {} for {}".format(page, t)) 300 | 301 | form = forms.searchForm(request.form) 302 | error = None 303 | page_size = 50 304 | offset = (page-1) * page_size 305 | url = "/{}/{}/".format(t,page) 306 | search_url = "" 307 | results_text = "" 308 | try: 309 | 310 | 311 | #this is the default query for actors in ES, i'd imagine this will be recently added/modified actors 312 | es_query = { 313 | "query": { 314 | "match_all": {} 315 | }, 316 | "size": page_size, 317 | "from" : offset, 318 | "sort": { 319 | "last_updated_s": { 320 | "order": "desc" 321 | } 322 | } 323 | } 324 | 325 | #pull the query out of the url 326 | query_string = request.args.get("q") 327 | 328 | #someone is searching for something 329 | if request.method == 'POST' and not query_string: 330 | if form.validate(): 331 | print("VALID SEARCH OPERATION DETECTED, redirecting...") 332 | 333 | #get the value 334 | value = form.query.data 335 | 336 | 337 | log.info(value) 338 | 339 | #redirect to this same page, but setting the query value in the url 340 | return redirect("/{}/1/?q={}".format(t,quote_plus(value)), code=307) 341 | 342 | else: 343 | #if there was an error print the error dictionary to the console 344 | # temporary help, these should also appear under the form field 345 | print(form.errors) 346 | 347 | elif query_string: 348 | #now that the query_string is provided as ?q=, perform the search 349 | print("VALID SEARCH OPERATION DETECTED") 350 | 351 | #do some searching... 352 | es_query = { 353 | "query": { 354 | "query_string": { 355 | "query" : query_string 356 | } 357 | }, 358 | "size": page_size, 359 | "from" : offset 360 | } 361 | 362 | search_url = "?q=" + query_string 363 | #set the form query value to what the user is searching for 364 | form.query.data = query_string 365 | 366 | ''' 367 | Fetch the data from ES 368 | ''' 369 | 370 | data = {} 371 | data['hits'] = {} 372 | data['hits']['hits'] = [] 373 | 374 | if t == 'actor': 375 | index = ES_PREFIX + 'threat_actors' 376 | doc_type = 'actor' 377 | salt = SALTS['actor'] 378 | link_prefix = 'actor' 379 | data_header = 'Actors' 380 | field_header = 'Actor Name' 381 | elif t == 'report': 382 | index = ES_PREFIX + 'threat_reports' 383 | doc_type = 'report' 384 | salt = SALTS['report'] 385 | link_prefix = 'report' 386 | data_header = 'Reports' 387 | field_header = 'Report Title' 388 | elif t == 'ttp': 389 | index = ES_PREFIX + 'threat_ttps' 390 | doc_type = 'ttp' 391 | salt = SALTS['ttp'] 392 | link_prefix = 'ttp' 393 | data_header = 'TTPs' 394 | field_header = 'TTP Name' 395 | else: 396 | raise Exception("Unknown type {}".format(t)) 397 | 398 | try: 399 | data = get_es().search(index, doc_type, es_query) 400 | num_hits = len(data['hits']['hits']) 401 | 402 | #set up previous link 403 | if page == 1: 404 | prev_url = None 405 | else: 406 | prev_url = "/{}/{}/{}".format(t,(page-1),search_url) 407 | 408 | if ((page-1)*page_size) + num_hits < data['hits']['total']: 409 | next_url = "/{}/{}/{}".format(t,(page+1),search_url) 410 | else: 411 | next_url = None 412 | 413 | url += search_url 414 | 415 | for d in data['hits']['hits']: 416 | s = salt + d['_id'] 417 | hash_object = hashlib.sha256(s.encode('utf-8')) 418 | hex_dig = hash_object.hexdigest() 419 | d["_source"]['id_hash'] = hex_dig 420 | 421 | if num_hits == 0: 422 | results_text = "" 423 | else: 424 | f = ( (page-1) * page_size ) + 1 425 | l = f + (num_hits-1) 426 | results_text = "Showing {} to {} of {} total results".format(f,l,data['hits']['total']) 427 | 428 | except TransportError as te: 429 | 430 | #if the index was not found, this is most likely becuase theres no data there 431 | if te.status_code == 404: 432 | log.warning("Index '{}' was not found".format(index)) 433 | else: 434 | error = "There was an error fetching {}. Details: {}".format(t, te) 435 | flash(error,'danger') 436 | log.exception(logging_prefix + error) 437 | 438 | except Exception as e: 439 | error = "The was an error fetching {}. Error: {}".format(t,e) 440 | log.exception(error) 441 | flash(error, "danger") 442 | 443 | 444 | except Exception as e: 445 | error = "There was an error completing your request. Details: {}".format(e) 446 | log.exception(error) 447 | flash(error, "danger") 448 | return redirect("/") 449 | 450 | return render_template("view_all.html", 451 | page_title="View All", 452 | form=form, 453 | data_header=data_header, 454 | results_text=results_text, 455 | field_header=field_header, 456 | data=data, 457 | link_prefix=link_prefix, 458 | prev_url=prev_url, 459 | next_url=next_url, 460 | url = quote_plus(url) 461 | ) 462 | 463 | -------------------------------------------------------------------------------- /app/templates/ttp.html: -------------------------------------------------------------------------------- 1 | {% extends "layouts/base.html" %} 2 | {% block container %} 3 | 4 | 5 | {% if role=='VIEW' %} 6 |
7 |
8 |

TTP

9 |
10 |
11 | 12 | {% if session['write'] %} 13 | 14 | {% endif %} 15 |
16 |
17 | {% elif role=='EDIT'%} 18 |
19 |
20 |

TTP

21 |
22 |
23 | 24 | 25 |
26 |
27 | {% else %} 28 | 29 |

TTP

30 | 31 | {% endif %} 32 | 33 | 34 | 51 | 52 |
53 |
54 |
55 | {{ form.hidden_tag() }} 56 | {% if form.errors %} 57 |
58 |
59 | There were errors submitting the form! 60 | {% if 'csrf_token' in form.errors %} 61 | Invalid CSRF Token, try submitting the form again 62 | {% endif %} 63 |
64 | 67 |
68 | {% endif %} 69 | 70 |
71 | 72 | {% if role=='ADD' or role=='EDIT' %} 73 | {{ form.ttp_name(class="form-control") }} 74 | {% else %} 75 | {{ form.ttp_name(class="form-control", disabled=true) }} 76 | {% endif %} 77 | {% if form.ttp_name.errors %} 78 |
{% for error in form.ttp_name.errors %}{{ error }}
{% endfor %}
79 | {% endif %} 80 |
81 | 82 |
83 | 84 | {% if role=='ADD' or role=='EDIT' %} 85 | {{ form.ttp_first_observed(class="form-control") }} 86 | {% else %} 87 | {{ form.ttp_first_observed(class="form-control", disabled=true) }} 88 | {% endif %} 89 | {% if form.ttp_first_observed.errors %} 90 |
{% for error in form.ttp_first_observed.errors %}{{ error }}
{% endfor %}
91 | {% endif %} 92 |
93 | 94 |
95 | 96 | {% if role=='ADD' or role=='EDIT' %} 97 | {{ form.ttp_description(class="form-control") }} 98 | {% else %} 99 | {{ form.ttp_description(class="form-control", disabled=true) }} 100 | {% endif %} 101 | {% if form.ttp_description.errors %} 102 |
{% for error in form.ttp_description.errors %}{{ error }}
{% endfor %}
103 | {% endif %} 104 |
105 | 106 |
107 | 108 | {% if role=='ADD' or role=='EDIT' %} 109 | {{ form.ttp_criticality(class="form-control") }} 110 | {% else %} 111 | {{ form.ttp_criticality(class="form-control", disabled=true) }} 112 | {% endif %} 113 | {% if form.ttp_criticality.errors %} 114 |
{% for error in form.ttp_criticality.errors %}{{ error }}
{% endfor %}
115 | {% endif %} 116 |
117 | 118 |
119 |
120 |
121 | {% if role=='ADD' or role=='EDIT' %} 122 |
123 |
124 | 125 |
126 |
127 | 128 |
129 |
130 | 131 | {% for l in form.ttp_class %} 132 |
133 |
134 | {{ l.form.a_family(class="form-control", onchange="javascript:selectChanged(this.id,'tpx_classification')", **{'data-change-target':'true'} ) }} 135 |
136 |
137 | {{ l.form.a_id(class="form-control dynamic-target") }} 138 |
139 |
140 | 141 |
142 |
143 | {% endfor %} 144 | 145 |
146 |
147 | 148 |
149 |
150 | 151 | {% else %} 152 |
153 |
154 | 155 |
156 |
157 | 158 |
159 |
160 | 161 | {% for l in form.ttp_class %} 162 |
163 |
164 | {{ l.form.a_family(class="form-control", disabled=true) }} 165 |
166 |
167 | {{ l.form.a_id(class="form-control", disabled=true) }} 168 |
169 |
170 | {% endfor %} 171 | 172 | {% endif %} 173 | 174 | {% if form.ttp_class.errors %} 175 |
{% for error in form.ttp_class.errors %}{{ error }}
{% endfor %}
176 | {% endif %} 177 |
178 |
179 |
180 | 181 |
182 | 183 |
184 | 185 | 227 |
228 | 229 |
230 | 231 | 273 |
274 | 275 |
276 | 277 | 319 |
320 | 321 |
322 | {% if role=='ADD' %} 323 | 324 | {% elif role=='EDIT' %} 325 | 326 | {% else %} 327 |   328 | {% endif %} 329 |
330 | 331 |
332 |
333 | 334 | {% endblock %} 335 | 336 | {% block javascript %} 337 | 340 | {% endblock %} 341 | -------------------------------------------------------------------------------- /app/core/blueprints/ttp.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import Blueprint 4 | from flask import flash 5 | from flask import request 6 | from flask import redirect 7 | from flask import jsonify 8 | from flask import Markup 9 | from flask import g 10 | from flask import session 11 | 12 | import functools 13 | import json 14 | import hashlib 15 | import logging 16 | import os 17 | import sys 18 | import time 19 | import uuid 20 | from datetime import datetime 21 | from operator import itemgetter 22 | from urllib.parse import quote_plus 23 | 24 | from config.settings import * 25 | from core.decorators import authentication 26 | from core.forms import forms 27 | from app import log, get_es, get_mysql 28 | from utils.elasticsearch import * 29 | from utils.functions import * 30 | 31 | ttp_blueprint = Blueprint('ttp', __name__, url_prefix="/ttp") 32 | logger_prefix = "ttp.py:" 33 | 34 | def es_to_form(ttp_id): 35 | form = forms.ttpForm() 36 | 37 | #get the values from ES 38 | results = get_es().get(ES_PREFIX + "threat_ttps", doc_type="ttp", id=ttp_id) 39 | 40 | 41 | #store certain fields from ES, so this form can be used in an update 42 | form.doc_index.data = results['_index'] 43 | form.doc_type.data = results['_type'] 44 | 45 | ttp_data = results['_source'] 46 | 47 | form.ttp_name.data = ttp_data['name'] 48 | form.ttp_first_observed.data = datetime.strptime(ttp_data['created_s'],"%Y-%m-%dT%H:%M:%S") 49 | form.ttp_description.data = ttp_data['description'] 50 | form.ttp_criticality.data = int(ttp_data['criticality']) 51 | 52 | idx = 0 53 | for entry in range(len(form.ttp_class.entries)): form.ttp_class.pop_entry() 54 | for i in multikeysort(ttp_data['classification'], ['family', 'id']): 55 | ttp_class_form = forms.TPXClassificationForm() 56 | ttp_class_form.a_family = i['family'] 57 | ttp_class_form.a_id = i['id'] 58 | 59 | form.ttp_class.append_entry(ttp_class_form) 60 | 61 | #set the options since this select is dynamic 62 | form.ttp_class[idx].a_id.choices = fetch_child_data('tpx_classification',i['family']) 63 | idx += 1 64 | 65 | if ttp_data['related_actor']: 66 | for entry in range(len(form.ttp_actors.entries)): form.ttp_actors.pop_entry() 67 | for i in multikeysort(ttp_data['related_actor'], ['name', 'id']): 68 | sub_form = forms.RelatedActorsForm() 69 | sub_form.data = i['id'] + ":::" + i['name'] 70 | 71 | form.ttp_actors.append_entry(sub_form) 72 | 73 | if ttp_data['related_report']: 74 | for entry in range(len(form.ttp_reports.entries)): form.ttp_reports.pop_entry() 75 | for i in multikeysort(ttp_data['related_report'], ['name', 'id']): 76 | sub_form = forms.RelatedReportsForm() 77 | sub_form.data = i['id'] + ":::" + i['name'] 78 | 79 | form.ttp_reports.append_entry(sub_form) 80 | 81 | if ttp_data['related_ttp']: 82 | for entry in range(len(form.ttp_ttps.entries)): form.ttp_ttps.pop_entry() 83 | for i in multikeysort(ttp_data['related_ttp'], ['name', 'id']): 84 | sub_form = forms.RelatedTTPsForm() 85 | sub_form.data = i['id'] + ":::" + i['name'] 86 | 87 | form.ttp_ttps.append_entry(sub_form) 88 | 89 | #convert editor dictionary of ids and times to names and times 90 | editors = get_editor_names(get_mysql(), ttp_data['editor']) 91 | 92 | return form, editors 93 | 94 | def es_to_tpx(ttp_id): 95 | ''' 96 | Build the TPX file from the data stored in Elasticsearch 97 | ''' 98 | element_observables = {} 99 | 100 | results = get_es().get(ES_PREFIX + "threat_ttps", doc_type="ttp", id=ttp_id) 101 | ttp_data = results['_source'] 102 | 103 | tpx = {} 104 | tpx["schema_version_s"] = "2.2.0" 105 | tpx["provider_s"] = "LookingGlass" 106 | tpx["list_name_s"] = "Threat Actor" 107 | tpx["created_t"] = ttp_data['created_milli'] 108 | tpx["created_s"] = ttp_data['created_s'] 109 | tpx["last_updated_t"] = ttp_data['last_updated_milli'] 110 | tpx["last_updated_s"] = ttp_data['last_updated_s'] 111 | tpx["score_i"] = 95 112 | tpx["source_observable_s"] = "Cyveillance Threat Actor" 113 | tpx["source_description_s"] = "This feed provides threat actor or threat actor group profiles and characterizations created by the LookingGlass Cyber Threat Intelligence Group" 114 | 115 | tpx["observable_dictionary_c_array"] = [] 116 | 117 | observable_dict = {} 118 | observable_dict["ttp_uuid_s"] = ttp_id 119 | observable_dict["observable_id_s"] = ttp_data['name'] 120 | observable_dict["description_s"] = ttp_data['description'] 121 | observable_dict["criticality_i"] = ttp_data['criticality'] 122 | 123 | observable_dict["classification_c_array"] = [] 124 | 125 | class_dict = {} 126 | class_dict["score_i"] = 70 127 | class_dict["classification_id_s"] = "Intel" 128 | class_dict["classification_family_s"] = "TTP" 129 | observable_dict["classification_c_array"].append(class_dict) 130 | 131 | for i in ttp_data['classification']: 132 | class_dict = {} 133 | class_dict["score_i"] = i["score"] 134 | class_dict["classification_id_s"] = i["id"] 135 | class_dict["classification_family_s"] = i["family"] 136 | 137 | if class_dict not in observable_dict["classification_c_array"]: 138 | observable_dict["classification_c_array"].append(class_dict) 139 | 140 | observable_dict["related_ttps_c_array"] = [] 141 | for i in ttp_data['related_ttp']: 142 | if i['name']: 143 | observable_dict["related_ttps_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] }) 144 | 145 | 146 | observable_dict["related_actors_c_array"] = [] 147 | for i in ttp_data['related_actor']: 148 | if i['name']: 149 | observable_dict["related_actors_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] }) 150 | 151 | observable_dict["related_reports_c_array"] = [] 152 | for i in ttp_data['related_report']: 153 | if i['name']: 154 | observable_dict["related_reports_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] }) 155 | 156 | 157 | ''' 158 | Related elements 159 | ''' 160 | 161 | relate_element_name_map = { 162 | "FQDN" : "subject_fqdn_s", 163 | "IPv4" : "subject_ipv4_s", 164 | "TTP" : "subject_ttp_s", 165 | "CommAddr" : "subject_address_s" 166 | } 167 | 168 | 169 | 170 | tpx["observable_dictionary_c_array"].append(observable_dict) 171 | 172 | return tpx 173 | 174 | def form_to_es(form, ttp_id): 175 | logging_prefix = logger_prefix + "form_to_es() - " 176 | log.info(logging_prefix + "Converting Form to ES for {}".format(ttp_id)) 177 | 178 | doc = {} 179 | 180 | created_t = int(time.mktime(form.ttp_first_observed.data.timetuple())) * 1000 181 | created_s = form.ttp_first_observed.data.strftime("%Y-%m-%dT%H:%M:%S") 182 | now_t = int(time.time()) * 1000 183 | now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") 184 | 185 | doc["created_milli"] = created_t 186 | doc["created_s"] = created_s 187 | doc["last_updated_milli"] = now_t 188 | doc["last_updated_s"] = now_s 189 | 190 | doc['name'] = escape(form.ttp_name.data) 191 | doc['description'] = escape(form.ttp_description.data) 192 | doc['criticality'] = int(escape(form.ttp_criticality.data)) 193 | 194 | doc['classification'] = [] 195 | for sub_form in form.ttp_class.entries: 196 | classification_dict = {} 197 | classification_dict['score'] = int(get_score(sub_form.data['a_family'], sub_form.data['a_id'])) 198 | classification_dict['id'] = escape(sub_form.data['a_id']) 199 | classification_dict['family'] = escape(sub_form.data['a_family']) 200 | 201 | if classification_dict not in doc['classification']: 202 | doc['classification'].append(classification_dict) 203 | 204 | #Links to actors and reports 205 | doc['related_actor'] = [] 206 | for sub_form in form.ttp_actors.entries: 207 | r_dict = {} 208 | data = escape(sub_form.data.data) 209 | 210 | if data == "_NONE_": 211 | continue 212 | 213 | data_array = data.split(":::") 214 | 215 | r_dict['id'] = escape(data_array[0]) 216 | r_dict['name'] = escape(data_array[1]) 217 | #this is gonna be a nightmare to maintain, 218 | # but it make sense to have this for searches 219 | 220 | if r_dict not in doc['related_actor']: 221 | doc['related_actor'].append(r_dict) 222 | 223 | doc['related_report'] = [] 224 | for sub_form in form.ttp_reports.entries: 225 | r_dict = {} 226 | data = escape(sub_form.data.data) 227 | 228 | if data == "_NONE_": 229 | continue 230 | 231 | data_array = data.split(":::") 232 | 233 | r_dict['id'] = escape(data_array[0]) 234 | r_dict['name'] = escape(data_array[1]) 235 | #this is gonna be a nightmare to maintain, 236 | # but it make sense to have this for searches 237 | 238 | if r_dict not in doc['related_report']: 239 | doc['related_report'].append(r_dict) 240 | 241 | doc['related_ttp'] = [] 242 | for sub_form in form.ttp_ttps.entries: 243 | r_dict = {} 244 | data = escape(sub_form.data.data) 245 | 246 | if data == "_NONE_": 247 | continue 248 | 249 | data_array = data.split(":::") 250 | 251 | r_dict['id'] = escape(data_array[0]) 252 | r_dict['name'] = escape(data_array[1]) 253 | #this is gonna be a nightmare to maintain, 254 | # but it make sense to have this for searches 255 | 256 | if r_dict not in doc['related_ttp']: 257 | doc['related_ttp'].append(r_dict) 258 | 259 | ''' 260 | Edit Tracking 261 | ''' 262 | 263 | doc['editor'] = get_editor_list( 264 | es=get_es(), 265 | index=ES_PREFIX + "threat_ttps", 266 | doc_type="ttp", 267 | item_id=ttp_id, 268 | user_id=session.get('id',None) 269 | ) 270 | 271 | #print_tpx(doc) 272 | 273 | #index the doc 274 | log.info(logging_prefix + "Start Indexing of {}".format(ttp_id)) 275 | response = get_es().index(ES_PREFIX + "threat_ttps", "ttp", doc, ttp_id) 276 | log.info(logging_prefix + "Done Indexing of {}".format(ttp_id)) 277 | 278 | return response, doc 279 | 280 | ''' 281 | TTP Pages 282 | ''' 283 | 284 | @ttp_blueprint.route("/add", methods = ['GET','POST']) 285 | @ttp_blueprint.route("/add/", methods = ['GET','POST']) 286 | @ttp_blueprint.route("/add/