├── .gitignore ├── README.md ├── actor.wsgi ├── app ├── __init__.py ├── config │ └── settings.py ├── core │ ├── __init__.py │ ├── blueprints │ │ ├── actor.py │ │ ├── admin.py │ │ ├── ajax.py │ │ ├── index.py │ │ ├── report.py │ │ ├── ttp.py │ │ └── user.py │ ├── decorators │ │ └── authentication.py │ └── forms │ │ └── forms.py ├── static │ ├── css │ │ ├── bootstrap.css │ │ ├── bootstrap.min.css │ │ └── shop-item.css │ ├── fonts │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ └── glyphicons-halflings-regular.woff2 │ └── js │ │ ├── bootstrap.js │ │ ├── bootstrap.min.js │ │ └── jquery.js ├── templates │ ├── about.html │ ├── account_reverify.html │ ├── actor.html │ ├── admin.html │ ├── admin_choices.html │ ├── contact.html │ ├── error_403.html │ ├── index.html │ ├── issues.html │ ├── layouts │ │ └── base.html │ ├── login.html │ ├── password_reset.html │ ├── register.html │ ├── report.html │ ├── ttp.html │ └── view_all.html └── utils │ ├── __init__.py │ ├── elasticsearch.py │ └── functions.py ├── favicon.ico ├── license.txt ├── robots.txt ├── setup ├── delete_create_index.py ├── load_data.py ├── schema.sql └── setup_steps.sh └── updates └── 2016_05_24 ├── mappings_file.py └── update_mappings.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ActorTrackr 2 | 3 | This repo has moved to https://github.com/jalewis/actortrackr 4 | -------------------------------------------------------------------------------- /actor.wsgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4 2 | import os 3 | import sys 4 | 5 | # set PATH so imports are correct 6 | TOP_DIR = os.path.dirname(os.path.realpath(__file__)) 7 | APP_PATH = TOP_DIR+"/app" 8 | 9 | sys.path.insert(0, TOP_DIR) 10 | sys.path.insert(0, APP_PATH) 11 | 12 | # Fire up our application 13 | from app import app as application -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.4 2 | if __name__ == "__main__": 3 | import os 4 | import sys 5 | 6 | TOP_DIR = os.path.dirname(os.path.realpath(__file__)) 7 | APP_PATH = TOP_DIR+"/app" 8 | 9 | sys.path.insert(0, TOP_DIR) 10 | sys.path.insert(0, APP_PATH) 11 | 12 | from config.settings import * 13 | 14 | import logging 15 | from logging.handlers import TimedRotatingFileHandler 16 | from logging import StreamHandler 17 | log = logging.getLogger(__name__) 18 | 19 | log.setLevel(logging.getLevelName(LOG_LEVEL)) 20 | 21 | #log formatter 22 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 23 | 24 | # add a rotating handler 25 | handler = TimedRotatingFileHandler(LOG_FILE, when='d', interval=1, backupCount=5) #creates daily logs for 5 days 26 | handler.setFormatter(formatter) 27 | log.addHandler(handler) 28 | 29 | # add a console hander 30 | if LOG_TO_CONSOLE: 31 | consoleHandler = StreamHandler() 32 | consoleHandler.setFormatter(formatter) 33 | log.addHandler(consoleHandler) 34 | 35 | try: 36 | from flask import Flask 37 | from flask import render_template 38 | from flask import flash 39 | from flask import request 40 | from flask import redirect 41 | from flask import jsonify 42 | from flask import Markup 43 | from flask import g 44 | except Exception as e: 45 | print("Error: {}\nFlask is not installed, try 'pip install flask'".format(e)) 46 | exit(1) 47 | 48 | try: 49 | from flask.ext.compress import Compress 50 | except Exception as e: 51 | print("Error: {}\Flask compress library is not installed, try 'pip install flask-compress'".format(e)) 52 | exit(1) 53 | try: 54 | from elasticsearch import Elasticsearch 55 | from elasticsearch import exceptions 56 | except Exception as e: 57 | print("Error: {}\nElasticsearch library is not installed, try 'pip install elasticsearch'".format(e)) 58 | exit(1) 59 | 60 | 61 | try: 62 | import pymysql 63 | from pymysql.cursors import DictCursor 64 | except Exception as e: 65 | print("Error: {}\PyMySQL library is not installed, try 'pip install PyMySQL'".format(e)) 66 | exit(1) 67 | 68 | #if you want a lot of elastic logs uncomment this section 69 | ''' 70 | es_logger = logging.getLogger('elasticsearch') 71 | es_logger.propagate = False 72 | es_logger.setLevel(logging.DEBUG) 73 | es_logger_handler=logging.StreamHandler() 74 | es_logger.addHandler(es_logger_handler) 75 | 76 | es_tracer = logging.getLogger('elasticsearch.trace') 77 | es_tracer.propagate = False 78 | es_tracer.setLevel(logging.INFO) 79 | es_tracer_handler=logging.StreamHandler() 80 | es_tracer.addHandler(es_tracer_handler) 81 | ''' 82 | 83 | app = Flask(__name__) 84 | app.secret_key = "Fgtqweds5ywDJsQW87uQnL" 85 | 86 | #configure gzip compression 87 | app.config['COMPRESS_LEVEL'] = 9 88 | app.config['COMPRESS_MIN_SIZE'] = 1 89 | Compress(app) 90 | 91 | def get_es(): 92 | try: 93 | db = getattr(g, 'es', None) 94 | if db is None: 95 | db = g.es = Elasticsearch(ES_HOSTS) 96 | except RuntimeError as rte: 97 | db = Elasticsearch(ES_HOSTS) 98 | 99 | return db 100 | 101 | def get_mysql(): 102 | try: 103 | db = getattr(g, 'mysql', None) 104 | if db is None: 105 | db = g.mysql = pymysql.connect(user=MYSQL_USER,passwd=MYSQL_PASSWD,db=MYSQL_DB, cursorclass=DictCursor) 106 | except RuntimeError as rte: 107 | db = g.mysql = pymysql.connect(user=MYSQL_USER,passwd=MYSQL_PASSWD,db=MYSQL_DB, cursorclass=DictCursor) 108 | 109 | return db 110 | 111 | @app.before_request 112 | def before_request(): 113 | if MAINTENANCE_MODE: 114 | # Or alternatively, dont redirect 115 | return 'Sorry, off for maintenance! Be back in 5', 503 116 | 117 | g.es = get_es() 118 | g.mysql = get_mysql() 119 | 120 | @app.teardown_request 121 | def teardown_request(exception): 122 | get_mysql().close() 123 | pass 124 | 125 | 126 | from core.blueprints.actor import actor_blueprint 127 | from core.blueprints.admin import admin_blueprint 128 | from core.blueprints.ajax import ajax_blueprint 129 | from core.blueprints.index import index_blueprint 130 | from core.blueprints.report import report_blueprint 131 | from core.blueprints.ttp import ttp_blueprint 132 | from core.blueprints.user import user_blueprint 133 | 134 | app.register_blueprint(actor_blueprint) 135 | app.register_blueprint(admin_blueprint) 136 | app.register_blueprint(ajax_blueprint) 137 | app.register_blueprint(index_blueprint) 138 | app.register_blueprint(report_blueprint) 139 | app.register_blueprint(ttp_blueprint) 140 | app.register_blueprint(user_blueprint) 141 | 142 | 143 | 144 | if __name__ == "__main__": 145 | 146 | #start flask 147 | app.run( 148 | host = '0.0.0.0', 149 | port = 8888, 150 | threaded=True, 151 | debug=True 152 | ) 153 | -------------------------------------------------------------------------------- /app/config/settings.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Elasticsearch Settings 3 | ''' 4 | 5 | ES_PREFIX = "tp-" 6 | ES_HOSTS = ['http://localhost:9200',] 7 | 8 | ''' 9 | MySQL Settings 10 | ''' 11 | 12 | MYSQL_USER = '' 13 | MYSQL_PASSWD = '' 14 | MYSQL_DB = 'threat_actors' 15 | 16 | ''' 17 | Log Settings 18 | ''' 19 | 20 | LOG_FILE = "/var/log/actortrackr/actortrackr.log" 21 | LOG_TO_CONSOLE = True 22 | LOG_LEVEL = "DEBUG" #NOSET, DEBUG, INFO, WARNING, ERROR, CRITICAL 23 | 24 | ''' 25 | Email Settings 26 | ''' 27 | 28 | #the email address of the sender 29 | EMAIL_SENDER = "noreply_ctig@lookingglasscyber.com" 30 | 31 | #alerts go to these addresses 32 | EMAIL_ADDRESSES = [ "ctig@lgscout.com", ] 33 | 34 | ''' 35 | Application Settings 36 | ''' 37 | 38 | MAINTENANCE_MODE = False 39 | 40 | APPLICATION_DOMAIN = "http://actortrackr.com/" 41 | APPLICATION_ORG = "Lookingglass" 42 | APPLICATION_NAME = "ActorTrackr" 43 | 44 | TLPS = [ 45 | ("0", "White"), 46 | ("1", "Green"), 47 | ("2", "Amber"), 48 | ("3", "Red"), 49 | ("4", "Black") 50 | ] 51 | 52 | SOURCE_RELIABILITY = [ 53 | ("A", "A. Reliable - No doubt about the source's authenticity, trustworthiness, or competency. History of complete reliability."), 54 | ("B", "B. Usually Reliable - Minor doubts. History of mostly valid information."), 55 | ("C", "C. Fairly Reliable - Doubts. Provided valid information in the past."), 56 | ("D", "D. Not Usually Reliable - Significant doubts. Provided valid information in the past."), 57 | ("E", "E. Unreliable - Lacks authenticity, trustworthiness, and competency. History of invalid information."), 58 | ("F", "F. Can’t Be Judged - Insufficient information to evaluate reliability. May or may not be reliable.") 59 | ] 60 | 61 | INFORMATION_RELIABILITY = [ 62 | ("1", "1. Confirmed - Logical, consistent with other relevant information, confirmed by independent sources."), 63 | ("2", "2. Probably True - Logical, consistent with other relevant information, not confirmed by independent sources."), 64 | ("3", "3. Possibly True - Reasonably logical, agrees with some relevant information, not confirmed."), 65 | ("4", "4. Doubtfully True - Not logical but possible, no other information on the subject, not confirmed."), 66 | ("5", "5. Improbable - Not logical, contradicted by other relevant information."), 67 | ("6", "6. Can’t Be Judged - The validity of the information can not be determined.") 68 | ] 69 | 70 | SALTS = { 71 | "actor" : "salt", 72 | "report" : "salt", 73 | "ttp" : "salt", 74 | "user" : "salt", 75 | "email_verification" : "salt" 76 | } 77 | 78 | SESSION_EXPIRE = -1 # in seconds, -1 to disable 79 | 80 | ''' 81 | Recaptcha Settings 82 | ''' 83 | 84 | RECAPTCHA_ENABLED = True 85 | RECAPTCHA_PUBLIC_KEY = '' 86 | RECAPTCHA_PRIVATE_KEY = '' 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /app/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dougiep16/actortrackr/0f82140850fceb8dbe5516c755cffbcb10c86340/app/core/__init__.py -------------------------------------------------------------------------------- /app/core/blueprints/admin.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import Blueprint 4 | from flask import flash 5 | from flask import request 6 | from flask import redirect 7 | from flask import jsonify 8 | from flask import Markup 9 | from flask import g 10 | 11 | import functools 12 | import json 13 | import hashlib 14 | import logging 15 | import os 16 | import requests 17 | import sys 18 | import time 19 | import uuid 20 | from datetime import datetime 21 | from pymysql.cursors import DictCursor 22 | from operator import itemgetter 23 | from urllib.parse import unquote_plus, quote_plus 24 | 25 | from config.settings import * 26 | from core.decorators import authentication 27 | from core.forms import forms 28 | from app import log, get_es, get_mysql 29 | from utils.elasticsearch import * 30 | from utils.functions import * 31 | 32 | #blue print def 33 | admin_blueprint = Blueprint('admin', __name__, url_prefix="/admin") 34 | logger_prefix = "admin.py:" 35 | 36 | @admin_blueprint.route("/user////", methods = ['GET']) 37 | @admin_blueprint.route("/user/////", methods = ['GET']) 38 | @authentication.access(authentication.ADMIN) 39 | def edit_user_permissions(action,value,user_id,user_c): 40 | logging_prefix = logger_prefix + "edit_user_permissions({},{},{},{}) - ".format(action,value,user_id,user_c) 41 | log.info(logging_prefix + "Starting") 42 | 43 | action_column_map = {} 44 | action_column_map['approve'] = "approved" 45 | action_column_map['write_perm'] = "write_permission" 46 | action_column_map['delete_perm'] = "delete_permission" 47 | action_column_map['admin_perm'] = "admin" 48 | 49 | success = 1 50 | try: 51 | #make sure the value is valid 52 | if value not in ["0","1"]: 53 | raise Exception("Invald value: {}".format(value)) 54 | 55 | #make sure the action is valid 56 | try: 57 | column = action_column_map[action] 58 | except Exception as f: 59 | log.warning(logging_prefix + "Action '{}' not found in action_column_map".format(action)) 60 | raise f 61 | 62 | #check the hash 63 | if user_c == sha256(SALTS['user'], str(user_id)): 64 | 65 | #if action is approve, emails need to be sent 66 | if action == "approve": 67 | conn = get_mysql().cursor(DictCursor) 68 | conn.execute("SELECT name, email FROM users WHERE id = %s", (user_id,)) 69 | user = conn.fetchone() 70 | conn.close() 71 | 72 | if value == "1": 73 | log.info(logging_prefix + "Setting approved=1 for user {}".format(user_id)) 74 | sendAccountApprovedEmail(user['email']) 75 | else: 76 | log.info(logging_prefix + "Setting approved=0 for user {}".format(user_id)) 77 | sendAccountDisapprovedEmail(user['email']) 78 | 79 | #now update the desired setting 80 | conn = get_mysql().cursor() 81 | conn.execute("UPDATE users SET "+column+" = %s WHERE id = %s", (value,user_id)) 82 | get_mysql().commit() 83 | conn.close() 84 | log.info(logging_prefix + "Successfully update {} to {} for user id {}".format(column,value,user_id)) 85 | 86 | else: 87 | log.warning(logging_prefix + "Hash mismatch {} {}".format(user_id, user_c)) 88 | except Exception as e: 89 | success = 0 90 | error = "There was an error completing your request. Details: {}".format(e) 91 | log.exception(logging_prefix + error) 92 | 93 | return jsonify({ "success" : success, "new_value" : value }) 94 | 95 | @admin_blueprint.route("/user/delete//", methods = ['GET']) 96 | @admin_blueprint.route("/user/delete///", methods = ['GET']) 97 | @authentication.access(authentication.ADMIN) 98 | def user_delete(user_id, user_id_hash): 99 | logging_prefix = logger_prefix + "user_delete({},{}) - ".format(user_id, user_id_hash) 100 | log.info(logging_prefix + "Starting") 101 | 102 | redirect_url = "/admin/" 103 | try: 104 | redirect_url = request.args.get("_r") 105 | if not redirect_url: 106 | log.warning(logging_prefix + "redirect_url not set, using default") 107 | redirect_url = "/admin/" 108 | 109 | #check user_id against user_id_hash and perform delete if match 110 | if user_id_hash == sha256( SALTS['user'], user_id ): 111 | #now delete the user 112 | conn=get_mysql().cursor(DictCursor) 113 | conn.execute("DELETE FROM users WHERE id=%s", (user_id,)) 114 | conn.close() 115 | flash("The user has been deleted", "success") 116 | else: 117 | flash("Unable to delete user", "danger") 118 | 119 | except Exception as e: 120 | error = "There was an error completing your request. Details: {}".format(e) 121 | flash(error,'danger') 122 | log.exception(logging_prefix + error) 123 | 124 | return redirect(redirect_url) 125 | 126 | 127 | @admin_blueprint.route("/email", methods = ['POST']) 128 | @admin_blueprint.route("/email/", methods = ['POST']) 129 | @authentication.access(authentication.ADMIN) 130 | def email_user(): 131 | 132 | logging_prefix = logger_prefix + "email_user() - " 133 | log.info(logging_prefix + "Starting") 134 | 135 | try: 136 | email = request.form['email'] 137 | subject = request.form['subject'] 138 | body = request.form['body'] 139 | 140 | log.debug("Email: {}, Subject: {}, Body: {}".format(email, subject, body)) 141 | sendCustomEmail(email, subject, body) 142 | except Exception as e: 143 | error = "There was an error completing your request. Details: {}".format(e) 144 | log.exception(logging_prefix + error) 145 | 146 | return jsonify({ 'success' : False, 'error' : str(e) }) 147 | 148 | return jsonify({ 'success' : True }) 149 | 150 | ''' 151 | Admin Pages 152 | ''' 153 | 154 | @admin_blueprint.route("/", methods = ['GET','POST']) 155 | @admin_blueprint.route("", methods = ['GET','POST']) 156 | @authentication.access(authentication.ADMIN) 157 | def main(): 158 | logging_prefix = logger_prefix + "main() - " 159 | log.info(logging_prefix + "Starting") 160 | 161 | try: 162 | conn=get_mysql().cursor(DictCursor) 163 | conn.execute("SELECT id, name, email, company, justification, email_verified, approved, write_permission, delete_permission, admin, created, last_login FROM users ORDER BY created DESC") 164 | 165 | users = conn.fetchall() 166 | for user in users: 167 | user['id_hash'] = sha256( SALTS['user'], str(user['id']) ) 168 | 169 | conn.close() 170 | 171 | email_user_form = forms.sendUserEmailForm() 172 | 173 | except Exception as e: 174 | error = "There was an error completing your request. Details: {}".format(e) 175 | flash(error,'danger') 176 | log.exception(logging_prefix + error) 177 | 178 | 179 | return render_template("admin.html", 180 | page_title="Admin", 181 | url = "/admin/", 182 | users=users, 183 | email_user_form = email_user_form 184 | ) 185 | @admin_blueprint.route("/choices", methods = ['GET','POST']) 186 | @admin_blueprint.route("/choices/", methods = ['GET','POST']) 187 | @authentication.access(authentication.ADMIN) 188 | def choices(): 189 | logging_prefix = logger_prefix + "choices() - " 190 | log.info(logging_prefix + "Starting") 191 | 192 | simple_choices = None 193 | try: 194 | 195 | body = { 196 | "query" : { 197 | "match_all" : {} 198 | }, 199 | "size" : 1000 200 | } 201 | 202 | results = get_es().search(ES_PREFIX + 'threat_actor_simple', 'data', body) 203 | 204 | parsed_results = [] 205 | for r in results['hits']['hits']: 206 | d = {} 207 | d['type'] = r['_source']['type'] 208 | d['value'] = r['_source']['value'] 209 | d['id'] = r['_id'] 210 | 211 | #determine how many actor profiles use this value 212 | 213 | query_string = None 214 | if d['type'] == "classification": 215 | query_string = "type:\"" + escape(d['value']) + "\"" 216 | elif d['type'] == "communication": 217 | query_string = "communication_address.type:\"" + escape(d['value']) + "\"" 218 | elif d['type'] == "country": 219 | query_string = "country_affiliation:\"" + escape(d['value']) + "\" origin:\"" + escape(d['value']) + "\"" 220 | 221 | if query_string: 222 | body = { 223 | "query" : { 224 | "query_string" : { 225 | "query" : query_string 226 | } 227 | }, 228 | "size" : 0 229 | } 230 | 231 | count = get_es().search(ES_PREFIX + 'threat_actors', 'actor', body) 232 | 233 | d['count'] = count['hits']['total'] 234 | d['q'] = quote_plus(query_string) 235 | else: 236 | d['count'] = "-" 237 | d['q'] = "" 238 | 239 | parsed_results.append(d) 240 | 241 | simple_choices = multikeysort(parsed_results, ['type', 'value']) 242 | 243 | except Exception as e: 244 | error = "There was an error completing your request. Details: {}".format(e) 245 | flash(error,'danger') 246 | log.exception(logging_prefix + error) 247 | 248 | return render_template("admin_choices.html", 249 | page_title="Admin", 250 | simple_choices = simple_choices 251 | ) 252 | 253 | 254 | -------------------------------------------------------------------------------- /app/core/blueprints/ajax.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import Blueprint 4 | from flask import flash 5 | from flask import request 6 | from flask import redirect 7 | from flask import jsonify 8 | from flask import Markup 9 | from flask import g 10 | 11 | import functools 12 | import json 13 | import hashlib 14 | import logging 15 | import os 16 | import sys 17 | import time 18 | import uuid 19 | from datetime import datetime 20 | from operator import itemgetter 21 | from urllib.parse import quote_plus 22 | 23 | from config.settings import * 24 | from core.forms import forms 25 | from app import log, get_es, get_mysql 26 | from utils.elasticsearch import * 27 | from utils.functions import * 28 | 29 | #blue print def 30 | ajax_blueprint = Blueprint('ajax', __name__, url_prefix="/ajax") 31 | logger_prefix = "ajax.py:" 32 | 33 | #dynamic select populator 34 | @ajax_blueprint.route("/fetch/<_type>/", methods = ['GET']) 35 | def fetch(_type, value): 36 | logging_prefix = logger_prefix + "fetch({},{}) - ".format(_type,value) 37 | log.info(logging_prefix + "Starting") 38 | 39 | r = fetch_child_data(_type,value) 40 | return jsonify(r), 200 41 | 42 | 43 | #dynamic select populator 44 | @ajax_blueprint.route("/related/<_type>", methods = ['GET']) 45 | @ajax_blueprint.route("/related/<_type>/", methods = ['GET']) 46 | def populate_related_elements(_type): 47 | logging_prefix = logger_prefix + "populate_related_elements({}) - ".format(_type) 48 | log.info(logging_prefix + "Starting") 49 | 50 | r = fetch_related_elements(_type) 51 | return jsonify(r), 200 -------------------------------------------------------------------------------- /app/core/blueprints/index.py: -------------------------------------------------------------------------------- 1 | 2 | from flask import Flask 3 | from flask import render_template 4 | from flask import Blueprint 5 | from flask import flash 6 | from flask import request 7 | from flask import redirect 8 | from flask import make_response 9 | from flask import jsonify 10 | from flask import Markup 11 | from flask import g 12 | 13 | from elasticsearch import TransportError 14 | from elasticsearch.helpers import scan 15 | 16 | import functools 17 | import json 18 | import hashlib 19 | import logging 20 | import os 21 | import sys 22 | import time 23 | import uuid 24 | from datetime import datetime 25 | from operator import itemgetter 26 | from urllib.parse import quote_plus 27 | 28 | from config.settings import * 29 | from core.decorators import authentication 30 | from core.forms import forms 31 | from app import log, get_es, get_mysql 32 | 33 | 34 | #blue print def 35 | index_blueprint = Blueprint('index', __name__, url_prefix="") 36 | logger_prefix = "index.py:" 37 | 38 | 39 | @index_blueprint.route("/about", methods = ['GET']) 40 | @authentication.access(authentication.PUBLIC) 41 | def about(): 42 | search_form = forms.searchForm() 43 | 44 | return render_template("about.html", 45 | page_title="About", 46 | search_form = search_form 47 | ) 48 | 49 | @index_blueprint.route("/contact", methods = ['GET']) 50 | @authentication.access(authentication.PUBLIC) 51 | def contact(): 52 | search_form = forms.searchForm() 53 | 54 | return render_template("contact.html", 55 | page_title="Contact", 56 | search_form = search_form 57 | ) 58 | 59 | @index_blueprint.route("/", methods = ['GET','POST']) 60 | @authentication.access(authentication.PUBLIC) 61 | def index(): 62 | logging_prefix = logger_prefix + "index() - " 63 | log.info(logging_prefix + "Loading home page") 64 | 65 | form = forms.searchForm(request.form) 66 | error = None 67 | url = "/" 68 | query_string_url = "" 69 | try: 70 | #this is the default query for actors in ES, i'd imagine this will be recently added/modified actors 71 | es_query = { 72 | "query": { 73 | "match_all": {} 74 | }, 75 | "size": 10, 76 | "sort": { 77 | "last_updated_s": { 78 | "order": "desc" 79 | } 80 | } 81 | } 82 | 83 | #pull the query out of the url 84 | query_string = request.args.get("q") 85 | 86 | #someone is searching for something 87 | if request.method == 'POST' and not query_string: 88 | if form.validate(): 89 | 90 | #get the value 91 | value = form.query.data 92 | 93 | #redirect to this same page, but setting the query value in the url 94 | return redirect("/?q={}".format(quote_plus(value)), code=307) 95 | 96 | else: 97 | #if there was an error print the error dictionary to the console 98 | # temporary help, these should also appear under the form field 99 | print(form.errors) 100 | 101 | elif query_string: 102 | #now that the query_string is provided as ?q=, perform the search 103 | print("VALID SEARCH OPERATION DETECTED") 104 | 105 | #do some searching... 106 | es_query = { 107 | "query": { 108 | "query_string": { 109 | "query" : query_string 110 | } 111 | }, 112 | "size": 10 113 | } 114 | 115 | url += "?q=" + query_string 116 | query_string_url = "?q=" + query_string 117 | 118 | #set the form query value to what the user is searching for 119 | form.query.data = query_string 120 | 121 | ''' 122 | Fetch the data from ES 123 | ''' 124 | 125 | actors = {} 126 | actors['hits'] = {} 127 | actors['hits']['hits'] = [] 128 | 129 | reports = dict(actors) 130 | 131 | ttps = dict(actors) 132 | 133 | try: 134 | actors = get_es().search(ES_PREFIX + 'threat_actors', 'actor', es_query) 135 | except TransportError as te: 136 | #if the index was not found, this is most likely becuase theres no data there 137 | if te.status_code == 404: 138 | log.warning("Index 'threat_actors' was not found") 139 | else: 140 | error = "There was an error fetching actors. Details: {}".format(te) 141 | flash(error,'danger') 142 | log.exception(logging_prefix + error) 143 | except Exception as e: 144 | error = "The was an error fetching Actors. Error: {}".format(e) 145 | log.exception(error) 146 | flash(error, "danger") 147 | 148 | try: 149 | reports = get_es().search(ES_PREFIX + 'threat_reports', 'report', es_query) 150 | except TransportError as te: 151 | #if the index was not found, this is most likely becuase theres no data there 152 | if te.status_code == 404: 153 | log.warning("Index 'threat_reports' was not found") 154 | else: 155 | error = "There was an error fetching reports. Details: {}".format(te) 156 | flash(error,'danger') 157 | log.exception(logging_prefix + error) 158 | except Exception as e: 159 | error = "The was an error fetching Reports. Error: {}".format(e) 160 | log.exception(error) 161 | flash(error, "danger") 162 | 163 | try: 164 | ttps = get_es().search(ES_PREFIX + 'threat_ttps', 'ttp', es_query) 165 | except TransportError as te: 166 | #if the index was not found, this is most likely becuase theres no data there 167 | if te.status_code == 404: 168 | log.warning("Index 'threat_ttps' was not found") 169 | else: 170 | error = "There was an error ttps. Details: {}".format(te) 171 | flash(error,'danger') 172 | log.exception(logging_prefix + error) 173 | except Exception as e: 174 | error = "The was an error fetching TTPs. Error: {}".format(e) 175 | log.exception(error) 176 | flash(error, "danger") 177 | 178 | ''' 179 | Modify the data as needed 180 | ''' 181 | 182 | for actor in actors['hits']['hits']: 183 | s = SALTS['actor'] + actor['_id'] 184 | hash_object = hashlib.sha256(s.encode('utf-8')) 185 | hex_dig = hash_object.hexdigest() 186 | actor["_source"]['id_hash'] = hex_dig 187 | 188 | for report in reports['hits']['hits']: 189 | s = SALTS['report'] + report['_id'] 190 | hash_object = hashlib.sha256(s.encode('utf-8')) 191 | hex_dig = hash_object.hexdigest() 192 | report["_source"]['id_hash'] = hex_dig 193 | 194 | for ttp in ttps['hits']['hits']: 195 | s = SALTS['ttp'] + ttp['_id'] 196 | hash_object = hashlib.sha256(s.encode('utf-8')) 197 | hex_dig = hash_object.hexdigest() 198 | ttp["_source"]['id_hash'] = hex_dig 199 | 200 | except Exception as e: 201 | error = "There was an error completing your request. Details: {}".format(e) 202 | log.exception(error) 203 | flash(error, "danger") 204 | 205 | #render the template, passing the variables we need 206 | # templates live in the templates folder 207 | return render_template("index.html", 208 | page_title="ActorTrackr", 209 | form=form, 210 | query_string_url=query_string_url, 211 | actors=actors, 212 | reports=reports, 213 | ttps=ttps, 214 | url = quote_plus(url) 215 | ) 216 | 217 | @index_blueprint.route("/export", methods = ['GET']) 218 | @index_blueprint.route("/export/", methods = ['GET']) 219 | @authentication.access(authentication.PUBLIC) 220 | def export_all_the_data(): 221 | logging_prefix = logger_prefix + "export_all_the_data() - " 222 | log.info(logging_prefix + "Exporting") 223 | 224 | try: 225 | dump = {} 226 | dump['actors'] = [] 227 | dump['reports'] = [] 228 | dump['ttps'] = [] 229 | dump['choices'] = {} 230 | 231 | query = { 232 | "query" : { 233 | "match_all" : {} 234 | } 235 | } 236 | 237 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actors",doc_type="actor") 238 | 239 | 240 | for i in results: 241 | dump['actors'].append(i) 242 | 243 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_reports",doc_type="report") 244 | 245 | for i in results: 246 | dump['reports'].append(i) 247 | 248 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_ttps",doc_type="ttp") 249 | 250 | for i in results: 251 | dump['ttps'].append(i) 252 | 253 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actor_pc",doc_type="parent") 254 | 255 | dump['choices']['parents'] = [] 256 | for i in results: 257 | dump['choices']['parents'].append(i) 258 | 259 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actor_pc",doc_type="child") 260 | 261 | dump['choices']['children'] = [] 262 | for i in results: 263 | dump['choices']['children'].append(i) 264 | 265 | results = scan(get_es(),query=query,index=ES_PREFIX + "threat_actor_simple",doc_type="data") 266 | 267 | dump['choices']['simple'] = [] 268 | for i in results: 269 | dump['choices']['simple'].append(i) 270 | 271 | # We need to modify the response, so the first thing we 272 | # need to do is create a response out of the Dictionary 273 | response = make_response(json.dumps(dump)) 274 | 275 | # This is the key: Set the right header for the response 276 | # to be downloaded, instead of just printed on the browser 277 | response.headers["Content-Disposition"] = "attachment; filename=export.json" 278 | response.headers["Content-Type"] = "text/json; charset=utf-8" 279 | 280 | return response 281 | 282 | except Exception as e: 283 | error = "There was an error completing your request. Details: {}".format(e) 284 | log.exception(error) 285 | flash(error, "danger") 286 | return redirect("/") 287 | 288 | @index_blueprint.route("/", methods = ['GET','POST']) 289 | @index_blueprint.route("//", methods = ['GET','POST']) 290 | @index_blueprint.route("//", methods = ['GET','POST']) 291 | @index_blueprint.route("///", methods = ['GET','POST']) 292 | def view_all(t,page=1): 293 | if t == 'favicon.ico': 294 | return jsonify({}),404 295 | 296 | page = int(page) 297 | 298 | logging_prefix = logger_prefix + "view_all() - " 299 | log.info(logging_prefix + "Loading view all page {} for {}".format(page, t)) 300 | 301 | form = forms.searchForm(request.form) 302 | error = None 303 | page_size = 50 304 | offset = (page-1) * page_size 305 | url = "/{}/{}/".format(t,page) 306 | search_url = "" 307 | results_text = "" 308 | try: 309 | 310 | 311 | #this is the default query for actors in ES, i'd imagine this will be recently added/modified actors 312 | es_query = { 313 | "query": { 314 | "match_all": {} 315 | }, 316 | "size": page_size, 317 | "from" : offset, 318 | "sort": { 319 | "last_updated_s": { 320 | "order": "desc" 321 | } 322 | } 323 | } 324 | 325 | #pull the query out of the url 326 | query_string = request.args.get("q") 327 | 328 | #someone is searching for something 329 | if request.method == 'POST' and not query_string: 330 | if form.validate(): 331 | print("VALID SEARCH OPERATION DETECTED, redirecting...") 332 | 333 | #get the value 334 | value = form.query.data 335 | 336 | 337 | log.info(value) 338 | 339 | #redirect to this same page, but setting the query value in the url 340 | return redirect("/{}/1/?q={}".format(t,quote_plus(value)), code=307) 341 | 342 | else: 343 | #if there was an error print the error dictionary to the console 344 | # temporary help, these should also appear under the form field 345 | print(form.errors) 346 | 347 | elif query_string: 348 | #now that the query_string is provided as ?q=, perform the search 349 | print("VALID SEARCH OPERATION DETECTED") 350 | 351 | #do some searching... 352 | es_query = { 353 | "query": { 354 | "query_string": { 355 | "query" : query_string 356 | } 357 | }, 358 | "size": page_size, 359 | "from" : offset 360 | } 361 | 362 | search_url = "?q=" + query_string 363 | #set the form query value to what the user is searching for 364 | form.query.data = query_string 365 | 366 | ''' 367 | Fetch the data from ES 368 | ''' 369 | 370 | data = {} 371 | data['hits'] = {} 372 | data['hits']['hits'] = [] 373 | 374 | if t == 'actor': 375 | index = ES_PREFIX + 'threat_actors' 376 | doc_type = 'actor' 377 | salt = SALTS['actor'] 378 | link_prefix = 'actor' 379 | data_header = 'Actors' 380 | field_header = 'Actor Name' 381 | elif t == 'report': 382 | index = ES_PREFIX + 'threat_reports' 383 | doc_type = 'report' 384 | salt = SALTS['report'] 385 | link_prefix = 'report' 386 | data_header = 'Reports' 387 | field_header = 'Report Title' 388 | elif t == 'ttp': 389 | index = ES_PREFIX + 'threat_ttps' 390 | doc_type = 'ttp' 391 | salt = SALTS['ttp'] 392 | link_prefix = 'ttp' 393 | data_header = 'TTPs' 394 | field_header = 'TTP Name' 395 | else: 396 | raise Exception("Unknown type {}".format(t)) 397 | 398 | try: 399 | data = get_es().search(index, doc_type, es_query) 400 | num_hits = len(data['hits']['hits']) 401 | 402 | #set up previous link 403 | if page == 1: 404 | prev_url = None 405 | else: 406 | prev_url = "/{}/{}/{}".format(t,(page-1),search_url) 407 | 408 | if ((page-1)*page_size) + num_hits < data['hits']['total']: 409 | next_url = "/{}/{}/{}".format(t,(page+1),search_url) 410 | else: 411 | next_url = None 412 | 413 | url += search_url 414 | 415 | for d in data['hits']['hits']: 416 | s = salt + d['_id'] 417 | hash_object = hashlib.sha256(s.encode('utf-8')) 418 | hex_dig = hash_object.hexdigest() 419 | d["_source"]['id_hash'] = hex_dig 420 | 421 | if num_hits == 0: 422 | results_text = "" 423 | else: 424 | f = ( (page-1) * page_size ) + 1 425 | l = f + (num_hits-1) 426 | results_text = "Showing {} to {} of {} total results".format(f,l,data['hits']['total']) 427 | 428 | except TransportError as te: 429 | 430 | #if the index was not found, this is most likely becuase theres no data there 431 | if te.status_code == 404: 432 | log.warning("Index '{}' was not found".format(index)) 433 | else: 434 | error = "There was an error fetching {}. Details: {}".format(t, te) 435 | flash(error,'danger') 436 | log.exception(logging_prefix + error) 437 | 438 | except Exception as e: 439 | error = "The was an error fetching {}. Error: {}".format(t,e) 440 | log.exception(error) 441 | flash(error, "danger") 442 | 443 | 444 | except Exception as e: 445 | error = "There was an error completing your request. Details: {}".format(e) 446 | log.exception(error) 447 | flash(error, "danger") 448 | return redirect("/") 449 | 450 | return render_template("view_all.html", 451 | page_title="View All", 452 | form=form, 453 | data_header=data_header, 454 | results_text=results_text, 455 | field_header=field_header, 456 | data=data, 457 | link_prefix=link_prefix, 458 | prev_url=prev_url, 459 | next_url=next_url, 460 | url = quote_plus(url) 461 | ) 462 | 463 | -------------------------------------------------------------------------------- /app/core/blueprints/ttp.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask import render_template 3 | from flask import Blueprint 4 | from flask import flash 5 | from flask import request 6 | from flask import redirect 7 | from flask import jsonify 8 | from flask import Markup 9 | from flask import g 10 | from flask import session 11 | 12 | import functools 13 | import json 14 | import hashlib 15 | import logging 16 | import os 17 | import sys 18 | import time 19 | import uuid 20 | from datetime import datetime 21 | from operator import itemgetter 22 | from urllib.parse import quote_plus 23 | 24 | from config.settings import * 25 | from core.decorators import authentication 26 | from core.forms import forms 27 | from app import log, get_es, get_mysql 28 | from utils.elasticsearch import * 29 | from utils.functions import * 30 | 31 | ttp_blueprint = Blueprint('ttp', __name__, url_prefix="/ttp") 32 | logger_prefix = "ttp.py:" 33 | 34 | def es_to_form(ttp_id): 35 | form = forms.ttpForm() 36 | 37 | #get the values from ES 38 | results = get_es().get(ES_PREFIX + "threat_ttps", doc_type="ttp", id=ttp_id) 39 | 40 | 41 | #store certain fields from ES, so this form can be used in an update 42 | form.doc_index.data = results['_index'] 43 | form.doc_type.data = results['_type'] 44 | 45 | ttp_data = results['_source'] 46 | 47 | form.ttp_name.data = ttp_data['name'] 48 | form.ttp_first_observed.data = datetime.strptime(ttp_data['created_s'],"%Y-%m-%dT%H:%M:%S") 49 | form.ttp_description.data = ttp_data['description'] 50 | form.ttp_criticality.data = int(ttp_data['criticality']) 51 | 52 | idx = 0 53 | for entry in range(len(form.ttp_class.entries)): form.ttp_class.pop_entry() 54 | for i in multikeysort(ttp_data['classification'], ['family', 'id']): 55 | ttp_class_form = forms.TPXClassificationForm() 56 | ttp_class_form.a_family = i['family'] 57 | ttp_class_form.a_id = i['id'] 58 | 59 | form.ttp_class.append_entry(ttp_class_form) 60 | 61 | #set the options since this select is dynamic 62 | form.ttp_class[idx].a_id.choices = fetch_child_data('tpx_classification',i['family']) 63 | idx += 1 64 | 65 | if ttp_data['related_actor']: 66 | for entry in range(len(form.ttp_actors.entries)): form.ttp_actors.pop_entry() 67 | for i in multikeysort(ttp_data['related_actor'], ['name', 'id']): 68 | sub_form = forms.RelatedActorsForm() 69 | sub_form.data = i['id'] + ":::" + i['name'] 70 | 71 | form.ttp_actors.append_entry(sub_form) 72 | 73 | if ttp_data['related_report']: 74 | for entry in range(len(form.ttp_reports.entries)): form.ttp_reports.pop_entry() 75 | for i in multikeysort(ttp_data['related_report'], ['name', 'id']): 76 | sub_form = forms.RelatedReportsForm() 77 | sub_form.data = i['id'] + ":::" + i['name'] 78 | 79 | form.ttp_reports.append_entry(sub_form) 80 | 81 | if ttp_data['related_ttp']: 82 | for entry in range(len(form.ttp_ttps.entries)): form.ttp_ttps.pop_entry() 83 | for i in multikeysort(ttp_data['related_ttp'], ['name', 'id']): 84 | sub_form = forms.RelatedTTPsForm() 85 | sub_form.data = i['id'] + ":::" + i['name'] 86 | 87 | form.ttp_ttps.append_entry(sub_form) 88 | 89 | #convert editor dictionary of ids and times to names and times 90 | editors = get_editor_names(get_mysql(), ttp_data['editor']) 91 | 92 | return form, editors 93 | 94 | def es_to_tpx(ttp_id): 95 | ''' 96 | Build the TPX file from the data stored in Elasticsearch 97 | ''' 98 | element_observables = {} 99 | 100 | results = get_es().get(ES_PREFIX + "threat_ttps", doc_type="ttp", id=ttp_id) 101 | ttp_data = results['_source'] 102 | 103 | tpx = {} 104 | tpx["schema_version_s"] = "2.2.0" 105 | tpx["provider_s"] = "LookingGlass" 106 | tpx["list_name_s"] = "Threat Actor" 107 | tpx["created_t"] = ttp_data['created_milli'] 108 | tpx["created_s"] = ttp_data['created_s'] 109 | tpx["last_updated_t"] = ttp_data['last_updated_milli'] 110 | tpx["last_updated_s"] = ttp_data['last_updated_s'] 111 | tpx["score_i"] = 95 112 | tpx["source_observable_s"] = "Cyveillance Threat Actor" 113 | tpx["source_description_s"] = "This feed provides threat actor or threat actor group profiles and characterizations created by the LookingGlass Cyber Threat Intelligence Group" 114 | 115 | tpx["observable_dictionary_c_array"] = [] 116 | 117 | observable_dict = {} 118 | observable_dict["ttp_uuid_s"] = ttp_id 119 | observable_dict["observable_id_s"] = ttp_data['name'] 120 | observable_dict["description_s"] = ttp_data['description'] 121 | observable_dict["criticality_i"] = ttp_data['criticality'] 122 | 123 | observable_dict["classification_c_array"] = [] 124 | 125 | class_dict = {} 126 | class_dict["score_i"] = 70 127 | class_dict["classification_id_s"] = "Intel" 128 | class_dict["classification_family_s"] = "TTP" 129 | observable_dict["classification_c_array"].append(class_dict) 130 | 131 | for i in ttp_data['classification']: 132 | class_dict = {} 133 | class_dict["score_i"] = i["score"] 134 | class_dict["classification_id_s"] = i["id"] 135 | class_dict["classification_family_s"] = i["family"] 136 | 137 | if class_dict not in observable_dict["classification_c_array"]: 138 | observable_dict["classification_c_array"].append(class_dict) 139 | 140 | observable_dict["related_ttps_c_array"] = [] 141 | for i in ttp_data['related_ttp']: 142 | if i['name']: 143 | observable_dict["related_ttps_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] }) 144 | 145 | 146 | observable_dict["related_actors_c_array"] = [] 147 | for i in ttp_data['related_actor']: 148 | if i['name']: 149 | observable_dict["related_actors_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] }) 150 | 151 | observable_dict["related_reports_c_array"] = [] 152 | for i in ttp_data['related_report']: 153 | if i['name']: 154 | observable_dict["related_reports_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] }) 155 | 156 | 157 | ''' 158 | Related elements 159 | ''' 160 | 161 | relate_element_name_map = { 162 | "FQDN" : "subject_fqdn_s", 163 | "IPv4" : "subject_ipv4_s", 164 | "TTP" : "subject_ttp_s", 165 | "CommAddr" : "subject_address_s" 166 | } 167 | 168 | 169 | 170 | tpx["observable_dictionary_c_array"].append(observable_dict) 171 | 172 | return tpx 173 | 174 | def form_to_es(form, ttp_id): 175 | logging_prefix = logger_prefix + "form_to_es() - " 176 | log.info(logging_prefix + "Converting Form to ES for {}".format(ttp_id)) 177 | 178 | doc = {} 179 | 180 | created_t = int(time.mktime(form.ttp_first_observed.data.timetuple())) * 1000 181 | created_s = form.ttp_first_observed.data.strftime("%Y-%m-%dT%H:%M:%S") 182 | now_t = int(time.time()) * 1000 183 | now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") 184 | 185 | doc["created_milli"] = created_t 186 | doc["created_s"] = created_s 187 | doc["last_updated_milli"] = now_t 188 | doc["last_updated_s"] = now_s 189 | 190 | doc['name'] = escape(form.ttp_name.data) 191 | doc['description'] = escape(form.ttp_description.data) 192 | doc['criticality'] = int(escape(form.ttp_criticality.data)) 193 | 194 | doc['classification'] = [] 195 | for sub_form in form.ttp_class.entries: 196 | classification_dict = {} 197 | classification_dict['score'] = int(get_score(sub_form.data['a_family'], sub_form.data['a_id'])) 198 | classification_dict['id'] = escape(sub_form.data['a_id']) 199 | classification_dict['family'] = escape(sub_form.data['a_family']) 200 | 201 | if classification_dict not in doc['classification']: 202 | doc['classification'].append(classification_dict) 203 | 204 | #Links to actors and reports 205 | doc['related_actor'] = [] 206 | for sub_form in form.ttp_actors.entries: 207 | r_dict = {} 208 | data = escape(sub_form.data.data) 209 | 210 | if data == "_NONE_": 211 | continue 212 | 213 | data_array = data.split(":::") 214 | 215 | r_dict['id'] = escape(data_array[0]) 216 | r_dict['name'] = escape(data_array[1]) 217 | #this is gonna be a nightmare to maintain, 218 | # but it make sense to have this for searches 219 | 220 | if r_dict not in doc['related_actor']: 221 | doc['related_actor'].append(r_dict) 222 | 223 | doc['related_report'] = [] 224 | for sub_form in form.ttp_reports.entries: 225 | r_dict = {} 226 | data = escape(sub_form.data.data) 227 | 228 | if data == "_NONE_": 229 | continue 230 | 231 | data_array = data.split(":::") 232 | 233 | r_dict['id'] = escape(data_array[0]) 234 | r_dict['name'] = escape(data_array[1]) 235 | #this is gonna be a nightmare to maintain, 236 | # but it make sense to have this for searches 237 | 238 | if r_dict not in doc['related_report']: 239 | doc['related_report'].append(r_dict) 240 | 241 | doc['related_ttp'] = [] 242 | for sub_form in form.ttp_ttps.entries: 243 | r_dict = {} 244 | data = escape(sub_form.data.data) 245 | 246 | if data == "_NONE_": 247 | continue 248 | 249 | data_array = data.split(":::") 250 | 251 | r_dict['id'] = escape(data_array[0]) 252 | r_dict['name'] = escape(data_array[1]) 253 | #this is gonna be a nightmare to maintain, 254 | # but it make sense to have this for searches 255 | 256 | if r_dict not in doc['related_ttp']: 257 | doc['related_ttp'].append(r_dict) 258 | 259 | ''' 260 | Edit Tracking 261 | ''' 262 | 263 | doc['editor'] = get_editor_list( 264 | es=get_es(), 265 | index=ES_PREFIX + "threat_ttps", 266 | doc_type="ttp", 267 | item_id=ttp_id, 268 | user_id=session.get('id',None) 269 | ) 270 | 271 | #print_tpx(doc) 272 | 273 | #index the doc 274 | log.info(logging_prefix + "Start Indexing of {}".format(ttp_id)) 275 | response = get_es().index(ES_PREFIX + "threat_ttps", "ttp", doc, ttp_id) 276 | log.info(logging_prefix + "Done Indexing of {}".format(ttp_id)) 277 | 278 | return response, doc 279 | 280 | ''' 281 | TTP Pages 282 | ''' 283 | 284 | @ttp_blueprint.route("/add", methods = ['GET','POST']) 285 | @ttp_blueprint.route("/add/", methods = ['GET','POST']) 286 | @ttp_blueprint.route("/add/