├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── docs ├── LAUNCHING.md ├── PLUGINS.md ├── RULES.md └── TESTING.md ├── requirements.txt ├── sam ├── ConfigEnvy.py ├── __init__.py ├── common.py ├── constants.py ├── default.cfg ├── errors.py ├── httpserver.py ├── importers │ ├── __init__.py │ ├── import_asasyslog.py │ ├── import_aws.py │ ├── import_base.py │ ├── import_netflow.py │ ├── import_paloalto.py │ ├── import_tcpdump.py │ ├── import_tshark.py │ └── netflow_collector.py ├── integrity.py ├── launcher.py ├── local │ ├── __init__.py │ ├── en.py │ ├── fr.py │ └── rv.py ├── models │ ├── __init__.py │ ├── base.py │ ├── datasources.py │ ├── details.py │ ├── filters.py │ ├── links.py │ ├── livekeys.py │ ├── nodes.py │ ├── ports.py │ ├── security │ │ ├── __init__.py │ │ ├── alerts.py │ │ ├── anomaly_plugin.py │ │ ├── rule.py │ │ ├── rule_parser.py │ │ ├── rule_template.py │ │ ├── rules.py │ │ ├── ruling_process.py │ │ └── warnings.py │ ├── settings.py │ ├── strings.py │ ├── subscriptions.py │ ├── tables.py │ ├── upload.py │ ├── user.py │ └── whois.py ├── pages │ ├── __init__.py │ ├── alerts.py │ ├── anomaly_plugin.py │ ├── base.py │ ├── details.py │ ├── links.py │ ├── login.py │ ├── logout.py │ ├── map.py │ ├── metadata.py │ ├── nodes.py │ ├── portinfo.py │ ├── rules.py │ ├── sec_dashboard.py │ ├── settings.py │ ├── settings_page.py │ ├── stats.py │ └── table.py ├── preprocess.py ├── rule_templates │ ├── compromised.yml │ ├── compromised_hosts.txt │ ├── dos.yml │ ├── netscan.yml │ ├── portscan.yml │ └── suspicious.yml ├── server_aggregator.py ├── server_collector.py ├── server_webserver.py ├── sql │ ├── default_port_data.json │ ├── delete_staging_data.sql │ ├── drop_datasource.sql │ ├── drop_subscription.sql │ ├── profiling.sql │ ├── sessions.sql │ ├── setup_datasource_mysql.sql │ ├── setup_datasource_sqlite.sql │ ├── setup_shared_tables_mysql.sql │ ├── setup_shared_tables_sqlite.sql │ ├── setup_subscription_tables_mysql.sql │ ├── setup_subscription_tables_sqlite.sql │ └── test_data.sql ├── static │ ├── css │ │ ├── general.css │ │ ├── map.css │ │ ├── security.css │ │ ├── table.css │ │ └── timerange.css │ ├── img │ │ ├── logo_dark.png │ │ └── logo_light.png │ ├── js │ │ ├── jquery-3.1.0.min.js │ │ ├── liveUpdate.js │ │ ├── local │ │ │ ├── en.js │ │ │ ├── fr.js │ │ │ └── rv.js │ │ ├── map.js │ │ ├── map_events.js │ │ ├── map_links.js │ │ ├── map_node.js │ │ ├── map_ports.js │ │ ├── map_render.js │ │ ├── map_selection.js │ │ ├── metadata.js │ │ ├── security.js │ │ ├── settings.js │ │ ├── table.js │ │ ├── table_filters.js │ │ └── timerange.js │ ├── nouislider │ │ ├── nouislider-dark.css │ │ ├── nouislider.css │ │ ├── nouislider.min.js │ │ └── nouislider.pips.css │ └── semantic │ │ ├── LICENSE │ │ ├── semantic.css │ │ ├── semantic.js │ │ ├── semantic.min.css │ │ ├── semantic.min.js │ │ └── themes │ │ └── default │ │ └── assets │ │ ├── fonts │ │ ├── icons.eot │ │ ├── icons.otf │ │ ├── icons.svg │ │ ├── icons.ttf │ │ ├── icons.woff │ │ └── icons.woff2 │ │ └── images │ │ └── flags.png ├── templates │ ├── _head.html │ ├── _tail.html │ ├── en │ │ ├── _footer.html │ │ ├── _header.html │ │ ├── dashboard.html │ │ ├── login.html │ │ ├── map.html │ │ ├── metadata.html │ │ ├── settings.html │ │ ├── stats.html │ │ └── table.html │ ├── fr │ │ ├── _footer.html │ │ ├── _header.html │ │ ├── dashboard.html │ │ ├── login.html │ │ ├── map.html │ │ ├── metadata.html │ │ ├── settings.html │ │ ├── stats.html │ │ └── table.html │ └── rv │ │ ├── _footer.html │ │ ├── _header.html │ │ ├── dashboard.html │ │ ├── login.html │ │ ├── map.html │ │ ├── metadata.html │ │ ├── settings.html │ │ ├── stats.html │ │ └── table.html └── update_portLUT.py ├── setup.cfg ├── setup.py └── spec ├── __init__.py ├── browser ├── __init__.py ├── conftest.py ├── manual.py ├── test_all_pages.py ├── test_dashboard.py ├── test_metadata.py └── test_settings.py ├── javascripts ├── helpers │ ├── node_tree.js │ ├── ports.js │ └── test_maker.py ├── map_events_spec.js ├── map_links_spec.js ├── map_node_spec.js ├── map_ports_spec.js ├── map_render_spec.js ├── map_selection_spec.js ├── map_spec.js ├── metadata_spec.js ├── support │ └── jasmine.yml ├── table_filters_spec.js └── timerange_spec.js └── python ├── __init__.py ├── db_connection.py ├── dummy_content.txt ├── importers ├── __init__.py ├── nfcapd_test ├── test_import_asasyslog.py ├── test_import_aws.py ├── test_import_base.py ├── test_import_netflow.py ├── test_import_paloalto.py ├── test_import_tcpdump.py ├── test_import_tshark.py └── test_neflow_collector.py ├── mock_router.py ├── models ├── __init__.py ├── security │ ├── __init__.py │ ├── test_alerts.py │ ├── test_anomaly_plugin.py │ ├── test_rule.py │ ├── test_rule_parser.py │ ├── test_rule_template.py │ ├── test_rules.py │ ├── test_ruling_process.py │ └── test_warnings.py ├── test_datasources.py ├── test_details.py ├── test_filters.py ├── test_links.py ├── test_livekeys.py ├── test_nodes.py ├── test_ports.py ├── test_settings.py ├── test_subscriptions.py ├── test_tables.py ├── test_upload.py ├── test_user.py └── test_whois.py ├── pages ├── __init__.py ├── test_alerts.py ├── test_anomaly_plugin.py ├── test_base.py ├── test_dashboard.py ├── test_details.py ├── test_links.py ├── test_login.py ├── test_logout.py ├── test_map.py ├── test_metadata.py ├── test_nodes.py ├── test_portinfo.py ├── test_rules.py ├── test_settings.py ├── test_settings_page.py ├── test_stats.py └── test_table.py ├── plugins ├── test_plugin1 │ ├── __init__.py │ ├── models │ │ ├── __init__.py │ │ └── test_model.py │ ├── sam_installer.py │ └── sql │ │ └── create_test_table.sql └── test_plugin2 │ ├── __init__.py │ ├── pages │ ├── __init__.py │ └── test_url.py │ ├── sam_installer.py │ └── templates │ ├── en │ └── test_template.html │ └── fr │ └── test_template.html ├── portlut_mock.csv ├── rule_templates ├── test_hosts.txt └── test_rule.yml ├── test_aggregator.py ├── test_collector.py ├── test_common.py ├── test_constants.py ├── test_integrity.py ├── test_launcher.py ├── test_localization.py ├── test_server.py ├── test_sql.sql └── test_update_portLUT.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior, in case people don't have core.autocrlf set. 2 | * text=auto 3 | 4 | # Explicitly declare text files you want to always be normalized and converted 5 | # to native line endings on checkout. 6 | *.py text 7 | *.html text 8 | 9 | # Declare files that will always have CRLF line endings on checkout. 10 | *.sln text eol=crlf 11 | 12 | # Denote all files that are truly binary and should not be modified. 13 | *.png binary 14 | *.jpg binary -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # PyCharm project settings 92 | .idea/ 93 | 94 | # database root access 95 | dbconfig_local.* 96 | 97 | # data dump folder 98 | data/ 99 | 100 | temp.* 101 | 102 | # certificates and keys 103 | *.pem 104 | 105 | # don't include plugins by default 106 | plugins_omit/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SAM - System Architecture Mapper 2 | 3 | SAM is a tool designed to map a network based on the data log of a router. 4 | It runs as a local python-based server and displays the a map and statistics on the browser. 5 | 6 | Check out the [website](http://www.samapper.com) for details about the project and a demo! 7 | 8 | ## Quickstart (using pip) 9 | 10 | install SAM with pip: 11 | 12 | pip install samapper 13 | 14 | Collect network data with tcpdump and run the http server: 15 | 16 | sudo tcpdump -i any -f --immediate-mode -l -n -Q inout -tt | samapper --local --whois --format=tcpdump 17 | 18 | * tcpdump will probably need to be run with sudo to allow it to capture network traffic from your devices. 19 | * Only tcpdump format works locally via pipe at the moment. 20 | 21 | Or, run the http server without collecting data: 22 | 23 | samapper --local --whois --format=none 24 | 25 | 26 | #### Known issue: 27 | When running samapper in local mode using sqlite (the default) the database will sometimes 28 | lock up when the collector is inserting and you are viewing the display. If this is happening, 29 | just run the collector for a while, stop it, and run the http server on its own. 30 | 31 | 32 | ## Installation (using git) 33 | 34 | ### Prerequisites 35 | (optional) mysql - database software that will work better for this than sqlite. 36 | 37 | apt-get install mysql-server libmysqlclient-dev python python-dev 38 | 39 | pip - to install python packages 40 | 41 | apt-get install python-pip 42 | 43 | ### Installing 44 | 1. Clone this repository 45 | 2. Run `pip install -r requirements.txt` from within the directory to install necessary packages. 46 | 3. Set environment variables for credentials and settings. See sam/default.cfg. 47 | 48 | ```bash 49 | e.g: 50 | export SAM__DATABASE__DBN=mysql 51 | export SAM__DATABASE__USER=root 52 | export SAM__DATABASE__PW=mypassword 53 | ``` 54 | 55 | ### Usage 56 | 57 | 1. Start the server locally by running: `python -m sam.launcher --target=webserver` For a more robust deployment, SAM supports the WSGI interface (`python sam/server_webserver.py`) and can be run through a different web server. 58 | 59 | 2. Create a data source to use in the settings page, or use the default empty data source provided. 60 | 61 | 3. For static analysis, import your log files into the database by running the following scripts, where log_file is the path to your log file and destination is the name of the data source you wish to fill. 62 | 63 | `python -m sam.launcher --target=import --format= --dest= ` 64 | 65 | Log formats currently supported include: 66 | 67 | 1. paloalto: The [paloalto syslog](https://www.paloaltonetworks.com/documentation/61/pan-os/pan-os/reports-and-logging/syslog-field-descriptions.html) format is expected. 68 | 2. nfdump: Binary files from **nfcapd** are expected. nfdump must be installed. 69 | 3. asa: Cisco ASA logs, Partial support. Thanks to Emre for contributing. 70 | 4. aws: AWS VPC Flow logs: Partial support. Thanks to Emre for contributing. [VPC log spec](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/flow-logs.html#flow-log-records) 71 | 5. tcpdump: Designed to work with live local mode. See quickstart above 72 | 6. tshark: Partial support. 73 | 74 | 4. For live analysis, 75 | 76 | 1. On the settings page, choose a data source for your live data to be funneled into then create a Access Key 77 | 2. Edit default.cfg or set an environment variable (SAM__COLLECTOR__UPLOAD_KEY) to your new access key 78 | 3. Start the aggregator (this loads log data into the database) 79 | * `python -m sam.launcher --target=aggregator` 80 | 4. Start the collector (this listens to port 514 and translates syslog lines) 81 | * `python -m sam.launcher --target=collector` 82 | * You will need priviledges to bind to system port 514. 83 | * It should print "Testing connection... Succeeded." 84 | 5. Tell your router to output it's log files to that freshly opened socket. 85 | 86 | 5. Navigate your browser to localhost:8080 and explore your network! 87 | -------------------------------------------------------------------------------- /docs/TESTING.md: -------------------------------------------------------------------------------- 1 | # Python Unit Tests 2 | Python unit tests are written for pytest 3 | They can be run from this directory via the command: 4 | 5 | ```bash 6 | pytest spec/python/ 7 | ``` 8 | or for more details on a particular test: 9 | ```bash 10 | pytest -xv spec/python/pages/test_table.py 11 | ``` 12 | 13 | # Javascript Unit Tests 14 | Javascript unit tests are written for use with jasmine. Port specification is optional. 15 | ```bash 16 | jasmine [-p 8888] 17 | ``` 18 | Navigate your browser to `http://localhost:8888/` to see the results of the tests. 19 | Every page refresh runs the tests again. 20 | 21 | # Browser-based Tests 22 | Browser tests are run in firefox by default, through selenium, controller by python. 23 | 24 | Selenium must be installed (v3.50 used) 25 | ```bash 26 | pip install selenium 27 | ``` 28 | 29 | Browser controller must be installed as well (and accessible to bash; in the PATH). Firefox driver available here: 30 | 31 | [https://github.com/mozilla/geckodriver/releases](https://github.com/mozilla/geckodriver/releases) 32 | 33 | e.g. extract it to `/home/{user}/bin/` 34 | and add that folder to `PATH` 35 | such that `whereis geckodriver` works. 36 | 37 | run tests as python unit tests in the _browser_ directory: 38 | ```bash 39 | pytest spec/browser/ 40 | ``` 41 | or for more details on a particular test: 42 | ```bash 43 | pytest -xv spec/browser/test_table.py 44 | ``` 45 | 46 | # Python Code Coverage 47 | This runs the browser and unit tests for python files 48 | and describes how many statements exist and how many statements were run during the tests. 49 | 50 | Javascript, SQL, and other files are not indicated here. 51 | 52 | Disclaimer: Code coverage here is about as useful an indicator of testing as BMI is for fitness 53 | ```bash 54 | pip install pytest-cov 55 | # from the root project folder (that contains folders 'sam' and 'spec'): 56 | pytest --cov sam spec 57 | ``` 58 | 59 | # Test a rule template 60 | Testing your own custom rule template can be done by using the launcher with the "template" target. This should provide enough information to make sure a template is ready for use. 61 | 62 | ```bash 63 | python sam/launcher.py --target=template sam/rule_templates/dos.yml 64 | ``` 65 | 66 | The positive successful output might look something like this: 67 | 68 | ``` 69 | 70 | ====================================================================== 71 | Rule name: High Traffic 72 | Rule type: periodic 73 | Rule subject: dst 74 | Rule inclusions: None 75 | Exposed parameters: 76 | (passed) threshold: 77 | default: 600 78 | regex (valid): ^\d+$ 79 | label: This rule will trigger on a host if it receives more than this number of inbound connections over a period of 5 minutes. 80 | format: text 81 | Action defaults: None 82 | Trigger condition: 83 | having conn[links] > $threshold 84 | Translated condition: 85 | having ('conn', 'links') > 600 86 | 87 | Rule SQL: 88 | SELECT timestamp, dst, COUNT(DISTINCT dst) AS 'src[hosts]', COUNT(DISTINCT port) AS 'conn[ports]', COUNT(DISTINCT protocol) AS 'conn[protocol]', SUM(links) AS 'conn[links]' 89 | FROM s1_ds1_Links 90 | GROUP BY timestamp, dst 91 | HAVING `conn[links]` > '600' 92 | 93 | Test SQL execution: Success 94 | 95 | ========================== Rule is valid =========================== 96 | ``` -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | MySQL-python>=1.2.5 2 | web.py>=0.38 3 | requests>=2.18 4 | PyYAML>=3.10 5 | pytest>=3.2 6 | jasmine>=2.8 7 | -------------------------------------------------------------------------------- /sam/ConfigEnvy.py: -------------------------------------------------------------------------------- 1 | from ConfigParser import SafeConfigParser 2 | import os 3 | 4 | 5 | class ConfigEnvy(SafeConfigParser, object): 6 | default_file_name = 'default.cfg' 7 | 8 | def __init__(self, namespace, defaults=None, filenames=None): 9 | self.namespace = namespace 10 | super(ConfigEnvy, self).__init__(defaults) 11 | BASE_PATH = os.path.dirname(__file__) 12 | default_file_name = os.path.join(BASE_PATH, self.default_file_name) 13 | if os.path.isfile(default_file_name): 14 | if filenames: 15 | filenames.push(default_file_name) 16 | else: 17 | filenames = default_file_name 18 | if filenames is not None: 19 | super(ConfigEnvy, self).read(filenames) 20 | 21 | def get(self, section, option, raw=False, vars=None, default=None): 22 | """Get an option value for a given section. 23 | Almost like ConfigParser, but look in the Environment variables first 24 | Env variables take the form __
__