├── .gitignore ├── 01_GettingStarted ├── README.md └── images │ ├── ambari.png │ ├── metron_alerts.png │ ├── metron_login.png │ └── metron_management.png ├── 02_ParsingSquid ├── README.md ├── es │ └── squid_template.json ├── images │ ├── alerts_ui_mysquid.png │ ├── kibana_create_es_template.png │ ├── kibana_get_working.png │ ├── mac_set_proxy.png │ ├── metron_alerts_details.png │ ├── metron_alerts_squid.png │ ├── mgmt_with_mysquid.png │ ├── mysquid_create_sensor.png │ ├── mysquid_grok.png │ ├── mysquid_grok_sample.png │ ├── nifi_01.png │ ├── nifi_02.png │ ├── nifi_03.png │ ├── nifi_04.png │ ├── nifi_05.png │ ├── nifi_06.png │ ├── nifi_complete_mysquid_started.png │ ├── nifi_input_annotated.png │ ├── solr_collections.png │ ├── solr_timestamp_01.png │ ├── solr_timestamp_02.png │ ├── solr_timestamp_03.png │ └── timestamp_parser_config.png └── solr │ ├── schema.xml │ └── solrconfig.xml ├── 03_EnrichingSquid ├── README.md └── images │ ├── alerts_with transformations.png │ ├── edit_advanced_raw_json.png │ ├── edit_ip_dst_addr_enrich.png │ ├── edit_mysquid.png │ ├── edit_schema.png │ ├── geo_code_enrich_detail.png │ ├── ip_dst_addr_geo.png │ ├── transformation_config_before.png │ └── transformations_config_after.png ├── 03a_ThreatIntel ├── README.md └── images │ ├── malicious_domain_alert_detail.png │ ├── malicious_domain_alerts.png │ ├── malicious_ip_alert_detail.png │ ├── malicious_ip_alerts.png │ ├── threat_intel_config_after.png │ └── threat_intel_config_before.png ├── 04_TriagingSquid ├── README.md ├── images │ ├── alerts_sort_desc_by_score.png │ ├── configure_columns.png │ ├── domain_whitelist.png │ ├── enrichments_after.png │ ├── enrichments_before.png │ ├── filter_alerts.png │ ├── metron_no_cnn_alert.png │ ├── remove_filter.png │ ├── scored_alerts.png │ └── triage_fields.png └── scripts │ ├── domain_whitelist.csv │ ├── domain_whitelist_extractor_config.json │ ├── generate_typosquats.sh │ ├── import_domain_whitelist.sh │ └── typosquat_domains.txt ├── 05_ProfilerBasics └── README.md ├── 06_ProfilingAuthLogs ├── README.md ├── config │ ├── create_auth_solr_collection.sh │ ├── install_auth.sh │ ├── load_auth_config.sh │ ├── patch_profiler_config.sh │ ├── schema │ │ └── auth │ │ │ ├── schema.xml │ │ │ └── solrconfig.xml │ └── zookeeper │ │ ├── enrichments │ │ └── auth.json │ │ ├── indexing │ │ ├── auth.json │ │ └── profiler.json │ │ ├── parsers │ │ └── auth.json │ │ └── profiler.json ├── images │ ├── 01_alerts_ui_auth_alerts.png │ ├── 02_select_auth_fields.png │ ├── 03_show_auth_measurements.png │ ├── 04_auth_high_score_alert_details.png │ ├── 05_auth_triage_details.png │ ├── 06_auth_enrichment.png │ ├── 07_auth_triage.png │ ├── continuous_anomaly.png │ ├── intermittent_anomaly.png │ ├── lanl.png │ └── typical_user.png ├── notebooks │ └── compromised_account_analysis.json └── scripts │ ├── build_demo_loader.sh │ ├── config.json │ ├── demo_loader.sh │ ├── download_auth_data.sh │ ├── host_filter.txt │ └── run.sh ├── 07_ExploringEventHistory ├── README.md ├── images │ ├── 00_zoom_dashboard.png │ ├── 01_starting_zeppelin.png │ ├── 02_zeppelin_no_user.png │ ├── 03_zeppelin_login.png │ ├── 04_zeppelin_select_notebook.png │ ├── 05_auth_investigation_notebook.png │ ├── 06_show_output.png │ ├── 07_indexed_data_file_structure.png │ ├── 08_create_table_from files.png │ ├── 09_create_graphs.png │ ├── 10_new_paragraph.png │ ├── 11_enter_run_query.png │ ├── 12_show_graph_sql.png │ ├── banana_01.png │ ├── banana_02.png │ ├── banana_03.png │ ├── banana_04.png │ ├── banana_05.png │ ├── banana_06.png │ ├── banana_07.png │ ├── banana_08.png │ ├── banana_09.png │ ├── banana_10.png │ ├── banana_11.png │ ├── banana_12.png │ ├── banana_13.png │ ├── banana_14.png │ ├── banana_15.png │ ├── banana_16.png │ ├── banana_17.png │ ├── banana_18.png │ ├── banana_19.png │ ├── banana_20.png │ ├── banana_21.png │ ├── banana_22.png │ ├── banana_23.png │ ├── banana_24.png │ ├── banana_25.png │ └── banana_26.png └── notebooks │ └── Auth Investigation.json ├── 08_DGADetectionModelAsService ├── README.md ├── dga_model │ ├── dga_model_alexa_counts.model │ ├── dga_model_alexa_vectorizor.model │ ├── dga_model_dict_counts.model │ ├── dga_model_dict_vectorizor.model │ ├── dga_model_random_forest.model │ ├── model.py │ ├── rest.py │ └── rest.sh ├── images │ ├── add_is_malicious.png │ ├── alerts_with_is_malicious.png │ ├── both_scores.png │ ├── dga_events.png │ ├── model_python.png │ ├── one_score.png │ ├── rest_python.png │ └── threat_intel_config.png └── scripts │ ├── setup_dga_maas.sh │ └── start_dga_model.sh ├── README.md ├── build_single ├── .gitignore ├── README.md ├── build_single.sh ├── deploy_single_node.sh ├── download_banana_schema.sh ├── extract_blueprint_variables.sh ├── hcp_metron_rest_db.sql ├── hostmapping.json ├── install_banana_schema.sh ├── install_sensors.sh ├── metron_squidtokafka.xml ├── mysql_init_users.sql ├── nifi_get_id.py ├── nifi_get_root_node.py ├── nifi_get_variable_client_id.py ├── nifi_get_variable_client_version.py ├── nifi_process_from_template.sh ├── nifi_set_variable.sh ├── post_install_setup.sh ├── prepare_ambari_config.sh ├── register_blueprint.sh ├── retrieve_blueprint.sh ├── sensor_install │ ├── .gitignore │ ├── sensor_install_funcs.sh │ └── sensor_install_phase.sh ├── sensors │ ├── general │ │ └── post_ambari.sh │ └── squid │ │ ├── .gitignore │ │ ├── create_typosquat_10k_filter.sh │ │ ├── extractor_count.json.template │ │ ├── extractor_filter.json.template │ │ ├── post_ambari.sh │ │ └── pre_ambari.sh ├── sethostname.sh ├── single-node-es-blueprint-19-variables.txt ├── single-node-es-blueprint-19.json ├── single-node-es-sensor-config.txt ├── single-node-hostmapping.json ├── single-node-solr-blueprint-19-variables.txt ├── single-node-solr-blueprint-19.json ├── start_service.sh └── typosquat │ ├── create_es_collection.sh │ ├── create_solr_collection.sh │ ├── create_typosquat_10k_filter.sh │ ├── delete_solr_collection.sh │ ├── extractor_count.json │ ├── extractor_filter.json │ ├── install_squid_enrichments.sh │ └── squid_enrichments.json └── cloudbreak ├── .gitignore ├── blueprint-19.json └── recipes ├── create_schemas.sh ├── enable_squid_nifi.sh ├── epelrelease.sh ├── init_squid_sensor.sh ├── install_bro.sh ├── install_squid.sh ├── install_tcpreplay.sh ├── nodejs.sh ├── requestsupgrade.sh └── setup_repos.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.jar 2 | -------------------------------------------------------------------------------- /01_GettingStarted/README.md: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | # Getting started with Metron 5 | ## Objectives 6 | After this lab you will be able to: 7 | 8 | 1. Connect your browser to your metron instance. 9 | 2. Describe the main UIs for Metron. 10 | 3. Describe the information contained in each UI. 11 | 4. Open the UI and view the content. 12 | 13 | ## Setting up a host entry for easy course navigation 14 | 15 | ### For Linux/Unix/OSX 16 | 17 | Edit your `hosts` file and add a entry to your AWS instance. First find your host IP adress that was provided to you. On your laptop do: 18 | 19 | 1. `sudo vi /etc/hosts` 20 | 2. Add `host_ip mobius.local`, where you replace *host_ip* with the real IP and save the file. 21 | 3. You should now be able to access Ambari at http://mobius.local:8080/ 22 | 23 | ### For Windows 24 | 25 | For Windows 10 and 8: 26 | 27 | 1. Press the Windows key. 28 | 2. Type Notepad in the search field. 29 | 3. In the search results, right-click Notepad and select Run as administrator. 30 | 4. From Notepad, open the following file: c:\Windows\System32\Drivers\etc\hosts 31 | 5. Add `*host ip* mobius.local`, where you replace *host ip* with the real IP and save the file. 32 | 7. Click File > Save to save your changes. 33 | 8. You should now be able to access Ambari at http://mobius.local:8080/ 34 | 35 | ## Opening the Metron Management UI 36 | 37 | 1. The management UI shows the sensors configured in Metron and their operational state (running/stopped and latency). From the management UI you can create new sensors, view sensor configuration, browse raw logs sent by the sensor, and change the configuration of sensors. 38 | 2. Open your browser to http://mobius.local:4200/ . 39 | 3. When prompted for the password enter metron for the user and metron for the password. 40 | ![Metron Login Screen](images/metron_login.png) 41 | 4. The Metron Management UI opens showing the sensors configured in Metron and their operational state (running/stopped and latency). 42 | ![Metron Management Screen](images/metron_management.png) 43 | 44 | ## Opening the Metron Alerts UI 45 | 1. The Metron Alerts UI shows enriched and triaged events stored in the Metron index. From the Alerts UI, you can filter events, view event fields, and sort events. You can also escalate, add comments to, or group events. 46 | 6. Open your browser to http://mobius.local:4201/. 47 | 7. When prompted for the password enter metron for the user and metron for the password. 48 | ![Metron Login Screen](images/metron_login.png) 49 | 8. The Metron Alerts UI opens listing the events reported to Metron. From the alerts UI you can search, filter and view the details of alerts. 50 | ![Metron Alerts Screen](images/metron_alerts.png) 51 | 9. Consult the [Viewing Alerts User Guide](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/user-guide/content/viewing_alerts.html) for more information on how to use the Alerts UI. 52 | ## Opening the Ambari UI 53 | 1. Open Ambari to view the status of the Metron infrastructure and start, stop, and restart services. Ambari also configures, installs and upgrades Metron software. 54 | 2. Open your browser to http://mobius.local:8080/. 55 | 3. When prompted for the password enter admin for the user and admin for the password. 56 | 4. The Ambari management pages open. 57 | ![Ambari Screen](images/ambari.png) 58 | 5. The Quicklinks Ambari menu item launches the UIs for a service. For example, select the Management UI and Alerts UI by selecting Services from the upper right and then Metron from the left. Pull down quick links and select Management UI or Alerts UI to launch the Metron UIs. 59 | 60 | ## Single Node Metron Instance Quick Reference 61 | For the workshop you will be provided with a single node version Metron installation. 62 | 63 | |Credential| Value | 64 | |--|--| 65 | |Metron Host Name | Provided to you | 66 | |Metron Management UI|http://mobius.local:4200| 67 | |Metron Alerts UI|http://mobius.local:4201| 68 | |Metron Mgmt/Alerts Credentials|metron:metron| 69 | |Ambari URL|http://mobius.local:8080| 70 | |Ambari credentials|admin:admin| 71 | |SSH key| Provided to you| 72 | |Ssh| ssh -i *ssh_key.pem* centos@*metron_host_name* 73 | |mysql console|mysql -u *user_name* -p| 74 | |mysqldb root credentials|root:| 75 | |mysqldb metron credentials|metron:Myp@ssw0rd | 76 | |zoomdata url|http://mobius.local:6430| 77 | |zoomdata credentials|admin:Metron1234!| 78 | |solr admin UI|http://mobius.local:8983/solr/#/| 79 | 80 | ## Next Lab 81 | [Parsing squid logs.](../02_ParsingSquid/README.md) 82 | -------------------------------------------------------------------------------- /01_GettingStarted/images/ambari.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/01_GettingStarted/images/ambari.png -------------------------------------------------------------------------------- /01_GettingStarted/images/metron_alerts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/01_GettingStarted/images/metron_alerts.png -------------------------------------------------------------------------------- /01_GettingStarted/images/metron_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/01_GettingStarted/images/metron_login.png -------------------------------------------------------------------------------- /01_GettingStarted/images/metron_management.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/01_GettingStarted/images/metron_management.png -------------------------------------------------------------------------------- /02_ParsingSquid/README.md: -------------------------------------------------------------------------------- 1 | # Parsing Squid logs 2 | ## Objectives 3 | After this lab you will be able to: 4 | 1. Collect a squid proxy log and move it to Kafka using Nifi. 5 | 2. Create a Metron squid parser. 6 | 3. Add some simple transformations to the events. 7 | 4. View the parsed logs in the Metron Alerts UI. 8 | ## Generate Squid Access Events 9 | 1. Your Metron host comes with a squid proxy installed. Configure your proxy settings to use your squid proxy. If you are not able to change your proxy settings, skip this step and use curl commands to generate traffic. 10 | 11 | ### Chrome Proxy 12 | 13 | The easiest way to control your proxy from Chrome is to install and extension such as Proxy SwitchySharp. An extension enables you to select a menu item in Chrome to switch between your regular settings and your Metron proxy easily. 14 | 15 | On Mac open Chrome and select Chrome > Preferences from the main menu. Scroll to the bottom of the settings page and click the Show advanced settings link. Locate the Network section and click the Change proxy settings button. Check Web Proxy (HTTP) and enter `mobius.local` and the squid port 3128. Check Secure Web Proxy (HTTPS) and enter your Metron name and the squid port 3128. Click the OK button and then the Apply button to save your changes. 16 | 17 | ### OSX Safari Proxy 18 | On OSX go to System Preferences > Network > Wi-Fi. Click the Advanced button. Select the Proxies Tab, and check the Web Proxy (HTTP) and enter `mobius.local` and the squid port 3128. 19 | 20 | ### Windows Proxy 21 | 22 | On Windows, follow the manual instructions on [this page](http://www.dummies.com/computers/operating-systems/windows-10/how-to-set-up-a-proxy-in-windows-10/) 23 | 24 | 2. In the browser configured with the Metron squid proxy, open a web site that generates web requests such as google.com or a news website. 25 | 26 | To generate data using curl, open a command line tool and enter: 27 | ``` 28 | curl -I --proxy mobius.local:3128 *web_url* 29 | ``` 30 | 31 | For example to access google using a proxy, 32 | ``` 33 | curl -I --proxy mobius.local:3128 http://google.com 34 | ``` 35 | 36 | 3. Open the Metron alerts ui by entering http://mobius.local:4201 in the browser. 37 | 38 | By default you will see the most recent events at the top of the UI. At this point you should see some squid events with recent timestamps in the alerts ui. 39 | ![Metron Alerts Squid](images/metron_alerts_squid.png) 40 | 41 | If you click in the space between the columns, the full details of the event will open on the right side of the browser. 42 | ![Metron Alerts Details](images/metron_alerts_details.png) 43 | 44 | ## Adding the mysquid sensor 45 | Now we will build a mysquid sensor from scratch using a grok parser. 46 | 1. Open the Metron Management UI. 47 | 2. Click on the + button on the lower right corner to add a new sensor. 48 | 3. Enter the following: 49 | ``` 50 | Name: mysquid 51 | Kafka Topic: mysquid 52 | Parser Type: Grok 53 | ``` 54 | Don't worry if you see the "No Matching Kafka Topic", the Kafka Topic will be created automatically on save. 55 | 56 | 57 | 4. Click to expand the Grok Statement. Paste the sample squid raw log entry below into the Sample field. 58 | 59 | ``` 60 | 1528766038.123 70328 75.133.181.135 TCP_TUNNEL/200 420 CONNECT data.cnn.com:443 - HIER_DIRECT/2.20.22.7 - 61 | ``` 62 | 63 | ![My Squid With Sample](images/mysquid_grok_sample.png) 64 | 65 | 5. Copy and paste the grok expression below to the right of the MYSQUID grok expression name: 66 | 67 | ``` 68 | %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})? 69 | ``` 70 | 71 | 6. Press the Test button. The Preview section will update with the fields parsed from the raw log. If there is an error and the preview does not appear, verify the sample and grok expression field values. Adjust the sample or grok expression field values so the preview appears. 72 | 73 | ![My Squid Sensor](images/mysquid_grok.png) 74 | 75 | 7. Click Save. 76 | 8. Verify that sensor name and topic name are "mysquid" with NO extra spaces or special characters. 77 | 9. Click Save on the mysquid sensor. 78 | 10. The mysquid sensor appears in the management UI. Click on the pencil icon to edit the mysquid sensor. The mysquid configuration opens. 79 | 11. Scroll down to the Parser Config section. 80 | 12. In the enter field point, enter timestampField. 81 | 13. In enter value, enter timestamp. 82 | 83 | 84 | 85 | 14. Click the Save button. 86 | 87 | ## Installing the mysquid index template 88 | After events are enriched and triaged, metron stores the events in an index. The index template specifies how to interpret the metron events and how to index strings using either a keyword or full text search. 89 | 90 | Determine the index you are using (Solr or Elastic Search - ask your instructor if you are not sure) and follow the instrutions in the corresponding section below. 91 | 92 | ### Creating a Collection in Solr 93 | SKIP this section if you are using Elastic Search. 94 | 95 | 1. SSH into the metron host. If you are logging in from windows, ssh using putty. 96 | 97 | ``` 98 | ssh -i centos@mobius.local 99 | ``` 100 | 2. Run the create_solr_collection command to create a new collection and install the collection schema. The schema defines the types and search capabilities of each field in the events stored in the index. 101 | 102 | ``` 103 | ./create_solr_collection.sh mysquid 104 | ``` 105 | 3. To see the definition of the schema use the more command on the metron host. Press enter to advance one line. Press space to advance one page. To exit the more command enter 'q': 106 | 107 | ``` 108 | more /usr/hcp/current/metron/config/schema/mysquid/schema.xml 109 | ``` 110 | 111 | 4. The solrconfig.xml file defines the configuration of the index. The solr config does not vary by the index content. When creating a new index, use the metron solrconfig.xml as is. 112 | 113 | ``` 114 | more /usr/hcp/current/metron/config/schema/mysquid/solrconfig.xml 115 | ``` 116 | 117 | 5. In the browser, open solr collections interface to see the mysquid collection: 118 | http://mobius.local:8983/solr/#/~collections 119 | 120 | 121 | 122 | 123 | ### Creating an index in Elastic Search 124 | 125 | SKIP this section if you using Solr. 126 | 127 | 1. Enter the Kibana url in the browser: 128 | http://mobius.local:5000 129 | 2. Select Dev Tools from the left hand side of the kibana page. The Dev Tools console is an easy way to interact with the index REST api. If the Welcome window appears, click the Get to work button. 130 | 131 | 132 | 133 | 3. Paste the following command into the left side of Dev Tools window: 134 | ![Kibana Dev Tools](images/kibana_create_es_template.png) 135 | 136 | ``` 137 | PUT _template/mysquid 138 | { 139 | "template": "mysquid_index*", 140 | "settings": {}, 141 | "mappings": { 142 | "mysquid_doc": { 143 | "dynamic_templates": [ 144 | { 145 | "geo_location_point": { 146 | "match": "enrichments:geo:*:location_point", 147 | "match_mapping_type": "*", 148 | "mapping": { 149 | "type": "geo_point" 150 | } 151 | } 152 | }, 153 | { 154 | "geo_country": { 155 | "match": "enrichments:geo:*:country", 156 | "match_mapping_type": "*", 157 | "mapping": { 158 | "type": "keyword" 159 | } 160 | } 161 | }, 162 | { 163 | "geo_city": { 164 | "match": "enrichments:geo:*:city", 165 | "match_mapping_type": "*", 166 | "mapping": { 167 | "type": "keyword" 168 | } 169 | } 170 | }, 171 | { 172 | "geo_location_id": { 173 | "match": "enrichments:geo:*:locID", 174 | "match_mapping_type": "*", 175 | "mapping": { 176 | "type": "keyword" 177 | } 178 | } 179 | }, 180 | { 181 | "geo_dma_code": { 182 | "match": "enrichments:geo:*:dmaCode", 183 | "match_mapping_type": "*", 184 | "mapping": { 185 | "type": "keyword" 186 | } 187 | } 188 | }, 189 | { 190 | "geo_postal_code": { 191 | "match": "enrichments:geo:*:postalCode", 192 | "match_mapping_type": "*", 193 | "mapping": { 194 | "type": "keyword" 195 | } 196 | } 197 | }, 198 | { 199 | "geo_latitude": { 200 | "match": "enrichments:geo:*:latitude", 201 | "match_mapping_type": "*", 202 | "mapping": { 203 | "type": "float" 204 | } 205 | } 206 | }, 207 | { 208 | "geo_longitude": { 209 | "match": "enrichments:geo:*:longitude", 210 | "match_mapping_type": "*", 211 | "mapping": { 212 | "type": "float" 213 | } 214 | } 215 | }, 216 | { 217 | "timestamps": { 218 | "match": "*:ts", 219 | "match_mapping_type": "*", 220 | "mapping": { 221 | "type": "date", 222 | "format": "epoch_millis" 223 | } 224 | } 225 | }, 226 | { 227 | "threat_triage_score": { 228 | "mapping": { 229 | "type": "float" 230 | }, 231 | "match": "threat:triage:*score", 232 | "match_mapping_type": "*" 233 | } 234 | }, 235 | { 236 | "threat_triage_reason": { 237 | "mapping": { 238 | "type": "text", 239 | "fielddata": "true" 240 | }, 241 | "match": "threat:triage:rules:*:reason", 242 | "match_mapping_type": "*" 243 | } 244 | } 245 | ], 246 | "properties": { 247 | "action": { 248 | "type": "keyword" 249 | }, 250 | "bytes": { 251 | "type": "long" 252 | }, 253 | "code": { 254 | "type": "long" 255 | }, 256 | "domain_without_subdomains": { 257 | "type": "keyword" 258 | }, 259 | "elapsed": { 260 | "type": "long" 261 | }, 262 | "full_hostname": { 263 | "type": "keyword" 264 | }, 265 | "guid": { 266 | "type": "keyword" 267 | }, 268 | "ip_dst_addr": { 269 | "type": "ip" 270 | }, 271 | "ip_src_addr": { 272 | "type": "ip" 273 | }, 274 | "is_alert": { 275 | "type": "keyword" 276 | }, 277 | "is_potential_typosquat": { 278 | "type": "boolean" 279 | }, 280 | "method": { 281 | "type": "keyword" 282 | }, 283 | "original_text": { 284 | "type": "text" 285 | }, 286 | "source:type": { 287 | "type": "keyword" 288 | }, 289 | "timestamp": { 290 | "type": "date", 291 | "format": "epoch_millis" 292 | }, 293 | "url": { 294 | "type": "keyword" 295 | }, 296 | "alert": { 297 | "type": "nested" 298 | } 299 | } 300 | } 301 | } 302 | } 303 | ``` 304 | 305 | 4. Press the green play button. The result on the right hand side of the screen will show "acknowledged" : true 306 | ## Starting the mysquid sensor 307 | 1. Return to the Metron Management UI. 308 | 2. Push the play button on the mysquid sensor to start the parser. 309 | ## Moving squid access.log lines to Kafka for processing by Metron 310 | 1. Enter the Nifi URL in your browser: 311 | http://mobius.local:9090/nifi/ 312 | 2. The Nifi flow on the canvas tails the squid access.log and sends the lines to the PublishKafka. PublishKafka breaks the log sample into individual lines and writes each line as a Kafka message to the squid topic. 313 | 314 | 315 | 3. Right click on the PublishSquidToMetron processor and select Copy. 316 | 317 | 4. Right click on an empty area of the canvas and select Paste. 318 | ![Pasted Nifi PublishSquidToMetron](images/nifi_02.png) 319 | 320 | 5. Right click on the copy of PublishSquidToMetron and select Configure. 321 | 322 | 6. Click on the Settings tab and change the name to PublishMySquidToMetron. 323 | 324 | 325 | 7. Click the Properties tab and change Topic Name to mysquid. 326 | 327 | 328 | 8. Click Apply to save the changes. 329 | 330 | 9. Hover the cursor over the Read Squid Log processor until the connect icon appears. Click and drag the flow to the middle of the PublishMySquidToMetron. Drop the cursor and the Create Connection dialog appears. 331 | 332 | 333 | 10. Click Add and make sure you Start the processor by clicking Start in the Operate panel. 334 | 335 | 11. The flow should look as below: 336 | 337 | 338 | 12. The flow can now write squid data to both topics or either topic. It will be easier to see what is happening if we stop writing to the squid topic. 339 | 13. Right click on PublishSquidToMetron and select Stop. 340 | 14. Right click on PublishMySquidToMetron and select Start. 341 | 342 | 343 | 15. Set the browser to use the Metron proxy to start data flowing to the mysquid topic. Enter google or a news site URL or any other URL that creates web request traffic to the squid proxy. 344 | 16. When log lines are flowing to the mysquid sensor, the In field of the PublishMySquidToMetron processor should be greater than zero. 345 | 346 | 347 | 348 | 17. Open Metron Alerts UI. In a few minutes the Metron Alerts UI show mysquid events. If you don't see event with mysquid source, go to the Metron Configuration UI, stop mysquid and start it again. It might take a minute or two for events to start flowing. 349 | 350 | 351 | 352 | 353 | 18. Congratulations! You are now ingesting squid log events. The next step is to add enrichments. 354 | 355 | ## Next Lab 356 | [Enriching squid logs.](../03_EnrichingSquid/README.md) 357 | 358 | ## References 359 | 360 | [Metron Run Book](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/runbook/cybersecurity-runbook.pdf) 361 | 362 | [Metron Source Code](https://github.com/apache/metron) 363 | -------------------------------------------------------------------------------- /02_ParsingSquid/es/squid_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "template": "mysquid_index*", 3 | "settings": {}, 4 | "mappings": { 5 | "mysquid_doc": { 6 | "dynamic_templates": [ 7 | { 8 | "geo_location_point": { 9 | "match": "enrichments:geo:*:location_point", 10 | "match_mapping_type": "*", 11 | "mapping": { 12 | "type": "geo_point" 13 | } 14 | } 15 | }, 16 | { 17 | "geo_country": { 18 | "match": "enrichments:geo:*:country", 19 | "match_mapping_type": "*", 20 | "mapping": { 21 | "type": "keyword" 22 | } 23 | } 24 | }, 25 | { 26 | "geo_city": { 27 | "match": "enrichments:geo:*:city", 28 | "match_mapping_type": "*", 29 | "mapping": { 30 | "type": "keyword" 31 | } 32 | } 33 | }, 34 | { 35 | "geo_location_id": { 36 | "match": "enrichments:geo:*:locID", 37 | "match_mapping_type": "*", 38 | "mapping": { 39 | "type": "keyword" 40 | } 41 | } 42 | }, 43 | { 44 | "geo_dma_code": { 45 | "match": "enrichments:geo:*:dmaCode", 46 | "match_mapping_type": "*", 47 | "mapping": { 48 | "type": "keyword" 49 | } 50 | } 51 | }, 52 | { 53 | "geo_postal_code": { 54 | "match": "enrichments:geo:*:postalCode", 55 | "match_mapping_type": "*", 56 | "mapping": { 57 | "type": "keyword" 58 | } 59 | } 60 | }, 61 | { 62 | "geo_latitude": { 63 | "match": "enrichments:geo:*:latitude", 64 | "match_mapping_type": "*", 65 | "mapping": { 66 | "type": "float" 67 | } 68 | } 69 | }, 70 | { 71 | "geo_longitude": { 72 | "match": "enrichments:geo:*:longitude", 73 | "match_mapping_type": "*", 74 | "mapping": { 75 | "type": "float" 76 | } 77 | } 78 | }, 79 | { 80 | "timestamps": { 81 | "match": "*:ts", 82 | "match_mapping_type": "*", 83 | "mapping": { 84 | "type": "date", 85 | "format": "epoch_millis" 86 | } 87 | } 88 | }, 89 | { 90 | "threat_triage_score": { 91 | "mapping": { 92 | "type": "float" 93 | }, 94 | "match": "threat:triage:*score", 95 | "match_mapping_type": "*" 96 | } 97 | }, 98 | { 99 | "threat_triage_reason": { 100 | "mapping": { 101 | "type": "text", 102 | "fielddata": "true" 103 | }, 104 | "match": "threat:triage:rules:*:reason", 105 | "match_mapping_type": "*" 106 | } 107 | } 108 | ], 109 | "properties": { 110 | "action": { 111 | "type": "keyword" 112 | }, 113 | "bytes": { 114 | "type": "long" 115 | }, 116 | "code": { 117 | "type": "long" 118 | }, 119 | "domain_without_subdomains": { 120 | "type": "keyword" 121 | }, 122 | "elapsed": { 123 | "type": "long" 124 | }, 125 | "full_hostname": { 126 | "type": "keyword" 127 | }, 128 | "guid": { 129 | "type": "keyword" 130 | }, 131 | "ip_dst_addr": { 132 | "type": "ip" 133 | }, 134 | "ip_src_addr": { 135 | "type": "ip" 136 | }, 137 | "is_alert": { 138 | "type": "keyword" 139 | }, 140 | "is_potential_typosquat": { 141 | "type": "boolean" 142 | }, 143 | "method": { 144 | "type": "keyword" 145 | }, 146 | "original_text": { 147 | "type": "text" 148 | }, 149 | "source:type": { 150 | "type": "keyword" 151 | }, 152 | "timestamp": { 153 | "type": "date", 154 | "format": "epoch_millis" 155 | }, 156 | "url": { 157 | "type": "keyword" 158 | }, 159 | "alert": { 160 | "type": "nested" 161 | } 162 | } 163 | } 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /02_ParsingSquid/images/alerts_ui_mysquid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/alerts_ui_mysquid.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/kibana_create_es_template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/kibana_create_es_template.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/kibana_get_working.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/kibana_get_working.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/mac_set_proxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/mac_set_proxy.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/metron_alerts_details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/metron_alerts_details.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/metron_alerts_squid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/metron_alerts_squid.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/mgmt_with_mysquid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/mgmt_with_mysquid.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/mysquid_create_sensor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/mysquid_create_sensor.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/mysquid_grok.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/mysquid_grok.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/mysquid_grok_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/mysquid_grok_sample.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_01.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_02.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_03.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_04.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_05.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_06.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_complete_mysquid_started.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_complete_mysquid_started.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/nifi_input_annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/nifi_input_annotated.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/solr_collections.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/solr_collections.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/solr_timestamp_01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/solr_timestamp_01.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/solr_timestamp_02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/solr_timestamp_02.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/solr_timestamp_03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/solr_timestamp_03.png -------------------------------------------------------------------------------- /02_ParsingSquid/images/timestamp_parser_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/02_ParsingSquid/images/timestamp_parser_config.png -------------------------------------------------------------------------------- /02_ParsingSquid/solr/schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 27 | 28 | 29 | 30 | 33 | 34 | 35 | 36 | 37 | 38 | guid 39 | 40 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /03_EnrichingSquid/README.md: -------------------------------------------------------------------------------- 1 | # Enriching Squid logs 2 | ## Objectives 3 | After this lab you will be able to: 4 | 1. Enrich an event with the geo location of ip addresses accessed through the squid proxy. 5 | 2. Enrich an event with field transformations. 6 | ## Adding a geocode enrichment 7 | The geo enrichment applied to an ip address instructs Metron to find the physical location of an ip address and add the location as enrichments to the event. The ip locations can be used in triaging, hunting, map visualizations, and other analytics. 8 | 1. Open the Metron Configuration UI. 9 | 2. Click on the pencil icon to the right of the mysquid sensor to change the sensor configuration. The mysquid configuration opens. 10 | 11 | ![Edit mysquid sensor](images/edit_mysquid.png) 12 | 13 | 3. Click on the >> icon to the right of the Schema section. 14 | 15 | 16 | 17 | 4. Click the pencil icon to the right of the ip_dst_addr field. 18 | 19 | ![Edit IP dst addr enrichments](images/edit_ip_dst_addr_enrich.png) 20 | 21 | 5. Select geo from the Enrichments drop down. 22 | 23 | 24 | 25 | 6. Click Save. 26 | 7. Scroll to the bottom of the screen and click Save. 27 | 8. Configure your proxy to send web requests to the metron proxy or use curl. Open google.com 28 | 9. Open the Metron Alerts UI. The events will now have geo locations on their destination ip addresses. 29 | 10. Click between the columns to open the event detail. The components of the geo enrichments begin with enrichments.geo.ip_dst_addr if using Solr or enrichments:geo:ip_dst_addr if using Elastic Search indices. The geocoding includes the following attributes for each known ip address: city, country, latitude, longitude, postal code, DMA code, location ID, and location point. 30 | 31 | 32 | 33 | ## Adding field transformations 34 | Field transformations add new fields to the event that are derived from existing fields. Transformations prepare data for triaging, applying machine learning models or further analytics in the 35 | 1. Open the Metron Configuration UI. 36 | 2. Click the pencil icon to the right of the mysquid sensor to change the sensor configuration. The mysquid configuration opens. 37 | 38 | ![Edit mysquid sensor](images/edit_mysquid.png) 39 | 40 | 3. Metron configuration files are in json format. Simple transformations can be added in the Configuration UI but our example requires more complex transformations. Transformations are written in a domain specific language called [Stellar](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/stellar-quick-ref/content/introduction_to_stellar_language.html). 41 | 42 | Click the >> icon to the right of Raw Json in the Advanced section. 43 | 44 | 45 | 46 | ![config before transformations](images/transformation_config_before.png) 47 | 48 | 4. On the right side, in the Sensor Parser Config, replace the text: 49 | ``` 50 | "fieldTransformations": [], 51 | ``` 52 | with the following text: 53 | ``` 54 | "fieldTransformations": [ 55 | { 56 | "input": [], 57 | "output": [ 58 | "full_hostname", 59 | "domain_without_subdomains", 60 | "timestamp_solr" 61 | ], 62 | "transformation": "STELLAR", 63 | "config": { 64 | "full_hostname": "IF (IS_URL(url)) THEN URL_TO_HOST(url) ELSE GET_FIRST(SPLIT(url, ':'))", 65 | "domain_without_subdomains": "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)", 66 | "timestamp_solr": "DATE_FORMAT('yyyy-MM-dd\\'T\\'HH:mm:ss\\'Z\\'',timestamp)" 67 | } 68 | } 69 | ], 70 | ``` 71 | 72 | 73 | 74 | 5. Click Save under the raw json editors. 75 | 6. Click Save on the lower left of the Sensor configuration file. 76 | 7. Generate more squid log entries. 77 | 8. Open the Metron Alerts UI. The latest squid events will have new fields called full_hostname and domain_without_subdomains. The fields are derived from the url extracted from the original log entry. The timestamp_solr field is the event time stored in an ISO date format for building dashboards with the Banana UI. To duplicate the columns shown below click on the gear icon and check the fields shown below. Uncheck the fields that are not shown. 78 | 79 | 80 | 81 | 9. Congratulations! You have enriched events! The next step is to triage the events. 82 | 83 | ## Next Lab 84 | [Triaging squid events.](../04_TriagingSquid/README.md) 85 | 86 | ## References 87 | 88 | [Stellar Quick Reference](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/stellar-quick-ref/content/introduction_to_stellar_language.html) 89 | 90 | -------------------------------------------------------------------------------- /03_EnrichingSquid/images/alerts_with transformations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/alerts_with transformations.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/edit_advanced_raw_json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/edit_advanced_raw_json.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/edit_ip_dst_addr_enrich.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/edit_ip_dst_addr_enrich.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/edit_mysquid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/edit_mysquid.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/edit_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/edit_schema.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/geo_code_enrich_detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/geo_code_enrich_detail.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/ip_dst_addr_geo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/ip_dst_addr_geo.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/transformation_config_before.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/transformation_config_before.png -------------------------------------------------------------------------------- /03_EnrichingSquid/images/transformations_config_after.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03_EnrichingSquid/images/transformations_config_after.png -------------------------------------------------------------------------------- /03a_ThreatIntel/README.md: -------------------------------------------------------------------------------- 1 | # Applying Threat Intelligence to Squid logs 2 | ## Objectives 3 | After this lab you will be able to: 4 | 1. Load threat intelligence stored in a file. 5 | 2. Load threat intelligence through a streaming writer. 6 | 2. Apply threat intelligence to a field in an event. 7 | 8 | ## Overview 9 | 10 | Machine-Readable Threat Intelligence consists of a set of indicators of compromise in the current threat landscape. For example, known malicious ip and domain names. Metron automatically applies threat intelligence to fields of incoming events and sets the is_alert field to true if there is a match. 11 | 12 | ## Loading malicious ip list threat intelligence from a file 13 | 14 | 1. Ssh into mobius.local. Perform the following commands in the centos home directory. 15 | 16 | 2. Download the ip blocklist. 17 | 18 | ``` 19 | [centos@mobius ~]$ curl https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist | grep -v "^#" | grep -v "^$" > malicious_ips.csv 20 | % Total % Received % Xferd Average Speed Time Time Time Current 21 | Dload Upload Total Spent Left Speed 22 | 100 1999 100 1999 0 0 16618 0 --:--:-- --:--:-- --:--:-- 16658 23 | ``` 24 | 25 | 3. Verify that the malicious_ips.csv file contains a list of ip addresses. 26 | 27 | ``` 28 | [centos@mobius ~]$ head malicious_ips.csv 29 | 101.200.81.187 30 | 103.19.89.118 31 | 103.230.84.239 32 | 103.4.52.150 33 | 103.7.59.135 34 | 104.247.219.41 35 | 109.127.8.242 36 | 109.229.210.250 37 | 109.229.36.65 38 | 113.29.230.24 39 | ``` 40 | 41 | 4. Open a new file called malicious_ip_extractor_config.json in a text editor such as vi. 42 | 43 | ``` 44 | vi malicious_ip_extractor_config.json 45 | ``` 46 | 47 | 5. Add extractor config file json below to the file and save it. The extractor config specifies how Metron converts the columns of the CSV into name value pairs in HBase. The type is the type of threat intelligence, i.e. malicious_host vs malicious_domain. 48 | 49 | ``` 50 | { 51 | "config" : { 52 | "zk_quorum" : "localhost:2181" 53 | ,"columns" : { 54 | "ip" : 0 55 | } 56 | ,"indicator_column" : "ip" 57 | ,"type" : "malicious_ip" 58 | ,"separator" : "," 59 | } 60 | , "extractor" : "CSV" 61 | } 62 | ``` 63 | 64 | 6. Verify the extractor config file. 65 | 66 | ``` 67 | [centos@mobius ~]$ cat malicious_ip_extractor_config.json 68 | { 69 | "config" : { 70 | "zk_quorum" : "localhost:2181" 71 | ,"columns" : { 72 | "ip" : 0 73 | } 74 | ,"indicator_column" : "ip" 75 | ,"type" : "malicious_ip" 76 | ,"separator" : "," 77 | } 78 | , "extractor" : "CSV" 79 | } 80 | ``` 81 | 82 | 7. Use the flatfile_loader.sh to load the malicious ips into HBase. 83 | 84 | ``` 85 | /usr/hcp/current/metron/bin/flatfile_loader.sh -e malicious_ip_extractor_config.json -t threatintel -i malicious_ips.csv -c t 86 | ``` 87 | 88 | 8. Display a list of malicious_ips loaded into HBase. 89 | 90 | ``` 91 | [centos@mobius ~]$ head malicious_ips.csv 92 | 101.200.81.187 93 | 103.19.89.118 94 | 103.230.84.239 95 | 103.4.52.150 96 | 103.7.59.135 97 | 104.247.219.41 98 | 109.127.8.242 99 | 109.229.210.250 100 | 109.229.36.65 101 | 113.29.230.24 102 | ``` 103 | 104 | 9. Choose one of the ip addresses loaded into threat intelligence. Open the Stellar shell and enter the command, ENRICHMENT_GET command below. The first parameter ('malicious_ip') is the type of the indictor specified in the extractor config file. Enter a domain in the malicious_ips.csv file as the second parameter. The last two parameters are the table and column family specified as parameters to the flatfile_loader. The value returned from the Stellar command should be json with an ip attribute that is equal to the ip passed in. 105 | 106 | ``` 107 | [centos@mobius ~]$ /usr/hcp/current/metron/bin/stellar -z localhost:2181 108 | SLF4J: Class path contains multiple SLF4J bindings. 109 | SLF4J: Found binding in [jar:file:/usr/hcp/1.9.0.0-9/metron/lib/metron-profiler-repl-0.7.1.1.9.0.0-9.jar!/org/slf4j/impl/StaticLoggerBinder.class] 110 | SLF4J: Found binding in [jar:file:/usr/hdp/2.6.5.1100-53/hadoop/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] 111 | SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. 112 | SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] 113 | Stellar, Go! 114 | Functions are loading lazily in the background and will be unavailable until loaded fully. 115 | {es.clustername=metron, es.ip=node1:9300, es.date.format=yyyy.MM.dd.HH, parser.error.topic=indexing, update.hbase.table=metron_update, update.hbase.cf=t, es.client.settings={}, solr.zookeeper=mobius.local.localdomain:2181/solr, profiler.client.period.duration=1, profiler.client.period.duration.units=MINUTES, user.settings.hbase.table=user_settings, user.settings.hbase.cf=cf, bootstrap.servers=mobius.local.localdomain:6667, source.type.field=source.type, threat.triage.score.field=threat.triage.score, enrichment.writer.batchSize=15, enrichment.writer.batchTimeout=0, profiler.writer.batchSize=15, profiler.writer.batchTimeout=0, geo.hdfs.file=/apps/metron/geo/1553998428743/GeoLite2-City.tar.gz, asn.hdfs.file=/apps/metron/asn/1553998428743/GeoLite2-ASN.tar.gz} 116 | [Stellar]>>> ENRICHMENT_GET( 'malicious_ip', '101.200.81.187', 'threatintel', 't') 117 | {ip=101.200.81.187} 118 | [Stellar]>>> quit 119 | ``` 120 | 121 | ## Loading malicious domain list threat intelligence from a stream 122 | 123 | 1. Ssh into mobius.local. Download a malicious domain list and store it in a CSV file. 124 | 125 | ``` 126 | [centos@mobius ~]$ curl https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist | grep -v "^#" | grep -v "^$" | grep -v "^https" | awk '{print $1",abuse.ch"}' > malicious_domains.csv 127 | % Total % Received % Xferd Average Speed Time Time Time Current 128 | Dload Upload Total Spent Left Speed 129 | 100 7295 100 7295 0 0 63476 0 --:--:-- --:--:-- --:--:-- 63991 130 | ``` 131 | 132 | 2. Verify that the malicious domain list downloaded properly. 133 | 134 | ``` 135 | [centos@mobius ~]$ head malicious_domains.csv 136 | 039b1ee.netsolhost.com,abuse.ch 137 | 03a6b7a.netsolhost.com,abuse.ch 138 | 03a6f57.netsolhost.com,abuse.ch 139 | 03bbec4.netsolhost.com,abuse.ch 140 | 0if1nl6.org,abuse.ch 141 | 0x.x.gg,abuse.ch 142 | 54g35546-5g6hbggffhb.tk,abuse.ch 143 | 76tguy6hh6tgftrt7tg.su,abuse.ch 144 | afobal.cl,abuse.ch 145 | ahmedashid.com,abuse.ch 146 | ``` 147 | 148 | 3. In the Metron Config UI, create a new sensor by pressing the + icon on the lower right hand corner. 149 | 4. Set the Name field to malicious_domains. Set the Kafka Topic field to malicious_domains. Select Parser Type CSV. 150 | 151 | 5. Click Save. 152 | 153 | 6. Click on the malicious_domains sensor pencil icon to open the sensor config. 154 | 7. Go to the Advanced section and click the >>> on Raw Json to expand the Json configs. 155 | 156 | 8. Set the parser config to the following JSON: 157 | 158 | ``` 159 | { 160 | "parserClassName": "org.apache.metron.parsers.csv.CSVParser", 161 | "filterClassName": null, 162 | "sensorTopic": "malicious_domains", 163 | "outputTopic": null, 164 | "errorTopic": null, 165 | "writerClassName": "org.apache.metron.enrichment.writer.SimpleHbaseEnrichmentWriter", 166 | "errorWriterClassName": null, 167 | "readMetadata": false, 168 | "mergeMetadata": false, 169 | "numWorkers": null, 170 | "numAckers": null, 171 | "spoutParallelism": 1, 172 | "spoutNumTasks": 1, 173 | "parserParallelism": 1, 174 | "parserNumTasks": 1, 175 | "errorWriterParallelism": 1, 176 | "errorWriterNumTasks": 1, 177 | "spoutConfig": {}, 178 | "securityProtocol": null, 179 | "stormConfig": {}, 180 | "parserConfig": { 181 | "shew.table": "threatintel", 182 | "shew.cf": "t", 183 | "shew.keyColumns": "domain", 184 | "shew.enrichmentType": "malicious_domain", 185 | "columns": { 186 | "domain": 0, 187 | "source": 1 188 | } 189 | }, 190 | "fieldTransformations": [], 191 | "cacheConfig": {}, 192 | "rawMessageStrategy": "DEFAULT", 193 | "rawMessageStrategyConfig": {} 194 | } 195 | ``` 196 | 197 | Note the writerClassName attribute in the malicious_domains sensor and HBase information in the parserConfig. 198 | 199 | 11. Click Save below the Json configs. Click Save on the Sensor config. 200 | 201 | 10. Click the stop button on all sensor EXCEPT mysquid. This ensures the cluster will have enough resources to run the malicious_domains sensor. Click the play button on the malicious_domain sensor to start the sensor. 202 | 203 | 11. In the mobius.local console send some threat intelligence to the malicious_domain sensor. 204 | 205 | ``` 206 | [centos@mobius ~]$ head malicious_domains.csv | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list localhost:6667 --topic malicious_domains 207 | >>>>>>>>>>> 208 | ``` 209 | 210 | 12. Verify that the threat intelligence loaded properly. 211 | 212 | ``` 213 | [centos@mobius ~]$ /usr/hcp/current/metron/bin/stellar -z localhost:2181 214 | SLF4J: Class path contains multiple SLF4J bindings. 215 | SLF4J: Found binding in [jar:file:/usr/hcp/1.9.0.0-9/metron/lib/metron-profiler-repl-0.7.1.1.9.0.0-9.jar!/org/slf4j/impl/StaticLoggerBinder.class] 216 | SLF4J: Found binding in [jar:file:/usr/hdp/2.6.5.1100-53/hadoop/lib/slf4j-log4j12-1.7.10.jar!/org/slf4j/impl/StaticLoggerBinder.class] 217 | SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation. 218 | SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] 219 | Stellar, Go! 220 | Functions are loading lazily in the background and will be unavailable until loaded fully. 221 | {es.clustername=metron, es.ip=node1:9300, es.date.format=yyyy.MM.dd.HH, parser.error.topic=indexing, update.hbase.table=metron_update, update.hbase.cf=t, es.client.settings={}, solr.zookeeper=mobius.local.localdomain:2181/solr, profiler.client.period.duration=1, profiler.client.period.duration.units=MINUTES, user.settings.hbase.table=user_settings, user.settings.hbase.cf=cf, bootstrap.servers=mobius.local.localdomain:6667, source.type.field=source.type, threat.triage.score.field=threat.triage.score, enrichment.writer.batchSize=15, enrichment.writer.batchTimeout=0, profiler.writer.batchSize=15, profiler.writer.batchTimeout=0, geo.hdfs.file=/apps/metron/geo/1553998428743/GeoLite2-City.tar.gz, asn.hdfs.file=/apps/metron/asn/1553998428743/GeoLite2-ASN.tar.gz} 222 | [Stellar]>>> ENRICHMENT_GET( 'malicious_domain', '039b1ee.netsolhost.com', 'threatintel', 't') 223 | {original_string=039b1ee.netsolhost.com,abuse.ch, guid=a7542547-13c3-474a-8205-f497ea82a8e4, source=abuse.ch, timestamp=1555497608044, source.type=malicious_domains} 224 | [Stellar]>>> quit 225 | ``` 226 | 227 | ## Applying malicious_domain and malicious_ip threat intelligence to squid events. 228 | 229 | 1. Open the Metron config ui. 230 | 231 | 2. Open the mysquid sensor and navigate to the Raw Json. 232 | 233 | 3. In the sensor enrichment config section replace the threat intelligence fieldMap and fieldToTypeMap with the text below: 234 | 235 | ``` 236 | "fieldMap": { 237 | "hbaseThreatIntel": [ 238 | "full_hostname", 239 | "ip_dst_addr" 240 | ] 241 | }, 242 | "fieldToTypeMap": { 243 | "full_hostname": [ 244 | "malicious_domain" 245 | ], 246 | "ip_dst_addr": [ 247 | "malicious_ip" 248 | ] 249 | }, 250 | ``` 251 | 252 | 253 | 254 | 255 | 256 | 4. Generate some traffic that triggers the malicious_domain threat intelligence. 257 | 258 | ``` 259 | today=`date +'%s.000'` 260 | head malicious_domains.csv | awk -v todays_date=$today 'BEGIN {FS =","} {print todays_date" 66025 208.54.147.129 TCP_TUNNEL/200 7699 CONNECT " $1 ":443 - HIER_DIRECT/72.21.206.140 -"}' | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --topic mysquid --broker-list localhost:6667 261 | ``` 262 | 263 | 5. Open the Metron Alerts UI. Enter is_alert:true in the search. Click the search button. There will be some alerts. 264 | 265 | 266 | 267 | 6. Click on the alert. Scroll down to the bottom of the alert details. The threat intel causing the alert is indicated by the threatintels.hbaseThreatIntel.full_hostname.malicious_domain field. 268 | 269 | 270 | 271 | 7. Generate some traffic that triggers the malicious_ip threat intelligence. 272 | 273 | ``` 274 | today=`date +'%s.000'` 275 | head malicious_ips.csv | awk -v todays_date=$today 'BEGIN {FS =","} {print todays_date" 66025 208.54.147.129 TCP_TUNNEL/200 7699 CONNECT my.example.com:443 - HIER_DIRECT/" $1 " -"}' | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --topic mysquid --broker-list localhost:6667 276 | ``` 277 | 278 | 5. Click the search button with is_alert:true in the search field to refresh the events. The new malicious ip alerts will appear. 279 | 280 | 281 | 282 | 6. Click on the alert. Scroll down to the bottom of the alert details. The threat intel causing the alert is indicated by the threatintels.hbaseThreatIntel.ip_dst_addr.malicious_ip field. 283 | 284 | 285 | 286 | 287 | -------------------------------------------------------------------------------- /03a_ThreatIntel/images/malicious_domain_alert_detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03a_ThreatIntel/images/malicious_domain_alert_detail.png -------------------------------------------------------------------------------- /03a_ThreatIntel/images/malicious_domain_alerts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03a_ThreatIntel/images/malicious_domain_alerts.png -------------------------------------------------------------------------------- /03a_ThreatIntel/images/malicious_ip_alert_detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03a_ThreatIntel/images/malicious_ip_alert_detail.png -------------------------------------------------------------------------------- /03a_ThreatIntel/images/malicious_ip_alerts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03a_ThreatIntel/images/malicious_ip_alerts.png -------------------------------------------------------------------------------- /03a_ThreatIntel/images/threat_intel_config_after.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03a_ThreatIntel/images/threat_intel_config_after.png -------------------------------------------------------------------------------- /03a_ThreatIntel/images/threat_intel_config_before.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/03a_ThreatIntel/images/threat_intel_config_before.png -------------------------------------------------------------------------------- /04_TriagingSquid/README.md: -------------------------------------------------------------------------------- 1 | # Triaging Squid events 2 | ## Objectives 3 | After this lab you will be able to: 4 | 1. Determine which squid events are alerts. 5 | 2. Assign a score to squid alerts using [typosquatting](https://metron.apache.org/current-book/use-cases/typosquat_detection/index.html) detection. 6 | 3. Find the highest score alerts using the Metron Alerts UI. 7 | ## Triaging Background 8 | Security event triage rules determine which events require further follow up and which events can be archived without further investigation. Metron will process many events every day so effective triage helps analysts focus on the most important events. The two components of triage are: 9 | 1. Determine if the event is an alert. 10 | 2. If the event is an alert, assign a score. If the event is not an alert, it is not scored. 11 | For this example, we use a very simple triage rule to detect [typosquatting](https://en.wikipedia.org/wiki/Typosquatting), installing malicious web content using common domain name misspellings. 12 | ## Triaging Squid using the typosquatting algorithm 13 | 1. Open the Metron Configuration UI in the browser. 14 | 2. Click on the pencil icon next to the mysquid sensor. The sensor configuration form opens. 15 | 2. In the Advanced section, click on the >> next to the Raw json field. 16 | 17 | 18 | 19 | 3. Replace the Json in the Sensor Enrichment Config section with the json below: 20 | 21 | 22 | ``` 23 | { 24 | "enrichment": { 25 | "fieldMap": { 26 | "geo": [ 27 | "ip_dst_addr" 28 | ], 29 | "stellar": { 30 | "config": [ 31 | "domain_without_tld := DOMAIN_REMOVE_TLD(domain_without_subdomains)", 32 | "is_potential_typosquat := BLOOM_EXISTS(OBJECT_GET('/tmp/reference/alexa10k_filter.ser'), domain_without_tld)", 33 | "domain_without_tld := null" 34 | ] 35 | } 36 | }, 37 | "fieldToTypeMap": {}, 38 | "config": {} 39 | }, 40 | "threatIntel": { 41 | "fieldMap": { 42 | "stellar": { 43 | "config": [ 44 | "is_alert := (exists(is_alert) && is_alert) || is_potential_typosquat" 45 | ] 46 | } 47 | }, 48 | "fieldToTypeMap": {}, 49 | "config": {}, 50 | "triageConfig": { 51 | "riskLevelRules": [ 52 | { 53 | "name": "Alexa 10k Typosquat Bloom", 54 | "comment": "Inspect a bloom filter with potentially typosquatted domains from the top Alexa 10k", 55 | "rule": "is_potential_typosquat != null && is_potential_typosquat", 56 | "score": 50, 57 | "reason": "FORMAT('%s is a potential typosquatted domain from the top 10k domains from alexa', domain_without_subdomains)" 58 | } 59 | ], 60 | "aggregator": "MAX", 61 | "aggregationConfig": {} 62 | } 63 | }, 64 | "configuration": {} 65 | } 66 | ``` 67 | 68 | 69 | 70 | 4. Click the Save button below the json. 71 | 5. Click the Save button at the bottom of the mysquid sensor configuration. 72 | 6. Enter cnn.com or npr.com in the browser connected to the Metron proxy. 73 | 7. Open the Metron Alerts UI. 74 | 8. In the Score column you will see events with non-zero scores and the is_alert field set to true. 75 | 76 | 77 | 78 | 9. The event fields shown in the columns of the UI are configurable. If you want to view the columns as they appear in the screen shot, click the gear icon to the left of the Actions button. Check the Score, id, timestamp, source:type, domain_without_subdomains and is_alert fields. Remove the check from all other columns. Click Save. By default, the columns after the score, id and timestamp are displayed from left to right in alphabetical order by field name. You can change the order by clicking on the up or down button to move the columns displayed up or down. After changing the order, click Save. 79 | 80 | 81 | 82 | 9. Click on the Score header to sort the events ascending by Score. Click again to sort descending by Score. A downward arrow appears next to the Score header when sorted descending by Score. 83 | 84 | 85 | 86 | 10. Click between the columns of one of the Scored alerts to view the alert details. The fields beginning with threat:triage:rules show the results of all the triage rules. The threat:triage:score field is the aggregated score of the event. If there were more than one triage rule, this field would contain the score combining the results from all the rules. The is_alert field is set only if the triage rules indicate the event is an alert. 87 | 88 | 89 | 90 | 11. To see all the alerts for a particular domain, click on the domain name. The Alerts UI displays only the alerts with the selected domain name. 91 | 92 | 93 | 94 | 12. To remove the filter, hover over the search criteria that you want to remove and click on the x. To view all events, click the x on the Searches field. 95 | 96 | 97 | 98 | ## Improving Scoring with a Domain Whitelist 99 | Once a potential typosquatted domain is identified, investigated and found to be legitimate, stop future alerts by using a domain whitelist enrichment. 100 | 1. Open the Metron Configuration UI. 101 | 2. Click on the pencil icon next to the mysquid sensor. The sensor configuration form opens. 102 | 3. In the Advanced section, click on the >> next to the Raw json field. 103 | 4. Replace the is_potential_typosquat field with the value below: 104 | 105 | ``` 106 | "is_potential_typosquat := not (ENRICHMENT_EXISTS('domain_whitelist', domain_without_tld, 'enrichment', 't')) && BLOOM_EXISTS(OBJECT_GET('/tmp/reference/alexa10k_filter.ser'), domain_without_tld)", 107 | ``` 108 | 109 | 110 | 5. Click Save under the Raw Json section. 111 | 6. Click Save on the sensor configuration. 112 | 7. Open cnn.com and npr.org in browser connected to the Metron proxy. 113 | 8. Open the Metron Alerts UI. Click on the timestamp column header until the events are sorted descending by timestamp. Proxy events to cnn.com and npr.org are no longer alerts. 114 | 115 | 116 | 117 | ## Next Lab 118 | [Profiler basics.](../05_ProfilerBasics/README.md) 119 | 120 | ## References 121 | [Metron typosquat detection use case](https://metron.apache.org/current-book/use-cases/typosquat_detection/index.html) 122 | -------------------------------------------------------------------------------- /04_TriagingSquid/images/alerts_sort_desc_by_score.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/alerts_sort_desc_by_score.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/configure_columns.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/configure_columns.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/domain_whitelist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/domain_whitelist.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/enrichments_after.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/enrichments_after.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/enrichments_before.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/enrichments_before.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/filter_alerts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/filter_alerts.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/metron_no_cnn_alert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/metron_no_cnn_alert.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/remove_filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/remove_filter.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/scored_alerts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/scored_alerts.png -------------------------------------------------------------------------------- /04_TriagingSquid/images/triage_fields.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/04_TriagingSquid/images/triage_fields.png -------------------------------------------------------------------------------- /04_TriagingSquid/scripts/domain_whitelist.csv: -------------------------------------------------------------------------------- 1 | cnn,known domains 2 | npr,known domains 3 | -------------------------------------------------------------------------------- /04_TriagingSquid/scripts/domain_whitelist_extractor_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "config" : { 3 | "zk_quorum" : "localhost:2181" 4 | ,"columns" : { 5 | "domain" : 0 6 | ,"source" : 1 7 | } 8 | ,"indicator_column" : "domain" 9 | ,"type" : "domain_whitelist" 10 | ,"separator" : "," 11 | } 12 | , "extractor" : "CSV" 13 | } 14 | -------------------------------------------------------------------------------- /04_TriagingSquid/scripts/generate_typosquats.sh: -------------------------------------------------------------------------------- 1 | source /etc/default/metron 2 | 3 | today=`date +'%s.000'` 4 | cat typosquat_domains.txt | awk -v todays_date=$today 'BEGIN {FS =","} {print todays_date" 66025 208.54.147.129 TCP_TUNNEL/200 7699 CONNECT " $1 ":443 - HIER_DIRECT/72.21.206.140 -"}' | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --topic squid --broker-list $BROKERLIST 5 | -------------------------------------------------------------------------------- /04_TriagingSquid/scripts/import_domain_whitelist.sh: -------------------------------------------------------------------------------- 1 | /usr/hcp/current/metron/bin/flatfile_loader.sh -e domain_whitelist_extractor_config.json -t enrichment -i domain_whitelist.csv -c t 2 | -------------------------------------------------------------------------------- /04_TriagingSquid/scripts/typosquat_domains.txt: -------------------------------------------------------------------------------- 1 | gooogle.com 2 | googleq.com 3 | googler.com 4 | googles.com 5 | googlet.com 6 | googleu.com 7 | -------------------------------------------------------------------------------- /05_ProfilerBasics/README.md: -------------------------------------------------------------------------------- 1 | # Creating Profiles 2 | 3 | This section will describe how to create your very first “Hello, World” profile. It will also outline a useful workflow for creating, testing, and deploying profiles. 4 | 5 | Creating and refining profiles is an iterative process. Iterating against a live stream of data is slow, difficult and error prone. The Profile Debugger was created to provide a controlled and isolated execution environment to create, refine and troubleshoot profiles. 6 | 7 | Metron leverages the Stellar DSL for enhanced productivity. [Stellar Reference Guide](https://metron.apache.org/current-book/metron-stellar/stellar-common/index.html) 8 | 9 | ## Launch the Stellar Shell. We will leverage the Profiler Debugger from within the Stellar Shell. 10 | 11 | ``` 12 | [root@node1 ~]# cd /usr/hcp/current/metron/ 13 | [root@node1 ~]# ./bin/stellar 14 | Stellar, Go! 15 | [Stellar]>>> %functions PROFILER 16 | PROFILER_APPLY, PROFILER_FLUSH, PROFILER_INIT 17 | ``` 18 | 19 | Create a simple hello-world profile that will count the number of messages for each ip_src_addr. The SHELL_EDIT function will open an editor in which you can copy/paste the following Profiler configuration. 20 | 21 | ``` 22 | [Stellar]>>> conf := SHELL_EDIT() 23 | [Stellar]>>> conf 24 | { 25 | "profiles": [ 26 | { 27 | "profile": "hello-world", 28 | "onlyif": "exists(ip_src_addr)", 29 | "foreach": "ip_src_addr", 30 | "init": { "count": "0" }, 31 | "update": { "count": "count + 1" }, 32 | "result": "count" 33 | } 34 | ] 35 | } 36 | ``` 37 | 38 | ## Create a Profile execution environment; the Profile Debugger. 39 | 40 | The Profiler will output the number of profiles that have been defined, the number of messages that have been applied and the number of routes that have been followed. 41 | 42 | A route is defined when a message is applied to a specific profile. 43 | 44 | If a message is not needed by any profile, then there are no routes. 45 | If a message is needed by one profile, then one route has been followed. 46 | If a message is needed by two profiles, then two routes have been followed. 47 | 48 | ``` 49 | [Stellar]>>> profiler := PROFILER_INIT(conf) 50 | [Stellar]>>> profiler 51 | Profiler{1 profile(s), 0 messages(s), 0 route(s)} 52 | ``` 53 | 54 | Create a message that mimics the telemetry that your profile will consume. 55 | 56 | This message can be as simple or complex as you like. For the hello-world profile, all you need is a message containing an ip_src_addr field. 57 | 58 | ``` 59 | [Stellar]>>> msg := SHELL_EDIT() 60 | [Stellar]>>> msg 61 | { 62 | "ip_src_addr": "10.0.0.1" 63 | } 64 | ``` 65 | 66 | Apply the message to your Profiler, as many times as you like. 67 | 68 | ``` 69 | [Stellar]>>> PROFILER_APPLY(msg, profiler) 70 | Profiler{1 profile(s), 1 messages(s), 1 route(s)} 71 | [Stellar]>>> PROFILER_APPLY(msg, profiler) 72 | Profiler{1 profile(s), 2 messages(s), 2 route(s)} 73 | ``` 74 | 75 | # Flush the Profiler. 76 | 77 | A flush is what occurs at the end of each profiler interval, for example a 15 minute period, in the Profiler. The result is a list of Profile Measurements. Each measurement is a map containing detailed information about the profile data that has been generated. The value field is what is written to HBase when running this profile in the Profiler topology. 78 | 79 | There will always be one measurement for each [profile, entity] pair. This profile simply counts the number of messages by IP source address. Notice that the value is ‘3’ for the entity ‘10.0.0.1’ as we applied 3 messages with an ‘ip_src_addr’ of ’10.0.0.1’. 80 | 81 | ``` 82 | [Stellar]>>> values := PROFILER_FLUSH(profiler) 83 | [Stellar]>>> values 84 | [{period={duration=900000, period=1669628, start=1502665200000, end=1502666100000}, 85 | profile=hello-world, groups=[], value=3, entity=10.0.0.1}] 86 | ``` 87 | 88 | # Apply real, live telemetry to your profile. 89 | 90 | Once you are happy with your profile against a controlled data set, it can be useful to introduce more complex, live data. This example extracts 10 messages of live, enriched telemetry to test your profile(s). 91 | 92 | ``` 93 | [Stellar]>>> %define bootstrap.servers := "localhost:6667" 94 | [Stellar]>>> msgs := KAFKA_GET("indexing", 10) 95 | [Stellar]>>> LENGTH(msgs) 96 | 10 97 | ``` 98 | 99 | If you don't get any messages, fire off a few squid client commands: 100 | 101 | ``` 102 | curl -I --proxy ec2-54-215-245-197.us-west-1.compute.amazonaws.com:3128 https://cnn.com 103 | ``` 104 | 105 | Apply those 10 messages to your profile(s). 106 | 107 | ``` 108 | [Stellar]>>> PROFILER_APPLY(msgs, profiler) 109 | Profiler{1 profile(s), 10 messages(s), 10 route(s)} 110 | ``` 111 | 112 | # Access deployed profiles 113 | 114 | If you have earlier deployed profiles, you can get their values by `PROFILE_GET`. For example: 115 | 116 | ``` 117 | PROFILE_GET( "locations_by_user", "user1", PROFILE_FIXED(120, "DAYS")) 118 | ``` 119 | 120 | # Deploy you first profile 121 | 122 | Load the Metron environment variables: `source /etc/default/metron` 123 | 124 | Edit `/usr/hcp/current/metron/config/zookeeper/profiler.json` using your favorite editor (e.g. vi or emacs) 125 | 126 | Add the `hello-world` profile and save the file 127 | 128 | ``` 129 | { 130 | "profile": "hello-world", 131 | "onlyif": "exists(ip_src_addr)", 132 | "foreach": "ip_src_addr", 133 | "init": { "count": "0" }, 134 | "update": { "count": "count + 1" }, 135 | "result": "count" 136 | } 137 | ``` 138 | 139 | Push the updated profiler configuration 140 | 141 | ``` 142 | $ cd $METRON_HOME 143 | $ bin/zk_load_configs.sh -m PUSH -i config/zookeeper/ -z localhost:2181 144 | ``` 145 | ## Next Lab 146 | [User and Entity Behavior Analytics(UEBA) with User Authentication Events](../06_ProfilingAuthLogs/README.md) 147 | 148 | # References 149 | 150 | [Profile examples](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/analytics/content/profile_examples.html) 151 | 152 | [Accessing profiles](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/analytics/content/accessing_profiles.html) 153 | 154 | [Auth profile presentation by Casey Stella](https://www.slideshare.net/Hadoop_Summit/just-the-sketch-advanced-streaming-analytics-in-apache-metron) 155 | 156 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/README.md: -------------------------------------------------------------------------------- 1 | # User and Entity Behavior Analytics(UEBA) with User Authentication Events 2 | ## Objectives 3 | After this lab you will be able to: 4 | 1. Describe the components of a Metron profile. 5 | 2. Explain how to detect anomalous logins and triage events using Metron profiles. 6 | ## Background 7 | The Metron profiler collects time series measurements using high throughput algorithms to capture the behavior of an entity. Metron profiles are flexible and use an extension language to define the measurement to capture. Using statistics we can compare the measurements for an entity to the measurements for its peers. If the behavior of the entity is significantly different from its peers, we can trigger an alert. 8 | 9 | For example, the chart below shows the user C553's profile measurements. The light blue line is the upper bound of the typical behavior. The dark blue line is the user's behavior. The orange line is the score of the events. User C553 demonstrates typical behavior because the user's behavior (dark blue) is below the maximum upper bound (light blue). 10 | 11 | 12 | 13 | User C1114 demonstrates intermittent anomalous behavior. When the user's behavior (dark blue) intermittently goes above the maximum typical behavior (light blue) a scored alert is reported. The orange spikes show the scored alerts. 14 | 15 | 16 | 17 | User U66 demonstrates consistent anomalous behavior because the user's behavior (dark blue) is consistently above the maximum typical behavior (light blue). Almost every event is above typical behavior. 18 | 19 | 20 | 21 | In this lab we walk through the auth profiles, the measurements captured by those profiles, and the alerts triggered by the profile triage rules. 22 | 23 | ## Browse the LANL Auth data set in Metron Alerts UI 24 | 25 | 1. Let's take a look at the source authentication logs. The log data is real authentication data containing breaches captured by the from Los Alamos National Labs during test activities : 26 | 27 | http://csr.lanl.gov/data/cyber1/ 28 | 29 | 30 | 31 | 2. Because it takes some time to ingest the large event data set, your Metron test instance is already populated with the triaged LANL auth events. Open the metron alerts UI: 32 | 33 | http://mobius.local:4201 34 | 35 | Enter source.type:auth in the Searches field if using Solr or source:type:auth if using Elastic Search. Click the magnifying glass icon. The alerts UI shows only auth alerts. 36 | 37 | 38 | 39 | 3. Click on the gear icon. Uncheck all fields EXCEPT the following: 40 | Score, Timestamp, source.type, distinct auth_attempts, distinct_auth_attempts_median, distinct_auth_attempts_stddev, and user. 41 | 42 | 43 | 44 | 45 | 4. Click Save. 46 | 47 | 5. The Alerts UI now shows the measurements derived from the profiles that determine if a user is logging into more distinct hosts than other peer users. 48 | 49 | 50 | 51 | 6. Click on the arrows to the right of the score to sort ascending by score. Click again to sort descending by score. You should see red alerts scored 100 at the top. Click in the whitespace between the columns to bring up the alert details. 52 | 53 | 54 | 55 | 7. Scroll to the triage section and read the description of the score. For example: "The distinct number of machines that user U66 attempted to login to (13) is more than 5 standard deviations (1.39) from the median (1.00)" 56 | 57 | 58 | 59 | 8. Before ingesting the auth data, the following JSON profile config files were installed using the [zk_config_load_configs.sh PUSH command](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/runbook/content/creating_a_profile.html). For each profile, the configuration specifies: 60 | * profile: unique identifier of the profile 61 | * foreach: peer group of the profile. For example, one per entity or global or a peer group such as users in a department. 62 | * onlyif: if the boolean expression evaluates to true, apply the message to the profile and record a measurement. 63 | * init: executed once per time window. 64 | * update: executed each time a message is applied to the profile 65 | * result: actions at the end of a time window. Profile defines the value sent to the profile. Triage defines the data sent as a profile message to the triaging pipeline. 66 | 67 | For the auth example, we define three profiles: 68 | 69 | * distinct_auth_attempts_by_user: Counts the number of distinct hosts the user logs into in the time period using the [HyperLogLogPlus](https://metron.apache.org/current-book/metron-analytics/metron-statistics/HLLP.html) algorithm. 70 | 71 | ``` 72 | { 73 | "profile": "distinct_auth_attempts_by_user", 74 | "foreach": "user", 75 | "onlyif": "source.type == 'auth' && auth_orientation != null && auth_orientation == 'LogOn' && user != null && LENGTH(user) > 0 && ip_dst_host != null && ip_dst_host != '?'", 76 | "init": { 77 | "total": "HLLP_INIT(5,6)" 78 | }, 79 | "update": { 80 | "total": "HLLP_ADD(total, ip_dst_host)" 81 | }, 82 | "result": { 83 | "profile": "total", 84 | "triage": { 85 | "total_count": "HLLP_CARDINALITY(total)" 86 | } 87 | } 88 | } 89 | ``` 90 | 91 | * distinct_auth_attempts_by_user_distribution: Combines the total_count measurements captured by the distinct_auth_attempts_by_user measurements into a global distribution of all users. 92 | ``` 93 | { 94 | "profile": "distinct_auth_attempts_by_user_distribution", 95 | "foreach": "'global'", 96 | "onlyif": "source.type == 'profiler' && profile == 'distinct_auth_attempts_by_user'", 97 | "init": { 98 | "s": "STATS_INIT()" 99 | }, 100 | "update": { 101 | "s": "STATS_ADD(s, total_count)" 102 | }, 103 | "result": "s" 104 | } 105 | ``` 106 | 107 | 9. The enrichment rules retrieve and merge the HLLP structures of the distinct_auth_attempts_by_user profile together to determine the distinct_auth_attempts for the user logging on. The enrichment rules retreive and merge the statistics collected by the distinct_auth_attempts_by_user_distribution profile to determine the distinct_auth_attempts_median and distinct_auth_attempts_stddev. 108 | 109 | 110 | 111 | 10. The triaging rules set the is_alert value to true if the user is more than five standard deviations away from the median. If the is_alert field is set to true, the event is scored using the triage rules. The score is lower for an administrative user than a regular user. The reason shows the human readable explanation for the score. 112 | 113 | 114 | 115 | 11. You can define similar patterns to detect other types of anomalies with user logins or other entities. 116 | 117 | # Next Labs 118 | [Applying DGA Detection Machine Learning Predictions to Squid logs](../08_DGADetectionModelAsService/README.md) 119 | 120 | [Exploring Event History - Dashboards and Run Books for Analysis, Threat Hunting and Investigations](../07_ExploringEventHistory/README.md) 121 | 122 | # References 123 | [Metron Statistics Stellar Functions](https://metron.apache.org/current-book/metron-analytics/metron-statistics/index.html) 124 | 125 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/create_auth_solr_collection.sh: -------------------------------------------------------------------------------- 1 | source /etc/default/metron 2 | 3 | # install the auth schema 4 | sudo cp -r schema/auth $METRON_HOME/config/schema/ 5 | ../../build_single/typosquat/create_solr_collection.sh auth 6 | 7 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/install_auth.sh: -------------------------------------------------------------------------------- 1 | ./create_auth_solr_collection.sh 2 | ./load_auth_config.sh 3 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/load_auth_config.sh: -------------------------------------------------------------------------------- 1 | source /etc/default/metron 2 | 3 | # install the auth configs 4 | sudo $METRON_HOME/bin/zk_load_configs.sh -m PULL -z $ZOOKEEPER -o /tmp/zkconfig -f 5 | sudo cp zookeeper/enrichments/auth.json /tmp/zkconfig/enrichments/ 6 | sudo cp zookeeper/parsers/auth.json /tmp/zkconfig/parsers/ 7 | sudo cp zookeeper/indexing/auth.json /tmp/zkconfig/indexing/ 8 | 9 | PROFILER_CONFIG_FILE=/tmp/zkconfig/profiler.json 10 | PROFILER_PATCH_NEEDED=1 11 | if [ ! -f $PROFILER_CONFIG_FILE ]; then 12 | sudo cp zookeeper/profiler.json /tmp/zkconfig/profiler.json 13 | echo "no profiler.json found. adding auth profilers" 14 | PROFILER_PATCH_NEEDED=0 15 | fi 16 | 17 | sudo $METRON_HOME/bin/zk_load_configs.sh -m PUSH -z $ZOOKEEPER -i /tmp/zkconfig 18 | 19 | if ((PROFILER_PATCH_NEEDED)); then 20 | echo "profiler.json found. patching profilers" 21 | ./patch_profiler_config.sh 22 | fi 23 | 24 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/patch_profiler_config.sh: -------------------------------------------------------------------------------- 1 | wget -P /tmp/zkprof https://github.com/simonellistonball/metron-field-demos/raw/master/profiler_patch.py 2 | sudo -- bash -c 'cat zookeeper/profiler.json | python /tmp/zkprof/profiler_patch.py > /tmp/zkconfig/profiler_patch.json' 3 | sudo $METRON_HOME/bin/zk_load_configs.sh -z $ZOOKEEPER -m PATCH -c PROFILER -pf /tmp/zkconfig/profiler_patch.json 4 | 5 | 6 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/schema/auth/schema.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 27 | 28 | 29 | 30 | 33 | 34 | 35 | 36 | 37 | 38 | guid 39 | 40 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/zookeeper/enrichments/auth.json: -------------------------------------------------------------------------------- 1 | {"enrichment":{"fieldMap":{"stellar":{"config":{"distinct_auth_attempts":["window := PROFILE_WINDOW('from 6 minutes ago')","profile := PROFILE_GET('distinct_auth_attempts_by_user', user, window)","distinct_auth_attempts := STATS_MEAN( REDUCE(profile, (s, x) -> STATS_ADD(s, HLLP_CARDINALITY(x)) ,STATS_INIT()))","profile := null","window := null"],"distinct_auth_attempts_stats":["window := PROFILE_WINDOW('from 6 minutes ago')","profile := PROFILE_GET('distinct_auth_attempts_by_user_distribution', 'global', window)","stats := STATS_MERGE(profile)","distinct_auth_attempts_median := STATS_PERCENTILE(stats, 0.5)","distinct_auth_attempts_stddev := STATS_SD(stats)","profile := null","stats := null","window := null"],"num_alerts_by_user":["window := PROFILE_WINDOW('from 10 minutes ago')","profile := PROFILE_GET('num_alerts_by_user', user, window)","num_alerts_previous := REDUCE(profile, (s, x) -> s + x, 0)","profile := null","window := null"]}}},"fieldToTypeMap":{},"config":{}},"threatIntel":{"fieldMap":{"stellar":{"config":["distinct_auth_attempts_alert := distinct_auth_attempts_stddev > 0 && distinct_auth_attempts > 0 && ABS(distinct_auth_attempts - distinct_auth_attempts_median) > 5*distinct_auth_attempts_stddev","is_regular_user := is_system_user == null || (not(is_system_user) && user != 'ANONYMOUS LOGON')","is_alert := num_alerts_previous == 0 && auth_orientation == 'LogOn' && distinct_auth_attempts_alert"]}},"fieldToTypeMap":{},"config":{},"triageConfig":{"riskLevelRules":[{"name":"Too many distinct auth attempts: non-regular user","comment":null,"rule":"is_regular_user != null && not(is_regular_user)","score":10,"reason":"FORMAT('The distinct number of machines that user %s attempted to login to (%d) is more than 5 standard deviations (%.2f) from the median (%.2f)', user, TO_INTEGER(distinct_auth_attempts), distinct_auth_attempts_stddev, distinct_auth_attempts_median)"},{"name":"Too many distinct auth attempts: regular user","comment":null,"rule":"is_regular_user != null && is_regular_user","score":100,"reason":"FORMAT('The distinct number of machines that user %s attempted to login to (%d) is more than 5 standard deviations (%.2f) from the median (%.2f)', user, TO_INTEGER(distinct_auth_attempts), distinct_auth_attempts_stddev, distinct_auth_attempts_median)"}],"aggregator":"MAX","aggregationConfig":{}}},"configuration":{}} -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/zookeeper/indexing/auth.json: -------------------------------------------------------------------------------- 1 | {"hdfs":{"batchSize":300,"enabled":true,"index":"auth"},"elasticsearch":{"batchSize":300,"enabled":true,"index":"auth"},"solr":{"batchSize":300,"enabled":true,"index":"auth"}} 2 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/zookeeper/indexing/profiler.json: -------------------------------------------------------------------------------- 1 | {"hdfs":{"batchSize":1,"enabled":false,"index":"profiler"},"elasticsearch":{"batchSize":1,"enabled":false,"index":"profiler"},"solr":{"batchSize":1,"enabled":false,"index":"profiler"}} 2 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/zookeeper/parsers/auth.json: -------------------------------------------------------------------------------- 1 | {"parserClassName":"org.apache.metron.parsers.json.JSONMapParser","filterClassName":null,"sensorTopic":"auth","outputTopic":null,"errorTopic":null,"writerClassName":null,"errorWriterClassName":null,"readMetadata":false,"mergeMetadata":false,"numWorkers":null,"numAckers":null,"spoutParallelism":1,"spoutNumTasks":1,"parserParallelism":1,"parserNumTasks":1,"errorWriterParallelism":1,"errorWriterNumTasks":1,"spoutConfig":{},"securityProtocol":null,"stormConfig":{},"parserConfig":{},"fieldTransformations":[{"input":[],"output":["user","success","is_system_user","timestamp_solr"],"transformation":"STELLAR","config":{"user":"GET_FIRST(SPLIT(GET_FIRST(SPLIT(source_user, '@')),'$'))","success":"success == 'Success'","is_system_user":"STARTS_WITH(user, 'C')","timestamp_solr":"DATE_FORMAT('yyyy-MM-dd\\'T\\'HH:mm:ss\\'Z\\'',timestamp)"}}],"cacheConfig":{},"rawMessageStrategy":"DEFAULT","rawMessageStrategyConfig":{}} 2 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/config/zookeeper/profiler.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles" : [ { 3 | "profile": "distinct_auth_attempts_by_user", 4 | "foreach": "user", 5 | "onlyif": "source.type == 'auth' && auth_orientation != null && auth_orientation == 'LogOn' && user != null && LENGTH(user) > 0 && ip_dst_host != null && ip_dst_host != '?'", 6 | "init": { 7 | "total": "HLLP_INIT(5,6)" 8 | }, 9 | "update": { 10 | "total": "HLLP_ADD(total, ip_dst_host)" 11 | }, 12 | "result": { 13 | "profile": "total", 14 | "triage": { 15 | "total_count": "HLLP_CARDINALITY(total)" 16 | } 17 | } 18 | }, 19 | { 20 | "profile": "distinct_auth_attempts_by_user_distribution", 21 | "foreach": "'global'", 22 | "onlyif": "source.type == 'profiler' && profile == 'distinct_auth_attempts_by_user'", 23 | "init": { 24 | "s": "STATS_INIT()" 25 | }, 26 | "update": { 27 | "s": "STATS_ADD(s, total_count)" 28 | }, 29 | "result": "s" 30 | }, 31 | { 32 | "profile": "num_alerts_by_user", 33 | "foreach": "user", 34 | "onlyif": "source.type == 'auth' && is_alert != null && (is_alert == true || is_alert == 'true')", 35 | "init": { 36 | "count": "0" 37 | }, 38 | "update": { 39 | "count": "count + 1" 40 | }, 41 | "result": "count" 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/01_alerts_ui_auth_alerts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/01_alerts_ui_auth_alerts.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/02_select_auth_fields.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/02_select_auth_fields.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/03_show_auth_measurements.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/03_show_auth_measurements.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/04_auth_high_score_alert_details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/04_auth_high_score_alert_details.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/05_auth_triage_details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/05_auth_triage_details.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/06_auth_enrichment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/06_auth_enrichment.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/07_auth_triage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/07_auth_triage.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/continuous_anomaly.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/continuous_anomaly.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/intermittent_anomaly.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/intermittent_anomaly.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/lanl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/lanl.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/images/typical_user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/06_ProfilingAuthLogs/images/typical_user.png -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/scripts/build_demo_loader.sh: -------------------------------------------------------------------------------- 1 | wget -P /home/centos http://ftp.naz.com/apache/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz 2 | cd ~ 3 | gunzip apache-maven-3.6.0-bin.tar.gz 4 | tar xvf apache-maven-3.6.0-bin.tar.gz 5 | export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.201.b09-2.el7_6.x86_64 6 | export PATH=/home/centos/apache-maven-3.6.0/bin:$PATH 7 | git clone https://github.com/simonellistonball/metron-field-demos.git 8 | cd metron-field-demos/auth/ 9 | mvn package 10 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/scripts/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "stepTimeMs" : 5000, 3 | "timeOffset" : 1493837313104, 4 | "sources" : [ 5 | { 6 | "inputFile" : "auth.txt", 7 | "outputTopic" : "auth", 8 | "filter" : "source_user == 'U66@DOM1' || (ip_src_host not in [ null, 'TGT', '?'] && (not(exists(hostname_filter)) || ip_src_host in hostname_filter) ) || (ip_dst_host not in [ null, 'TGT', '?' ] && (not(exists(hostname_filter)) || ip_dst_host in hostname_filter))", 9 | "config" : { 10 | "columns" : [ "timestamp" 11 | , "source_user" 12 | , "dest_user" 13 | , "ip_src_host" 14 | , "ip_dst_host" 15 | , "auth_type" 16 | , "logon_type" 17 | , "auth_orientation" 18 | , "success" 19 | ] 20 | } 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/scripts/demo_loader.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default} 21 | [ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase 22 | 23 | # Autodetect JAVA_HOME if not defined 24 | if [ -e /usr/libexec/bigtop-detect-javahome ]; then 25 | . /usr/libexec/bigtop-detect-javahome 26 | elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then 27 | . /usr/lib/bigtop-utils/bigtop-detect-javahome 28 | fi 29 | 30 | 31 | source /etc/default/metron 32 | 33 | export CLASSNAME="com.hortonworks.metron.loader.csv.DemoLoader" 34 | 35 | CP="demo-loader-0.4.1.1.3.2.0-10-uber.jar" 36 | java $METRON_JVMFLAGS -cp $CP $CLASSNAME "$@" 37 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/scripts/download_auth_data.sh: -------------------------------------------------------------------------------- 1 | wget https://csr.lanl.gov/data/cyber1/auth.txt.gz 2 | gunzip auth.text.gz 3 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/scripts/host_filter.txt: -------------------------------------------------------------------------------- 1 | C22571 2 | C23660 3 | C23542 4 | C409 5 | C529 6 | C528 7 | C1295 8 | C12406 9 | C768 10 | C11557 11 | C15001 12 | C523 13 | C3235 14 | C15003 15 | C15007 16 | C13290 17 | C22698 18 | C1065 19 | C13629 20 | C16561 21 | C17651 22 | C19952 23 | C13988 24 | C5785 25 | C12414 26 | C12654 27 | C17536 28 | C15914 29 | C4581 30 | C14270 31 | C18191 32 | C5638 33 | C17682 34 | C10925 35 | C12547 36 | C12427 37 | C15266 38 | C13998 39 | C11336 40 | C5878 41 | C11455 42 | C14049 43 | C20738 44 | C19509 45 | C2493 46 | C4559 47 | C22462 48 | C14619 49 | C12318 50 | C19730 51 | C17552 52 | C436 53 | C553 54 | C22907 55 | C22902 56 | C22903 57 | C14296 58 | C9696 59 | C14053 60 | C22901 61 | C15160 62 | C14868 63 | C14866 64 | C2229 65 | C14744 66 | C2106 67 | C14984 68 | C14500 69 | C16811 70 | C12682 71 | C20515 72 | C2908 73 | C5186 74 | C14186 75 | C22483 76 | C23330 77 | C17693 78 | C14519 79 | C3440 80 | C1947 81 | C4651 82 | C13667 83 | C13303 84 | C3204 85 | C17338 86 | C453 87 | C14510 88 | C14993 89 | C13661 90 | C21711 91 | C1357 92 | C15180 93 | C4508 94 | C20097 95 | C14529 96 | C5279 97 | C13316 98 | C12347 99 | C20091 100 | C467 101 | C3534 102 | C586 103 | C101 104 | C13795 105 | C19308 106 | C13311 107 | C17926 108 | C4077 109 | C5045 110 | C20084 111 | C22023 112 | C21176 113 | C1243 114 | C14777 115 | C354 116 | C15855 117 | C16945 118 | C21739 119 | C23911 120 | C23912 121 | C14090 122 | C22027 123 | C26896 124 | C1453 125 | C3198 126 | C3990 127 | C920 128 | C2548 129 | C13457 130 | C2668 131 | C122 132 | C15767 133 | C17823 134 | C484 135 | C14540 136 | C21527 137 | C21524 138 | C21405 139 | C21075 140 | C3405 141 | C4858 142 | C13907 143 | C19795 144 | C1108 145 | C12257 146 | C14557 147 | C15753 148 | C13583 149 | C16969 150 | C23815 151 | C5154 152 | C1905 153 | C23812 154 | C709 155 | C21661 156 | C21782 157 | C13911 158 | C5358 159 | C1435 160 | C14202 161 | C21428 162 | C21302 163 | C23398 164 | C2893 165 | C20681 166 | C16065 167 | C17835 168 | C274 169 | C14692 170 | C12393 171 | C21535 172 | C3715 173 | C608 174 | C14903 175 | C2868 176 | C2989 177 | C14229 178 | C11996 179 | C15445 180 | C17744 181 | C13378 182 | C14225 183 | C18715 184 | C15207 185 | C14222 186 | C161 187 | C282 188 | C3163 189 | C13492 190 | C21202 191 | C1423 192 | C23297 193 | C1300 194 | C1420 195 | C2871 196 | C14917 197 | C13948 198 | C11768 199 | C19 200 | C4377 201 | C612 202 | C17611 203 | C3049 204 | C2514 205 | C15556 206 | C15678 207 | C14477 208 | C20345 209 | C23979 210 | C22526 211 | C21676 212 | C18170 213 | C21582 214 | C17082 215 | C13717 216 | C18179 217 | C625 218 | C988 219 | C5794 220 | C14804 221 | C3496 222 | C12745 223 | C2725 224 | C1515 225 | C19945 226 | C1754 227 | C17887 228 | C17407 229 | C14122 230 | C3021 231 | C1085 232 | C23767 233 | C17090 234 | C23990 235 | C2850 236 | C20000 237 | C18163 238 | C3143 239 | C4232 240 | C515 241 | C4233 242 | C17633 243 | C1645 244 | C992 245 | C13960 246 | C12630 247 | C14494 248 | C22668 249 | C22666 250 | -------------------------------------------------------------------------------- /06_ProfilingAuthLogs/scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | source /etc/default/metron 4 | 5 | ./demo_loader.sh -e 1848158 -c config.json -z $ZOOKEEPER -hf host_filter.txt 6 | -------------------------------------------------------------------------------- /07_ExploringEventHistory/README.md: -------------------------------------------------------------------------------- 1 | # Exploring Event History - Dashboards and Run Books for Analysis, Threat Hunting and Investigations 2 | # Objectives 3 | After this lab you will be able to: 4 | 1. Describe how to access the events stored in the index and HDFS to create dashboards, analytics for threat hunting, and run books for investigations. 5 | # Background 6 | After an event is fully enriched and triaged, Metron automatically stores the event in an index (Solr or Elastic Search) and in HDFS. HDFS stores the entire history of events for the specified retention period. The index typically stores several months of the most recent events. The retention period for each storage mechanism is configurable and there are several cost effective options for storing and analyzing years of events. 7 | 8 | Apache Metron and the Hadoop ecosystem offer many options for analyzing stored cybersecurity events. For example: 9 | 1. Metron Alerts UI 10 | 2. Apache Zeppelin Notebooks using Apache Spark 11 | 3. Banana and Kibana Dashboards 12 | 4. Third party business intelligence tools 13 | 14 | # Metron Alerts UI 15 | The Metron Alerts UI reads events from the index which provides a very flexible search API that quickly filters the haystack to a subset of events. See the [Getting started](../01_GettingStarted/README.md) lab for instructions on opening the Metron Alerts UI. Once the UI is open, enter a simple or complex query in the UI and the filtered events will display in the UI. Click on the headers to sort ascending or descending by an event field. Complete the earlier labs to see the Alerts UI in action. 16 | 17 | # Apache Zeppelin and Spark 18 | [Apache Zeppelin](https://zeppelin.apache.org/) is an open source web based notebook for recording and sharing analytics and visualizations of event data stored in both HDFS and the index. Zeppelin offers a number of interpeters including shell commands, Apache Spark and SQL queries. Different interpreters can be combined in the same notebook as can python, scala and R code. 19 | 20 | [Apache Spark](https://spark.apache.org/) is a distributed analytics engine capable of analyzing the entire haystack and building machine learning models. Spark supports scala, python, java, and R programming languages. 21 | 22 | Spark code included in a notebook records commands for common cyber security analytics tasks such as dashboard visualizations, investigation run books, and threat hunting queries and visualizations. 23 | 24 | ## Exploring a Zeppelin Run Book 25 | 26 | 1. Open Ambari in the browser: 27 | http://mobius.local:8080/ 28 | 2. Select Zeppelin Notebook on the left hand side of the browser window. 29 | 3. Pull down the Quick Links menu to the right of the Summary and Configs tab and select Zeppelin UI. 30 | 31 | 32 | 33 | 4. The Zeppelin page appears. Click the Login button on the upper right side of the page. 34 | 35 | 36 | 37 | 5. Enter the Zeppelin credentials (admin/admin). Click Login. 38 | 39 | 40 | 41 | 6. The Welcome to Zeppelin page opens. Click on the Auth Investigation notebook link. 42 | 43 | 44 | 45 | 7. The Auth Investigation notebook is an example of a run book to assist a SOC analyst when investigating an authentication alert. A notebook consists of a set of paragraphs. Each paragraph starts with a % followed by an interpreter name. For example the first paragraph uses the shell interpreter. The second paragraph uses the spark2 interpreter. The third paragraph uses the spark2.sql interpreter. Following the interpreter is some commands or code in the language required by the interpreter. 46 | 47 | 48 | 49 | 8. For better readability, the notebook author can show or hide the commands and command output independently. The first paragraph lists the auth events files stored in HDFS. The output is hidden. Click on the Show output icon to the right of the "Find auth events in HDFS paragraph". 50 | 51 | 52 | 53 | 9. Metron groups events together and writes the fully enriched and triaged events to HDFS files in the directory /apps/metron/indexing/indexed/. The hdfs dfs -ls shell command lists the auth events stored in HDFS. 54 | 55 | 56 | 57 | 10. A quick way to explore the events is to load them into a Spark dataframe, create a temporary table, and use SQL to access the events in the table. The spark.read.json function reads all the json auth event files in the /apps/metron/indexing/indexed/auth HDFS directory and loads them into a DataFrame. The createOrReplaceTempView function creates a temporary Spark SQL table called auth_events. Once the table is created, we can query the auth events using SQL. 58 | 59 | Click the Run button to the right of the "Register auth events as table so we can query" paragraph to reload the temporary table. 60 | 61 | In the spark2.sql "Which users log into many distinct hosts?" paragraph, Zeppelin sends the query over the auth_events temporary table to Spark and displays the results in a table. 62 | 63 | 64 | 65 | 11. Zeppelin supports other query result visualizations including bar charts, pie charts, line charts, area charts, line charts and scatter charts. Scroll down to the two line charts "Distinct host logons by time" and "Number of Logons by Time". 66 | 67 | 68 | 69 | 12. Click on the Show editor button on the upper right of the "Distinct host logons by time" paragraph to show the SQL that produced the points in the line graph below. The value of the user in the where clause is parameterized. Zeppelin replaces ${user} in the query with the value entered in the user field text entry (U66). Parameterized queries are helpful in run books or for threat hunting when you want to use the same visualization for different entities. 70 | 71 | To change the visualization, click on the tool bar below the user text entry field. 72 | 73 | 74 | 75 | 13. Scroll through the visualizations in the rest of the notebook. Go to the bottom of the notebook. You will see an empty new paragraph. 76 | 77 | 78 | 79 | 14. Enter the commands below and click Run. The number of events stored in HDFS should display in a table below. 80 | 81 | ``` 82 | %spark2.sql 83 | select count(*) from auth_events 84 | ``` 85 | 86 | If you see the error below, scroll back up to the top of the notebook and click the run button on the "Register auth events as table so we can query paragraph". Then run the query again: 87 | 88 | ``` 89 | Table or view not found: auth_events; line 1 pos 21 90 | set zeppelin.spark.sql.stacktrace = true to see full stacktrace 91 | ``` 92 | 93 | 94 | 95 | 15. Notebooks automate many common analytics tasks. Once the data is loaded into a Spark DataFrame, you can perform common queries for threat hunting or other explorations. The data loaded into a dataframe can also be used for training machine learning models. 96 | 97 | ## Creating a Solr Banana Dashboard 98 | 99 | Banana is another option for visualizing data stored in Solr indices. 100 | 101 | 1. In the browser, enter the Banana url: 102 | 103 | http://mobius.local.localdomain:8983/solr/banana 104 | 105 | 2. The Basic Dashboard with Pointers appears. 106 | 107 | 3. Click on the New icon in the upper right. Select Time-series dashboard. 108 | 109 | 110 | 111 | 4. The New Dashboard Settings dialog appears. Enter mysquid in the Collection Name. Enter timestamp_solr in the Time Field. Click Create. 112 | 113 | 114 | 115 | 5. The New Time Series Dashboard appears with default visualizations. The dashboard consists of horizontal rows. Each row has one or more panels that display data. 116 | 117 | 118 | 119 | 6. Locate Total Hits in the upper right of the dashboard. The total hits is zero. Click on the gear icon in the Total Hits panel. 120 | 121 | 122 | 123 | 7. The Hits Setting dialog opens. 124 | 125 | 126 | 127 | 8. Click on the Panel tab. 128 | 129 | 9. Enter guid in the Field. Click Close. 130 | 131 | 132 | 133 | 10. The Total Hits will now display the number of mysquid events in the time period for the dashboard. 134 | 135 | 136 | 137 | 11. The Time Window in the upper left controls the time period of the data displayed in the dashboard. Time periods are relative to the current time by default. 138 | 139 | 140 | 141 | 12. Click on the disk icon to save the dashboard. Enter squid in the text entry field. Click the disk icon to save. 142 | 143 | 144 | 145 | 13. To select the dashboard displayed, select the folder icon. Click on the squid link at the bottom of the dialog. 146 | 147 | 148 | 149 | 14. Check Auto-refresh and the dashboard will read the latest data from the index and update the visualization. 150 | 151 | 152 | 153 | 15. Scroll to the bottom of the window and click + Add a row. 154 | 155 | 156 | 157 | 16. Enter Map in the Title and 300px in the Height. Click Create Row. 158 | 159 | 160 | 161 | 17. Click the up arrow for the Map row to move the Map row above the Table. Map should now be the second to last row. Click Close. 162 | 163 | 164 | 165 | 166 | 167 | 18. An empty row now appears underneath the Event Counts histograms. 168 | 169 | 170 | 171 | 19. Click Add panel to emptry row. 172 | 173 | 20. Click the + to the right of Panels. 174 | 175 | 176 | 177 | 21. Select map for the Panel Type. Enter Destinations in Title. Select 6 for the Span. The Span controls the width of the panel. A panel extending across the full width of the row is Span 12. Enter enrichments.geo.ip_dst_addr.country. Click Add Panel. 178 | 179 | 180 | 181 | 22. Select sunburst for the Panel Type. Enter Cities in the Title. Select 6 for the Span. Enter enrichments.geo.ip_dst_addr.country,enrichments.geo.ip_dst_addr.city in the Facet Pivot String. Click Add Panel. 182 | 183 | 184 | 185 | 23. Click Close. 186 | 187 | 188 | 189 | 24. The dashboard now shows the location of the events on a map and in a sunburst. The countries with more events have darker colors. Hover over the country and the number of events destined to the country pops up. 190 | 191 | 192 | 193 | 194 | 195 | 25. Hover over the inner ring of the sunburst to see the breakdown of the number of events destined for each country. The outer ring of the sunburst shows the breakdown of the events destined for each city. 196 | 197 | 198 | 199 | 26. Go to the top right and click the save button. 200 | 201 | 27. Add a new row called Bytes and move it above the Table. Click Close. 202 | 203 | 204 | 205 | 28. Scroll the dashboard up and locate the new empty row. Add a histgram panel. Enter Bytes for the Title. Enter 12 in the Span. For Mode select values. Enter Value Field bytes. 206 | 207 | 208 | 209 | 29. Scroll down to Chart Settings. Check Lines and uncheck Bars. Click Add Panel. 210 | 211 | 212 | 213 | 30. A line graph of the bytes over time is now shown on the dashboard. 214 | 215 | 216 | 217 | 31. Click Save to save the Dashboard. 218 | 219 | 220 | ## Using a Business Intelligence Tool 221 | 222 | Many business intelligence tools include connectors to Solr, Elastic Search and Spark SQL. For example, [ZoomData](https://www.zoomdata.com/) is convenient for creating visualizations and dashboards from Metron events with no programming. 223 | 224 | 225 | 226 | # References 227 | 228 | ## Tutorials 229 | [Spark Dataframe and DataSet Tutorial](https://hortonworks.com/tutorial/dataframe-and-dataset-examples-in-spark-repl/) 230 | 231 | [Getting started with Apache Zeppelin Tutorial](https://hortonworks.com/tutorial/getting-started-with-apache-zeppelin/) 232 | 233 | [Intro to Machine Learning with Apache Spark and Apache Zeppelin Tutorial](https://hortonworks.com/tutorial/intro-to-machine-learning-with-apache-spark-and-apache-zeppelin/) 234 | 235 | ## Documentation 236 | [Apache Spark Component Guide](https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.5/bk_spark-component-guide/content/ch_introduction-spark.html) 237 | 238 | [Apache Zeppelin Component Guide](https://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.6.5/bk_zeppelin-component-guide/content/ch_overview.html) 239 | 240 | -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/00_zoom_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/00_zoom_dashboard.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/01_starting_zeppelin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/01_starting_zeppelin.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/02_zeppelin_no_user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/02_zeppelin_no_user.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/03_zeppelin_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/03_zeppelin_login.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/04_zeppelin_select_notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/04_zeppelin_select_notebook.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/05_auth_investigation_notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/05_auth_investigation_notebook.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/06_show_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/06_show_output.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/07_indexed_data_file_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/07_indexed_data_file_structure.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/08_create_table_from files.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/08_create_table_from files.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/09_create_graphs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/09_create_graphs.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/10_new_paragraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/10_new_paragraph.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/11_enter_run_query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/11_enter_run_query.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/12_show_graph_sql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/12_show_graph_sql.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_01.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_02.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_03.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_04.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_05.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_06.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_07.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_07.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_08.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_08.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_09.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_09.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_10.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_11.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_12.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_13.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_14.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_15.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_16.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_17.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_18.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_19.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_20.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_21.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_22.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_23.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_23.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_24.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_25.png -------------------------------------------------------------------------------- /07_ExploringEventHistory/images/banana_26.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/07_ExploringEventHistory/images/banana_26.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/README.md: -------------------------------------------------------------------------------- 1 | # Applying DGA Detection Machine Learning Predictions to Squid logs 2 | ## Objectives 3 | After this lab you will be able to: 4 | 1. Enrich an event with the predictions from a machine learning model. 5 | ## Overview 6 | Machine learning models are algorithms that return a prediction or classification given a set of inputs. Machine learning models can be obtained from open source or a third party or developed by data scientists using your Metron prepared data to train the model. To use a machine learning model predition or classification as an enrichment: 7 | 8 | 1. Implement a rest service that follows the Metron Model as a Service interface API conventions. 9 | 2. Register the model with Metron Model as a Service. 10 | 3. Add an enrichment equal to the model result given a set of input fields from the event. 11 | 4. Use the model results in triaging. 12 | 13 | This lab uses an example from [Brian Wylie's](https://www.linkedin.com/in/briford/) BSides DFW 2013 conference presentation [Data Hacking Mad Scientist Style](http://www.securitybsides.com/w/page/68749447/BSidesDFW%202013%20Full%20Track%202%20Abstracts). The [DGA detection model](https://github.com/SuperCowPowers/data_hacking/tree/master/dga_detection) accepts a top level domain name and returns a legitimate or dga classification of the domain. To learn more about the model training model was developed, check out the [notebook](https://nbviewer.jupyter.org/github/SuperCowPowers/data_hacking/blob/master/dga_detection/DGA_Domain_Detection.ipynb). 14 | 15 | ## Creating a Model Rest Service 16 | The [model python implementation](https://github.com/carolynduby/ApacheMetronWorkshop/tree/master/08_DGADetectionModelAsService/dga_model) consists of two main components: [model.py](https://github.com/carolynduby/ApacheMetronWorkshop/blob/master/08_DGADetectionModelAsService/dga_model/model.py) and [rest.py](https://github.com/carolynduby/ApacheMetronWorkshop/blob/master/08_DGADetectionModelAsService/dga_model/rest.py). 17 | 18 | The __init__ function in model.py (line 7) loads the model definitions, a machine learning model algorithm type and its mathematical parameters. The __evalute_domain__ function (line 14) accepts a domain name, passes the domain name into the model, and returns a legit or dga classifcation returned by the model. 19 | 20 | 21 | 22 | The rest module uses the python Flask library to start a REST service endpoint. The rest service calls the model evaluate_domain method (line 14) and returns the is_malicious DGA classification in JSON format (line 17). 23 | 24 | 25 | 26 | ## Deploying DGA Detection Model for use with Metron 27 | 1. Ssh into the mobius.local.localdomain host as the centos user using the .pem key file provided. 28 | 29 | ``` 30 | ssh -i .pem centos@mobius.local.localdomain 31 | ``` 32 | 33 | 2. Run the start_dga_model script. The script runs the maas_service command to start the model as a service discovery service. Then it calls maas_deploy to start a new dga detection model rest service and register the model with the name dga and version 1.0. 34 | 35 | ``` 36 | [centos@mobius ~]$ cat start_dga_model.sh 37 | sudo /usr/hcp/current/metron/bin/maas_service.sh -zq localhost:2181 38 | sudo /usr/hcp/current/metron/bin/maas_deploy.sh -hmp /user/root/models -mo ADD -m 750 -n dga -v 1.0 -ni 1 -zq localhost:2181 -lmp /root/dga_model 39 | [centos@mobius ~]$ 40 | [centos@mobius ~]$ ./start_dga_model.sh 41 | 19/04/09 23:48:34 INFO service.Client: Initializing Client 42 | 19/04/09 23:48:34 INFO service.Client: Running Client 43 | 19/04/09 23:48:34 INFO client.RMProxy: Connecting to ResourceManager at mobius.local.localdomain/127.0.0.1:8050 44 | 19/04/09 23:48:35 INFO client.AHSProxy: Connecting to Application History server at mobius.local.localdomain/127.0.0.1:10200 45 | .... 46 | ``` 47 | 48 | 3. Model as a service deploys the rest service and registers a URL endpoint. You can find information about the DGA model service host and url by using the MAAS_GET_ENDPOINT stellar function: 49 | 50 | ``` 51 | [centos@mobius ~]$ /usr/hcp/current/metron/bin/stellar -z localhost:2181 52 | .... 53 | [Stellar]>>> MAAS_ 54 | MAAS_GET_ENDPOINT( MAAS_MODEL_APPLY( 55 | [Stellar]>>> MAAS_GET_ENDPOINT( 'dga') 56 | {name=dga, endpoint:apply=apply, version=1.0, url=http://mobius.local.localdomain:33731} 57 | [Stellar]>>> quit 58 | ``` 59 | 60 | 4. Copy the URL returned by MAAS_GET_ENDPOINT and test the rest endpoint using curl. For example, in the previous step, the url for the DGA rest model endpoint is http://mobius.local.localdomain:33731. 61 | 62 | ``` 63 | ## the domain google is classified as legitimate 64 | [centos@mobius ~]$ curl http://mobius.local.localdomain:YOUR_PORT_HERE/apply?host=google 65 | {"is_malicious":"legit"} 66 | 67 | ## the domain zaze is classified as dga 68 | [centos@mobius ~]$ curl http://mobius.local.localdomain:YOUR_PORT_HERE/apply?host=zaze 69 | {"is_malicious":"dga"} 70 | ``` 71 | 5. Open the Metron Configuration UI. (http://mobius.local.localdomain:4200) 72 | 73 | 6. Click on the pencil icon to edit the mysquid sensor configuration. Click Raw JSON in the Advanced section to modify the mysquid configuration. 74 | 75 | 7. Add the is_malicious enrichment after the is_potential typosquat enrichment. Set is_malicious as follows: 76 | 77 | “is_malicious := MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host' : domain_without_subdomains}))” 78 | 79 | 80 | 81 | 8. Click Save below the json. 82 | 83 | 9. Click Save under the sensor configuration. 84 | 85 | 10. Configure your browser to use the proxy on the mobius.local.localdomain host port 3128. Browse to some web sites. Open the Metron UI and look at the latest events. Events have a new is_malicious field set to either legit or dga. 86 | 87 | 88 | 89 | 11. Open the mysquid sensor configuration and select Raw JSON in the Advanced section to modify the mysquid configuration. 90 | 91 | 12. In the Sensor Enrichment Config replace the threatIntel section with the text below. The configuration adds another triage rule which sets the event score to 75 if the is_malicious field equals dga. Note that events with that trigger both a typosquat and a dga alert will be scored at the maximum of the two scores because the aggregator is MAX. 92 | 93 | ``` 94 | "threatIntel": { 95 | "fieldMap": { 96 | "stellar": { 97 | "config": [ 98 | "is_alert := (exists(is_alert) && is_alert) || is_potential_typosquat || is_malicious == 'dga'" 99 | ] 100 | } 101 | }, 102 | "fieldToTypeMap": {}, 103 | "config": {}, 104 | "triageConfig": { 105 | "riskLevelRules": [ 106 | { 107 | "name": "Alexa 10k Typosquat Bloom", 108 | "comment": "Inspect a bloom filter with potentially typosquatted domains from the top Alexa 10k", 109 | "rule": "is_potential_typosquat != null && is_potential_typosquat", 110 | "reason": "FORMAT('%s is a potential typosquatted domain from the top 10k domains from alexa', domain_without_subdomains)", 111 | "score": "50" 112 | }, 113 | { 114 | "name": "DGA Detection", 115 | "comment": "DGA Model classified domain as generated", 116 | "rule": "is_malicious == 'dga'", 117 | "reason": "FORMAT('%s is a potential generated domain name', domain_without_subdomains)", 118 | "score": "75" 119 | } 120 | ], 121 | "aggregator": "MAX", 122 | "aggregationConfig": {} 123 | } 124 | }, 125 | ``` 126 | 127 | 128 | 129 | 13. Click Save below the json. 130 | 131 | 14. Click Save under the sensor configuration. 132 | 133 | 15. Configure your browser to use the proxy on the mobiu.local.localdomain host port 3128. Browse to gigya.com and other sites such as a news site. Eventually you should see some alerts with a red score of 75. Enter is_malicious:dga in the search field and click the search button. The UI displays only dga alerts. 134 | 135 | 136 | 137 | For alerts with is_malicious equal to dga and is_potential_typosquat equal to false, the alert will only have one score from the dga detection: 138 | 139 | 140 | 141 | Alerts with is_malicious equal to dga and is_potential_typosquat equal to true have two scores - one score from the typosquat and a second score from dga detection. The overall score is the maximum of the two scores because the aggregator is set to MAX: 142 | 143 | 144 | 145 | 16. Excellent work! You applied an off the shelf machine learning model classification to squid events and used the classification in scoring. 146 | 147 | # Next Lab 148 | 149 | [Exploring Event History - Dashboards and Run Books for Analysis, Threat Hunting and Investigations](../07_ExploringEventHistory/README.md) 150 | 151 | 152 | ## References 153 | [Domain Generation Algorithm](https://en.wikipedia.org/wiki/Domain_generation_algorithm) 154 | 155 | [Model as Service: Modern Streaming Data Science with Apache Metron](https://hortonworks.com/blog/model-service-modern-streaming-data-science-apache-metron/) 156 | -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/dga_model_alexa_counts.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/dga_model/dga_model_alexa_counts.model -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/dga_model_alexa_vectorizor.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/dga_model/dga_model_alexa_vectorizor.model -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/dga_model_dict_counts.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/dga_model/dga_model_dict_counts.model -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/dga_model_dict_vectorizor.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/dga_model/dga_model_dict_vectorizor.model -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/dga_model_random_forest.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/dga_model/dga_model_random_forest.model -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/model.py: -------------------------------------------------------------------------------- 1 | import sklearn 2 | import numpy as np 3 | import math 4 | import pickle 5 | import collections 6 | class DGA: 7 | def __init__(self): 8 | self.model = { 'clf': pickle.loads(open('./dga_model_random_forest.model','rb').read()) 9 | , 'alexa_vc': pickle.loads(open('./dga_model_alexa_vectorizor.model','rb').read()) 10 | , 'alexa_counts': pickle.loads(open('./dga_model_alexa_counts.model','rb').read()) 11 | , 'dict_vc': pickle.loads(open('./dga_model_dict_vectorizor.model','rb').read()) 12 | , 'dict_counts': pickle.loads(open('./dga_model_dict_counts.model','rb').read()) } 13 | 14 | def evaluate_domain(self, domain): 15 | alexa_match = self.model['alexa_counts'] * self.model['alexa_vc'].transform([domain]).T 16 | dict_match = self.model['dict_counts'] * self.model['dict_vc'].transform([domain]).T 17 | 18 | # Assemble feature matrix (for just one domain) 19 | X = [len(domain), self.entropy(domain), alexa_match, dict_match] 20 | y_pred = self.model['clf'].predict([ X ])[0] 21 | return y_pred 22 | 23 | def entropy(self, s): 24 | p, lns = collections.Counter(s), float(len(s)) 25 | return -sum( count/lns * math.log(count/lns, 2) for count in p.values()) -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/rest.py: -------------------------------------------------------------------------------- 1 | import json 2 | import model 3 | from flask import Flask 4 | from flask import request,jsonify 5 | import socket 6 | 7 | app = Flask(__name__) 8 | 9 | @app.route("/apply", methods=['GET']) 10 | def predict(): 11 | # We expect one argument, the hostname without TLD. 12 | h = request.args.get('host') 13 | r = {} 14 | r['is_malicious'] = model.evaluate_domain(h) 15 | # We will return a JSON map with one field, 'is_malicious' which will be 16 | # 'legit' or 'dga', the two possible outputs of our model. 17 | return jsonify(r) 18 | 19 | if __name__ == "__main__": 20 | # Create my model object that I want to expose. 21 | model = model.DGA() 22 | # In order to register with model as a service, we need to bind to a port 23 | # and inform the discovery service of the endpoint. Therefore, 24 | # we will bind to a port and close the socket to reserve it. 25 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 26 | sock.bind(('localhost', 0)) 27 | port = sock.getsockname()[1] 28 | sock.close() 29 | with open("endpoint.dat", "w") as text_file: 30 | # To inform the discovery service, we need to write a file with a simple 31 | # JSON Map indicating the full URL that we've bound to. 32 | text_file.write("{\"url\" : \"http://0.0.0.0:%d\"}" % port) 33 | # Make sure flask uses the port we reserved 34 | app.run(threaded=True, host="0.0.0.0", port=port) -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/dga_model/rest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PYTHONPATH="${PYTHONPATH}:." /usr/bin/python rest.py 3 | -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/add_is_malicious.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/add_is_malicious.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/alerts_with_is_malicious.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/alerts_with_is_malicious.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/both_scores.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/both_scores.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/dga_events.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/dga_events.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/model_python.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/model_python.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/one_score.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/one_score.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/rest_python.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/rest_python.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/images/threat_intel_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carolynduby/ApacheMetronWorkshop/26dca9d37a7ece58eaaa12176a9cddebda47f88e/08_DGADetectionModelAsService/images/threat_intel_config.png -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/scripts/setup_dga_maas.sh: -------------------------------------------------------------------------------- 1 | # install the python packages required by python model 2 | sudo yum -y install python-pip 3 | sudo pip install pandas 4 | sudo pip install sklearn 5 | sudo pip install spicy 6 | sudo pip install tldextract 7 | sudo pip install spicy 8 | sudo pip install Flask 9 | sudo pip install bumpy 10 | 11 | # create the root user in hdfs 12 | sudo -u hdfs hdfs dfs -mkdir /user/root 13 | sudo -u hdfs hdfs dfs -chown root /user/root 14 | sudo -u hdfs hdfs dfs -ls /user/ 15 | 16 | # install the model in the root home directory 17 | sudo cp -r ../dga_model/ /root/ 18 | 19 | # move the start script to home 20 | cp ./start_dga_model.sh /home/centos 21 | -------------------------------------------------------------------------------- /08_DGADetectionModelAsService/scripts/start_dga_model.sh: -------------------------------------------------------------------------------- 1 | sudo /usr/hcp/current/metron/bin/maas_service.sh -zq localhost:2181 2 | sudo /usr/hcp/current/metron/bin/maas_deploy.sh -hmp /user/root/models -mo ADD -m 750 -n dga -v 1.0 -ni 1 -zq localhost:2181 -lmp /root/dga_model 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apache Metron Workshop 2 | 3 | ## Objectives 4 | 5 | After this workshop you will be able to: 6 | 7 | 1. [Open and describe the purpose of the Metron UIs.](01_GettingStarted/README.md) 8 | 2. [Parse and normalize squid log format using a Grok parser.](02_ParsingSquid/README.md) 9 | 3. [Enrich squid events with geocoding and field transformations.](03_EnrichingSquid/README.md) 10 | 4. [Triage squid events](04_TriagingSquid/README.md) 11 | 5. [Profiler Basics](05_ProfilerBasics/README.md) 12 | 6. [User and Entity Behavior Analytics(UEBA) with User Authentication Events](06_ProfilingAuthLogs/README.md) 13 | 7. [Exploring Event History - Dashboards and Run Books for Analysis, Threat Hunting and Investigations](07_ExploringEventHistory/README.md) 14 | 8. [Applying DGA Detection Machine Learning Predictions to Squid logs](08_DGADetectionModelAsService/README.md) 15 | 16 | ## Required Metron Version 17 | 18 | The labs are designed to work with Apache Metron 0.5.1 as packaged in Hortonworks Cyber Security Platform. For more information consult the [HCP Release Notes](https://docs.hortonworks.com/HDPDocuments/HCP1/HCP-1.6.1/release-notes/content/hortonworks_cybersecurity_platform_1_6.1_release_notes.html) 19 | -------------------------------------------------------------------------------- /build_single/.gitignore: -------------------------------------------------------------------------------- 1 | genconfig.*/ 2 | *mpack*.tar.gz 3 | *.out 4 | *.rpm 5 | build_single.log 6 | -------------------------------------------------------------------------------- /build_single/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | The scripts in this directory install and configure a single node metron from a Centos 7 host. 4 | 5 | # Provision the host 6 | 7 | Create a host running Centos 7. The user you use must be able to sudo. 8 | 9 | # Install the single node HCP 10 | 11 | To use the scripts: 12 | 13 | 1. Install git 14 | 15 | ``` 16 | sudo yum -y install git 17 | ``` 18 | 19 | 2. Get the scripts from github and change to the directory containing the scripts. 20 | 21 | ``` 22 | git clone https://github.com/carolynduby/ApacheMetronWorkshop.git 23 | cd ApacheMetronWorkshop/build_single/ 24 | ``` 25 | 26 | 3. If running in AWS and you want to set the hostname, run the sethostname script and allow the host to reboot. If not running in AWS, skip this step or use cloud specific scripts to set the desired host name. 27 | 28 | ``` 29 | ./sethostname.sh 30 | ``` 31 | 32 | 4. Select the index type to use (es or solr). Substitute the CHANGEME text with your passwords. Passwords requiring at least 12 characters are noted. 33 | 34 | ``` 35 | vi single-node--blueprint-19-variables.txt 36 | ``` 37 | 38 | 5. Log into the host and run the buld script. The build script output will print to the console and be logged to build_single.log. 39 | 40 | ``` 41 | cd ApacheMetronWorkshop/build_single/ 42 | bash -x ./build_single.sh 2>&1 | tee -a build_single.log 43 | ``` 44 | 45 | 6. Register the Blueprint and start the services. 46 | 47 | ``` 48 | ./deploy_single_node.sh 49 | ``` 50 | 51 | 7. Wait for the cluster install to finish. You can either open the Ambari UI or use the curl request below to check the status from the command line. 52 | 53 | ``` 54 | curl -H "X-Requested-By: ambari" -X GET -u admin:admin http://localhost:8080/api/v1/clusters/metron/requests/1 | grep -e status -e percent -e queue -e fail 55 | ``` 56 | 57 | 58 | -------------------------------------------------------------------------------- /build_single/build_single.sh: -------------------------------------------------------------------------------- 1 | #install epel repo 2 | sudo yum -y install epel-release 3 | 4 | # install prereqs 5 | sudo yum install wget -y 6 | sudo yum install net-tools -y 7 | 8 | #install nodejs 9 | curl --silent --location https://rpm.nodesource.com/setup_8.x | sudo bash - 10 | sudo yum install -y nodejs 11 | 12 | #install mysql 13 | wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm 14 | sudo rpm -ivh mysql-community-release-el7-5.noarch.rpm 15 | sudo yum -y install mysql-community-server 16 | sudo yum -y install mysql-connector-java 17 | sudo service mysqld start 18 | 19 | mysql -u root < mysql_init_users.sql 20 | echo "grant all privileges on *.* to 'hive'@'"`hostname`"' identified by 'admin';" | mysql -u root 21 | 22 | #change the root pw 23 | sudo mysqladmin -u root password root 24 | 25 | # correct the python-requests version so metron service status displays correct results 26 | sudo yum -y install python-pip 27 | sudo pip install requests==2.6.1 28 | sudo service ambari-agent restart 29 | 30 | ## add autorestart command 31 | sudo cp start_service.sh /root 32 | sudo sh -c "echo 'bash /root/start_service.sh' >> /etc/rc.local" 33 | sudo chmod a+x /etc/rc.d/rc.local 34 | 35 | # set umask 36 | sudo umask 0022 37 | 38 | # configure the machine and install ambari 39 | export ambari_version="2.6.2.2" 40 | export install_ambari_server=true 41 | curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh 42 | 43 | wget -nv http://public-repo-1.hortonworks.com/HCP/centos7/1.x/updates/1.9.1.0/tars/metron/hcp-ambari-mpack-1.9.1.0-6.tar.gz 44 | wget -nv http://public-repo-1.hortonworks.com/HCP/centos7/1.x/updates/1.9.1.0/tars/metron/elasticsearch_mpack-1.9.1.0-6.tar.gz 45 | wget -nv http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.1.2.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.1.2.0-7.tar.gz 46 | wget -nv http://public-repo-1.hortonworks.com/HDP-SOLR/hdp-solr-ambari-mp/solr-service-mpack-3.0.0.tar.gz 47 | 48 | sudo ambari-server install-mpack --mpack=hcp-ambari-mpack-1.9.1.0-6.tar.gz --verbose 49 | sudo ambari-server install-mpack --mpack=elasticsearch_mpack-1.9.1.0-6.tar.gz --verbose 50 | sudo ambari-server install-mpack --mpack=hdf-ambari-mpack-3.1.2.0-7.tar.gz --verbose 51 | sudo ambari-server install-mpack --mpack=solr-service-mpack-3.0.0.tar.gz --verbose 52 | sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar 53 | 54 | sudo ambari-server restart 55 | 56 | 57 | # install the demo content 58 | # don't run this yet, we need to setup metron with blueprint first 59 | # cd typosquat 60 | # ./install_squid_enrichments.sh 61 | -------------------------------------------------------------------------------- /build_single/deploy_single_node.sh: -------------------------------------------------------------------------------- 1 | if [ $# -ne 1 ]; then 2 | echo "Usage: " $0 " index_type(es|solr)" 3 | exit 1 4 | fi 5 | 6 | index_type=$1 7 | #index type must be es or solr 8 | if [ "$index_type" != "es" -a "$index_type" != "solr" ]; then 9 | echo "ERROR: index type argument must be es or solr." 10 | exit 1 11 | fi 12 | 13 | gendir=`mktemp -p . -d genconfig.XXXXXXXXXX` 14 | varfile=single-node-${index_type}-blueprint-19-variables.txt 15 | hostmap=single-node-hostmapping.json 16 | blueprint=single-node-${index_type}-blueprint-19.json 17 | 18 | ## replace the passwords in the blueprint with the variables 19 | ./prepare_ambari_config.sh $gendir $varfile $hostmap $blueprint 20 | 21 | ## print out the generated directory 22 | echo 'generated_directory = ' $gendir 23 | 24 | ## register the blueprint 25 | ./register_blueprint.sh $gendir $hostmap $blueprint 26 | -------------------------------------------------------------------------------- /build_single/download_banana_schema.sh: -------------------------------------------------------------------------------- 1 | mkdir -p /opt/lucidworks-hdpsearch/solr/banana-extra/resources/conf 2 | cd /opt/lucidworks-hdpsearch/solr/banana-extra/resources/conf 3 | wget https://raw.githubusercontent.com/lucidworks/banana/develop/resources/banana-int-solr-5.0/conf/schema.xml 4 | cp -p /usr/hcp/current/metron/config/schema/bro/solrconfig.xml . 5 | -------------------------------------------------------------------------------- /build_single/extract_blueprint_variables.sh: -------------------------------------------------------------------------------- 1 | grep -oP '(?<=\{\{\{).+?(?=\}\}\})' $1 | sort | uniq 2 | -------------------------------------------------------------------------------- /build_single/hcp_metron_rest_db.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS metronrest; 2 | CREATE USER 'metron'@'localhost' IDENTIFIED BY 'Myp@ssw0rd'; 3 | GRANT ALL PRIVILEGES ON metronrest.* TO 'metron'@'localhost'; 4 | use metronrest; 5 | create table if not exists users( 6 | username varchar(50) not null primary key, 7 | password varchar(50) not null, 8 | enabled boolean not null 9 | ); 10 | create table authorities ( 11 | username varchar(50) not null, 12 | authority varchar(50) not null, 13 | constraint fk_authorities_users foreign key(username) references 14 | users(username) 15 | ); 16 | create unique index ix_auth_username on authorities (username,authority); 17 | use metronrest; 18 | insert into users (username, password, enabled) values ('metron', 19 | 'metron',1); 20 | insert into authorities (username, authority) values ('metron', 21 | 'ROLE_USER'); 22 | 23 | -------------------------------------------------------------------------------- /build_single/hostmapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "blueprint" : "single-node-hcp-cluster", 3 | "default_password" : "admin", 4 | "host_groups" :[ 5 | { 6 | "name" : "host_group_1", 7 | "hosts" : [ 8 | { 9 | "fqdn" : "mobius.local.localdomain" 10 | } 11 | ] 12 | } 13 | ] 14 | } 15 | 16 | -------------------------------------------------------------------------------- /build_single/install_banana_schema.sh: -------------------------------------------------------------------------------- 1 | # do this as root 2 | sudo ./download_banana_schema.sh 3 | 4 | # do this as Solr 5 | cd /opt/lucidworks-hdpsearch/solr/bin/ 6 | sudo -u solr ./solr create_collection -c banana-int -d /opt/lucidworks-hdpsearch/solr/banana-extra/resources/conf -p 8983 7 | -------------------------------------------------------------------------------- /build_single/install_sensors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | pushd sensor_install 3 | ./sensor_install_phase.sh ../single-node-es-sensor-config.txt ../sensors pre_ambari.sh 4 | popd 5 | -------------------------------------------------------------------------------- /build_single/metron_squidtokafka.xml: -------------------------------------------------------------------------------- 1 | 2 |