├── .github ├── ISSUE_TEMPLATE │ └── bug_report.md ├── pull_request_template.md └── workflows │ ├── docker-release.yml │ ├── release.yml │ ├── static-analysis.yml │ └── test.yml ├── .gitignore ├── .goreleaser.yaml ├── Dockerfile ├── Formula └── threatest.rb ├── LICENSE ├── LICENSE-3rdparty.csv ├── Makefile ├── NOTICE ├── README.md ├── cmd └── threatest │ ├── lint.go │ ├── main.go │ └── run.go ├── examples ├── cli-usage │ ├── README.md │ └── scenarios.threatest.yaml └── programmatic-usage │ ├── README.md │ ├── cloudsiem_alerts_test.go │ ├── custom-aws-detonator-terratest │ ├── README.md │ ├── custom_aws_detonator_with_terratest_test.go │ ├── go.mod │ ├── go.sum │ └── terraform │ │ ├── .gitignore │ │ ├── .terraform.lock.hcl │ │ └── main.tf │ ├── custom_aws_detonator_test.go │ ├── cws_alerts_test.go │ └── local_detonator_test.go ├── go.mod ├── go.sum ├── logo.png ├── pkg └── threatest │ ├── detonators │ ├── aws_cli_detonator.go │ ├── aws_detonator.go │ ├── command_detonator.go │ ├── detonator.go │ ├── local_command_detonator.go │ ├── mocks │ │ └── Detonator.go │ ├── ssh_command_detonator.go │ ├── stratus_red_team.go │ └── utils.go │ ├── matchers │ ├── alert_matcher.go │ ├── datadog │ │ ├── datadog.go │ │ ├── datadog_test.go │ │ ├── mocks │ │ │ └── DatadogSecuritySignalsAPI.go │ │ └── types.go │ └── mocks │ │ └── AlertGeneratedMatcher.go │ ├── parser │ ├── main.go │ ├── parser.go │ └── parser_test.go │ ├── runner.go │ ├── runner_test.go │ └── scenario.go └── schemas ├── awsCliDetonator.schema.json ├── datadogSecuritySignal.schema.json ├── localDetonator.schema.json ├── remoteDetonator.schema.json ├── stratusRedTeamDetonator.schema.json └── threatest.schema.json /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Something is not working properly 4 | title: '' 5 | labels: bug 6 | assignees: christophetd 7 | 8 | --- 9 | 10 | **What is not working?** 11 | A clear and concise description of what the bug is. 12 | 13 | **What OS are you using?** 14 | Windows 10 / Ubuntu 18.04 / Mac OS X 15 | 16 | **What version of threatest are you using?** 17 | 18 | **Code to reproduce the issue** 19 | 20 | **Full output?** 21 | If applicable, please include the full output. 22 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ### What does this PR do? 2 | 3 | 9 | 10 | ### Motivation 11 | 12 | 16 | 17 | ### Checklist 18 | 19 | 22 | - [ ] Unit tests 23 | - [ ] Documentation 24 | -------------------------------------------------------------------------------- /.github/workflows/docker-release.yml: -------------------------------------------------------------------------------- 1 | name: Release docker image 2 | 3 | on: 4 | push: 5 | tags: 6 | - "*" 7 | 8 | env: 9 | REGISTRY: ghcr.io 10 | IMAGE_NAME: datadog/threatest 11 | 12 | permissions: 13 | contents: read 14 | 15 | jobs: 16 | docker-build-push: 17 | runs-on: ubuntu-latest 18 | permissions: 19 | contents: read 20 | packages: write 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v2.5.0 24 | with: 25 | fetch-depth: 0 26 | 27 | - name: Log into registry ${{ env.REGISTRY }} 28 | uses: docker/login-action@v2.1.0 29 | with: 30 | registry: ${{ env.REGISTRY }} 31 | username: ${{ github.actor }} 32 | password: ${{ secrets.GITHUB_TOKEN }} 33 | 34 | - name: Build and push Docker image 35 | uses: docker/build-push-action@v3.2.0 36 | with: 37 | context: . 38 | push: true 39 | build-args: | 40 | VERSION=${{ github.ref_name }} 41 | tags: | 42 | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} 43 | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | goreleaser: 13 | runs-on: ubuntu-latest 14 | permissions: 15 | contents: write 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v2.5.0 19 | with: 20 | fetch-depth: 0 21 | - name: Set up Go 22 | uses: actions/setup-go@v3.3.1 23 | with: 24 | go-version: 1.19 25 | - name: Run GoReleaser 26 | timeout-minutes: 60 27 | uses: goreleaser/goreleaser-action@v4.4.0 28 | with: 29 | distribution: goreleaser 30 | version: latest 31 | args: release --rm-dist --config .goreleaser.yaml 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | -------------------------------------------------------------------------------- /.github/workflows/static-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "go static analysis" 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | static-analysis: 13 | name: "Run Go static analysis" 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v1 17 | with: 18 | fetch-depth: 1 19 | - name: Set up Go 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: 1.18 23 | - uses: dominikh/staticcheck-action@v1.2.0 24 | with: 25 | version: "2022.1" 26 | install-go: false 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | unit-test: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout repository 19 | uses: actions/checkout@v2 20 | 21 | - name: Set up Go 22 | uses: actions/setup-go@v2 23 | with: 24 | go-version: 1.18 25 | 26 | - name: Run unit tests 27 | run: make test 28 | 29 | docker-build: 30 | runs-on: ubuntu-latest 31 | permissions: 32 | contents: read 33 | steps: 34 | - name: Checkout 35 | uses: actions/checkout@v2.5.0 36 | with: 37 | fetch-depth: 0 38 | - name: Build local Docker image 39 | run: docker build . -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | dist/ 3 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod tidy 4 | builds: 5 | - env: 6 | - CGO_ENABLED=0 7 | goos: 8 | - linux 9 | - windows 10 | - darwin 11 | goarch: 12 | - amd64 13 | - arm64 14 | ldflags: 15 | - -X main.BuildVersion={{.Version}} 16 | 17 | dir: cmd/threatest 18 | binary: threatest 19 | brews: 20 | - name: threatest 21 | repository: 22 | owner: datadog 23 | name: threatest 24 | folder: Formula 25 | url_template: "https://github.com/DataDog/threatest/releases/download/{{ .Tag }}/{{ .ArtifactName }}" 26 | license: Apache-2.0 27 | homepage: "https://github.com/DataDog/threatest" 28 | archives: 29 | - name_template: >- 30 | {{ .ProjectName }}_ 31 | {{- title .Os }}_ 32 | {{- if eq .Arch "amd64" }}x86_64 33 | {{- else if eq .Arch "386" }}i386 34 | {{- else }}{{ .Arch }}{{ end }} 35 | checksum: 36 | name_template: 'checksums.txt' 37 | snapshot: 38 | name_template: "{{ incpatch .Version }}-next" 39 | changelog: 40 | sort: asc 41 | filters: 42 | exclude: 43 | - '^docs:' 44 | - '^test:' 45 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.19-alpine3.16@sha256:4e2a54594cfe7002a98c483c28f6f3a78e5c7f4010c355a8cf960292a3fdecfe AS builder 2 | ARG VERSION=dev-snapshot 3 | RUN mkdir /build 4 | RUN apk add --update make gcc musl-dev 5 | WORKDIR /build 6 | COPY . /build 7 | RUN make BUILD_VERSION=${VERSION} 8 | 9 | FROM alpine:3.16@sha256:3d426b0bfc361d6e8303f51459f17782b219dece42a1c7fe463b6014b189c86d AS runner 10 | LABEL org.opencontainers.image.source="https://github.com/DataDog/threatest/" 11 | COPY --from=builder /build/dist/threatest /threatest 12 | ENTRYPOINT ["/threatest"] 13 | CMD ["--help"] -------------------------------------------------------------------------------- /Formula/threatest.rb: -------------------------------------------------------------------------------- 1 | # typed: false 2 | # frozen_string_literal: true 3 | 4 | # This file was generated by GoReleaser. DO NOT EDIT. 5 | class Threatest < Formula 6 | desc "" 7 | homepage "https://github.com/DataDog/threatest" 8 | version "1.2.4" 9 | license "Apache-2.0" 10 | 11 | on_macos do 12 | if Hardware::CPU.intel? 13 | url "https://github.com/DataDog/threatest/releases/download/v1.2.4/threatest_Darwin_x86_64.tar.gz" 14 | sha256 "f2393c42bfc7edcf864f2684aa0147a3e6652e8c95f13f7131c9e05e64c2e59a" 15 | 16 | def install 17 | bin.install "threatest" 18 | end 19 | end 20 | if Hardware::CPU.arm? 21 | url "https://github.com/DataDog/threatest/releases/download/v1.2.4/threatest_Darwin_arm64.tar.gz" 22 | sha256 "ebf12dbb8b308a9d353ea21ec3871ce33ee62b48b9f6f601b0d61c72854de847" 23 | 24 | def install 25 | bin.install "threatest" 26 | end 27 | end 28 | end 29 | 30 | on_linux do 31 | if Hardware::CPU.arm? && Hardware::CPU.is_64_bit? 32 | url "https://github.com/DataDog/threatest/releases/download/v1.2.4/threatest_Linux_arm64.tar.gz" 33 | sha256 "f83cb6cd17b25bb1f8dcbd2660d3133ce51ff01baad693b88ec1f7c7e06e213f" 34 | 35 | def install 36 | bin.install "threatest" 37 | end 38 | end 39 | if Hardware::CPU.intel? 40 | url "https://github.com/DataDog/threatest/releases/download/v1.2.4/threatest_Linux_x86_64.tar.gz" 41 | sha256 "86538a9f8e88c642d5991cde65c7d2bb9c4790d67b42085244c3a893c5c85d9a" 42 | 43 | def install 44 | bin.install "threatest" 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /LICENSE-3rdparty.csv: -------------------------------------------------------------------------------- 1 | github.com/Azure/azure-sdk-for-go/sdk/azcore,https://github.com/Azure/azure-sdk-for-go/blob/sdk/azcore/v1.0.0/sdk/azcore/LICENSE.txt,MIT 2 | github.com/Azure/azure-sdk-for-go/sdk/azidentity,https://github.com/Azure/azure-sdk-for-go/blob/sdk/azidentity/v1.0.0/sdk/azidentity/LICENSE.txt,MIT 3 | github.com/Azure/azure-sdk-for-go/sdk/internal,https://github.com/Azure/azure-sdk-for-go/blob/sdk/internal/v1.0.0/sdk/internal/LICENSE.txt,MIT 4 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute,https://github.com/Azure/azure-sdk-for-go/blob/sdk/resourcemanager/compute/armcompute/v1.0.0/sdk/resourcemanager/compute/armcompute/LICENSE.txt,MIT 5 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources,https://github.com/Azure/azure-sdk-for-go/blob/sdk/resourcemanager/resources/armresources/v1.0.0/sdk/resourcemanager/resources/armresources/LICENSE.txt,MIT 6 | github.com/AzureAD/microsoft-authentication-library-for-go/apps,https://github.com/AzureAD/microsoft-authentication-library-for-go/blob/v0.4.0/LICENSE,MIT 7 | github.com/aws/aws-sdk-go-v2,https://github.com/aws/aws-sdk-go-v2/blob/v1.16.7/LICENSE.txt,Apache-2.0 8 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream,https://github.com/aws/aws-sdk-go-v2/blob/aws/protocol/eventstream/v1.1.0/aws/protocol/eventstream/LICENSE.txt,Apache-2.0 9 | github.com/aws/aws-sdk-go-v2/config,https://github.com/aws/aws-sdk-go-v2/blob/config/v1.13.0/config/LICENSE.txt,Apache-2.0 10 | github.com/aws/aws-sdk-go-v2/credentials,https://github.com/aws/aws-sdk-go-v2/blob/credentials/v1.8.0/credentials/LICENSE.txt,Apache-2.0 11 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds,https://github.com/aws/aws-sdk-go-v2/blob/feature/ec2/imds/v1.10.0/feature/ec2/imds/LICENSE.txt,Apache-2.0 12 | github.com/aws/aws-sdk-go-v2/internal/configsources,https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.1.14/internal/configsources/LICENSE.txt,Apache-2.0 13 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2,https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.4.8/internal/endpoints/v2/LICENSE.txt,Apache-2.0 14 | github.com/aws/aws-sdk-go-v2/internal/ini,https://github.com/aws/aws-sdk-go-v2/blob/internal/ini/v1.3.4/internal/ini/LICENSE.txt,Apache-2.0 15 | github.com/aws/aws-sdk-go-v2/internal/sync/singleflight,https://github.com/aws/aws-sdk-go-v2/blob/v1.16.7/internal/sync/singleflight/LICENSE,BSD-3-Clause 16 | github.com/aws/aws-sdk-go-v2/service/cloudtrail,https://github.com/aws/aws-sdk-go-v2/blob/service/cloudtrail/v1.13.0/service/cloudtrail/LICENSE.txt,Apache-2.0 17 | github.com/aws/aws-sdk-go-v2/service/ec2,https://github.com/aws/aws-sdk-go-v2/blob/service/ec2/v1.26.0/service/ec2/LICENSE.txt,Apache-2.0 18 | github.com/aws/aws-sdk-go-v2/service/iam,https://github.com/aws/aws-sdk-go-v2/blob/service/iam/v1.14.0/service/iam/LICENSE.txt,Apache-2.0 19 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding,https://github.com/aws/aws-sdk-go-v2/blob/service/internal/accept-encoding/v1.6.0/service/internal/accept-encoding/LICENSE.txt,Apache-2.0 20 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url,https://github.com/aws/aws-sdk-go-v2/blob/service/internal/presigned-url/v1.7.0/service/internal/presigned-url/LICENSE.txt,Apache-2.0 21 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared,https://github.com/aws/aws-sdk-go-v2/blob/service/internal/s3shared/v1.10.0/service/internal/s3shared/LICENSE.txt,Apache-2.0 22 | github.com/aws/aws-sdk-go-v2/service/lambda,https://github.com/aws/aws-sdk-go-v2/blob/service/lambda/v1.17.0/service/lambda/LICENSE.txt,Apache-2.0 23 | github.com/aws/aws-sdk-go-v2/service/organizations,https://github.com/aws/aws-sdk-go-v2/blob/service/organizations/v1.12.0/service/organizations/LICENSE.txt,Apache-2.0 24 | github.com/aws/aws-sdk-go-v2/service/rds,https://github.com/aws/aws-sdk-go-v2/blob/service/rds/v1.16.0/service/rds/LICENSE.txt,Apache-2.0 25 | github.com/aws/aws-sdk-go-v2/service/rolesanywhere,https://github.com/aws/aws-sdk-go-v2/blob/service/rolesanywhere/v1.0.0/service/rolesanywhere/LICENSE.txt,Apache-2.0 26 | github.com/aws/aws-sdk-go-v2/service/s3,https://github.com/aws/aws-sdk-go-v2/blob/service/s3/v1.23.0/service/s3/LICENSE.txt,Apache-2.0 27 | github.com/aws/aws-sdk-go-v2/service/secretsmanager,https://github.com/aws/aws-sdk-go-v2/blob/service/secretsmanager/v1.13.0/service/secretsmanager/LICENSE.txt,Apache-2.0 28 | github.com/aws/aws-sdk-go-v2/service/ssm,https://github.com/aws/aws-sdk-go-v2/blob/service/ssm/v1.20.0/service/ssm/LICENSE.txt,Apache-2.0 29 | github.com/aws/aws-sdk-go-v2/service/sso,https://github.com/aws/aws-sdk-go-v2/blob/service/sso/v1.9.0/service/sso/LICENSE.txt,Apache-2.0 30 | github.com/aws/aws-sdk-go-v2/service/sts,https://github.com/aws/aws-sdk-go-v2/blob/service/sts/v1.14.0/service/sts/LICENSE.txt,Apache-2.0 31 | github.com/aws/smithy-go,https://github.com/aws/smithy-go/blob/v1.12.0/LICENSE,Apache-2.0 32 | github.com/datadog/stratus-red-team/v2,https://github.com/datadog/stratus-red-team/blob/v2.2.3/v2/LICENSE,Apache-2.0 33 | github.com/datadog/threatest/pkg/threatest,https://github.com/datadog/threatest/blob/HEAD/LICENSE,Apache-2.0 34 | github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC 35 | github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.0/LICENSE,Apache-2.0 36 | github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause 37 | github.com/golang-jwt/jwt,https://github.com/golang-jwt/jwt/blob/v3.2.2/LICENSE,MIT 38 | github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.2/LICENSE,BSD-3-Clause 39 | github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.5.9/LICENSE,BSD-3-Clause 40 | github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.1.0/LICENSE,Apache-2.0 41 | github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.0/LICENSE,BSD-3-Clause 42 | github.com/googleapis/gnostic,https://github.com/googleapis/gnostic/blob/v0.5.5/LICENSE,Apache-2.0 43 | github.com/hashicorp/go-cleanhttp,https://github.com/hashicorp/go-cleanhttp/blob/v0.5.2/LICENSE,MPL-2.0 44 | github.com/hashicorp/go-uuid,https://github.com/hashicorp/go-uuid/blob/v1.0.0/LICENSE,MPL-2.0 45 | github.com/hashicorp/go-version,https://github.com/hashicorp/go-version/blob/v1.4.0/LICENSE,MPL-2.0 46 | github.com/hashicorp/hc-install,https://github.com/hashicorp/hc-install/blob/v0.3.2/LICENSE,MPL-2.0 47 | github.com/hashicorp/terraform-exec,https://github.com/hashicorp/terraform-exec/blob/v0.15.0/LICENSE,MPL-2.0 48 | github.com/hashicorp/terraform-json,https://github.com/hashicorp/terraform-json/blob/v0.13.0/LICENSE,MPL-2.0 49 | github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.12/LICENSE,BSD-3-Clause 50 | github.com/jmespath/go-jmespath,https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE,Apache-2.0 51 | github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT 52 | github.com/kevinburke/ssh_config,https://github.com/kevinburke/ssh_config/blob/4977a11b4351/LICENSE,MIT 53 | github.com/kylelemons/godebug,https://github.com/kylelemons/godebug/blob/v1.1.0/LICENSE,Apache-2.0 54 | github.com/moby/spdystream,https://github.com/moby/spdystream/blob/v0.2.0/LICENSE,Apache-2.0 55 | github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 56 | github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 57 | github.com/pkg/browser,https://github.com/pkg/browser/blob/ce105d075bb4/LICENSE,BSD-2-Clause 58 | github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause 59 | github.com/zclconf/go-cty/cty,https://github.com/zclconf/go-cty/blob/v1.9.1/LICENSE,MIT 60 | golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.1.0:LICENSE,BSD-3-Clause 61 | golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.1.0:LICENSE,BSD-3-Clause 62 | golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/6fdb5e3d:LICENSE,BSD-3-Clause 63 | golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.1.0:LICENSE,BSD-3-Clause 64 | golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.1.0:LICENSE,BSD-3-Clause 65 | golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.4.0:LICENSE,BSD-3-Clause 66 | golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/579cf78f:LICENSE,BSD-3-Clause 67 | google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.28.1/LICENSE,BSD-3-Clause 68 | gopkg.in/alessio/shellescape.v1,https://github.com/alessio/shellescape/blob/52074bc9df61/LICENSE,MIT 69 | gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause 70 | gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 71 | gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT 72 | k8s.io/api,https://github.com/kubernetes/api/blob/v0.23.3/LICENSE,Apache-2.0 73 | k8s.io/apimachinery/pkg,https://github.com/kubernetes/apimachinery/blob/v0.23.3/LICENSE,Apache-2.0 74 | k8s.io/apimachinery/third_party/forked/golang,https://github.com/kubernetes/apimachinery/blob/v0.23.3/third_party/forked/golang/LICENSE,BSD-3-Clause 75 | k8s.io/client-go,https://github.com/kubernetes/client-go/blob/v0.23.3/LICENSE,Apache-2.0 76 | k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.80.1/LICENSE,Apache-2.0 77 | k8s.io/kube-openapi/pkg,https://github.com/kubernetes/kube-openapi/blob/e816edb12b65/LICENSE,Apache-2.0 78 | k8s.io/utils,https://github.com/kubernetes/utils/blob/6203023598ed/LICENSE,Apache-2.0 79 | k8s.io/utils/internal/third_party/forked/golang/net,https://github.com/kubernetes/utils/blob/6203023598ed/internal/third_party/forked/golang/LICENSE,BSD-3-Clause 80 | sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/c049b76a60c6/LICENSE,Apache-2.0 81 | sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.2.1/LICENSE,Apache-2.0 82 | sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.3.0/LICENSE,MIT 83 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: mocks parser 2 | 3 | MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) 4 | ROOT_DIR := $(dir $(MAKEFILE_PATH)) 5 | 6 | all: build 7 | 8 | build: 9 | mkdir -p dist 10 | go build -o dist/threatest cmd/threatest/*.go 11 | 12 | test: 13 | go test $(shell go list ./... | grep -v examples) 14 | 15 | thirdparty-licenses: 16 | go get github.com/google/go-licenses 17 | go install github.com/google/go-licenses 18 | $${GOPATH}/bin/go-licenses csv github.com/datadog/threatest/pkg/threatest | sort > $(ROOT_DIR)/LICENSE-3rdparty.csv 19 | 20 | mocks: 21 | mockery --name=Detonator --dir pkg/threatest/detonators/ --output pkg/threatest/detonators/mocks 22 | mockery --name=AlertGeneratedMatcher --dir pkg/threatest/matchers/ --output pkg/threatest/matchers/mocks 23 | mockery --name=DatadogSecuritySignalsAPI --dir pkg/threatest/matchers/datadog --output pkg/threatest/matchers/datadog/mocks 24 | 25 | parser: 26 | go get github.com/atombender/go-jsonschema/... 27 | go install github.com/atombender/go-jsonschema/cmd/gojsonschema@latest 28 | $${GOPATH}/bin/gojsonschema -p parser schemas/threatest.schema.json > pkg/threatest/parser/parser.go -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Threatest 2 | Copyright 2022-Present Datadog, Inc. 3 | 4 | This product includes software developed at Datadog ( 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Threatest 2 | 3 | ![unit tests](https://github.com/DataDog/threatest/actions/workflows/test.yml/badge.svg) 4 | ![static analysis](https://github.com/DataDog/threatest/actions/workflows/static-analysis.yml/badge.svg) 5 | 6 |

7 | Threatest 8 |

9 | 10 | Threatest is a CLI and Go framework for testing threat detection end-to-end. 11 | 12 | Threatest allows you to **detonate** an attack technique, and verify that the alert you expect was generated in your favorite security platform. 13 | 14 | Read the announcement blog post: https://securitylabs.datadoghq.com/articles/threatest-end-to-end-testing-threat-detection/ 15 | 16 | ## Concepts 17 | 18 | ### Detonators 19 | 20 | A **detonator** describes how and where an attack technique is executed. 21 | 22 | Supported detonators: 23 | * Local command execution 24 | * SSH command execution 25 | * Stratus Red Team 26 | * AWS CLI detonator 27 | * AWS detonator (programmatic only, does not work with the CLI) 28 | 29 | ### Alert matchers 30 | 31 | An **alert matcher** is a platform-specific integration that can check if an expected alert was triggered. 32 | 33 | Supported alert matchers: 34 | * Datadog security signals 35 | 36 | ### Detonation and alert correlation 37 | 38 | Each detonation is assigned a UUID. This UUID is reflected in the detonation and used to ensure that the matched alert corresponds exactly to this detonation. 39 | 40 | The way this is done depends on the detonator; for instance, Stratus Red Team and the AWS Detonator inject it in the user-agent; the SSH detonator uses a parent process containing the UUID. 41 | 42 | ## Usage 43 | 44 | ### Through the CLI 45 | 46 | Threatest comes with a CLI that you can use to run test scenarios described as YAML, following a specific [schema](./schemas/threatest.schema.json). You can configure this schema in your editor to benefit from in-IDE linting and autocompletion (see [documentation for VSCode](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml#associating-a-schema-to-a-glob-pattern-via-yaml.schemas) using the [YAML](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) extension). 47 | 48 | Install the CLI by downloading a [binary release](https://github.com/DataDog/threatest/releases) or with Homebrew: 49 | 50 | ``` 51 | brew tap datadog/threatest https://github.com/datadog/threatest 52 | brew install datadog/threatest/threatest 53 | ``` 54 | 55 | Sample usage: 56 | 57 | ```bash 58 | $ threatest lint scenarios.threatest.yaml 59 | All 6 scenarios are syntaxically valid 60 | 61 | # Local detonation 62 | $ threatest run local-scenarios.threatest.yaml 63 | 64 | # Remote detonation over SSH 65 | $ threatest run scenarios.threatest.yaml --ssh-host test-box --ssh-username vagrant 66 | 67 | # Alternatively, specify SSH parameters from environment variables 68 | $ export THREATEST_SSH_HOST=test-box 69 | $ export THREATEST_SSH_USERNAME=vagrant 70 | $ threatest run scenarios.threatest.yaml 71 | ``` 72 | 73 | **Sample scenario definition files** 74 | 75 | * Detonating over SSH 76 | 77 | ```yaml 78 | scenarios: 79 | # Remote detonation over SSH 80 | # Note: SSH configuration is provided using the --ssh-host, --ssh-username and --ssh-keyfile CLI arguments 81 | - name: curl metadata service 82 | detonate: 83 | remoteDetonator: 84 | commands: ["curl http://169.254.169.254 --connect-timeout 1"] 85 | expectations: 86 | - timeout: 1m 87 | datadogSecuritySignal: 88 | name: "Network utility accessed cloud metadata service" 89 | severity: medium 90 | ``` 91 | 92 | * Detonating using Stratus Red Team 93 | 94 | ```yaml 95 | scenarios: 96 | # Stratus Red Team detonation 97 | # Note: You must be authenticated to the relevant cloud provider before running it 98 | # The example below is equivalent to manually running "stratus detonate aws.exfiltration.ec2-security-group-open-port-22-ingress" 99 | - name: opening a security group to the Internet 100 | detonate: 101 | stratusRedTeamDetonator: 102 | attackTechnique: aws.exfiltration.ec2-security-group-open-port-22-ingress 103 | expectations: 104 | - timeout: 15m 105 | datadogSecuritySignal: 106 | name: "Potential administrative port open to the world via AWS security group" 107 | ``` 108 | 109 | 110 | * Detonating using AWS CLI commands 111 | 112 | ```yaml 113 | scenarios: 114 | # AWS CLI detonation 115 | # Note: You must be authenticated to AWS before running it and have the AWS CLI installed 116 | - name: opening a security group to the Internet 117 | detonate: 118 | awsCliDetonator: 119 | script: | 120 | set -e 121 | 122 | # Setup 123 | vpc=$(aws ec2 create-vpc --cidr-block 10.0.0.0/16 --query Vpc.VpcId --output text) 124 | sg=$(aws ec2 create-security-group --group-name sample-sg --description "Test security group" --vpc-id $vpc --query GroupId --output text) 125 | 126 | # Open security group 127 | aws ec2 authorize-security-group-ingress --group-id $sg --protocol tcp --port 22 --cidr 0.0.0.0/0 128 | 129 | # Cleanup 130 | aws ec2 delete-security-group --group-id $sg 131 | aws ec2 delete-vpc --vpc-id $vpc 132 | expectations: 133 | - timeout: 15m 134 | datadogSecuritySignal: 135 | name: "Potential administrative port open to the world via AWS security group" 136 | ``` 137 | 138 | 139 | You can output the test results to a JSON file: 140 | 141 | ``` 142 | $ threatest run scenarios.threatest.yaml --output test-results.json 143 | $ cat test-results.json 144 | [ 145 | { 146 | "description": "change user password", 147 | "isSuccess": true, 148 | "errorMessage": "", 149 | "durationSeconds": 22.046627348, 150 | "timeDetonated": "2022-11-15T22:26:14.182844+01:00" 151 | }, 152 | { 153 | "description": "adding an SSH key", 154 | "isSuccess": true, 155 | "errorMessage": "", 156 | "durationSeconds": 23.604699625, 157 | "timeDetonated": "2022-11-15T22:26:14.182832+01:00" 158 | }, 159 | { 160 | "description": "change user password", 161 | "isSuccess": false, 162 | "errorMessage": "At least one scenario failed:\n\nchange user password returned: change user password: 1 assertions did not pass\n =\u003e Did not find Datadog security signal 'bar'\n", 163 | "durationSeconds": 3.505294235, 164 | "timeDetonated": "2022-11-15T22:26:36.229349+01:00" 165 | } 166 | ] 167 | ``` 168 | 169 | By default, scenarios are run with a maximum parallelism of 5. You can increase this setting using the `--parallelism` argument. 170 | Note that when using remote SSH detonators, each scenario running establishes a new SSH connection. 171 | 172 | ### Using Threatest programmatically 173 | 174 | See [examples](./examples) for complete programmatic usage example. 175 | 176 | #### Testing Datadog Cloud SIEM signals triggered by Stratus Red Team 177 | 178 | ```go 179 | threatest := Threatest() 180 | 181 | threatest.Scenario("AWS console login"). 182 | WhenDetonating(StratusRedTeamTechnique("aws.initial-access.console-login-without-mfa")). 183 | Expect(DatadogSecuritySignal("AWS Console login without MFA").WithSeverity("medium")). 184 | WithTimeout(15 * time.Minute) 185 | 186 | assert.NoError(t, threatest.Run()) 187 | ``` 188 | 189 | ### Testing Datadog Cloud Workload Security signals triggered by running commands over SSH 190 | 191 | ```go 192 | ssh, _ := NewSSHCommandExecutor("test-box", "", "") 193 | 194 | threatest := Threatest() 195 | 196 | threatest.Scenario("curl to metadata service"). 197 | WhenDetonating(NewCommandDetonator(ssh, "curl http://169.254.169.254 --connect-timeout 1")). 198 | Expect(DatadogSecuritySignal("EC2 Instance Metadata Service Accessed via Network Utility")) 199 | 200 | assert.NoError(t, threatest.Run()) 201 | ``` 202 | -------------------------------------------------------------------------------- /cmd/threatest/lint.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/datadog/threatest/pkg/threatest" 7 | "github.com/datadog/threatest/pkg/threatest/parser" 8 | log "github.com/sirupsen/logrus" 9 | "github.com/spf13/cobra" 10 | "os" 11 | ) 12 | 13 | // LintCommand implements syntax verification of a Threatest scenario file 14 | type LintCommand struct { 15 | InputFiles []string 16 | } 17 | 18 | func (m *LintCommand) Do() error { 19 | if len(m.InputFiles) == 0 { 20 | return errors.New("please provide at least 1 scenario") 21 | } 22 | var numScenarios = 0 23 | for _, inputFile := range m.InputFiles { 24 | rawScenario, err := os.ReadFile(inputFile) 25 | if err != nil { 26 | return fmt.Errorf("unable to read input file %s: %v", inputFile, err) 27 | } 28 | scenarios, err := parser.Parse(rawScenario, "unused", "", "") 29 | if err != nil { 30 | return fmt.Errorf("unable to parse input file %s: %v", inputFile, err) 31 | } 32 | for _, scenario := range scenarios { 33 | if err := validateScenario(scenario); err != nil { 34 | return fmt.Errorf("invalid scenario '%s': %s", scenario.Name, err.Error()) 35 | } 36 | } 37 | numScenarios += len(scenarios) 38 | } 39 | log.Infof("All %d scenarios are syntaxically valid", numScenarios) 40 | return nil 41 | } 42 | 43 | func validateScenario(scenario *threatest.Scenario) error { 44 | if scenario.Detonator == nil { 45 | return errors.New("no detonator defined") 46 | } 47 | if len(scenario.Assertions) == 0 { 48 | return errors.New("no assertion defined") 49 | } 50 | return nil 51 | } 52 | 53 | func NewLintCommand() *cobra.Command { 54 | lintCmd := &cobra.Command{ 55 | Use: "lint", 56 | Short: "Validate the format of scenarios", 57 | SilenceUsage: true, 58 | Example: "lint /path/to/scenario/1 [/path/to/scenario/2]...", 59 | RunE: func(cmd *cobra.Command, args []string) error { 60 | command := LintCommand{ 61 | InputFiles: args, 62 | } 63 | return command.Do() 64 | }, 65 | } 66 | 67 | return lintCmd 68 | } 69 | -------------------------------------------------------------------------------- /cmd/threatest/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "github.com/spf13/cobra" 6 | "os" 7 | ) 8 | 9 | var rootCmd = &cobra.Command{ 10 | Use: "threatest", 11 | } 12 | 13 | func init() { 14 | rootCmd.AddCommand(NewRunCommand()) 15 | rootCmd.AddCommand(NewLintCommand()) 16 | } 17 | 18 | func main() { 19 | if os.Getenv("THREATEST_DEBUG") == "1" { 20 | log.SetLevel(log.DebugLevel) 21 | } 22 | if err := rootCmd.Execute(); err != nil { 23 | os.Exit(1) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cmd/threatest/run.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "github.com/datadog/threatest/pkg/threatest" 8 | "github.com/datadog/threatest/pkg/threatest/parser" 9 | log "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | "math" 12 | "os" 13 | "strconv" 14 | "time" 15 | ) 16 | 17 | // RunCommand implements the command to run Threatest test scenarios 18 | type RunCommand struct { 19 | SSHConfig *SSHConfiguration 20 | InputFiles []string 21 | Parallelism int 22 | JsonOutputFile string 23 | } 24 | 25 | type SSHConfiguration struct { 26 | SSHHost string 27 | SSHUsername string 28 | SSHKey string 29 | } 30 | 31 | type ScenarioRunResult struct { 32 | Description string `json:"description"` 33 | Success bool `json:"isSuccess"` 34 | ErrorMessage string `json:"errorMessage"` 35 | DurationSeconds float64 `json:"durationSeconds"` 36 | TimeDetonated time.Time `json:"timeDetonated"` 37 | //TODO: We possibly want to add some metadata about the kind of detonation 38 | } 39 | 40 | func NewRunCommand() *cobra.Command { 41 | var sshHost string 42 | var sshUsername string 43 | var sshKey string 44 | var parallelism int 45 | var jsonOutputFile string 46 | 47 | runCmd := &cobra.Command{ 48 | Use: "run", 49 | Short: "Run Threatest scenarios", 50 | SilenceUsage: true, 51 | Example: "run /path/to/scenario/1 [/path/to/scenario/2]...", 52 | RunE: func(cmd *cobra.Command, args []string) error { 53 | command := RunCommand{ 54 | InputFiles: args, 55 | Parallelism: parallelism, 56 | JsonOutputFile: jsonOutputFile, 57 | SSHConfig: &SSHConfiguration{ 58 | SSHHost: sshHost, 59 | SSHUsername: sshUsername, 60 | SSHKey: sshKey, 61 | }, 62 | } 63 | 64 | return command.Do() 65 | }, 66 | } 67 | 68 | runCmd.Flags().StringVarP(&sshHost, "ssh-host", "", os.Getenv("THREATEST_SSH_HOST"), "SSH host to connect to for remote command detonation. Can also be specified through THREATEST_SSH_HOST") 69 | runCmd.Flags().StringVarP(&sshUsername, "ssh-username", "", os.Getenv("THREATEST_SSH_USERNAME"), "SSH username to use for remote command detonation (leave empty to use system configuration). Can also be specified through THREATEST_SSH_USERNAME") 70 | runCmd.Flags().StringVarP(&sshKey, "ssh-key", "", os.Getenv("THREATEST_SSH_KEY"), "SSH keypair to use for remote command detonation (leave empty to use system configuration). Can also be specified through THREATEST_SSH_KEY. Only unencrypted keys are currently supported") 71 | runCmd.Flags().StringVarP(&jsonOutputFile, "output", "o", "", "Write JSON test results to the specified file") 72 | runCmd.Flags().IntVarP(¶llelism, "max-parallelism", "", getDefaultParallelism(), "Maximal parallelism to run the scenarios with. Can also be set through THREATEST_MAX_PARALLELISM") 73 | 74 | return runCmd 75 | } 76 | 77 | func getDefaultParallelism() int { 78 | const DefaultParallelism = 5 79 | if parallelism, isSet := os.LookupEnv("THREATEST_MAX_PARALLELISM"); isSet { 80 | parsedParallelism, err := strconv.Atoi(parallelism) 81 | if err != nil { 82 | log.Fatalf("unable to convert max parallelism '%s' to integer: %v", parallelism, err) 83 | } 84 | return parsedParallelism 85 | } 86 | return DefaultParallelism 87 | } 88 | 89 | func (m *RunCommand) Do() error { 90 | if err := m.Validate(); err != nil { 91 | return err 92 | } 93 | 94 | var allScenarios []*threatest.Scenario 95 | 96 | for _, inputFile := range m.InputFiles { 97 | rawScenario, err := os.ReadFile(inputFile) 98 | if err != nil { 99 | return fmt.Errorf("unable to read input file %s: %v", inputFile, err) 100 | } 101 | scenario, err := parser.Parse(rawScenario, m.SSHConfig.SSHHost, m.SSHConfig.SSHUsername, m.SSHConfig.SSHKey) 102 | if err != nil { 103 | return fmt.Errorf("unable to parse input file %s: %v", inputFile, err) 104 | } 105 | allScenarios = append(allScenarios, scenario...) 106 | } 107 | 108 | var hasError = false 109 | results := m.runScenariosParallel(allScenarios, func(result *ScenarioRunResult) { 110 | roundedDuration := math.Round(result.DurationSeconds*100) / 100 111 | if result.Success { 112 | log.Infof("Scenario '%s' passed in %.2f seconds", result.Description, roundedDuration) 113 | } else { 114 | hasError = true 115 | log.Errorf("Scenario '%s' failed in %.2f seconds: %s", result.Description, roundedDuration, result.ErrorMessage) 116 | } 117 | }) 118 | 119 | // Handle output file 120 | if m.JsonOutputFile != "" { 121 | if err := m.writeJsonOutput(results); err != nil { 122 | return err 123 | } 124 | log.Infof("Wrote scenario test results to %s", m.JsonOutputFile) 125 | } 126 | 127 | // Return an error to exit with a non-zero status code if at least one test failed 128 | if hasError { 129 | return fmt.Errorf("at least 1 scenario failed") 130 | } else { 131 | return nil 132 | } 133 | } 134 | 135 | func (m *RunCommand) Validate() error { 136 | if len(m.InputFiles) == 0 { 137 | return errors.New("please provide at least 1 scenario") 138 | } 139 | 140 | // If an SSH key is provided, check it exists 141 | if sshKey := m.SSHConfig.SSHKey; sshKey != "" { 142 | if _, err := os.Stat(sshKey); err != nil && sshKey != "" { 143 | return fmt.Errorf("invalid SSH key file %s: %v", sshKey, err) 144 | } 145 | } 146 | 147 | return nil 148 | } 149 | 150 | // runScenariosParallel runs all the provided scenarios in parallel, honoring the maximum parallelism 151 | // every time a test completes, the callback function is invoked 152 | func (m *RunCommand) runScenariosParallel(allScenarios []*threatest.Scenario, callback func(result *ScenarioRunResult)) []ScenarioRunResult { 153 | numWorkers := m.Parallelism 154 | // No point in having more workers than scenarios to run 155 | if numScenarios := len(allScenarios); numScenarios < numWorkers { 156 | numWorkers = numScenarios 157 | } 158 | 159 | log.Infof("Running %d scenarios with a parallelism of %d", len(allScenarios), numWorkers) 160 | 161 | // Channel to hold "tasks" (scenarios to run) 162 | scenarioChan := make(chan *threatest.Scenario, numWorkers) 163 | // Channel to hold test results 164 | resultsChan := make(chan *ScenarioRunResult) 165 | 166 | // Create 1 worker by desired parallelism unit 167 | for worker := 0; worker < numWorkers; worker++ { 168 | go m.runSingleScenario(scenarioChan, resultsChan) 169 | } 170 | 171 | // Submit each scenario 172 | for _, scenario := range allScenarios { 173 | scenarioChan <- scenario 174 | } 175 | 176 | // Retrieve results as they are produced 177 | var allResults []ScenarioRunResult 178 | for range allScenarios { 179 | result := <-resultsChan 180 | callback(result) 181 | allResults = append(allResults, *result) 182 | } 183 | 184 | return allResults 185 | } 186 | 187 | // runSingleScenario runs inside a goroutine and uses Threatest to run one scenario 188 | func (m *RunCommand) runSingleScenario(scenarios <-chan *threatest.Scenario, results chan<- *ScenarioRunResult) { 189 | for scenario := range scenarios { 190 | runner := threatest.Threatest() 191 | runner.Scenarios = append(runner.Scenarios, scenario) 192 | runner.Interval = 2 * time.Second 193 | 194 | start := time.Now() 195 | err := runner.Run() 196 | end := time.Now() 197 | 198 | var errorMessage = "" 199 | if err != nil { 200 | errorMessage = err.Error() 201 | } 202 | 203 | results <- &ScenarioRunResult{ 204 | Description: scenario.Name, 205 | ErrorMessage: errorMessage, 206 | Success: err == nil, 207 | DurationSeconds: end.Sub(start).Seconds(), 208 | TimeDetonated: start, 209 | } 210 | } 211 | } 212 | 213 | func (m *RunCommand) writeJsonOutput(results []ScenarioRunResult) error { 214 | outputBytes, err := json.MarshalIndent(results, "", " ") 215 | if err != nil { 216 | return fmt.Errorf("unable to convert scenario test results to JSON: %w", err) 217 | } 218 | 219 | if err := os.WriteFile(m.JsonOutputFile, outputBytes, 0600); err != nil { 220 | return fmt.Errorf("unable to write scenario test results to %s: %v", m.JsonOutputFile, err) 221 | } 222 | 223 | return nil 224 | } 225 | -------------------------------------------------------------------------------- /examples/cli-usage/README.md: -------------------------------------------------------------------------------- 1 | # Sample CLI usage 2 | 3 | Sample usage: 4 | 5 | ``` 6 | threatest run scenarios.threatest.yaml --ssh-host test-box --output test-results.json 7 | ``` 8 | 9 | Sample scenario test output file: 10 | 11 | ```json 12 | [ 13 | { 14 | "description": "curl metadata service", 15 | "isSuccess": true, 16 | "errorMessage": "", 17 | "durationSeconds": 20.175771331, 18 | "timeDetonated": "2022-11-15T22:41:28.137922+01:00" 19 | }, 20 | { 21 | "description": "opening a security group to the Internet", 22 | "isSuccess": true, 23 | "errorMessage": "", 24 | "durationSeconds": 114.920678743, 25 | "timeDetonated": "2022-11-15T22:41:28.137932+01:00" 26 | } 27 | ] 28 | ``` 29 | 30 | You can use Threatest's JSONSchema in your editor to benefit from in-IDE linting and autocompletion (see [documentation for VSCode](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml#associating-a-schema-to-a-glob-pattern-via-yaml.schemas) using the [YAML](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) extension). 31 | -------------------------------------------------------------------------------- /examples/cli-usage/scenarios.threatest.yaml: -------------------------------------------------------------------------------- 1 | scenarios: 2 | # Example 1: Remote detonation over SSH 3 | # Note: SSH configuration is provided using the --ssh-host, --ssh-username and --ssh-keyfile CLI arguments 4 | - name: curl metadata service 5 | detonate: 6 | remoteDetonator: 7 | commands: ["curl http://169.254.169.254 --connect-timeout 1"] 8 | expectations: 9 | - timeout: 1m 10 | datadogSecuritySignal: 11 | name: "Network utility accessed cloud metadata service" 12 | severity: medium 13 | 14 | # Example 2: Stratus Red Team detonation 15 | # Note: You must be authenticated to the relevant cloud provider before running it 16 | # The example below is equivalent to manually running "stratus detonate aws.exfiltration.ec2-security-group-open-port-22-ingress" 17 | - name: opening a security group to the Internet 18 | detonate: 19 | stratusRedTeamDetonator: 20 | attackTechnique: aws.exfiltration.ec2-security-group-open-port-22-ingress 21 | expectations: 22 | - timeout: 15m 23 | datadogSecuritySignal: 24 | name: "Potential administrative port open to the world via AWS security group" -------------------------------------------------------------------------------- /examples/programmatic-usage/README.md: -------------------------------------------------------------------------------- 1 | # Sample programmatic usage 2 | 3 | ## Testing Cloud SIEM rules 4 | 5 | [`cloudsiem_alerts_test.go`](cloudsiem_alerts_test.go) uses Stratus Red Team (through its programmatic interface) to detonate AWS attack ttechniques, then polls the Datadog API to verify that an expected Cloud SIEM signal was created. 6 | 7 | Sample usage: 8 | 9 | ``` 10 | go test -timeout 99999s cloudsiem_alerts_test.go -v 11 | ``` 12 | 13 | Sample output: 14 | 15 | ``` 16 | === RUN TestCloudSIEMAWSAlerts 17 | Detonating 'aws.initial-access.console-login-without-mfa' with Stratus Red Team 18 | 2022/06/16 16:31:08 AWS console login: Confirmed that the expected signal (Datadog security signal 'An IAM user was created') was created in Datadog (took 17 seconds). 19 | 2022/06/16 16:31:08 AWS console login: Confirmed that the expected signal (Datadog security signal 'AWS Console login without MFA') was created in Datadog (took 17 seconds). 20 | 2022/06/16 16:31:08 AWS console login: All assertions passed 21 | 22 | Detonating 'aws.persistence.iam-create-admin-user' with Stratus Red Team 23 | 2022/06/16 16:31:14 AWS persistence IAM user: Confirmed that the expected signal (Datadog security signal 'An IAM user was created') was created in Datadog (took 0 seconds). 24 | 2022/06/16 16:31:14 AWS persistence IAM user: All assertions passed 25 | --- PASS: TestCloudSIEMAWSAlerts (126.53s) 26 | PASS 27 | ``` 28 | 29 | ## Testing CWS rules 30 | 31 | [`cws_alerts_tests.go`](cws_alerts_test.go) assumes you have a machine `test-box` configured in your OpenSSH configuration, and running CWS (for instance using [datadog-security-monitoring-strater](https://github.com/DataDog/datadog-security-monitoring-starter/tree/main/1.virtual-machine)). 32 | 33 | It will detonate several commands through SSH on the machine, and poll the Datadog API to verify that the expected CWS signals were generated. 34 | 35 | Sample usage: 36 | 37 | ``` 38 | go test cws_alerts_test.go -v 39 | ``` 40 | 41 | Sample output: 42 | 43 | ``` 44 | === RUN TestCWSAlerts 45 | Connecting over SSH 46 | Connection succeeded 47 | 2022/06/16 16:25:20 curl to metadata service: Confirmed that the expected signal (Datadog security signal 'EC2 Instance Metadata Service Accessed via Network Utility') was created in Datadog (took 12 seconds). 48 | 2022/06/16 16:25:20 curl to metadata service: All assertions passed 49 | 2022/06/16 16:25:42 Java spawning shell: Confirmed that the expected signal (Datadog security signal 'Java process spawned shell/utility') was created in Datadog (took 19 seconds). 50 | 2022/06/16 16:25:42 Java spawning shell: All assertions passed 51 | --- PASS: TestCWSAlerts (45.64s) 52 | ``` 53 | 54 | ``` 55 | === RUN TestCWSAlertsV2 56 | Connecting over SSH 57 | Connection succeeded 58 | === RUN TestCWSAlertsV2/curl_to_metadata_service 59 | === PAUSE TestCWSAlertsV2/curl_to_metadata_service 60 | === RUN TestCWSAlertsV2/java_spawns_shell 61 | === PAUSE TestCWSAlertsV2/java_spawns_shell 62 | === CONT TestCWSAlertsV2/java_spawns_shell 63 | === CONT TestCWSAlertsV2/curl_to_metadata_service 64 | 2022/06/16 16:26:02 curl to metadata service: Confirmed that the expected signal (Datadog security signal 'EC2 Instance Metadata Service Accessed via Network Utility') was created in Datadog (took 11 seconds). 65 | 2022/06/16 16:26:02 curl to metadata service: All assertions passed 66 | 2022/06/16 16:26:02 java spawns shell: Confirmed that the expected signal (Datadog security signal 'Java process spawned shell/utility') was created in Datadog (took 17 seconds). 67 | 2022/06/16 16:26:02 java spawns shell: All assertions passed 68 | --- PASS: TestCWSAlertsV2 (0.06s) 69 | --- PASS: TestCWSAlertsV2/java_spawns_shell (20.12s) 70 | --- PASS: TestCWSAlertsV2/curl_to_metadata_service (20.24s) 71 | PASS 72 | ``` 73 | 74 | ## Using the custom AWS detonator and Terratest to prepare infrastructure 75 | 76 | See [custom-aws-detonator-terratest](custom-aws-detonator-terratest). 77 | 78 | ## Using the local detonator 79 | 80 | Setup: Export DD API key and App key as environment variables 81 | ``` 82 | export DD_API_KEY= 83 | export DD_APP_KEY= 84 | ``` 85 | 86 | Sample usage: 87 | ``` 88 | go test local_detonator_test.go -v 89 | ``` 90 | 91 | Sample output: 92 | ``` 93 | === RUN TestLocalDetonator 94 | Executing curl http://169.254.169.254 --connect-timeout 5 95 | Executing cp /bin/bash /tmp/java; /tmp/java -c "curl 1.1.1.1" 96 | Test failed: At least one scenario failed: 97 | 98 | curl to metadata service returned: curl to metadata service: 1 assertions did not pass 99 | => Did not find Datadog security signal 'Network utility executed' 100 | Java spawning shell returned: Java spawning shell: 1 assertions did not pass 101 | => Did not find Datadog security signal 'Java process spawned shell' 102 | 103 | --- FAIL: TestLocalDetonator (361.94s) 104 | FAIL 105 | FAIL command-line-arguments 361.954s 106 | FAIL 107 | ``` 108 | 109 | ``` 110 | === RUN TestLocalDetonator 111 | Executing curl http://169.254.169.254 --connect-timeout 5 112 | Executing cp /bin/bash /tmp/java; /tmp/java -c "curl 1.1.1.1" 113 | --- PASS: TestLocalDetonator (38.10s) 114 | PASS 115 | ok command-line-arguments 38.121s 116 | ``` 117 | -------------------------------------------------------------------------------- /examples/programmatic-usage/cloudsiem_alerts_test.go: -------------------------------------------------------------------------------- 1 | package programmatic_usage 2 | 3 | import ( 4 | _ "github.com/datadog/stratus-red-team/v2/pkg/stratus/loader" // Note: This import is needed 5 | . "github.com/datadog/threatest/pkg/threatest" 6 | . "github.com/datadog/threatest/pkg/threatest/detonators" 7 | . "github.com/datadog/threatest/pkg/threatest/matchers/datadog" 8 | "github.com/stretchr/testify/require" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | /*/ 14 | func TestCloudSIEMAWSAlerts(t *testing.T) { 15 | threatest := Threatest() 16 | threatest.Interval = 0 17 | 18 | threatest.Scenario("AWS console login"). 19 | WhenDetonating(StratusRedTeamTechnique("aws.initial-access.console-login-without-mfa")). 20 | Expect(DatadogSecuritySignal("AWS Console login without MFA").WithSeverity("medium")). 21 | WithTimeout(10 * time.Minute) 22 | 23 | threatest.Scenario("Opening port 22 of a security group to the Internet"). 24 | WhenDetonating(StratusRedTeamTechnique("aws.exfiltration.ec2-security-group-open-port-22-ingress")). 25 | Expect(DatadogSecuritySignal("Potential administrative port open to the world via AWS security group")). 26 | WithTimeout(10 * time.Minute) 27 | 28 | threatest.Scenario("Exfiltrating an EBS snapshot"). 29 | WhenDetonating(StratusRedTeamTechnique("aws.exfiltration.ec2-share-ebs-snapshot")). 30 | Expect(DatadogSecuritySignal("AWS EBS Snapshot possible exfiltration")). 31 | WithTimeout(10 * time.Minute) 32 | 33 | threatest.Scenario("Disabling CloudTrail through event selectors"). 34 | WhenDetonating(StratusRedTeamTechnique("aws.defense-evasion.cloudtrail-event-selectors")). 35 | Expect(DatadogSecuritySignal("AWS Disable Cloudtrail with event selectors")). 36 | WithTimeout(10 * time.Minute) 37 | 38 | require.Nil(t, threatest.Run()) 39 | } 40 | //*/ 41 | 42 | /* 43 | This function shows a way of writing data-driven Go tests, which has the nice property of parallelization 44 | and showing errors per test case. It should be a little less easy to write, but faster 45 | */ 46 | func TestCloudSIEMAWSAlertsParallel(t *testing.T) { 47 | testCases := []struct { 48 | StratusRedTeamTTP string 49 | ExpectedSignalName string 50 | }{ 51 | {"aws.initial-access.console-login-without-mfa", "AWS Console login without MFA"}, 52 | {"aws.exfiltration.ec2-security-group-open-port-22-ingress", "Potential administrative port open to the world via AWS security group"}, 53 | {"aws.exfiltration.ec2-share-ebs-snapshot", "AWS EBS Snapshot possible exfiltration"}, 54 | {"aws.defense-evasion.cloudtrail-event-selectors", "AWS Disable Cloudtrail with event selectors"}, 55 | } 56 | 57 | for i := range testCases { 58 | scenario := testCases[i] 59 | t.Run(scenario.StratusRedTeamTTP, func(t *testing.T) { 60 | t.Parallel() 61 | threatest := Threatest() 62 | threatest.Scenario(scenario.StratusRedTeamTTP). 63 | WhenDetonating(StratusRedTeamTechnique(scenario.StratusRedTeamTTP)). 64 | Expect(DatadogSecuritySignal(scenario.ExpectedSignalName)). 65 | WithTimeout(15 * time.Minute) 66 | 67 | require.Nil(t, threatest.Run()) 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /examples/programmatic-usage/custom-aws-detonator-terratest/README.md: -------------------------------------------------------------------------------- 1 | This examples shows how to use Threatest with pre-requisite infrastructure spun up by 2 | [Terratest](https://terratest.gruntwork.io/). 3 | 4 | Note that when the attack technique you want to simulate is supported by Stratus Red Team, 5 | it is simpler to use the Stratus Red Team detonator. 6 | However, the AWS Detonator allows you to detonate arbitrary code using the AWS SDK, for reproducing custom or more advanced attack techniques. 7 | 8 | The AWS detonator injects the detonation UUID inside of the AWS SDK user-agent, allowing to 9 | correlate the alert with the detonation. 10 | 11 | In this test, we attempt to change the S3 bucket of a running CloudTrail trail, simulating 12 | an attacker who attempts to disrupt CloudTrail logging. 13 | 14 | You need Terraform installed to run this test. 15 | 16 | ``` 17 | go test -v ./custom_aws_detonator_with_terratest_test.go 18 | ``` -------------------------------------------------------------------------------- /examples/programmatic-usage/custom-aws-detonator-terratest/custom_aws_detonator_with_terratest_test.go: -------------------------------------------------------------------------------- 1 | package examples 2 | 3 | import ( 4 | "context" 5 | "github.com/aws/aws-sdk-go-v2/aws" 6 | "github.com/aws/aws-sdk-go-v2/service/cloudtrail" 7 | . "github.com/datadog/threatest/pkg/threatest" 8 | . "github.com/datadog/threatest/pkg/threatest/detonators" 9 | . "github.com/datadog/threatest/pkg/threatest/matchers/datadog" 10 | "github.com/google/uuid" 11 | "github.com/stretchr/testify/assert" 12 | "time" 13 | 14 | "github.com/gruntwork-io/terratest/modules/terraform" 15 | "testing" 16 | ) 17 | 18 | func TestCustomAWSDetonatorWithTerratest(t *testing.T) { 19 | // Step 1: Use terratest to spin up our pre-requisite infrastructure 20 | terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ 21 | TerraformDir: "./terraform", 22 | }) 23 | defer terraform.Destroy(t, terraformOptions) 24 | terraform.InitAndApply(t, terraformOptions) 25 | trailName := terraform.Output(t, terraformOptions, "cloudtrail_trail_name") 26 | 27 | // Step 2: Test scenario 28 | threatest := Threatest() 29 | 30 | threatest.Scenario("stopping cloudtrail trail"). 31 | WhenDetonating(NewAWSDetonator(func(config aws.Config, _ uuid.UUID) error { 32 | // Threatest automatically injects the detonation UUID inside the AWS SDK user-agent 33 | // allowing to correlate the alert with the detonation 34 | cloudtrailClient := cloudtrail.NewFromConfig(config) 35 | cloudtrailClient.UpdateTrail(context.Background(), &cloudtrail.UpdateTrailInput{ 36 | Name: aws.String(trailName), 37 | S3BucketName: aws.String("nope"), 38 | }) 39 | return nil 40 | })). 41 | Expect(DatadogSecuritySignal("AWS CloudTrail configuration modified")). 42 | WithTimeout(15 * time.Minute) 43 | 44 | assert.NoError(t, threatest.Run()) 45 | } 46 | -------------------------------------------------------------------------------- /examples/programmatic-usage/custom-aws-detonator-terratest/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/datadog/threatest/examples/custom-aws-detonator-terratest 2 | 3 | require ( 4 | github.com/aws/aws-sdk-go-v2 v1.16.7 5 | github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.13.0 6 | github.com/datadog/threatest v0.0.0-20220727103622-b9af76ea2391 7 | github.com/google/uuid v1.3.0 8 | github.com/gruntwork-io/terratest v0.40.18 9 | github.com/stretchr/testify v1.7.0 10 | ) 11 | 12 | require ( 13 | cloud.google.com/go v0.99.0 // indirect 14 | cloud.google.com/go/storage v1.14.0 // indirect 15 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0 // indirect 16 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0 // indirect 17 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect 18 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 // indirect 19 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0 // indirect 20 | github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect 21 | github.com/DataDog/datadog-api-client-go v1.14.0 // indirect 22 | github.com/agext/levenshtein v1.2.3 // indirect 23 | github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect 24 | github.com/aws/aws-sdk-go v1.40.56 // indirect 25 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.1.0 // indirect 26 | github.com/aws/aws-sdk-go-v2/config v1.13.0 // indirect 27 | github.com/aws/aws-sdk-go-v2/credentials v1.8.0 // indirect 28 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 // indirect 29 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.14 // indirect 30 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8 // indirect 31 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.4 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/ec2 v1.26.0 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/iam v1.14.0 // indirect 34 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.6.0 // indirect 35 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 // indirect 36 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.10.0 // indirect 37 | github.com/aws/aws-sdk-go-v2/service/lambda v1.17.0 // indirect 38 | github.com/aws/aws-sdk-go-v2/service/organizations v1.12.0 // indirect 39 | github.com/aws/aws-sdk-go-v2/service/rds v1.16.0 // indirect 40 | github.com/aws/aws-sdk-go-v2/service/rolesanywhere v1.0.0 // indirect 41 | github.com/aws/aws-sdk-go-v2/service/s3 v1.23.0 // indirect 42 | github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.13.0 // indirect 43 | github.com/aws/aws-sdk-go-v2/service/ssm v1.20.0 // indirect 44 | github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 // indirect 45 | github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 // indirect 46 | github.com/aws/smithy-go v1.12.0 // indirect 47 | github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect 48 | github.com/datadog/stratus-red-team/v2 v2.2.3 // indirect 49 | github.com/davecgh/go-spew v1.1.1 // indirect 50 | github.com/go-logr/logr v1.2.0 // indirect 51 | github.com/gogo/protobuf v1.3.2 // indirect 52 | github.com/golang-jwt/jwt v3.2.2+incompatible // indirect 53 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 54 | github.com/golang/protobuf v1.5.2 // indirect 55 | github.com/golang/snappy v0.0.4 // indirect 56 | github.com/google/go-cmp v0.5.8 // indirect 57 | github.com/google/gofuzz v1.1.0 // indirect 58 | github.com/googleapis/gax-go/v2 v2.1.1 // indirect 59 | github.com/googleapis/gnostic v0.5.5 // indirect 60 | github.com/hashicorp/errwrap v1.0.0 // indirect 61 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 62 | github.com/hashicorp/go-getter v1.6.1 // indirect 63 | github.com/hashicorp/go-multierror v1.1.1 // indirect 64 | github.com/hashicorp/go-safetemp v1.0.0 // indirect 65 | github.com/hashicorp/go-uuid v1.0.0 // indirect 66 | github.com/hashicorp/go-version v1.4.0 // indirect 67 | github.com/hashicorp/hc-install v0.3.2 // indirect 68 | github.com/hashicorp/hcl/v2 v2.9.1 // indirect 69 | github.com/hashicorp/terraform-exec v0.15.0 // indirect 70 | github.com/hashicorp/terraform-json v0.13.0 // indirect 71 | github.com/imdario/mergo v0.3.12 // indirect 72 | github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect 73 | github.com/jmespath/go-jmespath v0.4.0 // indirect 74 | github.com/json-iterator/go v1.1.12 // indirect 75 | github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect 76 | github.com/klauspost/compress v1.13.0 // indirect 77 | github.com/kylelemons/godebug v1.1.0 // indirect 78 | github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect 79 | github.com/mitchellh/go-homedir v1.1.0 // indirect 80 | github.com/mitchellh/go-testing-interface v1.0.0 // indirect 81 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 82 | github.com/moby/spdystream v0.2.0 // indirect 83 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 84 | github.com/modern-go/reflect2 v1.0.2 // indirect 85 | github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect 86 | github.com/pmezard/go-difflib v1.0.0 // indirect 87 | github.com/spf13/pflag v1.0.5 // indirect 88 | github.com/tmccombs/hcl2json v0.3.3 // indirect 89 | github.com/ulikunitz/xz v0.5.8 // indirect 90 | github.com/zclconf/go-cty v1.9.1 // indirect 91 | go.opencensus.io v0.23.0 // indirect 92 | golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88 // indirect 93 | golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect 94 | golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect 95 | golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect 96 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect 97 | golang.org/x/text v0.3.7 // indirect 98 | golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect 99 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 100 | google.golang.org/api v0.63.0 // indirect 101 | google.golang.org/appengine v1.6.7 // indirect 102 | google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect 103 | google.golang.org/grpc v1.43.0 // indirect 104 | google.golang.org/protobuf v1.27.1 // indirect 105 | gopkg.in/alessio/shellescape.v1 v1.0.0-20170105083845-52074bc9df61 // indirect 106 | gopkg.in/inf.v0 v0.9.1 // indirect 107 | gopkg.in/yaml.v2 v2.4.0 // indirect 108 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 109 | k8s.io/api v0.23.3 // indirect 110 | k8s.io/apimachinery v0.23.3 // indirect 111 | k8s.io/client-go v0.23.3 // indirect 112 | k8s.io/klog/v2 v2.30.0 // indirect 113 | k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect 114 | k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect 115 | sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect 116 | sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect 117 | sigs.k8s.io/yaml v1.2.0 // indirect 118 | ) 119 | 120 | replace github.com/datadog/threatest v0.0.0-20220727103622-b9af76ea2391 => ../../ 121 | 122 | go 1.18 123 | -------------------------------------------------------------------------------- /examples/programmatic-usage/custom-aws-detonator-terratest/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform 2 | terraform.tfstate* 3 | -------------------------------------------------------------------------------- /examples/programmatic-usage/custom-aws-detonator-terratest/terraform/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.23.0" 6 | constraints = ">= 3.63.0" 7 | hashes = [ 8 | "h1:j6RGCfnoLBpzQVOKUbGyxf4EJtRvQClKplO+WdXL5O0=", 9 | "zh:17adbedc9a80afc571a8de7b9bfccbe2359e2b3ce1fffd02b456d92248ec9294", 10 | "zh:23d8956b031d78466de82a3d2bbe8c76cc58482c931af311580b8eaef4e6a38f", 11 | "zh:343fe19e9a9f3021e26f4af68ff7f4828582070f986b6e5e5b23d89df5514643", 12 | "zh:6b8ff83d884b161939b90a18a4da43dd464c4b984f54b5f537b2870ce6bd94bc", 13 | "zh:7777d614d5e9d589ad5508eecf4c6d8f47d50fcbaf5d40fa7921064240a6b440", 14 | "zh:82f4578861a6fd0cde9a04a1926920bd72d993d524e5b34d7738d4eff3634c44", 15 | "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", 16 | "zh:a08fefc153bbe0586389e814979cf7185c50fcddbb2082725991ed02742e7d1e", 17 | "zh:ae789c0e7cb777d98934387f8888090ccb2d8973ef10e5ece541e8b624e1fb00", 18 | "zh:b4608aab78b4dbb32c629595797107fc5a84d1b8f0682f183793d13837f0ecf0", 19 | "zh:ed2c791c2354764b565f9ba4be7fc845c619c1a32cefadd3154a5665b312ab00", 20 | "zh:f94ac0072a8545eebabf417bc0acbdc77c31c006ad8760834ee8ee5cdb64e743", 21 | ] 22 | } 23 | 24 | provider "registry.terraform.io/hashicorp/random" { 25 | version = "3.3.2" 26 | hashes = [ 27 | "h1:Fu0IKMy46WsO5Y6KfuH9IFkkuxZjE/gIcgtB7GWkTtc=", 28 | "zh:038293aebfede983e45ee55c328e3fde82ae2e5719c9bd233c324cfacc437f9c", 29 | "zh:07eaeab03a723d83ac1cc218f3a59fceb7bbf301b38e89a26807d1c93c81cef8", 30 | "zh:427611a4ce9d856b1c73bea986d841a969e4c2799c8ac7c18798d0cc42b78d32", 31 | "zh:49718d2da653c06a70ba81fd055e2b99dfd52dcb86820a6aeea620df22cd3b30", 32 | "zh:5574828d90b19ab762604c6306337e6cd430e65868e13ef6ddb4e25ddb9ad4c0", 33 | "zh:7222e16f7833199dabf1bc5401c56d708ec052b2a5870988bc89ff85b68a5388", 34 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 35 | "zh:b1b2d7d934784d2aee98b0f8f07a8ccfc0410de63493ae2bf2222c165becf938", 36 | "zh:b8f85b6a20bd264fcd0814866f415f0a368d1123cd7879c8ebbf905d370babc8", 37 | "zh:c3813133acc02bbebddf046d9942e8ba5c35fc99191e3eb057957dafc2929912", 38 | "zh:e7a41dbc919d1de800689a81c240c27eec6b9395564630764ebb323ea82ac8a9", 39 | "zh:ee6d23208449a8eaa6c4f203e33f5176fa795b4b9ecf32903dffe6e2574732c2", 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /examples/programmatic-usage/custom-aws-detonator-terratest/terraform/main.tf: -------------------------------------------------------------------------------- 1 | # Code taken from https://github.com/DataDog/stratus-red-team/blob/main/v2/internal/attacktechniques/aws/defense-evasion/cloudtrail-stop/main.tf 2 | 3 | resource "aws_cloudtrail" "trail" { 4 | name = "sample-cloudtrail-trail" 5 | s3_bucket_name = aws_s3_bucket.cloudtrail.id 6 | } 7 | 8 | resource "random_string" "suffix" { 9 | length = 16 10 | min_lower = 16 11 | special = false 12 | } 13 | 14 | locals { 15 | bucket-name = "my-cloudtrail-bucket-${random_string.suffix.result}" 16 | } 17 | resource "aws_s3_bucket" "cloudtrail" { 18 | bucket = local.bucket-name 19 | force_destroy = true 20 | 21 | policy = <= maxSignals { 47 | return nil, errors.New("unsupported: more than 1000 open signals") // todo: paginate response 48 | } 49 | return signals.Data, err 50 | } 51 | 52 | func (m *DatadogSecuritySignalsAPIImpl) CloseSignal(id string) error { 53 | payload, _ := json.Marshal(map[string]interface{}{ 54 | "state": "archived", 55 | "archiveReason": "testing_or_maintenance", 56 | "archiveComment": "End to end detection testing", 57 | }) 58 | path := fmt.Sprintf("api/v1/security_analytics/signals/%s/state", id) 59 | ddSite := (m.ctx.Value(datadog.ContextServerVariables).(map[string]string))["site"] 60 | req, err := http.NewRequest( 61 | http.MethodPatch, 62 | fmt.Sprintf("https://api.%s/%s", ddSite, path), 63 | bytes.NewBuffer(payload), 64 | ) 65 | 66 | if err != nil { 67 | return err 68 | } 69 | keys := m.ctx.Value(datadog.ContextAPIKeys).(map[string]datadog.APIKey) 70 | req.Header.Set("Content-Type", "application/json") 71 | req.Header.Set("DD-API-KEY", keys["apiKeyAuth"].Key) 72 | req.Header.Set("DD-APPLICATION-KEY", keys["appKeyAuth"].Key) 73 | 74 | client := &http.Client{} 75 | response, err := client.Do(req) 76 | if err != nil { 77 | return err 78 | } 79 | if response.StatusCode != 200 { 80 | return errors.New("unable to archive signal, got status code " + strconv.Itoa(response.StatusCode)) 81 | } 82 | return nil 83 | } 84 | 85 | func (m *DatadogAlertGeneratedAssertionBuilder) HasExpectedAlert(detonationUuid string) (bool, error) { 86 | return m.DatadogAlertGeneratedAssertion.HasExpectedAlert(detonationUuid) 87 | } 88 | 89 | func (m *DatadogAlertGeneratedAssertionBuilder) Cleanup(detonationUuid string) error { 90 | return m.DatadogAlertGeneratedAssertion.Cleanup(detonationUuid) 91 | } 92 | 93 | func (m *DatadogAlertGeneratedAssertion) HasExpectedAlert(detonationUuid string) (bool, error) { 94 | // Possible improvement: cache signal IDs and exclude them in the search to avoid checking multiple times the same signal 95 | query := m.buildDatadogSignalQuery() 96 | signals, err := m.SignalsAPI.SearchSignals(query) 97 | if err != nil { 98 | return false, errors.New("unable to search for Datadog security signal: " + err.Error()) 99 | } 100 | 101 | if len(signals) == 0 { 102 | return false, nil 103 | } 104 | 105 | for i := range signals { 106 | if m.signalMatchesExecution(signals[i], detonationUuid) { //TODO low-prio unify naming of "uuid"/"uid" 107 | return true, nil 108 | } 109 | } 110 | 111 | return false, nil 112 | } 113 | 114 | func (m *DatadogAlertGeneratedAssertion) String() string { 115 | return fmt.Sprintf("Datadog security signal '%s'", m.AlertFilter.RuleName) 116 | } 117 | 118 | func (m *DatadogAlertGeneratedAssertion) Cleanup(detonationUuid string) error { 119 | signals, err := m.SignalsAPI.SearchSignals(QueryAllOpenSignals) 120 | if err != nil { 121 | return errors.New("unable to search for Datadog security monitoring signals: " + err.Error()) 122 | } 123 | 124 | for i := range signals { 125 | if m.signalMatchesExecution(signals[i], detonationUuid) { 126 | if err := m.SignalsAPI.CloseSignal(*signals[i].Id); err != nil { 127 | return errors.New("unable to archive signal " + *signals[i].Id + ": " + err.Error()) 128 | } 129 | } 130 | } 131 | 132 | return nil 133 | } 134 | 135 | // TODO: Would probably make more sense to retrieve all open signal and iterate instead of doing 2 pass 136 | func (m *DatadogAlertGeneratedAssertion) buildDatadogSignalQuery() string { 137 | severityQuery := "" 138 | if m.AlertFilter.Severity != "" { 139 | severityQuery = fmt.Sprintf(QuerySeverity, m.AlertFilter.Severity) + " " 140 | } 141 | return fmt.Sprintf( 142 | QueryOpenSignalsByAlertNameAndSeverity, 143 | m.AlertFilter.RuleName, 144 | severityQuery, 145 | ) 146 | } 147 | 148 | func (m *DatadogAlertGeneratedAssertion) signalMatchesExecution(signal datadogV2.SecurityMonitoringSignal, uid string) bool { 149 | buf, _ := json.Marshal(signal.Attributes.Custom) 150 | rawSignal := string(buf) 151 | return strings.Contains(rawSignal, uid) 152 | } 153 | -------------------------------------------------------------------------------- /pkg/threatest/matchers/datadog/datadog_test.go: -------------------------------------------------------------------------------- 1 | package datadog 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" 9 | "github.com/aws/smithy-go/ptr" 10 | "github.com/datadog/threatest/pkg/threatest/matchers/datadog/mocks" 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/mock" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | // Utility function to returns a sample Datadog signal 17 | func sampleSignal(id int) *datadogV2.SecurityMonitoringSignal { 18 | signal := datadogV2.NewSecurityMonitoringSignal() 19 | signal.Id = ptr.String(strconv.Itoa(id)) 20 | signal.Attributes = &datadogV2.SecurityMonitoringSignalAttributes{Custom: map[string]interface{}{}} 21 | signal.Attributes.Custom["title"] = "Sample signal " + strconv.Itoa(id) 22 | return signal 23 | } 24 | 25 | // Utility function that generates a "universe of signals" that match either nothing, either the rule name + severity, either the 26 | // execution UID, either both 27 | func generateSignals(numSignalsMatchingNothing int, numSignalsMatchingRuleAndSeverity int, numSignalsMatchingUUID int, numSignalsMatchingBoth int, detonationUid string) ([]datadogV2.SecurityMonitoringSignal, []datadogV2.SecurityMonitoringSignal, []datadogV2.SecurityMonitoringSignal, []datadogV2.SecurityMonitoringSignal) { 28 | signalsMatchingDetonationUid := make([]datadogV2.SecurityMonitoringSignal, 0) 29 | signalsMatchingRuleAndSeverity := make([]datadogV2.SecurityMonitoringSignal, 0) 30 | signalsMatchingNothing := make([]datadogV2.SecurityMonitoringSignal, 0) 31 | signalsMatchingBoth := make([]datadogV2.SecurityMonitoringSignal, 0) 32 | 33 | for i := 0; i < numSignalsMatchingNothing; i++ { 34 | signalsMatchingNothing = append(signalsMatchingNothing, *sampleSignal(i)) 35 | } 36 | for i := 0; i < numSignalsMatchingUUID; i++ { 37 | signal := *sampleSignal(i + numSignalsMatchingNothing) 38 | signal.Attributes.Custom["foobar"] = detonationUid 39 | signalsMatchingDetonationUid = append(signalsMatchingDetonationUid, signal) 40 | } 41 | for i := 0; i < numSignalsMatchingRuleAndSeverity; i++ { 42 | signalsMatchingRuleAndSeverity = append(signalsMatchingRuleAndSeverity, *sampleSignal(i + numSignalsMatchingNothing + numSignalsMatchingUUID)) 43 | } 44 | for i := 0; i < numSignalsMatchingBoth; i++ { 45 | signal := *sampleSignal(i + numSignalsMatchingNothing + numSignalsMatchingUUID + numSignalsMatchingRuleAndSeverity) 46 | signal.Attributes.Custom["foobar"] = detonationUid 47 | signalsMatchingBoth = append(signalsMatchingBoth, signal) 48 | } 49 | 50 | return signalsMatchingNothing, signalsMatchingRuleAndSeverity, signalsMatchingDetonationUid, signalsMatchingBoth 51 | } 52 | 53 | func union(signals ...[]datadogV2.SecurityMonitoringSignal) []datadogV2.SecurityMonitoringSignal { 54 | result := make([]datadogV2.SecurityMonitoringSignal, 0) 55 | for _, signalSet := range signals { 56 | result = append(result, signalSet...) 57 | } 58 | return result 59 | } 60 | func TestDatadog(t *testing.T) { 61 | detonationUid := "my-detonation-uuid" 62 | tests := []struct { 63 | Name string 64 | NumSignalsMatchingNothing int // all signals matching neither rule/severity nor UID 65 | NumSignalsMatchingOnlyRuleAndSeverity int // signals matching only the rule name 66 | NumSignalsMatchingOnlyUUID int // signals matching only the detonation UUID 67 | NumSignalsMatchingBoth int // signals matching both 68 | ExpectMatch bool 69 | }{ 70 | { 71 | Name: "No matching at all", 72 | NumSignalsMatchingNothing: 0, 73 | NumSignalsMatchingOnlyRuleAndSeverity: 0, 74 | NumSignalsMatchingOnlyUUID: 0, 75 | NumSignalsMatchingBoth: 0, 76 | ExpectMatch: false, 77 | }, 78 | { 79 | Name: "No matching signal matching anything", 80 | NumSignalsMatchingNothing: 1, 81 | NumSignalsMatchingOnlyRuleAndSeverity: 0, 82 | NumSignalsMatchingOnlyUUID: 0, 83 | NumSignalsMatchingBoth: 0, 84 | ExpectMatch: false, 85 | }, 86 | { 87 | Name: "One signal matching alert name and severity, but not the detonation UID, should not be closed and not result in a match", 88 | NumSignalsMatchingNothing: 0, 89 | NumSignalsMatchingOnlyRuleAndSeverity: 1, 90 | NumSignalsMatchingOnlyUUID: 0, 91 | NumSignalsMatchingBoth: 0, 92 | ExpectMatch: false, 93 | }, 94 | { 95 | Name: "One signal matching the detonation UID, but not the alert name, should be closed without match", 96 | NumSignalsMatchingNothing: 0, 97 | NumSignalsMatchingOnlyRuleAndSeverity: 0, 98 | NumSignalsMatchingOnlyUUID: 1, 99 | NumSignalsMatchingBoth: 0, 100 | ExpectMatch: false, 101 | }, 102 | { 103 | Name: "One signal the detonation UID and the alert name should be closed with a match", 104 | NumSignalsMatchingNothing: 0, 105 | NumSignalsMatchingOnlyRuleAndSeverity: 0, 106 | NumSignalsMatchingOnlyUUID: 0, 107 | NumSignalsMatchingBoth: 1, 108 | ExpectMatch: true, 109 | }, 110 | { 111 | Name: "One signal matching everything, one signal matching rule name but not UID", 112 | NumSignalsMatchingNothing: 0, 113 | NumSignalsMatchingOnlyRuleAndSeverity: 0, 114 | NumSignalsMatchingOnlyUUID: 1, 115 | NumSignalsMatchingBoth: 1, 116 | ExpectMatch: true, 117 | }, 118 | { 119 | Name: "One signal matching everything, one signal matching rule name but not UID, one signal matching only UID", 120 | NumSignalsMatchingNothing: 0, 121 | NumSignalsMatchingOnlyRuleAndSeverity: 1, 122 | NumSignalsMatchingOnlyUUID: 1, 123 | NumSignalsMatchingBoth: 1, 124 | ExpectMatch: true, 125 | }, 126 | { 127 | Name: "One of each", 128 | NumSignalsMatchingNothing: 1, 129 | NumSignalsMatchingOnlyRuleAndSeverity: 1, 130 | NumSignalsMatchingOnlyUUID: 1, 131 | NumSignalsMatchingBoth: 1, 132 | ExpectMatch: true, 133 | }, 134 | } 135 | 136 | for _, test := range tests { 137 | t.Run(test.Name, func(t *testing.T) { 138 | mockDatadog := &mocks.DatadogSecuritySignalsAPI{} 139 | detonationUid = "my-uid" 140 | signalsMatchingNothing, signalsMatchingOnlyRuleAndSeverity, signalsMatchingOnlyDetonationUid, signalsMatchingBoth := generateSignals(test.NumSignalsMatchingNothing, test.NumSignalsMatchingOnlyRuleAndSeverity, test.NumSignalsMatchingOnlyUUID, test.NumSignalsMatchingBoth, detonationUid) 141 | allOpenSignals := union(signalsMatchingNothing, signalsMatchingOnlyRuleAndSeverity, signalsMatchingOnlyDetonationUid, signalsMatchingBoth) 142 | alertFilter := &DatadogAlertFilter{RuleName: "my-rule-name", Severity: "medium"} 143 | expectedQuery := fmt.Sprintf( 144 | QueryOpenSignalsByAlertNameAndSeverity, 145 | alertFilter.RuleName, 146 | fmt.Sprintf(QuerySeverity, alertFilter.Severity)+" ", 147 | ) 148 | 149 | mockDatadog.On("SearchSignals", QueryAllOpenSignals).Return(allOpenSignals, nil) 150 | mockDatadog.On("SearchSignals", expectedQuery).Return(union(signalsMatchingOnlyRuleAndSeverity, signalsMatchingBoth), nil) 151 | mockDatadog.On("CloseSignal", mock.AnythingOfType("string")).Return(nil) 152 | 153 | matcher := DatadogAlertGeneratedAssertion{ 154 | SignalsAPI: mockDatadog, 155 | AlertFilter: alertFilter, 156 | } 157 | 158 | matches, err := matcher.HasExpectedAlert(detonationUid) 159 | require.Nil(t, err) 160 | 161 | // Check expected match 162 | if test.ExpectMatch { 163 | assert.True(t, matches, "matcher should match the signal") 164 | } else { 165 | assert.False(t, matches, "matcher should not match the signal") 166 | } 167 | 168 | // Check if the Datadog API is queried with the expected query 169 | mockDatadog.AssertCalled(t, "SearchSignals", expectedQuery) // first query to find the relevant signals 170 | 171 | // CLEANUP 172 | //TODO split test? 173 | err = matcher.Cleanup(detonationUid) 174 | require.Nil(t, err) 175 | 176 | // Every signal matching the UID (independently of whether it matches the alert name) should be closed 177 | for i := 0; i < test.NumSignalsMatchingOnlyUUID; i++ { 178 | mockDatadog.AssertCalled(t, "CloseSignal", *signalsMatchingOnlyDetonationUid[i].Id) 179 | } 180 | for i := 0; i < test.NumSignalsMatchingBoth; i++ { 181 | mockDatadog.AssertCalled(t, "CloseSignal", *signalsMatchingBoth[i].Id) 182 | } 183 | }) 184 | } 185 | 186 | } 187 | -------------------------------------------------------------------------------- /pkg/threatest/matchers/datadog/mocks/DatadogSecuritySignalsAPI.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.13.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import ( 6 | v2datadog "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // DatadogSecuritySignalsAPI is an autogenerated mock type for the DatadogSecuritySignalsAPI type 11 | type DatadogSecuritySignalsAPI struct { 12 | mock.Mock 13 | } 14 | 15 | // CloseSignal provides a mock function with given fields: id 16 | func (_m *DatadogSecuritySignalsAPI) CloseSignal(id string) error { 17 | ret := _m.Called(id) 18 | 19 | var r0 error 20 | if rf, ok := ret.Get(0).(func(string) error); ok { 21 | r0 = rf(id) 22 | } else { 23 | r0 = ret.Error(0) 24 | } 25 | 26 | return r0 27 | } 28 | 29 | // SearchSignals provides a mock function with given fields: query 30 | func (_m *DatadogSecuritySignalsAPI) SearchSignals(query string) ([]v2datadog.SecurityMonitoringSignal, error) { 31 | ret := _m.Called(query) 32 | 33 | var r0 []v2datadog.SecurityMonitoringSignal 34 | if rf, ok := ret.Get(0).(func(string) []v2datadog.SecurityMonitoringSignal); ok { 35 | r0 = rf(query) 36 | } else { 37 | if ret.Get(0) != nil { 38 | r0 = ret.Get(0).([]v2datadog.SecurityMonitoringSignal) 39 | } 40 | } 41 | 42 | var r1 error 43 | if rf, ok := ret.Get(1).(func(string) error); ok { 44 | r1 = rf(query) 45 | } else { 46 | r1 = ret.Error(1) 47 | } 48 | 49 | return r0, r1 50 | } 51 | 52 | type mockConstructorTestingTNewDatadogSecuritySignalsAPI interface { 53 | mock.TestingT 54 | Cleanup(func()) 55 | } 56 | 57 | // NewDatadogSecuritySignalsAPI creates a new instance of DatadogSecuritySignalsAPI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 58 | func NewDatadogSecuritySignalsAPI(t mockConstructorTestingTNewDatadogSecuritySignalsAPI) *DatadogSecuritySignalsAPI { 59 | mock := &DatadogSecuritySignalsAPI{} 60 | mock.Mock.Test(t) 61 | 62 | t.Cleanup(func() { mock.AssertExpectations(t) }) 63 | 64 | return mock 65 | } 66 | -------------------------------------------------------------------------------- /pkg/threatest/matchers/datadog/types.go: -------------------------------------------------------------------------------- 1 | package datadog 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/DataDog/datadog-api-client-go/v2/api/datadog" 8 | "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" 9 | ) 10 | 11 | type DatadogAlertFilter struct { 12 | RuleName string `yaml:"rule-name"` 13 | Severity string 14 | // There might be other attributes in the future 15 | } 16 | 17 | type DatadogAlertGeneratedAssertion struct { 18 | SignalsAPI DatadogSecuritySignalsAPI 19 | AlertFilter *DatadogAlertFilter 20 | } 21 | 22 | // builder 23 | type DatadogAlertGeneratedAssertionBuilder struct { 24 | DatadogAlertGeneratedAssertion 25 | } 26 | 27 | func GetDDSite() string { 28 | if ddSite, isSet := os.LookupEnv("DD_SITE"); isSet { 29 | return ddSite 30 | } 31 | return "datadoghq.com" 32 | } 33 | 34 | func DatadogSecuritySignal(name string) *DatadogAlertGeneratedAssertionBuilder { 35 | builder := &DatadogAlertGeneratedAssertionBuilder{} 36 | ddApiKey := os.Getenv("DD_API_KEY") 37 | ddAppKey := os.Getenv("DD_APP_KEY") 38 | ctx := context.WithValue(context.Background(), datadog.ContextAPIKeys, map[string]datadog.APIKey{ 39 | "apiKeyAuth": {Key: ddApiKey}, 40 | "appKeyAuth": {Key: ddAppKey}, 41 | }) 42 | ctx = context.WithValue(ctx, datadog.ContextServerVariables, map[string]string{ 43 | "site": GetDDSite(), 44 | }) 45 | cfg := datadog.NewConfiguration() 46 | cfg.SetUnstableOperationEnabled("SearchSecurityMonitoringSignals", true) 47 | 48 | builder.SignalsAPI = &DatadogSecuritySignalsAPIImpl{ 49 | securityMonitoringAPI: datadogV2.NewSecurityMonitoringApi(datadog.NewAPIClient(cfg)), 50 | ctx: ctx, 51 | } 52 | builder.AlertFilter = &DatadogAlertFilter{RuleName: name} 53 | return builder 54 | } 55 | 56 | func (m *DatadogAlertGeneratedAssertionBuilder) WithSeverity(severity string) *DatadogAlertGeneratedAssertionBuilder { 57 | m.AlertFilter.Severity = severity 58 | return m 59 | } 60 | -------------------------------------------------------------------------------- /pkg/threatest/matchers/mocks/AlertGeneratedMatcher.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v2.13.1. DO NOT EDIT. 2 | 3 | package mocks 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | 7 | // AlertGeneratedMatcher is an autogenerated mock type for the AlertGeneratedMatcher type 8 | type AlertGeneratedMatcher struct { 9 | mock.Mock 10 | } 11 | 12 | // Cleanup provides a mock function with given fields: uuid 13 | func (_m *AlertGeneratedMatcher) Cleanup(uuid string) error { 14 | ret := _m.Called(uuid) 15 | 16 | var r0 error 17 | if rf, ok := ret.Get(0).(func(string) error); ok { 18 | r0 = rf(uuid) 19 | } else { 20 | r0 = ret.Error(0) 21 | } 22 | 23 | return r0 24 | } 25 | 26 | // HasExpectedAlert provides a mock function with given fields: uid 27 | func (_m *AlertGeneratedMatcher) HasExpectedAlert(uid string) (bool, error) { 28 | ret := _m.Called(uid) 29 | 30 | var r0 bool 31 | if rf, ok := ret.Get(0).(func(string) bool); ok { 32 | r0 = rf(uid) 33 | } else { 34 | r0 = ret.Get(0).(bool) 35 | } 36 | 37 | var r1 error 38 | if rf, ok := ret.Get(1).(func(string) error); ok { 39 | r1 = rf(uid) 40 | } else { 41 | r1 = ret.Error(1) 42 | } 43 | 44 | return r0, r1 45 | } 46 | 47 | // String provides a mock function with given fields: 48 | func (_m *AlertGeneratedMatcher) String() string { 49 | ret := _m.Called() 50 | 51 | var r0 string 52 | if rf, ok := ret.Get(0).(func() string); ok { 53 | r0 = rf() 54 | } else { 55 | r0 = ret.Get(0).(string) 56 | } 57 | 58 | return r0 59 | } 60 | 61 | type mockConstructorTestingTNewAlertGeneratedMatcher interface { 62 | mock.TestingT 63 | Cleanup(func()) 64 | } 65 | 66 | // NewAlertGeneratedMatcher creates a new instance of AlertGeneratedMatcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. 67 | func NewAlertGeneratedMatcher(t mockConstructorTestingTNewAlertGeneratedMatcher) *AlertGeneratedMatcher { 68 | mock := &AlertGeneratedMatcher{} 69 | mock.Mock.Test(t) 70 | 71 | t.Cleanup(func() { mock.AssertExpectations(t) }) 72 | 73 | return mock 74 | } 75 | -------------------------------------------------------------------------------- /pkg/threatest/parser/main.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "github.com/datadog/threatest/pkg/threatest" 6 | "github.com/datadog/threatest/pkg/threatest/detonators" 7 | "github.com/datadog/threatest/pkg/threatest/matchers/datadog" 8 | "sigs.k8s.io/yaml" // we use this library as it provides a handy "YAMLToJSON" function 9 | "strings" 10 | "time" 11 | ) 12 | 13 | // Parse turns a YAML input string into a list of Threatest scenarios 14 | // TODO: A SSH configuration shouldn't be required at this point 15 | func Parse(yamlInput []byte, sshHostname string, sshUsername string, sshKey string) ([]*threatest.Scenario, error) { 16 | jsonInput, err := yaml.YAMLToJSON(yamlInput) 17 | if err != nil { 18 | return nil, fmt.Errorf("unable to convert input YAML to JSON: %v", err) 19 | } 20 | 21 | parsed := ThreatestSchemaJson{} 22 | if err := parsed.UnmarshalJSON(jsonInput); err != nil { 23 | return nil, fmt.Errorf("unable to parse input: %v", err) 24 | } 25 | 26 | return buildScenarios(&parsed, sshHostname, sshUsername, sshKey) 27 | } 28 | 29 | func buildScenarios(parsed *ThreatestSchemaJson, sshHostname string, sshUsername string, sshKey string) ([]*threatest.Scenario, error) { 30 | scenarios := []*threatest.Scenario{} 31 | if len(parsed.Scenarios) == 0 { 32 | return nil, fmt.Errorf("input file has no scenarios defined") 33 | } 34 | 35 | for _, parsedScenario := range parsed.Scenarios { 36 | scenario := threatest.Scenario{} 37 | scenario.Name = parsedScenario.Name 38 | 39 | if !hasDetonation(parsedScenario) { 40 | return nil, fmt.Errorf("scenario '%s' has no detonation defined", parsedScenario.Name) 41 | } 42 | 43 | // Detonation 44 | if localDetonator := parsedScenario.Detonate.LocalDetonator; localDetonator != nil { 45 | commandToRun := strings.Join(parsedScenario.Detonate.LocalDetonator.Commands, "; ") 46 | scenario.Detonator = detonators.NewCommandDetonator(&detonators.LocalCommandExecutor{}, commandToRun) 47 | } else if remoteDetonator := parsedScenario.Detonate.RemoteDetonator; remoteDetonator != nil { 48 | commandToRun := strings.Join(remoteDetonator.Commands, "; ") 49 | //TODO: decouple 50 | //TODO: confirm 1 SSH executor per attack makes sense 51 | sshExecutor, err := detonators.NewSSHCommandExecutor(sshHostname, sshUsername, sshKey) 52 | if err != nil { 53 | return nil, fmt.Errorf("invalid SSH detonator configuration: %v", err) 54 | } 55 | scenario.Detonator = detonators.NewCommandDetonator(sshExecutor, commandToRun) 56 | } else if stratusRedTeamDetonator := parsedScenario.Detonate.StratusRedTeamDetonator; stratusRedTeamDetonator != nil { 57 | scenario.Detonator = detonators.StratusRedTeamTechnique(*stratusRedTeamDetonator.AttackTechnique) 58 | } else if awsCliDetonator := parsedScenario.Detonate.AwsCliDetonator; awsCliDetonator != nil { 59 | scenario.Detonator = detonators.NewAWSCLIDetonator(*awsCliDetonator.Script) 60 | } 61 | 62 | // Assertions 63 | if len(parsedScenario.Expectations) == 0 { 64 | return nil, fmt.Errorf("scenario '%s' has no assertions defined", parsedScenario.Name) 65 | } 66 | for _, parsedAssertion := range parsedScenario.Expectations { 67 | if datadogMatcher := parsedAssertion.DatadogSecuritySignal; datadogMatcher != nil { 68 | assertion := datadog.DatadogSecuritySignal(datadogMatcher.Name) 69 | if severity := datadogMatcher.Severity; severity != nil { 70 | assertion.WithSeverity(*severity) 71 | } 72 | scenario.Assertions = append(scenario.Assertions, assertion) 73 | } 74 | } 75 | 76 | //TODO: in the threatest core, the timeout should be part of each assertion (not scenario level) 77 | // We should probably define a default timeout at the CLI level 78 | rawTimeout := parsedScenario.Expectations[0].Timeout 79 | parsedDuration, err := time.ParseDuration(rawTimeout) 80 | if err != nil { 81 | return nil, fmt.Errorf("scenario '%s' has an invalid timeout '%s': '%v'", parsedScenario.Name, rawTimeout, err) 82 | } 83 | scenario.Timeout = parsedDuration 84 | 85 | scenarios = append(scenarios, &scenario) 86 | } 87 | return scenarios, nil 88 | } 89 | 90 | // hasDetonation returns true if the scenario has at least 1 detonation defined 91 | func hasDetonation(scenario ThreatestSchemaJsonScenariosElem) bool { 92 | detonations := scenario.Detonate 93 | return detonations.LocalDetonator != nil || 94 | detonations.RemoteDetonator != nil || 95 | detonations.StratusRedTeamDetonator != nil || 96 | detonations.AwsCliDetonator != nil 97 | } 98 | -------------------------------------------------------------------------------- /pkg/threatest/parser/parser.go: -------------------------------------------------------------------------------- 1 | // Code generated by github.com/atombender/go-jsonschema, DO NOT EDIT. 2 | 3 | package parser 4 | 5 | import "encoding/json" 6 | import "fmt" 7 | 8 | // UnmarshalJSON implements json.Unmarshaler. 9 | func (j *DatadogSecuritySignalSchemaJson) UnmarshalJSON(b []byte) error { 10 | var raw map[string]interface{} 11 | if err := json.Unmarshal(b, &raw); err != nil { 12 | return err 13 | } 14 | if v, ok := raw["name"]; !ok || v == nil { 15 | return fmt.Errorf("field name in DatadogSecuritySignalSchemaJson: required") 16 | } 17 | type Plain DatadogSecuritySignalSchemaJson 18 | var plain Plain 19 | if err := json.Unmarshal(b, &plain); err != nil { 20 | return err 21 | } 22 | *j = DatadogSecuritySignalSchemaJson(plain) 23 | return nil 24 | } 25 | 26 | // Definition of an AWS CLI detonation 27 | type AwsCliDetonatorSchemaJson struct { 28 | // Script corresponds to the JSON schema field "script". 29 | Script *string `json:"script,omitempty" yaml:"script,omitempty" mapstructure:"script,omitempty"` 30 | } 31 | 32 | // Matcher for a Datadog security signal 33 | type DatadogSecuritySignalSchemaJson struct { 34 | // Name of the Datadog signal to match on (exact match) 35 | Name string `json:"name" yaml:"name" mapstructure:"name"` 36 | 37 | // Severity of the Datadog signal to match on 38 | Severity *string `json:"severity,omitempty" yaml:"severity,omitempty" mapstructure:"severity,omitempty"` 39 | } 40 | 41 | // Definition of a local command detonation 42 | type LocalDetonatorSchemaJson struct { 43 | // Commands corresponds to the JSON schema field "commands". 44 | Commands []string `json:"commands,omitempty" yaml:"commands,omitempty" mapstructure:"commands,omitempty"` 45 | } 46 | 47 | // Definition of a remote command detonation 48 | type RemoteDetonatorSchemaJson struct { 49 | // Commands corresponds to the JSON schema field "commands". 50 | Commands []string `json:"commands,omitempty" yaml:"commands,omitempty" mapstructure:"commands,omitempty"` 51 | } 52 | 53 | // Definition of a Stratus Red Team detonator 54 | type StratusRedTeamDetonatorSchemaJson struct { 55 | // Attack technique ID of the Stratus Red Team technique to detonate (per 56 | // https://stratus-red-team.cloud/attack-techniques/list/) 57 | AttackTechnique *string `json:"attackTechnique,omitempty" yaml:"attackTechnique,omitempty" mapstructure:"attackTechnique,omitempty"` 58 | } 59 | 60 | // How to detonate the attack 61 | type ThreatestSchemaJsonScenariosElemDetonate struct { 62 | // AwsCliDetonator corresponds to the JSON schema field "awsCliDetonator". 63 | AwsCliDetonator *AwsCliDetonatorSchemaJson `json:"awsCliDetonator,omitempty" yaml:"awsCliDetonator,omitempty" mapstructure:"awsCliDetonator,omitempty"` 64 | 65 | // LocalDetonator corresponds to the JSON schema field "localDetonator". 66 | LocalDetonator *LocalDetonatorSchemaJson `json:"localDetonator,omitempty" yaml:"localDetonator,omitempty" mapstructure:"localDetonator,omitempty"` 67 | 68 | // RemoteDetonator corresponds to the JSON schema field "remoteDetonator". 69 | RemoteDetonator *RemoteDetonatorSchemaJson `json:"remoteDetonator,omitempty" yaml:"remoteDetonator,omitempty" mapstructure:"remoteDetonator,omitempty"` 70 | 71 | // StratusRedTeamDetonator corresponds to the JSON schema field 72 | // "stratusRedTeamDetonator". 73 | StratusRedTeamDetonator *StratusRedTeamDetonatorSchemaJson `json:"stratusRedTeamDetonator,omitempty" yaml:"stratusRedTeamDetonator,omitempty" mapstructure:"stratusRedTeamDetonator,omitempty"` 74 | } 75 | 76 | // Expectations 77 | type ThreatestSchemaJsonScenariosElemExpectationsElem struct { 78 | // DatadogSecuritySignal corresponds to the JSON schema field 79 | // "datadogSecuritySignal". 80 | DatadogSecuritySignal *DatadogSecuritySignalSchemaJson `json:"datadogSecuritySignal,omitempty" yaml:"datadogSecuritySignal,omitempty" mapstructure:"datadogSecuritySignal,omitempty"` 81 | 82 | // The maximal time to wait for the assertion, written as a Go duration (e.g. 5m) 83 | Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty" mapstructure:"timeout,omitempty"` 84 | } 85 | 86 | // UnmarshalJSON implements json.Unmarshaler. 87 | func (j *ThreatestSchemaJsonScenariosElemExpectationsElem) UnmarshalJSON(b []byte) error { 88 | var raw map[string]interface{} 89 | if err := json.Unmarshal(b, &raw); err != nil { 90 | return err 91 | } 92 | type Plain ThreatestSchemaJsonScenariosElemExpectationsElem 93 | var plain Plain 94 | if err := json.Unmarshal(b, &plain); err != nil { 95 | return err 96 | } 97 | if v, ok := raw["timeout"]; !ok || v == nil { 98 | plain.Timeout = "5m" 99 | } 100 | *j = ThreatestSchemaJsonScenariosElemExpectationsElem(plain) 101 | return nil 102 | } 103 | 104 | // The list of scenarios 105 | type ThreatestSchemaJsonScenariosElem struct { 106 | // How to detonate the attack 107 | Detonate ThreatestSchemaJsonScenariosElemDetonate `json:"detonate" yaml:"detonate" mapstructure:"detonate"` 108 | 109 | // Expectations corresponds to the JSON schema field "expectations". 110 | Expectations []ThreatestSchemaJsonScenariosElemExpectationsElem `json:"expectations" yaml:"expectations" mapstructure:"expectations"` 111 | 112 | // Description of the scenario 113 | Name string `json:"name" yaml:"name" mapstructure:"name"` 114 | } 115 | 116 | // UnmarshalJSON implements json.Unmarshaler. 117 | func (j *ThreatestSchemaJsonScenariosElem) UnmarshalJSON(b []byte) error { 118 | var raw map[string]interface{} 119 | if err := json.Unmarshal(b, &raw); err != nil { 120 | return err 121 | } 122 | if v, ok := raw["detonate"]; !ok || v == nil { 123 | return fmt.Errorf("field detonate in ThreatestSchemaJsonScenariosElem: required") 124 | } 125 | if v, ok := raw["expectations"]; !ok || v == nil { 126 | return fmt.Errorf("field expectations in ThreatestSchemaJsonScenariosElem: required") 127 | } 128 | if v, ok := raw["name"]; !ok || v == nil { 129 | return fmt.Errorf("field name in ThreatestSchemaJsonScenariosElem: required") 130 | } 131 | type Plain ThreatestSchemaJsonScenariosElem 132 | var plain Plain 133 | if err := json.Unmarshal(b, &plain); err != nil { 134 | return err 135 | } 136 | *j = ThreatestSchemaJsonScenariosElem(plain) 137 | return nil 138 | } 139 | 140 | // Schema for a Threatest test suite 141 | type ThreatestSchemaJson struct { 142 | // The display name of the vulnerability 143 | Scenarios []ThreatestSchemaJsonScenariosElem `json:"scenarios" yaml:"scenarios" mapstructure:"scenarios"` 144 | } 145 | 146 | // UnmarshalJSON implements json.Unmarshaler. 147 | func (j *ThreatestSchemaJson) UnmarshalJSON(b []byte) error { 148 | var raw map[string]interface{} 149 | if err := json.Unmarshal(b, &raw); err != nil { 150 | return err 151 | } 152 | if v, ok := raw["scenarios"]; !ok || v == nil { 153 | return fmt.Errorf("field scenarios in ThreatestSchemaJson: required") 154 | } 155 | type Plain ThreatestSchemaJson 156 | var plain Plain 157 | if err := json.Unmarshal(b, &plain); err != nil { 158 | return err 159 | } 160 | *j = ThreatestSchemaJson(plain) 161 | return nil 162 | } 163 | -------------------------------------------------------------------------------- /pkg/threatest/parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestParserCorrectlyParsesValidInput(t *testing.T) { 9 | validYaml := ` 10 | scenarios: 11 | # Example 1: Remote detonation over SSH 12 | # Note: SSH configuration is provided using the --ssh-host, --ssh-username and --ssh-keyfile CLI arguments 13 | - name: curl metadata service 14 | detonate: 15 | remoteDetonator: 16 | commands: ["curl http://169.254.169.254 --connect-timeout 1"] 17 | expectations: 18 | - timeout: 1m 19 | datadogSecuritySignal: 20 | name: "Network utility accessed cloud metadata service" 21 | severity: medium 22 | 23 | # Example 2: Stratus Red Team detonation 24 | # Note: You must be authenticated to the relevant cloud provider before running it 25 | # The example below is equivalent to manually running "stratus detonate aws.exfiltration.ec2-security-group-open-port-22-ingress" 26 | - name: opening a security group to the Internet 27 | detonate: 28 | stratusRedTeamDetonator: 29 | attackTechnique: aws.exfiltration.ec2-security-group-open-port-22-ingress 30 | expectations: 31 | - timeout: 15m 32 | datadogSecuritySignal: 33 | name: "Potential administrative port open to the world via AWS security group" 34 | ` 35 | scenarios, err := Parse([]byte(validYaml), "", "", "") 36 | assert.Nil(t, err, "parsing a valid YAML scenario file should not return an error") 37 | assert.Len(t, scenarios, 2) 38 | 39 | assert.Equal(t, scenarios[0].Name, "curl metadata service") 40 | assert.NotNil(t, scenarios[0].Detonator) 41 | assert.Len(t, scenarios[0].Assertions, 1) 42 | 43 | assert.Equal(t, scenarios[1].Name, "opening a security group to the Internet") 44 | assert.NotNil(t, scenarios[1].Detonator) 45 | assert.Len(t, scenarios[1].Assertions, 1) 46 | } 47 | -------------------------------------------------------------------------------- /pkg/threatest/runner.go: -------------------------------------------------------------------------------- 1 | package threatest 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/datadog/threatest/pkg/threatest/matchers" 7 | log "github.com/sirupsen/logrus" 8 | "strconv" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | type TestRunner struct { 14 | Builders []*ScenarioBuilder 15 | Scenarios []*Scenario 16 | Interval time.Duration 17 | } 18 | 19 | func Threatest() *TestRunner { 20 | return &TestRunner{Interval: 2 * time.Second} 21 | } 22 | 23 | func (m *TestRunner) Scenario(name string) *ScenarioBuilder { 24 | builder := ScenarioBuilder{} 25 | builder.Name = name 26 | builder.Timeout = 10 * time.Minute // default timeout 27 | m.Builders = append(m.Builders, &builder) 28 | return &builder 29 | } 30 | 31 | func (m *TestRunner) Add(scenario *ScenarioBuilder) { 32 | m.Scenarios = append(m.Scenarios, scenario.Build()) 33 | } 34 | 35 | func (m *TestRunner) Run() error { 36 | m.buildScenarios() 37 | 38 | // Run every scenario one by one 39 | failedScenarios := map[string]error{} 40 | for i := range m.Scenarios { 41 | scenario := m.Scenarios[i] 42 | if err := m.runScenario(scenario); err != nil { 43 | failedScenarios[scenario.Name] = err 44 | } 45 | } 46 | 47 | if len(failedScenarios) > 0 { 48 | var errorMessage strings.Builder 49 | errorMessage.WriteString("At least one scenario failed:\n\n") 50 | for scenario, err := range failedScenarios { 51 | errorMessage.WriteString(scenario) 52 | errorMessage.WriteString(" returned: ") 53 | errorMessage.WriteString(err.Error()) 54 | errorMessage.WriteRune('\n') 55 | } 56 | return errors.New(errorMessage.String()) 57 | } 58 | 59 | return nil 60 | } 61 | 62 | func (m *TestRunner) buildScenarios() { 63 | if len(m.Scenarios) == 0 { 64 | for i := range m.Builders { 65 | m.Scenarios = append(m.Scenarios, m.Builders[i].Build()) 66 | } 67 | } 68 | } 69 | 70 | func (m *TestRunner) runScenario(scenario *Scenario) error { 71 | detonationUid, err := scenario.Detonator.Detonate() 72 | if err != nil { 73 | return err 74 | } 75 | //TODO: When to clean? If we don't wait a bit, we risk missing signals that were generated after our assertion matched 76 | defer m.CleanupScenario(scenario, detonationUid) 77 | start := time.Now() 78 | 79 | if len(scenario.Assertions) == 0 { 80 | return nil 81 | } 82 | 83 | log.Debugf("Scenario '%s' detonated", scenario.Name) 84 | 85 | // Build a queue containing all assertions 86 | remainingAssertions := make(chan matchers.AlertGeneratedMatcher, len(scenario.Assertions)) 87 | for i := range scenario.Assertions { 88 | remainingAssertions <- scenario.Assertions[i] 89 | } 90 | log.Debugf("Waiting for %d assertions", len(scenario.Assertions)) 91 | hasDeadline := scenario.Timeout > 0 92 | deadline := start.Add(scenario.Timeout) 93 | for len(remainingAssertions) > 0 { 94 | if hasDeadline && time.Now().After(deadline) { 95 | log.Printf("%s: timeout exceeded waiting for alerts (%d alerts not generated)\n", scenario.Name, len(remainingAssertions)) 96 | break 97 | } 98 | 99 | assertion := <-remainingAssertions 100 | hasAlert, err := assertion.HasExpectedAlert(detonationUid) 101 | if err != nil { 102 | return err 103 | } 104 | if hasAlert { 105 | timeSpentStr := strconv.Itoa(int(time.Since(start).Seconds())) 106 | log.Printf("%s: Confirmed that the expected signal (%s) was created in Datadog (took %s seconds).\n", scenario.Name, assertion.String(), timeSpentStr) 107 | } else { 108 | // requeue assertion 109 | log.Debugf("Assertion %s did not pass, requeuing it", assertion.String()) 110 | remainingAssertions <- assertion 111 | time.Sleep(m.Interval) 112 | } 113 | } 114 | 115 | if numRemainingAssertions := len(remainingAssertions); numRemainingAssertions > 0 { 116 | errText := fmt.Sprintf("%s: %d assertions did not pass", scenario.Name, numRemainingAssertions) 117 | for i := 0; i < numRemainingAssertions; i++ { 118 | assertion := <-remainingAssertions 119 | errText += fmt.Sprintf("\n => Did not find %s", assertion) 120 | } 121 | return errors.New(errText) 122 | } else { 123 | log.Printf("%s: All assertions passed\n", scenario.Name) 124 | } 125 | 126 | return nil 127 | } 128 | 129 | func (m *TestRunner) CleanupScenario(scenario *Scenario, detonationUid string) { 130 | if len(scenario.Assertions) == 0 { 131 | return 132 | } 133 | 134 | err := scenario.Assertions[0].Cleanup(detonationUid) 135 | if err != nil { 136 | log.Warnf("warning: failed to clean up generated signals: %s", err.Error()) 137 | } 138 | // TODO (code smell): this shouldn't be specific to a single assertion? 139 | } 140 | -------------------------------------------------------------------------------- /pkg/threatest/runner_test.go: -------------------------------------------------------------------------------- 1 | package threatest 2 | 3 | import ( 4 | "errors" 5 | detonatorMocks "github.com/datadog/threatest/pkg/threatest/detonators/mocks" 6 | "github.com/datadog/threatest/pkg/threatest/matchers" 7 | matcherMocks "github.com/datadog/threatest/pkg/threatest/matchers/mocks" 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | //TODO nuke interval for tests 14 | 15 | func TestRunnerWorks(t *testing.T) { 16 | testCases := []struct { 17 | Name string 18 | AlertExistsSequence []bool 19 | HasNoAssertion bool 20 | ExpectError bool 21 | }{ 22 | {Name: "Alert exists from the beginning", AlertExistsSequence: []bool{true}}, 23 | {Name: "Alert doesn't exist then exists", AlertExistsSequence: []bool{false, true}}, 24 | {Name: "Alert never exists", AlertExistsSequence: []bool{false}, ExpectError: true}, 25 | {Name: "No assertion", HasNoAssertion: true}, 26 | } 27 | 28 | for i := range testCases { 29 | testCase := testCases[i] 30 | 31 | t.Run(testCase.Name, func(t *testing.T) { 32 | t.Parallel() 33 | mockDetonator := &detonatorMocks.Detonator{} 34 | mockDetonator.On("Detonate").Return("my-uid", nil) 35 | 36 | mockMatcher := &matcherMocks.AlertGeneratedMatcher{} 37 | if len(testCase.AlertExistsSequence) == 1 { 38 | mockMatcher.On("HasExpectedAlert", "my-uid").Return(testCase.AlertExistsSequence[0], nil) 39 | } else { 40 | for i := range testCase.AlertExistsSequence { 41 | mockMatcher.On("HasExpectedAlert", "my-uid").Return(testCase.AlertExistsSequence[i], nil).Once() 42 | } 43 | } 44 | mockMatcher.On("String").Return("sample") 45 | mockMatcher.On("Cleanup", "my-uid").Return(nil) 46 | 47 | var assertions []matchers.AlertGeneratedMatcher 48 | assertions = []matchers.AlertGeneratedMatcher{} 49 | if !testCase.HasNoAssertion { 50 | assertions = []matchers.AlertGeneratedMatcher{mockMatcher} 51 | } 52 | 53 | runner := TestRunner{ 54 | Scenarios: []*Scenario{ 55 | { 56 | Name: "test-scenario", 57 | Detonator: mockDetonator, 58 | Assertions: assertions, 59 | Timeout: 5 * time.Second, 60 | }, 61 | }, 62 | Interval: 0, 63 | } 64 | err := runner.Run() 65 | if testCase.ExpectError { 66 | assert.NotNil(t, err) 67 | } else { 68 | assert.Nil(t, err) 69 | } 70 | mockDetonator.AssertNumberOfCalls(t, "Detonate", 1) 71 | 72 | if !testCase.HasNoAssertion { 73 | mockMatcher.AssertCalled(t, "Cleanup", "my-uid") 74 | } 75 | 76 | }) 77 | } 78 | 79 | } 80 | 81 | func TestRunnerErrorHandling(t *testing.T) { 82 | 83 | mockDetonator := &detonatorMocks.Detonator{} 84 | mockDetonator.On("Detonate").Return("my-uid", nil) 85 | 86 | mockFailingDetonator := &detonatorMocks.Detonator{} 87 | mockFailingDetonator.On("Detonate").Return("", errors.New("foo")) 88 | 89 | mockMatcher := &matcherMocks.AlertGeneratedMatcher{} 90 | mockMatcher.On("String").Return("sample") 91 | mockMatcher.On("Cleanup", "my-uid").Return(nil) 92 | mockMatcher.On("HasExpectedAlert", "my-uid").Return(true, nil) 93 | 94 | runner := TestRunner{ 95 | Scenarios: []*Scenario{ 96 | { 97 | Name: "test-scenario1", 98 | Detonator: mockDetonator, 99 | Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, 100 | Timeout: 5 * time.Second, 101 | }, 102 | { 103 | Name: "test-scenario2-error", 104 | Detonator: mockFailingDetonator, 105 | Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, 106 | Timeout: 5 * time.Second, 107 | }, 108 | { 109 | Name: "test-scenario3", 110 | Detonator: mockDetonator, 111 | Assertions: []matchers.AlertGeneratedMatcher{mockMatcher}, 112 | Timeout: 5 * time.Second, 113 | }, 114 | }, 115 | Interval: 0, 116 | } 117 | err := runner.Run() 118 | assert.Error(t, err, "the runner should return an error when a scenario returns an error") 119 | 120 | // All scenarios should have been detonated, even if one returned an error 121 | mockDetonator.AssertNumberOfCalls(t, "Detonate", 2) 122 | mockFailingDetonator.AssertNumberOfCalls(t, "Detonate", 1) 123 | } 124 | -------------------------------------------------------------------------------- /pkg/threatest/scenario.go: -------------------------------------------------------------------------------- 1 | package threatest 2 | 3 | import ( 4 | "github.com/datadog/threatest/pkg/threatest/detonators" 5 | "github.com/datadog/threatest/pkg/threatest/matchers" 6 | "time" 7 | ) 8 | 9 | type Scenario struct { 10 | Name string 11 | Detonator detonators.Detonator 12 | Timeout time.Duration 13 | Assertions []matchers.AlertGeneratedMatcher 14 | } 15 | 16 | type ScenarioBuilder struct { 17 | Scenario 18 | } 19 | 20 | func (m *ScenarioBuilder) WhenDetonating(detonation detonators.Detonator) *ScenarioBuilder { 21 | m.Detonator = detonation 22 | return m 23 | } 24 | 25 | func (m *ScenarioBuilder) WithTimeout(timeout time.Duration) *ScenarioBuilder { 26 | m.Timeout = timeout 27 | return m 28 | } 29 | 30 | func (m *ScenarioBuilder) Expect(assertion matchers.AlertGeneratedMatcher) *ScenarioBuilder { 31 | m.Assertions = append(m.Assertions, assertion) 32 | return m 33 | } 34 | 35 | func (m *ScenarioBuilder) Build() *Scenario { 36 | return &Scenario{ 37 | Name: m.Name, 38 | Detonator: m.Detonator, 39 | Timeout: m.Timeout, 40 | Assertions: m.Assertions, 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /schemas/awsCliDetonator.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "description": "Definition of an AWS CLI detonation", 4 | "properties": { 5 | "script": { 6 | "type": "string" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /schemas/datadogSecuritySignal.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "description": "Matcher for a Datadog security signal", 4 | "required": ["name"], 5 | "properties": { 6 | "name": { 7 | "type": "string", 8 | "description": "Name of the Datadog signal to match on (exact match)" 9 | }, 10 | "severity": { 11 | "type": "string", 12 | "description": "Severity of the Datadog signal to match on" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /schemas/localDetonator.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "description": "Definition of a local command detonation", 4 | "properties": { 5 | "commands": { 6 | "type": "array", 7 | "items": {"type": "string"} 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /schemas/remoteDetonator.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "description": "Definition of a remote command detonation", 4 | "properties": { 5 | "commands": { 6 | "type": "array", 7 | "items": {"type": "string"} 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /schemas/stratusRedTeamDetonator.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "description": "Definition of a Stratus Red Team detonator", 4 | "properties": { 5 | "attackTechnique": { 6 | "type": "string", 7 | "description": "Attack technique ID of the Stratus Red Team technique to detonate (per https://stratus-red-team.cloud/attack-techniques/list/)" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /schemas/threatest.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json-schema.org/draft/2020-12/schema", 3 | "$id": "https://github.com/datadog/threatest/threatest.schema.json", 4 | "title": "Threatest Test Suite", 5 | "description": "Schema for a Threatest test suite", 6 | "type": "object", 7 | "required": [ 8 | "scenarios" 9 | ], 10 | "properties": { 11 | "scenarios": { 12 | "description": "The display name of the vulnerability", 13 | "type": "array", 14 | "items": { 15 | "type": "object", 16 | "description": "The list of scenarios", 17 | "required": [ 18 | "name", 19 | "detonate", 20 | "expectations" 21 | ], 22 | "properties": { 23 | "name": { 24 | "type": "string", 25 | "description": "Description of the scenario" 26 | }, 27 | "detonate": { 28 | "type": "object", 29 | "description": "How to detonate the attack", 30 | "oneOf": [ 31 | { 32 | "required": [ 33 | "localDetonator" 34 | ] 35 | }, 36 | { 37 | "required": [ 38 | "remoteDetonator" 39 | ] 40 | }, 41 | { 42 | "required": [ 43 | "stratusRedTeamDetonator" 44 | ] 45 | }, 46 | { 47 | "required": [ 48 | "awsCliDetonator" 49 | ] 50 | } 51 | ], 52 | "properties": { 53 | "localDetonator": { 54 | "$ref": "localDetonator.schema.json" 55 | }, 56 | "remoteDetonator": { 57 | "$ref": "remoteDetonator.schema.json" 58 | }, 59 | "stratusRedTeamDetonator": { 60 | "$ref": "stratusRedTeamDetonator.schema.json" 61 | }, 62 | "awsCliDetonator": { 63 | "$ref": "awsCliDetonator.schema.json" 64 | } 65 | } 66 | }, 67 | "expectations": { 68 | "type": "array", 69 | "items": { 70 | "type": "object", 71 | "description": "Expectations", 72 | "oneOf": [ 73 | { 74 | "required": [ 75 | "datadogSecuritySignal" 76 | ] 77 | } 78 | ], 79 | "properties": { 80 | "datadogSecuritySignal": { 81 | "$ref": "datadogSecuritySignal.schema.json" 82 | }, 83 | "timeout": { 84 | "type": "string", 85 | "default": "5m", 86 | "description": "The maximal time to wait for the assertion, written as a Go duration (e.g. 5m)" 87 | } 88 | } 89 | } 90 | } 91 | } 92 | } 93 | } 94 | } 95 | } --------------------------------------------------------------------------------