├── .gitattributes ├── AWS-agents ├── .gitignore ├── AWS_thumbnail.PNG ├── README.md ├── dashboards │ ├── AWS-extension.ndjson │ ├── CloudTrail Summary.ndjson │ └── VPCFlow Summary.ndjson ├── json_templates │ ├── aws_cw_integration.json │ ├── aws_integration.json │ ├── aws_rule_activation.json │ ├── default-policy.json │ ├── es_api_key.json │ ├── k8s_agent.yaml │ ├── k8s_cspm_integration.json │ ├── k8s_endpoint_integration.json │ └── k8s_integration.json └── terraform │ ├── ami_picker.tf │ ├── config.tftpl │ ├── ec_deployment.tf │ ├── elastic_agent.tf │ ├── main.tf │ └── variables.tf ├── AWS ├── .gitignore ├── AWS_thumbnail.PNG ├── README.md ├── dashboards │ ├── AWS-extension.ndjson │ ├── CloudTrail Summary.ndjson │ └── VPCFlow Summary.ndjson ├── json_templates │ ├── aws_cw_integration.json │ ├── aws_integration.json │ ├── aws_rule_activation.json │ ├── default-policy.json │ ├── es_api_key.json │ ├── k8s_agent.yaml │ ├── k8s_cspm_integration.json │ ├── k8s_endpoint_integration.json │ └── k8s_integration.json └── terraform │ ├── ami_picker.tf │ ├── cloud_formation.tf │ ├── config.tftpl │ ├── ec_deployment.tf │ ├── elastic_agent.tf │ ├── esf_formation.tf │ ├── logs-cloudwatch.tf │ ├── logs_cloudtrail.tf │ ├── logs_elb.tf │ ├── logs_s3.tf │ ├── logs_vpcflow.tf │ ├── main.tf │ ├── s3_bucket.tf │ └── variables.tf ├── Azure ├── .gitignore ├── AZURE_thumbnail.PNG ├── README.md ├── dashboards │ └── Azure-dashboards.ndjson ├── json_templates │ ├── azure-billing-integration.json │ ├── azure-logs-integration.json │ ├── azure-metrics-integration.json │ ├── billing-transform.json │ ├── default-policy.json │ ├── es_api_key.json │ └── es_rule_activation.json └── terraform │ ├── azure_environment.tf │ ├── ec_deployment.tf │ ├── elastic_agent.tf │ ├── logging.tf │ ├── main.tf │ └── variables.tf ├── GoogleCloud ├── .gitignore ├── Getting-started-with-Google-Cloud-Monitoring-Google-Slides.png ├── README.md ├── dashboards │ └── google_cloud_dashboards.ndjson ├── json_templates │ ├── default-policy.json │ ├── es_api_key.json │ ├── es_host_transform.json │ ├── es_repo_transform.json │ ├── es_rule_activation.json │ ├── es_vpc_flow_transform.json │ └── gcp_integration.json └── terraform │ ├── dataflow.tf │ ├── ec_deployment.tf │ ├── elastic_agent.tf │ ├── logging.tf │ ├── main.tf │ └── variables.tf ├── Kubernetes └── EKS │ └── terraform │ ├── aws_provider.tf │ ├── ec_deployment.tf │ ├── eks_agent_manifest.off │ ├── eks_integration.tf │ └── variables.tf ├── LICENSE ├── Monitoring ├── README.md └── terraform │ └── maint.tf ├── MultiCloud ├── .gitignore ├── README.md └── terraform │ ├── aws_provider.tf │ ├── aws_variables.tf │ ├── azure_provider.tf │ ├── azure_variables.tf │ ├── ec_variables.tf │ ├── gc_provider.tf │ ├── gc_variables.tf │ ├── main.tf │ └── modules.tf ├── README.md └── lib ├── elastic_api ├── es_api_key.sh ├── es_bulk.sh ├── es_create_ilm_policy.sh ├── es_create_index_template.sh ├── es_create_ingest_pipeline.sh ├── es_create_mapping.sh ├── es_create_transform.sh ├── es_start_transform.sh ├── kb_add_integration_to_policy.sh ├── kb_create_agent_policy.sh ├── kb_enable_detection_rules.sh ├── kb_load_detection_rules.sh └── kb_upload_saved_objects.sh └── scripts └── agent_install.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /AWS-agents/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .vs/slnx.sqlite 3 | *.tfstate 4 | *.backup 5 | terraform/*.tfstate 6 | terraform/.terraform 7 | *.hcl 8 | local_env -------------------------------------------------------------------------------- /AWS-agents/AWS_thumbnail.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felix-lessoer/elastic-terraform-examples/6eaddd225884752f7367be379b4795cac92fb314/AWS-agents/AWS_thumbnail.PNG -------------------------------------------------------------------------------- /AWS-agents/README.md: -------------------------------------------------------------------------------- 1 | # AWS install multiple agents 2 | 3 | This shows how to setup multiple agents at the same time. It also supports to run different version for the agent than for the Elastic environment. 4 | 5 | ```json 6 | { 7 | "aws_region" : "eu-west-2", 8 | "aws_access_key" : "", 9 | "aws_secret_key" : "" 10 | } 11 | ``` 12 | 13 | List of other optional parameters that can be added to terraform.tfvars.json 14 | | Parameter Name | Default value | Example | Description | 15 | | ------------- | ------------- | ------------- | ------------- | 16 | | elastic_version | latest | 8.4.1 | Used to define the Elastic Search version | 17 | | elastic_region | aws-eu-west-2 | aws-eu-west-2 | Used to set the Elastic Cloud region for the AWS deployment | 18 | | elastic_deployment_name | AWS Observe and Protect | AWS Observe and Protect | Used to define the name for the Elastic deployment | 19 | 20 | 21 | #### Create AWS Access credentials 22 | 23 | 1. Visit the [IAM Management Console](https://us-east-1.console.aws.amazon.com/iam/home) in AWS 24 | 2. Navigate to the user you want to use for the setup 25 | 3. Click on "Security credentials" 26 | 4. Click on "Create access key" and save the credentials in your `terraform.tfvars.json` file 27 | 28 | Hint: The credentials you choose here will also be used to authenticate the Elastic Agent against your AWS Environment. In production ready setups you might want to change that. Elastic also offers other authentication mechanisms for the Elastic Agent. This terraform script does not ATM. 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /AWS-agents/json_templates/aws_rule_activation.json: -------------------------------------------------------------------------------- 1 | {"action":"enable", "query": "alert.attributes.tags: \"AWS\""} -------------------------------------------------------------------------------- /AWS-agents/json_templates/default-policy.json: -------------------------------------------------------------------------------- 1 | { "name": "${policy_name}", "description": "Terraformed policy", "namespace": "default", "monitoring_enabled": [ "logs", "metrics" ]} -------------------------------------------------------------------------------- /AWS-agents/json_templates/es_api_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${elastic-api-key-name}" 3 | } 4 | -------------------------------------------------------------------------------- /AWS-agents/json_templates/k8s_agent.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-managed-by-fleet.html 3 | apiVersion: apps/v1 4 | kind: DaemonSet 5 | metadata: 6 | name: elastic-agent 7 | namespace: kube-system 8 | labels: 9 | app: elastic-agent 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: elastic-agent 14 | template: 15 | metadata: 16 | labels: 17 | app: elastic-agent 18 | spec: 19 | # Tolerations are needed to run Elastic Agent on Kubernetes master nodes. 20 | # Agents running on master nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes 21 | tolerations: 22 | - key: node-role.kubernetes.io/master 23 | effect: NoSchedule 24 | serviceAccountName: elastic-agent 25 | hostNetwork: true 26 | # 'hostPID: true' enables the Elastic Security integration to observe all process exec events on the host. 27 | # Sharing the host process ID namespace gives visibility of all processes running on the same host. 28 | hostPID: true 29 | dnsPolicy: ClusterFirstWithHostNet 30 | containers: 31 | - name: k8smd 32 | image: docker.elastic.co/endpoint/k8smd:8.4.0 33 | - name: endpoint-security 34 | image: docker.elastic.co/endpoint/endpoint-security:8.4.0 35 | securityContext: 36 | runAsUser: 0 37 | privileged: true 38 | volumeMounts: 39 | - name: boot 40 | mountPath: /boot 41 | - name: debug 42 | mountPath: /sys/kernel/debug 43 | - name: bpf 44 | mountPath: /sys/fs/bpf 45 | - name: etc-passwd 46 | mountPath: /mnt/host/etc/passwd 47 | readOnly: true 48 | - name: etc-group 49 | mountPath: /mnt/host/etc/group 50 | readOnly: true 51 | env: 52 | - name: ELASTIC_ENDPOINT_K8S 53 | value: "true" 54 | - name: elastic-agent 55 | image: docker.elastic.co/beats/elastic-agent:8.4.0 56 | env: 57 | - name: ELASTIC_ENDPOINT_K8S 58 | value: "true" 59 | # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode 60 | - name: FLEET_ENROLL 61 | value: "1" 62 | # Set to true to communicate with Fleet with either insecure HTTP or unverified HTTPS 63 | - name: FLEET_INSECURE 64 | value: "true" 65 | # Fleet Server URL to enroll the Elastic Agent into 66 | # FLEET_URL can be found in Kibana, go to Management > Fleet > Settings 67 | - name: FLEET_URL 68 | value: "${fleet_url}" 69 | # Elasticsearch API key used to enroll Elastic Agents in Fleet (https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens) 70 | # If FLEET_ENROLLMENT_TOKEN is empty then KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed 71 | - name: FLEET_ENROLLMENT_TOKEN 72 | value: "${enrollment_token}" 73 | - name: KIBANA_HOST 74 | value: "" 75 | # The basic authentication username used to connect to Kibana and retrieve a service_token to enable Fleet 76 | - name: KIBANA_FLEET_USERNAME 77 | value: "" # elastic 78 | # The basic authentication password used to connect to Kibana and retrieve a service_token to enable Fleet 79 | - name: KIBANA_FLEET_PASSWORD 80 | value: "" # changeme 81 | - name: NODE_NAME 82 | valueFrom: 83 | fieldRef: 84 | fieldPath: spec.nodeName 85 | - name: POD_NAME 86 | valueFrom: 87 | fieldRef: 88 | fieldPath: metadata.name 89 | securityContext: 90 | runAsUser: 0 91 | resources: 92 | limits: 93 | memory: 500Mi 94 | requests: 95 | cpu: 100m 96 | memory: 200Mi 97 | volumeMounts: 98 | - name: proc 99 | mountPath: /hostfs/proc 100 | readOnly: true 101 | - name: etc-kubernetes 102 | mountPath: /hostfs/etc/kubernetes 103 | readOnly: true 104 | - name: var-lib 105 | mountPath: /hostfs/var/lib 106 | readOnly: true 107 | - name: cgroup 108 | mountPath: /hostfs/sys/fs/cgroup 109 | readOnly: true 110 | - name: varlibdockercontainers 111 | mountPath: /var/lib/docker/containers 112 | readOnly: true 113 | - name: varlog 114 | mountPath: /var/log 115 | readOnly: true 116 | - name: passwd 117 | mountPath: /hostfs/etc/passwd 118 | readOnly: true 119 | - name: group 120 | mountPath: /hostfs/etc/group 121 | readOnly: true 122 | - name: etcsysmd 123 | mountPath: /hostfs/etc/systemd 124 | readOnly: true 125 | - name: etc-mid 126 | mountPath: /etc/machine-id 127 | readOnly: true 128 | volumes: 129 | - name: proc 130 | hostPath: 131 | path: /proc 132 | - name: cgroup 133 | hostPath: 134 | path: /sys/fs/cgroup 135 | - name: varlibdockercontainers 136 | hostPath: 137 | path: /var/lib/docker/containers 138 | - name: varlog 139 | hostPath: 140 | path: /var/log 141 | # Needed for cloudbeat 142 | - name: etc-kubernetes 143 | hostPath: 144 | path: /etc/kubernetes 145 | # Needed for cloudbeat 146 | - name: var-lib 147 | hostPath: 148 | path: /var/lib 149 | # Needed for cloudbeat 150 | - name: passwd 151 | hostPath: 152 | path: /etc/passwd 153 | # Needed for cloudbeat 154 | - name: group 155 | hostPath: 156 | path: /etc/group 157 | # Needed for cloudbeat 158 | - name: etcsysmd 159 | hostPath: 160 | path: /etc/systemd 161 | # Mount /etc/machine-id from the host to determine host ID 162 | # Needed for Elastic Security integration 163 | - name: etc-mid 164 | hostPath: 165 | path: /etc/machine-id 166 | type: File 167 | - name: etc-passwd 168 | hostPath: 169 | path: /etc/passwd 170 | type: File 171 | - name: etc-group 172 | hostPath: 173 | path: /etc/group 174 | type: File 175 | - name: boot 176 | hostPath: 177 | path: /boot 178 | - name: debug 179 | hostPath: 180 | path: /sys/kernel/debug 181 | - name: bpf 182 | hostPath: 183 | path: /sys/fs/bpf 184 | --- 185 | apiVersion: rbac.authorization.k8s.io/v1 186 | kind: ClusterRoleBinding 187 | metadata: 188 | name: elastic-agent 189 | subjects: 190 | - kind: ServiceAccount 191 | name: elastic-agent 192 | namespace: kube-system 193 | roleRef: 194 | kind: ClusterRole 195 | name: elastic-agent 196 | apiGroup: rbac.authorization.k8s.io 197 | --- 198 | apiVersion: rbac.authorization.k8s.io/v1 199 | kind: RoleBinding 200 | metadata: 201 | namespace: kube-system 202 | name: elastic-agent 203 | subjects: 204 | - kind: ServiceAccount 205 | name: elastic-agent 206 | namespace: kube-system 207 | roleRef: 208 | kind: Role 209 | name: elastic-agent 210 | apiGroup: rbac.authorization.k8s.io 211 | --- 212 | apiVersion: rbac.authorization.k8s.io/v1 213 | kind: RoleBinding 214 | metadata: 215 | name: elastic-agent-kubeadm-config 216 | namespace: kube-system 217 | subjects: 218 | - kind: ServiceAccount 219 | name: elastic-agent 220 | namespace: kube-system 221 | roleRef: 222 | kind: Role 223 | name: elastic-agent-kubeadm-config 224 | apiGroup: rbac.authorization.k8s.io 225 | --- 226 | apiVersion: rbac.authorization.k8s.io/v1 227 | kind: ClusterRole 228 | metadata: 229 | name: elastic-agent 230 | labels: 231 | k8s-app: elastic-agent 232 | rules: 233 | - apiGroups: [""] 234 | resources: 235 | - nodes 236 | - namespaces 237 | - events 238 | - pods 239 | - services 240 | - configmaps 241 | # Needed for cloudbeat 242 | - serviceaccounts 243 | - persistentvolumes 244 | - persistentvolumeclaims 245 | verbs: ["get", "list", "watch"] 246 | # Enable this rule only if planing to use kubernetes_secrets provider 247 | #- apiGroups: [""] 248 | # resources: 249 | # - secrets 250 | # verbs: ["get"] 251 | - apiGroups: ["extensions"] 252 | resources: 253 | - replicasets 254 | verbs: ["get", "list", "watch"] 255 | - apiGroups: ["apps"] 256 | resources: 257 | - statefulsets 258 | - deployments 259 | - replicasets 260 | - daemonsets 261 | verbs: ["get", "list", "watch"] 262 | - apiGroups: 263 | - "" 264 | resources: 265 | - nodes/stats 266 | verbs: 267 | - get 268 | - apiGroups: [ "batch" ] 269 | resources: 270 | - jobs 271 | - cronjobs 272 | verbs: [ "get", "list", "watch" ] 273 | # Needed for apiserver 274 | - nonResourceURLs: 275 | - "/metrics" 276 | verbs: 277 | - get 278 | # Needed for cloudbeat 279 | - apiGroups: ["rbac.authorization.k8s.io"] 280 | resources: 281 | - clusterrolebindings 282 | - clusterroles 283 | - rolebindings 284 | - roles 285 | verbs: ["get", "list", "watch"] 286 | # Needed for cloudbeat 287 | - apiGroups: ["policy"] 288 | resources: 289 | - podsecuritypolicies 290 | verbs: ["get", "list", "watch"] 291 | --- 292 | apiVersion: rbac.authorization.k8s.io/v1 293 | kind: Role 294 | metadata: 295 | name: elastic-agent 296 | # Should be the namespace where elastic-agent is running 297 | namespace: kube-system 298 | labels: 299 | k8s-app: elastic-agent 300 | rules: 301 | - apiGroups: 302 | - coordination.k8s.io 303 | resources: 304 | - leases 305 | verbs: ["get", "create", "update"] 306 | --- 307 | apiVersion: rbac.authorization.k8s.io/v1 308 | kind: Role 309 | metadata: 310 | name: elastic-agent-kubeadm-config 311 | namespace: kube-system 312 | labels: 313 | k8s-app: elastic-agent 314 | rules: 315 | - apiGroups: [""] 316 | resources: 317 | - configmaps 318 | resourceNames: 319 | - kubeadm-config 320 | verbs: ["get"] 321 | --- 322 | apiVersion: v1 323 | kind: ServiceAccount 324 | metadata: 325 | name: elastic-agent 326 | namespace: kube-system 327 | labels: 328 | k8s-app: elastic-agent 329 | --- -------------------------------------------------------------------------------- /AWS-agents/json_templates/k8s_cspm_integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloud_security_posture", 3 | "description": "Terraformed k8s integration", 4 | "namespace": "default", 5 | "policy_id": "${policy_id}", 6 | "enabled": true, 7 | "output_id": "", 8 | "inputs": [ 9 | { 10 | "type": "cloudbeat/vanilla", 11 | "policy_template": "kspm", 12 | "enabled": false, 13 | "streams": [ 14 | { 15 | "enabled": false, 16 | "data_stream": { 17 | "type": "logs", 18 | "dataset": "cloud_security_posture.findings" 19 | } 20 | } 21 | ] 22 | }, 23 | { 24 | "type": "cloudbeat/eks", 25 | "policy_template": "kspm", 26 | "enabled": true, 27 | "streams": [ 28 | { 29 | "enabled": true, 30 | "data_stream": { 31 | "type": "logs", 32 | "dataset": "cloud_security_posture.findings" 33 | }, 34 | "vars": { 35 | "access_key_id": { 36 | "type": "text", 37 | "value": "${access_key}" 38 | }, 39 | "secret_access_key": { 40 | "type": "text", 41 | "value": "${access_secret}" 42 | }, 43 | "session_token": { 44 | "type": "text" 45 | } 46 | } 47 | } 48 | ] 49 | } 50 | ], 51 | "package": { 52 | "name": "cloud_security_posture", 53 | "title": "Kubernetes Security Posture Management", 54 | "version": "0.0.26" 55 | }, 56 | "vars": { 57 | "dataYaml": { 58 | "type": "yaml" 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /AWS-agents/json_templates/k8s_endpoint_integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "k8s-endpoint", 3 | "description": "Terraformed k8s integration to protect your endpoints", 4 | "namespace": "default", 5 | "policy_id": "${policy_id}", 6 | "enabled": true, 7 | "output_id": "", 8 | "inputs": [], 9 | "package": { 10 | "name": "endpoint", 11 | "title": "Endpoint and Cloud Security", 12 | "version": "8.4.1" 13 | } 14 | } -------------------------------------------------------------------------------- /AWS-agents/terraform/ami_picker.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | map = { 3 | af-south-1: "ami-022666956ad401a1" 4 | ap-northeast-1: "ami-015f1a68ce825a8d2" 5 | ap-northeast-2: "ami-0be9734c9e68b99f4" 6 | ap-northeast-3: "ami-01cb3e73f8ef13fdc" 7 | ap-south-1: "ami-00aaac1f2ef4ce965" 8 | ap-southeast-1: "ami-0012ffabeb7413479" 9 | ap-southeast-2: "ami-03ec1fe05b3849c74" 10 | ca-central-1: "ami-04c56d394d31cdeac" 11 | eu-central-1: "ami-0980c5102b5ef10cc" 12 | me-south-1: "ami-03cc0b5db8321f2e5" 13 | ap-east-1: "ami-0c7e5903bee96ef81" 14 | eu-north-1: "ami-0663a4867a210287a" 15 | eu-south-1: "ami-035e213233577516f" 16 | eu-west-1: "ami-0213344887e47003a" 17 | eu-west-2: "ami-0add0a5a0cf9afc6c" 18 | eu-west-3: "ami-01019e7343a5f361d" 19 | sa-east-1: "ami-0312c74c38dc7bae6" 20 | us-east-1: "ami-0db6c6238a40c0681" 21 | us-east-2: "ami-03b6c8bd55e00d5ed" 22 | us-west-1: "ami-0f5868930cb63c89c" 23 | us-west-2: "ami-038a0ccaaedae6406" 24 | } 25 | 26 | ami = lookup(local.map, var.aws_region) 27 | } -------------------------------------------------------------------------------- /AWS-agents/terraform/config.tftpl: -------------------------------------------------------------------------------- 1 | inputs: 2 | %{ for values in s3-sqs-objs ~} 3 | - type: "s3-sqs" 4 | id: "${values.arn}" 5 | outputs: 6 | - type: "elasticsearch" 7 | args: 8 | elasticsearch_url: "${elasticsearch_url}" 9 | username: "${elasticsearch_user}" 10 | password: "${elasticsearch_password}" 11 | es_datastream_name: "${values.datastream}" 12 | batch_max_actions: 500 13 | batch_max_bytes: 10485760 14 | %{ endfor ~} 15 | # - type: "sqs" 16 | # id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" 17 | # outputs: 18 | # - type: "elasticsearch" 19 | # args: 20 | # elasticsearch_url: "${elasticsearch_url}" 21 | # username: "${elasticsearch_user}" 22 | # password: "${elasticsearch_password}" 23 | # es_datastream_name: "logs-generic-default" 24 | # batch_max_actions: 500 25 | # batch_max_bytes: 10485760 26 | # - type: "kinesis-data-stream" 27 | # id: "arn:aws:kinesis:%REGION%:%ACCOUNT%:stream/%STREAMNAME%" 28 | # outputs: 29 | # - type: "elasticsearch" 30 | # args: 31 | # elasticsearch_url: "${elasticsearch_url}" 32 | # username: "${elasticsearch_user}" 33 | # password: "${elasticsearch_password}" 34 | # es_datastream_name: "logs-generic-default" 35 | # batch_max_actions: 500 36 | # batch_max_bytes: 10485760 37 | %{ for arn in cw-logs-objs ~} 38 | - type: "cloudwatch-logs" 39 | id: "${arn}:*" 40 | outputs: 41 | - type: "elasticsearch" 42 | args: 43 | elasticsearch_url: "${elasticsearch_url}" 44 | username: "${elasticsearch_user}" 45 | password: "${elasticsearch_password}" 46 | es_datastream_name: "logs-aws.cloudwatch_logs-esf" 47 | batch_max_actions: 500 48 | batch_max_bytes: 10485760 49 | %{ endfor ~} -------------------------------------------------------------------------------- /AWS-agents/terraform/ec_deployment.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Deploy Elastic Cloud 3 | # ------------------------------------------------------------- 4 | data "ec_stack" "latest" { 5 | version_regex = "latest" 6 | region = var.elastic_region 7 | } 8 | 9 | resource "ec_deployment" "elastic_deployment" { 10 | name = var.elastic_deployment_name 11 | region = var.elastic_region 12 | version = var.elastic_version == "latest" ? data.ec_stack.latest.version : var.elastic_version 13 | deployment_template_id = var.elastic_deployment_template_id 14 | elasticsearch = { 15 | hot = { 16 | autoscaling = {} 17 | } 18 | } 19 | kibana = {} 20 | integrations_server = {} 21 | } 22 | 23 | output "elastic_cluster_id" { 24 | value = ec_deployment.elastic_deployment.id 25 | } 26 | 27 | output "elastic_cluster_alias" { 28 | value = ec_deployment.elastic_deployment.name 29 | } 30 | 31 | output "elastic_endpoint" { 32 | value = ec_deployment.elastic_deployment.kibana.https_endpoint 33 | } 34 | 35 | output "elastic_cloud_id" { 36 | value = ec_deployment.elastic_deployment.elasticsearch.cloud_id 37 | } 38 | 39 | output "elastic_username" { 40 | value = ec_deployment.elastic_deployment.elasticsearch_username 41 | } 42 | 43 | output "elastic_password" { 44 | value = ec_deployment.elastic_deployment.elasticsearch_password 45 | sensitive = true 46 | } 47 | 48 | 49 | # ------------------------------------------------------------- 50 | # Load Policy 51 | # ------------------------------------------------------------- 52 | 53 | data "external" "elastic_create_policy" { 54 | query = { 55 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 56 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 57 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 58 | elastic_json_body = templatefile("${path.module}/../json_templates/default-policy.json", {"policy_name": "AWS"}) 59 | } 60 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_create_agent_policy.sh" ] 61 | depends_on = [ec_deployment.elastic_deployment] 62 | } 63 | -------------------------------------------------------------------------------- /AWS-agents/terraform/elastic_agent.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Create EC2 instance + Elastic Agent 3 | # ------------------------------------------------------------- 4 | 5 | data "template_file" "install_agent" { 6 | template = file("../../lib/scripts/agent_install.sh") 7 | vars = { 8 | elastic_version = "8.8.0" 9 | elasticsearch_username = ec_deployment.elastic_deployment.elasticsearch_username 10 | elasticsearch_password = ec_deployment.elastic_deployment.elasticsearch_password 11 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 12 | integration_server_endpoint = ec_deployment.elastic_deployment.integrations_server.https_endpoint 13 | policy_id = data.external.elastic_create_policy.result.id 14 | } 15 | } 16 | 17 | resource "aws_security_group" "elastic-agent" { 18 | name = "elastic-mass-agent-test" 19 | description = "Allow traffic for elastic-agent" 20 | 21 | ingress { 22 | description = "TLS from VPC" 23 | from_port = 443 24 | to_port = 443 25 | protocol = "tcp" 26 | cidr_blocks = ["0.0.0.0/0"] 27 | ipv6_cidr_blocks = ["::/0"] 28 | } 29 | 30 | ingress { 31 | description = "SSH" 32 | from_port = 22 33 | to_port = 22 34 | protocol = "tcp" 35 | cidr_blocks = ["0.0.0.0/0"] 36 | ipv6_cidr_blocks = ["::/0"] 37 | } 38 | 39 | ingress { 40 | description = "ICMP" 41 | from_port = -1 42 | to_port = -1 43 | protocol = "icmp" 44 | cidr_blocks = ["0.0.0.0/0"] 45 | ipv6_cidr_blocks = ["::/0"] 46 | } 47 | 48 | ingress { 49 | description = "Other ports" 50 | from_port = 8000 51 | to_port = 9500 52 | protocol = "tcp" 53 | cidr_blocks = ["0.0.0.0/0"] 54 | ipv6_cidr_blocks = ["::/0"] 55 | } 56 | 57 | ingress { 58 | description = "Other ports" 59 | from_port = 6780 60 | to_port = 6800 61 | protocol = "tcp" 62 | cidr_blocks = ["0.0.0.0/0"] 63 | ipv6_cidr_blocks = ["::/0"] 64 | } 65 | 66 | egress { 67 | from_port = 0 68 | to_port = 0 69 | protocol = "-1" 70 | cidr_blocks = ["0.0.0.0/0"] 71 | ipv6_cidr_blocks = ["::/0"] 72 | } 73 | 74 | tags = { 75 | Name = "elastic-mass-agent-test" 76 | } 77 | } 78 | 79 | resource "aws_instance" "elastic-agent" { 80 | ami = local.ami # us-west-2 81 | instance_type = "t2.micro" 82 | associate_public_ip_address = true 83 | security_groups = [ aws_security_group.elastic-agent.name ] 84 | monitoring = true 85 | 86 | tags = { 87 | Name = "elastic-mass-agent-test" 88 | } 89 | 90 | user_data = "${data.template_file.install_agent.rendered}" 91 | 92 | count = 30 93 | } -------------------------------------------------------------------------------- /AWS-agents/terraform/main.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Terraform provider configuration 3 | # ------------------------------------------------------------- 4 | 5 | terraform { 6 | required_version = ">= 1.0.2" 7 | 8 | required_providers { 9 | ec = { 10 | source = "elastic/ec" 11 | version = ">= 0.4.1" 12 | } 13 | kubectl = { 14 | source = "gavinbunney/kubectl" 15 | version = ">= 1.7.0" 16 | } 17 | } 18 | } 19 | 20 | 21 | -------------------------------------------------------------------------------- /AWS-agents/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Elastic configuration 3 | # ------------------------------------------------------------- 4 | variable "elastic_version" { 5 | type = string 6 | default = "latest" 7 | } 8 | 9 | variable "elastic_region" { 10 | type = string 11 | default = "aws-eu-west-2" 12 | } 13 | 14 | variable "elastic_deployment_name" { 15 | type = string 16 | default = "AWS massive agent test" 17 | } 18 | 19 | variable "elastic_deployment_template_id" { 20 | type = string 21 | default = "aws-general-purpose-arm-v5" 22 | } 23 | 24 | variable "elastic_remotes" { 25 | type = list( 26 | object({ 27 | id = string 28 | alias = string 29 | }) 30 | ) 31 | default = [] 32 | } 33 | 34 | # ------------------------------------------------------------- 35 | # AWS configuration 36 | # ------------------------------------------------------------- 37 | 38 | variable "aws_region" { 39 | type = string 40 | default = "eu-west-1" 41 | } 42 | 43 | variable "aws_access_key" { 44 | type = string 45 | } 46 | 47 | variable "aws_secret_key" { 48 | type = string 49 | } 50 | 51 | variable "bucket_name" { 52 | type = string 53 | default = "elastic-sar-bucket" 54 | } 55 | 56 | variable "elb_names" { 57 | type = list 58 | default = [] 59 | } 60 | 61 | variable "eks_cluster" { 62 | type = string 63 | default = "" 64 | } 65 | -------------------------------------------------------------------------------- /AWS/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .vs/slnx.sqlite 3 | *.tfstate 4 | *.backup 5 | terraform/*.tfstate 6 | terraform/.terraform 7 | *.hcl 8 | local_env -------------------------------------------------------------------------------- /AWS/AWS_thumbnail.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felix-lessoer/elastic-terraform-examples/6eaddd225884752f7367be379b4795cac92fb314/AWS/AWS_thumbnail.PNG -------------------------------------------------------------------------------- /AWS/README.md: -------------------------------------------------------------------------------- 1 | # AWS environment 2 | 3 | ```json 4 | { 5 | "aws_region" : "eu-west-2", 6 | "aws_access_key" : "", 7 | "aws_secret_key" : "" 8 | } 9 | ``` 10 | 11 | List of other optional parameters that can be added to terraform.tfvars.json 12 | | Parameter Name | Default value | Example | Description | 13 | | ------------- | ------------- | ------------- | ------------- | 14 | | elastic_version | latest | 8.4.1 | Used to define the Elastic Search version | 15 | | elastic_region | aws-eu-west-2 | aws-eu-west-2 | Used to set the Elastic Cloud region for the AWS deployment | 16 | | elastic_deployment_name | AWS Observe and Protect | AWS Observe and Protect | Used to define the name for the Elastic deployment | 17 | 18 | 19 | #### Create AWS Access credentials 20 | 21 | 1. Visit the [IAM Management Console](https://us-east-1.console.aws.amazon.com/iam/home) in AWS 22 | 2. Navigate to the user you want to use for the setup 23 | 3. Click on "Security credentials" 24 | 4. Click on "Create access key" and save the credentials in your `terraform.tfvars.json` file 25 | 26 | Hint: The credentials you choose here will also be used to authenticate the Elastic Agent against your AWS Environment. In production ready setups you might want to change that. Elastic also offers other authentication mechanisms for the Elastic Agent. This terraform script does not ATM. 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /AWS/json_templates/aws_rule_activation.json: -------------------------------------------------------------------------------- 1 | {"action":"enable", "query": "alert.attributes.tags: \"AWS\""} -------------------------------------------------------------------------------- /AWS/json_templates/default-policy.json: -------------------------------------------------------------------------------- 1 | { "name": "${policy_name}", "description": "Terraformed policy", "namespace": "default", "monitoring_enabled": [ "logs", "metrics" ]} -------------------------------------------------------------------------------- /AWS/json_templates/es_api_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${elastic-api-key-name}" 3 | } 4 | -------------------------------------------------------------------------------- /AWS/json_templates/k8s_agent.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # For more information refer to https://www.elastic.co/guide/en/fleet/current/running-on-kubernetes-managed-by-fleet.html 3 | apiVersion: apps/v1 4 | kind: DaemonSet 5 | metadata: 6 | name: elastic-agent 7 | namespace: kube-system 8 | labels: 9 | app: elastic-agent 10 | spec: 11 | selector: 12 | matchLabels: 13 | app: elastic-agent 14 | template: 15 | metadata: 16 | labels: 17 | app: elastic-agent 18 | spec: 19 | # Tolerations are needed to run Elastic Agent on Kubernetes master nodes. 20 | # Agents running on master nodes collect metrics from the control plane components (scheduler, controller manager) of Kubernetes 21 | tolerations: 22 | - key: node-role.kubernetes.io/master 23 | effect: NoSchedule 24 | serviceAccountName: elastic-agent 25 | hostNetwork: true 26 | # 'hostPID: true' enables the Elastic Security integration to observe all process exec events on the host. 27 | # Sharing the host process ID namespace gives visibility of all processes running on the same host. 28 | hostPID: true 29 | dnsPolicy: ClusterFirstWithHostNet 30 | containers: 31 | - name: k8smd 32 | image: docker.elastic.co/endpoint/k8smd:8.4.0 33 | - name: endpoint-security 34 | image: docker.elastic.co/endpoint/endpoint-security:8.4.0 35 | securityContext: 36 | runAsUser: 0 37 | privileged: true 38 | volumeMounts: 39 | - name: boot 40 | mountPath: /boot 41 | - name: debug 42 | mountPath: /sys/kernel/debug 43 | - name: bpf 44 | mountPath: /sys/fs/bpf 45 | - name: etc-passwd 46 | mountPath: /mnt/host/etc/passwd 47 | readOnly: true 48 | - name: etc-group 49 | mountPath: /mnt/host/etc/group 50 | readOnly: true 51 | env: 52 | - name: ELASTIC_ENDPOINT_K8S 53 | value: "true" 54 | - name: elastic-agent 55 | image: docker.elastic.co/beats/elastic-agent:8.4.0 56 | env: 57 | - name: ELASTIC_ENDPOINT_K8S 58 | value: "true" 59 | # Set to 1 for enrollment into Fleet server. If not set, Elastic Agent is run in standalone mode 60 | - name: FLEET_ENROLL 61 | value: "1" 62 | # Set to true to communicate with Fleet with either insecure HTTP or unverified HTTPS 63 | - name: FLEET_INSECURE 64 | value: "true" 65 | # Fleet Server URL to enroll the Elastic Agent into 66 | # FLEET_URL can be found in Kibana, go to Management > Fleet > Settings 67 | - name: FLEET_URL 68 | value: "${fleet_url}" 69 | # Elasticsearch API key used to enroll Elastic Agents in Fleet (https://www.elastic.co/guide/en/fleet/current/fleet-enrollment-tokens.html#fleet-enrollment-tokens) 70 | # If FLEET_ENROLLMENT_TOKEN is empty then KIBANA_HOST, KIBANA_FLEET_USERNAME, KIBANA_FLEET_PASSWORD are needed 71 | - name: FLEET_ENROLLMENT_TOKEN 72 | value: "${enrollment_token}" 73 | - name: KIBANA_HOST 74 | value: "" 75 | # The basic authentication username used to connect to Kibana and retrieve a service_token to enable Fleet 76 | - name: KIBANA_FLEET_USERNAME 77 | value: "" # elastic 78 | # The basic authentication password used to connect to Kibana and retrieve a service_token to enable Fleet 79 | - name: KIBANA_FLEET_PASSWORD 80 | value: "" # changeme 81 | - name: NODE_NAME 82 | valueFrom: 83 | fieldRef: 84 | fieldPath: spec.nodeName 85 | - name: POD_NAME 86 | valueFrom: 87 | fieldRef: 88 | fieldPath: metadata.name 89 | securityContext: 90 | runAsUser: 0 91 | resources: 92 | limits: 93 | memory: 500Mi 94 | requests: 95 | cpu: 100m 96 | memory: 200Mi 97 | volumeMounts: 98 | - name: proc 99 | mountPath: /hostfs/proc 100 | readOnly: true 101 | - name: etc-kubernetes 102 | mountPath: /hostfs/etc/kubernetes 103 | readOnly: true 104 | - name: var-lib 105 | mountPath: /hostfs/var/lib 106 | readOnly: true 107 | - name: cgroup 108 | mountPath: /hostfs/sys/fs/cgroup 109 | readOnly: true 110 | - name: varlibdockercontainers 111 | mountPath: /var/lib/docker/containers 112 | readOnly: true 113 | - name: varlog 114 | mountPath: /var/log 115 | readOnly: true 116 | - name: passwd 117 | mountPath: /hostfs/etc/passwd 118 | readOnly: true 119 | - name: group 120 | mountPath: /hostfs/etc/group 121 | readOnly: true 122 | - name: etcsysmd 123 | mountPath: /hostfs/etc/systemd 124 | readOnly: true 125 | - name: etc-mid 126 | mountPath: /etc/machine-id 127 | readOnly: true 128 | volumes: 129 | - name: proc 130 | hostPath: 131 | path: /proc 132 | - name: cgroup 133 | hostPath: 134 | path: /sys/fs/cgroup 135 | - name: varlibdockercontainers 136 | hostPath: 137 | path: /var/lib/docker/containers 138 | - name: varlog 139 | hostPath: 140 | path: /var/log 141 | # Needed for cloudbeat 142 | - name: etc-kubernetes 143 | hostPath: 144 | path: /etc/kubernetes 145 | # Needed for cloudbeat 146 | - name: var-lib 147 | hostPath: 148 | path: /var/lib 149 | # Needed for cloudbeat 150 | - name: passwd 151 | hostPath: 152 | path: /etc/passwd 153 | # Needed for cloudbeat 154 | - name: group 155 | hostPath: 156 | path: /etc/group 157 | # Needed for cloudbeat 158 | - name: etcsysmd 159 | hostPath: 160 | path: /etc/systemd 161 | # Mount /etc/machine-id from the host to determine host ID 162 | # Needed for Elastic Security integration 163 | - name: etc-mid 164 | hostPath: 165 | path: /etc/machine-id 166 | type: File 167 | - name: etc-passwd 168 | hostPath: 169 | path: /etc/passwd 170 | type: File 171 | - name: etc-group 172 | hostPath: 173 | path: /etc/group 174 | type: File 175 | - name: boot 176 | hostPath: 177 | path: /boot 178 | - name: debug 179 | hostPath: 180 | path: /sys/kernel/debug 181 | - name: bpf 182 | hostPath: 183 | path: /sys/fs/bpf 184 | --- 185 | apiVersion: rbac.authorization.k8s.io/v1 186 | kind: ClusterRoleBinding 187 | metadata: 188 | name: elastic-agent 189 | subjects: 190 | - kind: ServiceAccount 191 | name: elastic-agent 192 | namespace: kube-system 193 | roleRef: 194 | kind: ClusterRole 195 | name: elastic-agent 196 | apiGroup: rbac.authorization.k8s.io 197 | --- 198 | apiVersion: rbac.authorization.k8s.io/v1 199 | kind: RoleBinding 200 | metadata: 201 | namespace: kube-system 202 | name: elastic-agent 203 | subjects: 204 | - kind: ServiceAccount 205 | name: elastic-agent 206 | namespace: kube-system 207 | roleRef: 208 | kind: Role 209 | name: elastic-agent 210 | apiGroup: rbac.authorization.k8s.io 211 | --- 212 | apiVersion: rbac.authorization.k8s.io/v1 213 | kind: RoleBinding 214 | metadata: 215 | name: elastic-agent-kubeadm-config 216 | namespace: kube-system 217 | subjects: 218 | - kind: ServiceAccount 219 | name: elastic-agent 220 | namespace: kube-system 221 | roleRef: 222 | kind: Role 223 | name: elastic-agent-kubeadm-config 224 | apiGroup: rbac.authorization.k8s.io 225 | --- 226 | apiVersion: rbac.authorization.k8s.io/v1 227 | kind: ClusterRole 228 | metadata: 229 | name: elastic-agent 230 | labels: 231 | k8s-app: elastic-agent 232 | rules: 233 | - apiGroups: [""] 234 | resources: 235 | - nodes 236 | - namespaces 237 | - events 238 | - pods 239 | - services 240 | - configmaps 241 | # Needed for cloudbeat 242 | - serviceaccounts 243 | - persistentvolumes 244 | - persistentvolumeclaims 245 | verbs: ["get", "list", "watch"] 246 | # Enable this rule only if planing to use kubernetes_secrets provider 247 | #- apiGroups: [""] 248 | # resources: 249 | # - secrets 250 | # verbs: ["get"] 251 | - apiGroups: ["extensions"] 252 | resources: 253 | - replicasets 254 | verbs: ["get", "list", "watch"] 255 | - apiGroups: ["apps"] 256 | resources: 257 | - statefulsets 258 | - deployments 259 | - replicasets 260 | - daemonsets 261 | verbs: ["get", "list", "watch"] 262 | - apiGroups: 263 | - "" 264 | resources: 265 | - nodes/stats 266 | verbs: 267 | - get 268 | - apiGroups: [ "batch" ] 269 | resources: 270 | - jobs 271 | - cronjobs 272 | verbs: [ "get", "list", "watch" ] 273 | # Needed for apiserver 274 | - nonResourceURLs: 275 | - "/metrics" 276 | verbs: 277 | - get 278 | # Needed for cloudbeat 279 | - apiGroups: ["rbac.authorization.k8s.io"] 280 | resources: 281 | - clusterrolebindings 282 | - clusterroles 283 | - rolebindings 284 | - roles 285 | verbs: ["get", "list", "watch"] 286 | # Needed for cloudbeat 287 | - apiGroups: ["policy"] 288 | resources: 289 | - podsecuritypolicies 290 | verbs: ["get", "list", "watch"] 291 | --- 292 | apiVersion: rbac.authorization.k8s.io/v1 293 | kind: Role 294 | metadata: 295 | name: elastic-agent 296 | # Should be the namespace where elastic-agent is running 297 | namespace: kube-system 298 | labels: 299 | k8s-app: elastic-agent 300 | rules: 301 | - apiGroups: 302 | - coordination.k8s.io 303 | resources: 304 | - leases 305 | verbs: ["get", "create", "update"] 306 | --- 307 | apiVersion: rbac.authorization.k8s.io/v1 308 | kind: Role 309 | metadata: 310 | name: elastic-agent-kubeadm-config 311 | namespace: kube-system 312 | labels: 313 | k8s-app: elastic-agent 314 | rules: 315 | - apiGroups: [""] 316 | resources: 317 | - configmaps 318 | resourceNames: 319 | - kubeadm-config 320 | verbs: ["get"] 321 | --- 322 | apiVersion: v1 323 | kind: ServiceAccount 324 | metadata: 325 | name: elastic-agent 326 | namespace: kube-system 327 | labels: 328 | k8s-app: elastic-agent 329 | --- -------------------------------------------------------------------------------- /AWS/json_templates/k8s_cspm_integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloud_security_posture", 3 | "description": "Terraformed k8s integration", 4 | "namespace": "default", 5 | "policy_id": "${policy_id}", 6 | "enabled": true, 7 | "output_id": "", 8 | "inputs": [ 9 | { 10 | "type": "cloudbeat/vanilla", 11 | "policy_template": "kspm", 12 | "enabled": false, 13 | "streams": [ 14 | { 15 | "enabled": false, 16 | "data_stream": { 17 | "type": "logs", 18 | "dataset": "cloud_security_posture.findings" 19 | } 20 | } 21 | ] 22 | }, 23 | { 24 | "type": "cloudbeat/eks", 25 | "policy_template": "kspm", 26 | "enabled": true, 27 | "streams": [ 28 | { 29 | "enabled": true, 30 | "data_stream": { 31 | "type": "logs", 32 | "dataset": "cloud_security_posture.findings" 33 | }, 34 | "vars": { 35 | "access_key_id": { 36 | "type": "text", 37 | "value": "${access_key}" 38 | }, 39 | "secret_access_key": { 40 | "type": "text", 41 | "value": "${access_secret}" 42 | }, 43 | "session_token": { 44 | "type": "text" 45 | } 46 | } 47 | } 48 | ] 49 | } 50 | ], 51 | "package": { 52 | "name": "cloud_security_posture", 53 | "title": "Kubernetes Security Posture Management", 54 | "version": "0.0.26" 55 | }, 56 | "vars": { 57 | "dataYaml": { 58 | "type": "yaml" 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /AWS/json_templates/k8s_endpoint_integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "k8s-endpoint", 3 | "description": "Terraformed k8s integration to protect your endpoints", 4 | "namespace": "default", 5 | "policy_id": "${policy_id}", 6 | "enabled": true, 7 | "output_id": "", 8 | "inputs": [], 9 | "package": { 10 | "name": "endpoint", 11 | "title": "Endpoint and Cloud Security", 12 | "version": "8.4.1" 13 | } 14 | } -------------------------------------------------------------------------------- /AWS/terraform/ami_picker.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | map = { 3 | af-south-1: "ami-022666956ad401a1" 4 | ap-northeast-1: "ami-015f1a68ce825a8d2" 5 | ap-northeast-2: "ami-0be9734c9e68b99f4" 6 | ap-northeast-3: "ami-01cb3e73f8ef13fdc" 7 | ap-south-1: "ami-00aaac1f2ef4ce965" 8 | ap-southeast-1: "ami-0012ffabeb7413479" 9 | ap-southeast-2: "ami-03ec1fe05b3849c74" 10 | ca-central-1: "ami-04c56d394d31cdeac" 11 | eu-central-1: "ami-0980c5102b5ef10cc" 12 | me-south-1: "ami-03cc0b5db8321f2e5" 13 | ap-east-1: "ami-0c7e5903bee96ef81" 14 | eu-north-1: "ami-0663a4867a210287a" 15 | eu-south-1: "ami-035e213233577516f" 16 | eu-west-1: "ami-0213344887e47003a" 17 | eu-west-2: "ami-0add0a5a0cf9afc6c" 18 | eu-west-3: "ami-01019e7343a5f361d" 19 | sa-east-1: "ami-0312c74c38dc7bae6" 20 | us-east-1: "ami-0db6c6238a40c0681" 21 | us-east-2: "ami-03b6c8bd55e00d5ed" 22 | us-west-1: "ami-0f5868930cb63c89c" 23 | us-west-2: "ami-038a0ccaaedae6406" 24 | } 25 | 26 | ami = lookup(local.map, var.aws_region) 27 | } -------------------------------------------------------------------------------- /AWS/terraform/cloud_formation.tf: -------------------------------------------------------------------------------- 1 | # resource "aws_cloudformation_stack" "elastic" { 2 | # name = "elastic-stack" 3 | # template_url = "https://mp-saas-integrations.s3.amazonaws.com/saas-elastic-cloud/main/templates/AgentInstallMain.yaml" 4 | # capabilities = ["CAPABILITY_NAMED_IAM"] 5 | # parameters = { 6 | # DeploymentName = var.elastic_aws_deployment_name 7 | # EC2HostName = "elastic-agent" 8 | # EC2InstanceType = "t2.micro" 9 | # GitBranchName = "main" 10 | # KeyPairName = "felix-putty" 11 | # PublicSubnet1ID = "subnet-00f145e4ab29d2d0c" 12 | # PublicSubnet2ID = "subnet-00f145e4ab29d2d0c" 13 | # QSS3BucketName = "mp-saas-integrations" 14 | # QSS3KeyPrefix = "saas-elastic-cloud/" 15 | # Region = "us-east-1" 16 | # RemoteAccessCIDR = "172.31.0.0/16" 17 | # RootVolumeSize = "10" 18 | # SecretName = "arn:aws:secretsmanager:us-east-1:644184947617:secret:ec_pme_dev_api_key-UCoLzg" 19 | # VPCID = "vpc-0a8d055883d1a19ac" 20 | # } 21 | 22 | # } -------------------------------------------------------------------------------- /AWS/terraform/config.tftpl: -------------------------------------------------------------------------------- 1 | inputs: 2 | %{ for values in s3-sqs-objs ~} 3 | - type: "s3-sqs" 4 | id: "${values.arn}" 5 | outputs: 6 | - type: "elasticsearch" 7 | args: 8 | elasticsearch_url: "${elasticsearch_url}" 9 | username: "${elasticsearch_user}" 10 | password: "${elasticsearch_password}" 11 | es_datastream_name: "${values.datastream}" 12 | batch_max_actions: 500 13 | batch_max_bytes: 10485760 14 | %{ endfor ~} 15 | # - type: "sqs" 16 | # id: "arn:aws:sqs:%REGION%:%ACCOUNT%:%QUEUENAME%" 17 | # outputs: 18 | # - type: "elasticsearch" 19 | # args: 20 | # elasticsearch_url: "${elasticsearch_url}" 21 | # username: "${elasticsearch_user}" 22 | # password: "${elasticsearch_password}" 23 | # es_datastream_name: "logs-generic-default" 24 | # batch_max_actions: 500 25 | # batch_max_bytes: 10485760 26 | # - type: "kinesis-data-stream" 27 | # id: "arn:aws:kinesis:%REGION%:%ACCOUNT%:stream/%STREAMNAME%" 28 | # outputs: 29 | # - type: "elasticsearch" 30 | # args: 31 | # elasticsearch_url: "${elasticsearch_url}" 32 | # username: "${elasticsearch_user}" 33 | # password: "${elasticsearch_password}" 34 | # es_datastream_name: "logs-generic-default" 35 | # batch_max_actions: 500 36 | # batch_max_bytes: 10485760 37 | %{ for arn in cw-logs-objs ~} 38 | - type: "cloudwatch-logs" 39 | id: "${arn}:*" 40 | outputs: 41 | - type: "elasticsearch" 42 | args: 43 | elasticsearch_url: "${elasticsearch_url}" 44 | username: "${elasticsearch_user}" 45 | password: "${elasticsearch_password}" 46 | es_datastream_name: "logs-aws.cloudwatch_logs-esf" 47 | batch_max_actions: 500 48 | batch_max_bytes: 10485760 49 | %{ endfor ~} -------------------------------------------------------------------------------- /AWS/terraform/ec_deployment.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Deploy Elastic Cloud 3 | # ------------------------------------------------------------- 4 | data "ec_stack" "latest" { 5 | version_regex = "latest" 6 | region = var.elastic_region 7 | } 8 | 9 | resource "ec_deployment" "elastic_deployment" { 10 | name = var.elastic_deployment_name 11 | region = var.elastic_region 12 | version = var.elastic_version == "latest" ? data.ec_stack.latest.version : var.elastic_version 13 | deployment_template_id = var.elastic_deployment_template_id 14 | elasticsearch = { 15 | hot = { 16 | autoscaling = {} 17 | } 18 | 19 | #"remote_cluster" = { 20 | # for_each = { for idx, remote in var.elastic_remotes : idx => remote } 21 | # deployment_id = each.value["id"] 22 | # alias = each.value["alias"] 23 | # } 24 | 25 | } 26 | kibana = {} 27 | integrations_server = {} 28 | } 29 | 30 | output "elastic_cluster_id" { 31 | value = ec_deployment.elastic_deployment.id 32 | } 33 | 34 | output "elastic_cluster_alias" { 35 | value = ec_deployment.elastic_deployment.name 36 | } 37 | 38 | output "elastic_endpoint" { 39 | value = ec_deployment.elastic_deployment.kibana.https_endpoint 40 | } 41 | 42 | output "elastic_cloud_id" { 43 | value = ec_deployment.elastic_deployment.elasticsearch.cloud_id 44 | } 45 | 46 | output "elastic_username" { 47 | value = ec_deployment.elastic_deployment.elasticsearch_username 48 | } 49 | 50 | output "elastic_password" { 51 | value = ec_deployment.elastic_deployment.elasticsearch_password 52 | sensitive = true 53 | } 54 | 55 | 56 | # ------------------------------------------------------------- 57 | # Load Policy 58 | # ------------------------------------------------------------- 59 | 60 | data "external" "elastic_create_policy" { 61 | query = { 62 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 63 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 64 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 65 | elastic_json_body = templatefile("${path.module}/../json_templates/default-policy.json", {"policy_name": "AWS"}) 66 | } 67 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_create_agent_policy.sh" ] 68 | depends_on = [ec_deployment.elastic_deployment] 69 | } 70 | 71 | data "external" "elastic_add_integration" { 72 | query = { 73 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 74 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 75 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 76 | elastic_json_body = templatefile("${path.module}/../json_templates/aws_integration.json", 77 | { 78 | "policy_id": data.external.elastic_create_policy.result.id, 79 | "access_key": var.aws_access_key, 80 | "access_secret": var.aws_secret_key, 81 | "aws_region": var.aws_region, 82 | } 83 | ) 84 | } 85 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 86 | depends_on = [data.external.elastic_create_policy] 87 | } 88 | 89 | # ------------------------------------------------------------- 90 | # Load Rules 91 | # ------------------------------------------------------------- 92 | 93 | data "external" "elastic_load_rules" { 94 | query = { 95 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 96 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 97 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 98 | } 99 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_load_detection_rules.sh" ] 100 | depends_on = [ec_deployment.elastic_deployment] 101 | } 102 | 103 | data "external" "elastic_enable_rules" { 104 | query = { 105 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 106 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 107 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 108 | elastic_json_body = templatefile("${path.module}/../json_templates/aws_rule_activation.json",{}) 109 | } 110 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_enable_detection_rules.sh" ] 111 | depends_on = [data.external.elastic_load_rules] 112 | } 113 | 114 | # output "elastic_enable_aws_rules" { 115 | # value = data.external.elastic_enable_rules.result 116 | # depends_on = [ 117 | # data.external.elastic_enable_rules 118 | # ] 119 | # } 120 | 121 | # ------------------------------------------------------------- 122 | # Create and Start transforms 123 | # ------------------------------------------------------------- 124 | 125 | # ------------------------------------------------------------- 126 | # Load Dashboards 127 | # ------------------------------------------------------------- 128 | data "external" "elastic_upload_saved_objects" { 129 | query = { 130 | elastic_http_method = "POST" 131 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 132 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 133 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 134 | so_file = "${path.module}/../dashboards/AWS-extension.ndjson" 135 | } 136 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_upload_saved_objects.sh" ] 137 | depends_on = [ec_deployment.elastic_deployment] 138 | } -------------------------------------------------------------------------------- /AWS/terraform/elastic_agent.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Create EC2 instance + Elastic Agent 3 | # ------------------------------------------------------------- 4 | 5 | data "template_file" "install_agent" { 6 | template = file("../../lib/scripts/agent_install.sh") 7 | vars = { 8 | elastic_version = var.elastic_version 9 | elasticsearch_username = ec_deployment.elastic_deployment.elasticsearch_username 10 | elasticsearch_password = ec_deployment.elastic_deployment.elasticsearch_password 11 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 12 | integration_server_endpoint = ec_deployment.elastic_deployment.integrations_server.https_endpoint 13 | policy_id = data.external.elastic_create_policy.result.id 14 | } 15 | } 16 | 17 | resource "aws_security_group" "elastic-agent" { 18 | name = "elastic-agent" 19 | description = "Allow traffic for elastic-agent" 20 | 21 | ingress { 22 | description = "TLS from VPC" 23 | from_port = 443 24 | to_port = 443 25 | protocol = "tcp" 26 | cidr_blocks = ["0.0.0.0/0"] 27 | ipv6_cidr_blocks = ["::/0"] 28 | } 29 | 30 | ingress { 31 | description = "SSH" 32 | from_port = 22 33 | to_port = 22 34 | protocol = "tcp" 35 | cidr_blocks = ["0.0.0.0/0"] 36 | ipv6_cidr_blocks = ["::/0"] 37 | } 38 | 39 | ingress { 40 | description = "ICMP" 41 | from_port = -1 42 | to_port = -1 43 | protocol = "icmp" 44 | cidr_blocks = ["0.0.0.0/0"] 45 | ipv6_cidr_blocks = ["::/0"] 46 | } 47 | 48 | ingress { 49 | description = "Other ports" 50 | from_port = 8000 51 | to_port = 9500 52 | protocol = "tcp" 53 | cidr_blocks = ["0.0.0.0/0"] 54 | ipv6_cidr_blocks = ["::/0"] 55 | } 56 | 57 | ingress { 58 | description = "Other ports" 59 | from_port = 6780 60 | to_port = 6800 61 | protocol = "tcp" 62 | cidr_blocks = ["0.0.0.0/0"] 63 | ipv6_cidr_blocks = ["::/0"] 64 | } 65 | 66 | egress { 67 | from_port = 0 68 | to_port = 0 69 | protocol = "-1" 70 | cidr_blocks = ["0.0.0.0/0"] 71 | ipv6_cidr_blocks = ["::/0"] 72 | } 73 | 74 | tags = { 75 | Name = "elastic-agent" 76 | } 77 | } 78 | 79 | resource "aws_instance" "elastic-agent" { 80 | ami = local.ami # us-west-2 81 | instance_type = "t2.micro" 82 | associate_public_ip_address = true 83 | security_groups = [ aws_security_group.elastic-agent.name ] 84 | monitoring = true 85 | #key_name = "felix-london" 86 | 87 | tags = { 88 | Name = "elastic-agent" 89 | } 90 | 91 | user_data = "${data.template_file.install_agent.rendered}" 92 | } -------------------------------------------------------------------------------- /AWS/terraform/esf_formation.tf: -------------------------------------------------------------------------------- 1 | resource "aws_s3_object" "config_file" { 2 | bucket = aws_s3_bucket.elastic_bucket.bucket 3 | key = "sar_config.yaml" 4 | content = templatefile("${path.module}/config.tftpl", 5 | { 6 | s3-sqs-objs = [ 7 | {arn= aws_sqs_queue.vpc-events.arn, datastream = "logs-aws.vpcflow-esf"}, 8 | {arn= aws_sqs_queue.cloudtrail-events.arn, datastream = "logs-aws.cloudtrail-esf"}, 9 | {arn= aws_sqs_queue.s3-events.arn, datastream = "logs-aws.s3access-esf"}, 10 | {arn= aws_sqs_queue.elb-events.arn, datastream = "logs-aws.elb-esf"} 11 | ] 12 | cw-logs-objs = data.aws_cloudwatch_log_groups.all.arns 13 | elasticsearch_url = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 14 | elasticsearch_user = ec_deployment.elastic_deployment.elasticsearch_username 15 | elasticsearch_password = ec_deployment.elastic_deployment.elasticsearch_password 16 | } 17 | ) 18 | } 19 | 20 | data "aws_serverlessapplicationrepository_application" "esf_sar" { 21 | application_id = "arn:aws:serverlessrepo:eu-central-1:267093732750:applications/elastic-serverless-forwarder" 22 | } 23 | 24 | resource "aws_serverlessapplicationrepository_cloudformation_stack" "esf_cf_stack" { 25 | name = "terraform-elastic-serverless-forwarder" 26 | application_id = data.aws_serverlessapplicationrepository_application.esf_sar.application_id 27 | semantic_version = data.aws_serverlessapplicationrepository_application.esf_sar.semantic_version 28 | capabilities = data.aws_serverlessapplicationrepository_application.esf_sar.required_capabilities 29 | 30 | parameters = { 31 | ElasticServerlessForwarderS3ConfigFile = "s3://${aws_s3_bucket.elastic_bucket.id}/sar_config.yaml" ## FILL WITH THE VALUE OF THE S3 URL IN THE FORMAT "s3://bucket-name/config-file-name" POINTING TO THE CONFIGURATION FILE FOR YOUR DEPLOYMENT OF THE ELASTIC SERVERLESS FORWARDER 32 | 33 | ElasticServerlessForwarderSSMSecrets = "" ## FILL WITH A COMMA DELIMITED LIST OF AWS SSM SECRETS ARNS REFERENCED IN THE CONFIG YAML FILE (IF ANY). 34 | 35 | ElasticServerlessForwarderKMSKeys = "" ## FILL WITH A COMMA DELIMITED LIST OF AWS KMS KEYS ARNS TO BE USED FOR DECRYPTING AWS SSM SECRETS REFERENCED IN THE CONFIG YAML FILE (IF ANY). 36 | 37 | ElasticServerlessForwarderSQSEvents = "" ## FILL WITH A COMMA DELIMITED LIST OF DIRECT SQS QUEUES ARNS TO SET AS EVENT TRIGGERS FOR THE LAMBDA (IF ANY). 38 | 39 | ElasticServerlessForwarderS3SQSEvents = "${aws_sqs_queue.vpc-events.arn},${aws_sqs_queue.cloudtrail-events.arn},${aws_sqs_queue.s3-events.arn},${aws_sqs_queue.elb-events.arn}" ## FILL WITH A COMMA DELIMITED LIST OF S3 SQS EVENT NOTIFICATIONS ARNS TO SET AS EVENT TRIGGERS FOR THE LAMBDA (IF ANY). 40 | 41 | ElasticServerlessForwarderKinesisEvents = "" ## FILL WITH A COMMA DELIMITED LIST OF KINESIS DATA STREAM ARNS TO SET AS EVENT TRIGGERS FOR THE LAMBDA (IF ANY). 42 | 43 | ElasticServerlessForwarderCloudWatchLogsEvents = "${join(",",data.aws_cloudwatch_log_groups.all.arns)}" ## FILL WITH A COMMA DELIMITED LIST OF CLOUDWATCH LOGS LOG GROUPS ARNS TO SET SUBSCRIPTION FILTERS ON THE LAMBDA FOR (IF ANY). 44 | 45 | ElasticServerlessForwarderS3Buckets = "${aws_s3_bucket.elastic_bucket.arn}" ## FILL WITH A COMMA DELIMITED LIST OF S3 BUCKETS ARNS THAT ARE THE SOURCES OF THE S3 SQS EVENT NOTIFICATIONS (IF ANY). 46 | } 47 | 48 | depends_on = [ 49 | data.aws_cloudwatch_log_groups.all 50 | ] 51 | } -------------------------------------------------------------------------------- /AWS/terraform/logs-cloudwatch.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Get all Log Groups 3 | # ------------------------------------------------------------- 4 | 5 | data "aws_cloudwatch_log_groups" "all" {} 6 | 7 | # ------------------------------------------------------------- 8 | # Data Collection 9 | # -- For Cloudwatch we use Elastic Agent to collect data from each log group 10 | # ------------------------------------------------------------- 11 | 12 | # data "external" "elastic_add_cw_integrations" { 13 | # for_each=data.aws_cloudwatch_log_groups.all.arns 14 | # query = { 15 | # kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 16 | # elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 17 | # elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 18 | # elastic_json_body = templatefile("${path.module}/../json_templates/aws_cw_integration.json", 19 | # { 20 | # "name_suffix": each.key, 21 | # //"log_group_name": each.value.log_group_names, 22 | # "log_group_arn": each.value, 23 | # "policy_id": data.external.elastic_create_policy.result.id, 24 | # "access_key": var.aws_access_key, 25 | # "access_secret": var.aws_secret_key, 26 | # } 27 | # ) 28 | # } 29 | # program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 30 | # depends_on = [data.external.elastic_create_policy, data.aws_cloudwatch_log_groups.all] 31 | # } 32 | 33 | # ------------------------------------------------------------- 34 | # Data Collection 35 | # -- using the serverless forwarder 36 | # ------------------------------------------------------------- 37 | 38 | -------------------------------------------------------------------------------- /AWS/terraform/logs_cloudtrail.tf: -------------------------------------------------------------------------------- 1 | data "aws_caller_identity" "current" {} 2 | 3 | resource "aws_cloudtrail" "management" { 4 | name = "tf-trail-elastic" 5 | s3_bucket_name = aws_s3_bucket.elastic_bucket.id 6 | include_global_service_events = true 7 | is_multi_region_trail = true 8 | enable_logging = true 9 | 10 | event_selector { 11 | read_write_type = "All" 12 | include_management_events = true 13 | 14 | data_resource { 15 | type = "AWS::Lambda::Function" 16 | values = ["arn:aws:lambda"] 17 | } 18 | 19 | data_resource { 20 | type = "AWS::S3::Object" 21 | values = ["arn:aws:s3"] 22 | } 23 | } 24 | 25 | depends_on = [aws_s3_bucket_policy.cloudtrail] 26 | } 27 | 28 | # ------------------------------------------------------------- 29 | # Event trigger 30 | # ------------------------------------------------------------- 31 | 32 | resource "aws_sqs_queue" "cloudtrail-events" { 33 | name = "s3-cloudtrail-event-notification-queue" 34 | visibility_timeout_seconds = 900 35 | policy = <", 7 | "azure_client_secret" : "", 8 | "azure_tenant_id": "", 9 | "azure_subscription_id": "" 10 | } 11 | ``` 12 | 13 | List of other optional parameters that can be added to terraform.tfvars.json 14 | | Parameter Name | Default value | Example | Description | 15 | | ------------- | ------------- | ------------- | ------------- | 16 | | elastic_version | latest | 8.4.1 | Used to define the Elastic Search version | 17 | | elastic_region | azure-westeurope | azure-westeurope | Used to set the Elastic Cloud region for the Azure deployment | 18 | | elastic_deployment_name | Azure Observe and Protect | Azure Observe and Protect | Used to define the name for the Elastic deployment | 19 | 20 | 21 | #### Create Azure Access credentials 22 | 23 | 1. Visit the [Azure Active Directory](https://portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps) config page in Azure 24 | 2. Create a new application 25 | 3. Click on "Client credentials" 26 | 4. Click on "New client secret" and save the credentials in your `azure.json` file 27 | 28 | Hint: The credentials you choose here will also be used to authenticate the Elastic Agent against your Azure Environment. In production ready setups you might want to change that. Elastic also offers other authentication mechanisms for the Elastic Agent. This terraform script does not ATM. 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /Azure/json_templates/azure-billing-integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "policy_id": "${policy_id}", 3 | "package": { 4 | "name": "azure_billing", 5 | "version": "1.1.1" 6 | }, 7 | "name": "azure_billing", 8 | "description": "", 9 | "namespace": "default", 10 | "inputs": { 11 | "billing-azure/metrics": { 12 | "enabled": true, 13 | "streams": { 14 | "azure.billing": { 15 | "enabled": true, 16 | "vars": { 17 | "period": "24h" 18 | } 19 | } 20 | } 21 | } 22 | }, 23 | "vars": { 24 | "client_id": "${client_id}", 25 | "client_secret": "${client_secret}", 26 | "tenant_id": "${tenant_id}", 27 | "subscription_id": "${subscription_id}" 28 | } 29 | } -------------------------------------------------------------------------------- /Azure/json_templates/azure-logs-integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure Logs", 3 | "description": "", 4 | "namespace": "default", 5 | "policy_id": "${policy_id}", 6 | "enabled": true, 7 | "output_id": "", 8 | "inputs": [ 9 | { 10 | "type": "azure-eventhub", 11 | "policy_template": "eventhub", 12 | "enabled": true, 13 | "streams": [ 14 | { 15 | "enabled": true, 16 | "data_stream": { 17 | "type": "logs", 18 | "dataset": "azure.eventhub" 19 | }, 20 | "vars": { 21 | "parse_message": { 22 | "value": false, 23 | "type": "bool" 24 | }, 25 | "preserve_original_event": { 26 | "value": false, 27 | "type": "bool" 28 | }, 29 | "data_stream.dataset": { 30 | "value": "azure.eventhub", 31 | "type": "text" 32 | }, 33 | "storage_account_container": { 34 | "type": "text" 35 | }, 36 | "tags": { 37 | "value": [ 38 | "azure-eventhub", 39 | "forwarded" 40 | ], 41 | "type": "text" 42 | }, 43 | "processors": { 44 | "type": "yaml" 45 | } 46 | } 47 | } 48 | ] 49 | }, 50 | { 51 | "type": "azure-eventhub", 52 | "policy_template": "adlogs", 53 | "enabled": true, 54 | "streams": [ 55 | { 56 | "enabled": true, 57 | "data_stream": { 58 | "type": "logs", 59 | "dataset": "azure.auditlogs" 60 | }, 61 | "vars": { 62 | "preserve_original_event": { 63 | "value": false, 64 | "type": "bool" 65 | }, 66 | "storage_account_container": { 67 | "type": "text" 68 | }, 69 | "tags": { 70 | "value": [ 71 | "azure-auditlogs", 72 | "forwarded" 73 | ], 74 | "type": "text" 75 | }, 76 | "processors": { 77 | "type": "yaml" 78 | } 79 | } 80 | }, 81 | { 82 | "enabled": false, 83 | "data_stream": { 84 | "type": "logs", 85 | "dataset": "azure.identity_protection" 86 | }, 87 | "vars": { 88 | "preserve_original_event": { 89 | "value": false, 90 | "type": "bool" 91 | }, 92 | "storage_account_container": { 93 | "type": "text" 94 | }, 95 | "tags": { 96 | "value": [ 97 | "azure-identity-protection-logs", 98 | "forwarded" 99 | ], 100 | "type": "text" 101 | }, 102 | "processors": { 103 | "type": "yaml" 104 | } 105 | } 106 | }, 107 | { 108 | "enabled": false, 109 | "data_stream": { 110 | "type": "logs", 111 | "dataset": "azure.provisioning" 112 | }, 113 | "vars": { 114 | "preserve_original_event": { 115 | "value": false, 116 | "type": "bool" 117 | }, 118 | "storage_account_container": { 119 | "type": "text" 120 | }, 121 | "tags": { 122 | "value": [ 123 | "azure-provisioning-logs", 124 | "forwarded" 125 | ], 126 | "type": "text" 127 | }, 128 | "processors": { 129 | "type": "yaml" 130 | } 131 | } 132 | }, 133 | { 134 | "enabled": true, 135 | "data_stream": { 136 | "type": "logs", 137 | "dataset": "azure.signinlogs" 138 | }, 139 | "vars": { 140 | "storage_account_container": { 141 | "type": "text" 142 | }, 143 | "tags": { 144 | "value": [ 145 | "azure-signinlogs", 146 | "forwarded" 147 | ], 148 | "type": "text" 149 | }, 150 | "preserve_original_event": { 151 | "value": false, 152 | "type": "bool" 153 | }, 154 | "processors": { 155 | "type": "yaml" 156 | } 157 | } 158 | } 159 | ] 160 | }, 161 | { 162 | "type": "azure-eventhub", 163 | "policy_template": "platformlogs", 164 | "enabled": true, 165 | "streams": [ 166 | { 167 | "enabled": true, 168 | "data_stream": { 169 | "type": "logs", 170 | "dataset": "azure.platformlogs" 171 | }, 172 | "vars": { 173 | "preserve_original_event": { 174 | "value": false, 175 | "type": "bool" 176 | }, 177 | "storage_account_container": { 178 | "type": "text" 179 | }, 180 | "tags": { 181 | "value": [ 182 | "azure-platformlogs", 183 | "forwarded" 184 | ], 185 | "type": "text" 186 | }, 187 | "processors": { 188 | "type": "yaml" 189 | } 190 | } 191 | } 192 | ] 193 | }, 194 | { 195 | "type": "azure-eventhub", 196 | "policy_template": "activitylogs", 197 | "enabled": true, 198 | "streams": [ 199 | { 200 | "enabled": true, 201 | "data_stream": { 202 | "type": "logs", 203 | "dataset": "azure.activitylogs" 204 | }, 205 | "vars": { 206 | "preserve_original_event": { 207 | "value": false, 208 | "type": "bool" 209 | }, 210 | "storage_account_container": { 211 | "type": "text" 212 | }, 213 | "tags": { 214 | "value": [ 215 | "azure-activitylogs", 216 | "forwarded" 217 | ], 218 | "type": "text" 219 | }, 220 | "processors": { 221 | "type": "yaml" 222 | } 223 | } 224 | } 225 | ] 226 | }, 227 | { 228 | "type": "azure-eventhub", 229 | "policy_template": "springcloudlogs", 230 | "enabled": true, 231 | "streams": [ 232 | { 233 | "enabled": true, 234 | "data_stream": { 235 | "type": "logs", 236 | "dataset": "azure.springcloudlogs" 237 | }, 238 | "vars": { 239 | "preserve_original_event": { 240 | "value": false, 241 | "type": "bool" 242 | }, 243 | "storage_account_container": { 244 | "type": "text" 245 | }, 246 | "tags": { 247 | "value": [ 248 | "azure-springcloudlogs", 249 | "forwarded" 250 | ], 251 | "type": "text" 252 | }, 253 | "processors": { 254 | "type": "yaml" 255 | } 256 | } 257 | } 258 | ] 259 | }, 260 | { 261 | "type": "azure-eventhub", 262 | "policy_template": "firewall_logs", 263 | "enabled": true, 264 | "streams": [ 265 | { 266 | "enabled": true, 267 | "data_stream": { 268 | "type": "logs", 269 | "dataset": "azure.firewall_logs" 270 | }, 271 | "vars": { 272 | "preserve_original_event": { 273 | "value": false, 274 | "type": "bool" 275 | }, 276 | "storage_account_container": { 277 | "type": "text" 278 | }, 279 | "tags": { 280 | "value": [ 281 | "azure-firewall", 282 | "forwarded" 283 | ], 284 | "type": "text" 285 | }, 286 | "processors": { 287 | "type": "yaml" 288 | } 289 | } 290 | } 291 | ] 292 | }, 293 | { 294 | "type": "azure-eventhub", 295 | "policy_template": "application_gateway", 296 | "enabled": true, 297 | "streams": [ 298 | { 299 | "enabled": true, 300 | "data_stream": { 301 | "type": "logs", 302 | "dataset": "azure.application_gateway" 303 | }, 304 | "vars": { 305 | "tags": { 306 | "value": [ 307 | "azure-application-gateway-logs", 308 | "forwarded" 309 | ], 310 | "type": "text" 311 | }, 312 | "preserve_original_event": { 313 | "value": false, 314 | "type": "bool" 315 | }, 316 | "processors": { 317 | "type": "yaml" 318 | } 319 | } 320 | } 321 | ] 322 | } 323 | ], 324 | "package": { 325 | "name": "azure", 326 | "title": "Azure Logs", 327 | "version": "1.5.2" 328 | }, 329 | "vars": { 330 | "eventhub": { 331 | "type": "text", 332 | "value": "${eventhub}" 333 | }, 334 | "consumer_group": { 335 | "value": "$Default", 336 | "type": "text" 337 | }, 338 | "connection_string": { 339 | "type": "text", 340 | "value": "${connection_string}" 341 | }, 342 | "storage_account": { 343 | "type": "text", 344 | "value": "${storage_account}" 345 | }, 346 | "storage_account_key": { 347 | "type": "text", 348 | "value": "${storage_account_key}" 349 | }, 350 | "resource_manager_endpoint": { 351 | "type": "text" 352 | } 353 | } 354 | } -------------------------------------------------------------------------------- /Azure/json_templates/azure-metrics-integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure Metrics", 3 | "description": "", 4 | "namespace": "default", 5 | "policy_id": "${policy_id}", 6 | "enabled": true, 7 | "output_id": "", 8 | "inputs": [ 9 | { 10 | "type": "azure/metrics", 11 | "policy_template": "monitor", 12 | "enabled": true, 13 | "streams": [ 14 | { 15 | "enabled": true, 16 | "data_stream": { 17 | "type": "metrics", 18 | "dataset": "azure.monitor" 19 | }, 20 | "vars": { 21 | "period": { 22 | "value": "300s", 23 | "type": "text" 24 | }, 25 | "resources": { 26 | "value": "- resource_query: \"resourceType eq 'Microsoft.DocumentDb/databaseAccounts'\"\n metrics:\n - name: [\"DataUsage\", \"DocumentCount\", \"DocumentQuota\"]\n namespace: \"Microsoft.DocumentDb/databaseAccounts\"\n", 27 | "type": "yaml" 28 | } 29 | } 30 | } 31 | ] 32 | }, 33 | { 34 | "type": "azure/metrics", 35 | "policy_template": "compute_vm", 36 | "enabled": true, 37 | "streams": [ 38 | { 39 | "enabled": true, 40 | "data_stream": { 41 | "type": "metrics", 42 | "dataset": "azure.compute_vm" 43 | }, 44 | "vars": { 45 | "period": { 46 | "value": "300s", 47 | "type": "text" 48 | }, 49 | "resource_groups": { 50 | "value": [], 51 | "type": "text" 52 | }, 53 | "resource_ids": { 54 | "value": [], 55 | "type": "text" 56 | } 57 | } 58 | } 59 | ] 60 | }, 61 | { 62 | "type": "azure/metrics", 63 | "policy_template": "compute_vm_scaleset", 64 | "enabled": true, 65 | "streams": [ 66 | { 67 | "enabled": true, 68 | "data_stream": { 69 | "type": "metrics", 70 | "dataset": "azure.compute_vm_scaleset" 71 | }, 72 | "vars": { 73 | "period": { 74 | "value": "300s", 75 | "type": "text" 76 | }, 77 | "resource_groups": { 78 | "value": [], 79 | "type": "text" 80 | }, 81 | "resource_ids": { 82 | "value": [], 83 | "type": "text" 84 | } 85 | } 86 | } 87 | ] 88 | }, 89 | { 90 | "type": "azure/metrics", 91 | "policy_template": "container_registry", 92 | "enabled": true, 93 | "streams": [ 94 | { 95 | "enabled": true, 96 | "data_stream": { 97 | "type": "metrics", 98 | "dataset": "azure.container_registry" 99 | }, 100 | "vars": { 101 | "period": { 102 | "value": "300s", 103 | "type": "text" 104 | }, 105 | "resource_groups": { 106 | "value": [], 107 | "type": "text" 108 | }, 109 | "resource_ids": { 110 | "value": [], 111 | "type": "text" 112 | } 113 | } 114 | } 115 | ] 116 | }, 117 | { 118 | "type": "azure/metrics", 119 | "policy_template": "container_instance", 120 | "enabled": true, 121 | "streams": [ 122 | { 123 | "enabled": true, 124 | "data_stream": { 125 | "type": "metrics", 126 | "dataset": "azure.container_instance" 127 | }, 128 | "vars": { 129 | "period": { 130 | "value": "300s", 131 | "type": "text" 132 | }, 133 | "resource_groups": { 134 | "value": [], 135 | "type": "text" 136 | }, 137 | "resource_ids": { 138 | "value": [], 139 | "type": "text" 140 | } 141 | } 142 | } 143 | ] 144 | }, 145 | { 146 | "type": "azure/metrics", 147 | "policy_template": "container_service", 148 | "enabled": true, 149 | "streams": [ 150 | { 151 | "enabled": true, 152 | "data_stream": { 153 | "type": "metrics", 154 | "dataset": "azure.container_service" 155 | }, 156 | "vars": { 157 | "period": { 158 | "value": "300s", 159 | "type": "text" 160 | }, 161 | "resource_groups": { 162 | "value": [], 163 | "type": "text" 164 | }, 165 | "resource_ids": { 166 | "value": [], 167 | "type": "text" 168 | } 169 | } 170 | } 171 | ] 172 | }, 173 | { 174 | "type": "azure/metrics", 175 | "policy_template": "database_account", 176 | "enabled": true, 177 | "streams": [ 178 | { 179 | "enabled": true, 180 | "data_stream": { 181 | "type": "metrics", 182 | "dataset": "azure.database_account" 183 | }, 184 | "vars": { 185 | "period": { 186 | "value": "300s", 187 | "type": "text" 188 | }, 189 | "resource_groups": { 190 | "value": [], 191 | "type": "text" 192 | }, 193 | "resource_ids": { 194 | "value": [], 195 | "type": "text" 196 | } 197 | } 198 | } 199 | ] 200 | }, 201 | { 202 | "type": "azure/metrics", 203 | "policy_template": "storage_account", 204 | "enabled": true, 205 | "streams": [ 206 | { 207 | "enabled": true, 208 | "data_stream": { 209 | "type": "metrics", 210 | "dataset": "azure.storage_account" 211 | }, 212 | "vars": { 213 | "period": { 214 | "value": "300s", 215 | "type": "text" 216 | }, 217 | "resource_groups": { 218 | "value": [], 219 | "type": "text" 220 | }, 221 | "resource_ids": { 222 | "value": [], 223 | "type": "text" 224 | }, 225 | "service_types": { 226 | "value": [], 227 | "type": "text" 228 | } 229 | } 230 | } 231 | ] 232 | } 233 | ], 234 | "package": { 235 | "name": "azure_metrics", 236 | "title": "Azure Resource Metrics", 237 | "version": "1.0.5" 238 | }, 239 | "vars": { 240 | "client_id": { 241 | "type": "text", 242 | "value": "${client_id}" 243 | }, 244 | "client_secret": { 245 | "type": "text", 246 | "value": "${client_secret}" 247 | }, 248 | "tenant_id": { 249 | "type": "text", 250 | "value": "${tenant_id}" 251 | }, 252 | "subscription_id": { 253 | "type": "text", 254 | "value": "${subscription_id}" 255 | }, 256 | "refresh_list_interval": { 257 | "value": "600s", 258 | "type": "text" 259 | }, 260 | "resource_manager_endpoint": { 261 | "type": "text" 262 | }, 263 | "active_directory_endpoint": { 264 | "type": "text" 265 | } 266 | } 267 | } -------------------------------------------------------------------------------- /Azure/json_templates/billing-transform.json: -------------------------------------------------------------------------------- 1 | { 2 | "source": { 3 | "index": [ 4 | "logs-azure*", 5 | "metrics-azure*" 6 | ], 7 | "query": { 8 | "bool": { 9 | "should": [ 10 | { 11 | "exists": { 12 | "field": "azure.resource.group" 13 | } 14 | } 15 | ], 16 | "minimum_should_match": 1 17 | } 18 | } 19 | }, 20 | "dest": { 21 | "index": "billing-azure" 22 | }, 23 | "frequency": "1m", 24 | "pivot": { 25 | "group_by": { 26 | "azure.resource.group": { 27 | "terms": { 28 | "field": "azure.resource.group" 29 | } 30 | } 31 | }, 32 | "aggregations": { 33 | "billing": { 34 | "filter": { 35 | "exists": { 36 | "field": "azure.billing.billing_period_id" 37 | } 38 | }, 39 | "aggs": { 40 | "forecast": { 41 | "sum": { 42 | "field": "azure.billing.forecast_cost" 43 | } 44 | }, 45 | "actual": { 46 | "sum": { 47 | "field": "azure.billing.actual_cost" 48 | } 49 | }, 50 | "pretax": { 51 | "sum": { 52 | "field": "azure.billing.pretax_cost" 53 | } 54 | } 55 | } 56 | }, 57 | "user_interaction": { 58 | "filter": { 59 | "exists": { 60 | "field": "user.name" 61 | } 62 | }, 63 | "aggs": { 64 | "last": { 65 | "max": { 66 | "field": "@timestamp" 67 | } 68 | }, 69 | "distinct_users": { 70 | "cardinality": { 71 | "field": "user.name" 72 | } 73 | }, 74 | "mail": { 75 | "top_metrics": { 76 | "metrics": [ 77 | { 78 | "field": "user.email" 79 | } 80 | ], 81 | "sort": { 82 | "@timestamp": "desc" 83 | } 84 | } 85 | }, 86 | "count": { 87 | "value_count": { 88 | "field": "@timestamp" 89 | } 90 | } 91 | } 92 | }, 93 | "bandwidth": { 94 | "filter": { 95 | "exists": { 96 | "field": "host.network.egress.bytes" 97 | } 98 | }, 99 | "aggs": { 100 | "host.network.egress.bytes.sum": { 101 | "sum": { 102 | "field": "host.network.egress.bytes" 103 | } 104 | } 105 | } 106 | }, 107 | "activitylogs": { 108 | "filter": { 109 | "term": { 110 | "event.dataset": "azure.activitylogs" 111 | } 112 | }, 113 | "aggs": { 114 | "count": { 115 | "value_count": { 116 | "field": "@timestamp" 117 | } 118 | }, 119 | "last": { 120 | "max": { 121 | "field": "@timestamp" 122 | } 123 | } 124 | } 125 | }, 126 | "platformlogs": { 127 | "filter": { 128 | "term": { 129 | "event.dataset": "azure.platformlogs" 130 | } 131 | }, 132 | "aggs": { 133 | "count": { 134 | "value_count": { 135 | "field": "@timestamp" 136 | } 137 | }, 138 | "last": { 139 | "max": { 140 | "field": "@timestamp" 141 | } 142 | } 143 | } 144 | } 145 | } 146 | }, 147 | "settings": { 148 | "max_page_search_size": 1500 149 | }, 150 | "sync": { 151 | "time": { 152 | "field": "@timestamp" 153 | } 154 | } 155 | } -------------------------------------------------------------------------------- /Azure/json_templates/default-policy.json: -------------------------------------------------------------------------------- 1 | { "name": "${policy_name}", "description": "Terraformed policy", "namespace": "default", "monitoring_enabled": [ "logs", "metrics" ]} -------------------------------------------------------------------------------- /Azure/json_templates/es_api_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${elastic-api-key-name}" 3 | } 4 | -------------------------------------------------------------------------------- /Azure/json_templates/es_rule_activation.json: -------------------------------------------------------------------------------- 1 | {"action":"enable", "query": "alert.attributes.tags: \"Azure\""} -------------------------------------------------------------------------------- /Azure/terraform/azure_environment.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "main" { 2 | name = var.azure_resource_group 3 | location = var.azure_region 4 | } 5 | 6 | data "azurerm_client_config" "current" {} 7 | 8 | data "azurerm_subscription" "current" {} 9 | 10 | ################# 11 | ## Eventhub to collect the Logs 12 | ################# 13 | 14 | resource "azurerm_eventhub_namespace" "elastic" { 15 | name = "azure-logs-to-elastic" 16 | location = azurerm_resource_group.main.location 17 | resource_group_name = azurerm_resource_group.main.name 18 | sku = "Standard" 19 | capacity = 1 20 | 21 | } 22 | 23 | resource "azurerm_eventhub" "elastic" { 24 | name = "azure-logs-to-elastic" 25 | namespace_name = azurerm_eventhub_namespace.elastic.name 26 | resource_group_name = azurerm_resource_group.main.name 27 | partition_count = 2 28 | message_retention = 1 29 | } 30 | 31 | resource "azurerm_eventhub_authorization_rule" "elastic" { 32 | name = "azure-logs-to-elastic-hub-rule" 33 | namespace_name = azurerm_eventhub_namespace.elastic.name 34 | eventhub_name = azurerm_eventhub.elastic.name 35 | resource_group_name = azurerm_resource_group.main.name 36 | listen = true 37 | send = true 38 | manage = false 39 | } 40 | 41 | resource "azurerm_storage_account" "elastic" { 42 | name = "azurelogs2elastic" 43 | resource_group_name = azurerm_resource_group.main.name 44 | location = azurerm_resource_group.main.location 45 | account_tier = "Standard" 46 | account_replication_type = "GRS" 47 | } 48 | 49 | ################# 50 | ## Create a vault + key for the Agent VM 51 | ################# 52 | 53 | # resource "azurerm_key_vault" "elastic" { 54 | # name = "elastic-key-vault-tf" 55 | # location = azurerm_resource_group.main.location 56 | # resource_group_name = azurerm_resource_group.main.name 57 | # enabled_for_disk_encryption = true 58 | # tenant_id = data.azurerm_client_config.current.tenant_id 59 | # soft_delete_retention_days = 7 60 | # purge_protection_enabled = false 61 | 62 | # sku_name = "standard" 63 | 64 | # access_policy { 65 | # tenant_id = data.azurerm_client_config.current.tenant_id 66 | # object_id = data.azurerm_client_config.current.object_id 67 | 68 | # key_permissions = [ 69 | # "Create", 70 | # "Get", 71 | # "Purge", 72 | # "Recover" 73 | # ] 74 | 75 | # secret_permissions = [ 76 | # "Get", 77 | # ] 78 | 79 | # storage_permissions = [ 80 | # "Get", 81 | # ] 82 | # } 83 | # } 84 | 85 | # resource "azurerm_key_vault_key" "generated" { 86 | # name = "elastic-agent-certificate" 87 | # key_vault_id = azurerm_key_vault.elastic.id 88 | # key_type = "RSA" 89 | # key_size = 2048 90 | 91 | # key_opts = [ 92 | # "decrypt", 93 | # "encrypt", 94 | # "sign", 95 | # "unwrapKey", 96 | # "verify", 97 | # "wrapKey", 98 | # ] 99 | # } -------------------------------------------------------------------------------- /Azure/terraform/ec_deployment.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Deploy Elastic Cloud 3 | # ------------------------------------------------------------- 4 | data "ec_stack" "latest" { 5 | version_regex = "latest" 6 | region = var.elastic_region 7 | } 8 | 9 | resource "ec_deployment" "elastic_deployment" { 10 | name = var.elastic_deployment_name 11 | region = var.elastic_region 12 | version = var.elastic_version == "latest" ? data.ec_stack.latest.version : var.elastic_version 13 | deployment_template_id = var.elastic_deployment_template_id 14 | elasticsearch = { 15 | hot = { 16 | autoscaling = {} 17 | } 18 | 19 | #"remote_cluster" = { 20 | # for_each = { for idx, remote in var.elastic_remotes : idx => remote } 21 | # deployment_id = each.value["id"] 22 | # alias = each.value["alias"] 23 | # } 24 | 25 | } 26 | kibana = {} 27 | integrations_server = {} 28 | } 29 | 30 | output "elastic_cluster_id" { 31 | value = ec_deployment.elastic_deployment.id 32 | } 33 | 34 | output "elastic_cluster_alias" { 35 | value = ec_deployment.elastic_deployment.name 36 | } 37 | 38 | output "elastic_endpoint" { 39 | value = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 40 | } 41 | 42 | output "elastic_password" { 43 | value = ec_deployment.elastic_deployment.elasticsearch_password 44 | sensitive=true 45 | } 46 | 47 | output "elastic_cloud_id" { 48 | value = ec_deployment.elastic_deployment.elasticsearch.cloud_id 49 | } 50 | 51 | output "elastic_username" { 52 | value = ec_deployment.elastic_deployment.elasticsearch_username 53 | } 54 | 55 | # ------------------------------------------------------------- 56 | # Load Policy 57 | # ------------------------------------------------------------- 58 | 59 | data "external" "elastic_create_policy" { 60 | query = { 61 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 62 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 63 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 64 | elastic_json_body = templatefile("${path.module}/../json_templates/default-policy.json", {"policy_name": "Azure"}) 65 | } 66 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_create_agent_policy.sh" ] 67 | depends_on = [ec_deployment.elastic_deployment] 68 | } 69 | 70 | data "external" "elastic_add_metrics_integration" { 71 | query = { 72 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 73 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 74 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 75 | elastic_json_body = templatefile("${path.module}/../json_templates/azure-metrics-integration.json", 76 | { 77 | "policy_id": data.external.elastic_create_policy.result.id, 78 | "client_id": var.azure_client_id, 79 | "client_secret": var.azure_client_secret, 80 | "tenant_id": var.azure_tenant_id, 81 | "subscription_id": var.azure_subscription_id 82 | } 83 | ) 84 | } 85 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 86 | depends_on = [data.external.elastic_create_policy] 87 | } 88 | 89 | data "external" "elastic_add_billing_integration" { 90 | query = { 91 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 92 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 93 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 94 | elastic_json_body = templatefile("${path.module}/../json_templates/azure-billing-integration.json", 95 | { 96 | "policy_id": data.external.elastic_create_policy.result.id, 97 | "client_id": var.azure_client_id, 98 | "client_secret": var.azure_client_secret, 99 | "tenant_id": var.azure_tenant_id, 100 | "subscription_id": var.azure_subscription_id 101 | } 102 | ) 103 | } 104 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 105 | depends_on = [data.external.elastic_create_policy] 106 | } 107 | 108 | data "external" "elastic_add_logs_integration" { 109 | query = { 110 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 111 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 112 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 113 | elastic_json_body = templatefile("${path.module}/../json_templates/azure-logs-integration.json", 114 | { 115 | "policy_id": data.external.elastic_create_policy.result.id, 116 | "eventhub": azurerm_eventhub.elastic.name, 117 | "connection_string": azurerm_eventhub_authorization_rule.elastic.primary_connection_string , 118 | "storage_account": azurerm_storage_account.elastic.name , 119 | "storage_account_key": azurerm_storage_account.elastic.primary_access_key 120 | } 121 | ) 122 | } 123 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 124 | depends_on = [ 125 | data.external.elastic_create_policy, 126 | azurerm_eventhub.elastic, 127 | azurerm_eventhub_authorization_rule.elastic, 128 | azurerm_storage_account.elastic 129 | ] 130 | } 131 | 132 | # ------------------------------------------------------------- 133 | # Load Rules 134 | # ------------------------------------------------------------- 135 | 136 | data "external" "elastic_load_rules" { 137 | query = { 138 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 139 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 140 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 141 | } 142 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_load_detection_rules.sh" ] 143 | depends_on = [ec_deployment.elastic_deployment] 144 | } 145 | 146 | data "external" "elastic_enable_rules" { 147 | query = { 148 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 149 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 150 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 151 | elastic_json_body = templatefile("${path.module}/../json_templates/es_rule_activation.json",{}) 152 | } 153 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_enable_detection_rules.sh" ] 154 | depends_on = [data.external.elastic_load_rules] 155 | } 156 | 157 | 158 | # ------------------------------------------------------------- 159 | # Create and Start transforms 160 | # ------------------------------------------------------------- 161 | 162 | data "external" "elastic_create_transforms" { 163 | query = { 164 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 165 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 166 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 167 | transform_name = "azure-billing-transform" 168 | elastic_json_body = templatefile("${path.module}/../json_templates/billing-transform.json",{}) 169 | } 170 | program = ["sh", "${path.module}/../../lib/elastic_api/es_create_transform.sh" ] 171 | depends_on = [ec_deployment.elastic_deployment] 172 | } 173 | 174 | data "external" "elastic_start_transforms" { 175 | query = { 176 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 177 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 178 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 179 | transform_name = "azure-billing-transform" 180 | } 181 | program = ["sh", "${path.module}/../../lib/elastic_api/es_start_transform.sh" ] 182 | depends_on = [data.external.elastic_create_transforms] 183 | } 184 | 185 | # ------------------------------------------------------------- 186 | # Load Dashboards 187 | # ------------------------------------------------------------- 188 | 189 | data "external" "elastic_upload_saved_objects" { 190 | query = { 191 | elastic_http_method = "POST" 192 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 193 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 194 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 195 | so_file = "${path.module}/../dashboards/Azure-dashboards.ndjson" 196 | } 197 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_upload_saved_objects.sh" ] 198 | depends_on = [ec_deployment.elastic_deployment] 199 | } 200 | -------------------------------------------------------------------------------- /Azure/terraform/elastic_agent.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Create VM + Elastic Agent 3 | # ------------------------------------------------------------- 4 | 5 | data "template_file" "install_agent" { 6 | template = file("../../lib/scripts/agent_install.sh") 7 | vars = { 8 | elastic_version = var.elastic_version == "latest" ? data.ec_stack.latest.version : var.elastic_version 9 | elasticsearch_username = ec_deployment.elastic_deployment.elasticsearch_username 10 | elasticsearch_password = ec_deployment.elastic_deployment.elasticsearch_password 11 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 12 | integration_server_endpoint = ec_deployment.elastic_deployment.integrations_server.https_endpoint 13 | policy_id = data.external.elastic_create_policy.result.id 14 | } 15 | } 16 | 17 | resource "azurerm_virtual_machine_extension" "elastic-agent" { 18 | name = var.elastic_agent_vm_name 19 | virtual_machine_id = "${azurerm_linux_virtual_machine.agent.id}" 20 | publisher = "Microsoft.Azure.Extensions" 21 | type = "CustomScript" 22 | type_handler_version = "2.1" 23 | 24 | settings = <", 9 | "google_cloud_service_account_path" : "/path/to/service/account/file" 10 | } 11 | ``` 12 | 13 | List of other optional parameters that can be added to terraform.tfvars.json 14 | | Parameter Name | Default value | Example | Description | 15 | | ------------- | ------------- | ------------- | ------------- | 16 | | elastic_version | latest | 8.4.1 | Used to define the Elastic Search version | 17 | | elastic_region | gcp-europe-west3 | gcp-europe-west3 | Used to set the Elastic Cloud region for the Google Cloud deployment | 18 | | elastic_deployment_name | Google Cloud Observe and Protect | Google Cloud Observe and Protect | Used to define the name for the Elastic deployment | 19 | | google_cloud_region | europe-west3 | europe-west3 | Used to change the region where the Google Cloud objects getting installed | 20 | | google_cloud_network | default | my-network | Used to change the network the Elastic Agent VM is installed in. (Network needs to be existent) | 21 | 22 | 23 | #### Create Google Cloud service account following this steps. 24 | 25 | ##### Create json for Google Cloud credentials. Follow the instractions here 26 | 27 | Use [Google Cloud Console](https://console.cloud.google.com/iam-admin/serviceaccounts) for the initial creation 28 | 29 | 30 | ##### Set permission for the Google Cloud service account 31 | We are using this service also to connect the Elastic Agent to your Google Cloud Project. 32 | Because of that you should also take care that your Service Account is following the Elastic Agent Integration docs. 33 | Meaning the service account need to have the following roles as well as the roles for creating the terraformed services 34 | 35 | - Elastic Agent integration roles needed 36 | - pubsub.subscriptions.consume 37 | - pubsub.subscriptions.create 38 | - pubsub.subscriptions.get 39 | - pubsub.topics.attachSubscription 40 | 41 | - Terraform installation roles need 42 | - resourcemanager.projectIamAdmin 43 | - roles/compute.instanceAdmin.v1 (To create compute instances) 44 | - roles/logging.admin (To create log sinks) 45 | - pubsub.editor (This one usually includes the roles the Elastic Agent needs) 46 | 47 | Example roles assignment via `gcloud` 48 | 49 | ```bash 50 | gcloud projects add-iam-policy-binding "[PUT YOUR GOOGLE CLOUD PROJECT NAME HERE]" \ 51 | --member=serviceAccount:[PUT YOUR SERVICE ACCOUNT MEMBER HERE] \ 52 | --role=roles/[PUT THE ROLE NAME IN HERE] 53 | ``` 54 | 55 | Example 56 | 57 | ```bash 58 | gcloud projects add-iam-policy-binding "my-project-name" \ 59 | --member=serviceAccount:terraform@elastic-product.iam.gserviceaccount.com \ 60 | --role=roles/pubsub.editor 61 | ``` 62 | 63 | - Verify permissions 64 | ```bash 65 | gcloud projects get-iam-policy "[PUT YOUR GOOGLE CLOUD PROJECT NAME HERE]" \ 66 | --flatten="bindings[].members" \ 67 | --format='table(bindings.role)' \ 68 | --filter="bindings.members:[PUT YOUR SERVICE ACCOUNT MEMBER HERE]"`` 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /GoogleCloud/json_templates/default-policy.json: -------------------------------------------------------------------------------- 1 | { "name": "${policy_name}", "description": "Terraformed policy", "namespace": "default", "monitoring_enabled": [ "logs", "metrics" ]} -------------------------------------------------------------------------------- /GoogleCloud/json_templates/es_api_key.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${elastic-api-key-name}" 3 | } 4 | -------------------------------------------------------------------------------- /GoogleCloud/json_templates/es_host_transform.json: -------------------------------------------------------------------------------- 1 | { 2 | "source": { 3 | "index": [ 4 | "metric*" 5 | ] 6 | }, 7 | "pivot": { 8 | "group_by": { 9 | "host.name": { 10 | "terms": { 11 | "field": "host.name" 12 | } 13 | } 14 | }, 15 | "aggregations": { 16 | "host.cpu.usage.max": { 17 | "max": { 18 | "field": "gcp.compute.instance.cpu.usage.pct" 19 | } 20 | }, 21 | "host.memory.usage.max": { 22 | "max": { 23 | "field": "gcp.compute.instance.memory.balloon.ram_used.value" 24 | } 25 | }, 26 | "host.disk.write.bytes.sum": { 27 | "sum": { 28 | "field": "gcp.compute.instance.disk.write.bytes" 29 | } 30 | }, 31 | "host.network.egress.bytes.sum": { 32 | "sum": { 33 | "field": "gcp.compute.instance.network.egress.bytes" 34 | } 35 | }, 36 | "last_data": { 37 | "max": { 38 | "field": "@timestamp" 39 | } 40 | } 41 | } 42 | }, 43 | "dest": { 44 | "index": "gcp-host-profile" 45 | }, 46 | "sync": { 47 | "time": { 48 | "field": "@timestamp" 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /GoogleCloud/json_templates/es_repo_transform.json: -------------------------------------------------------------------------------- 1 | { 2 | "source": { 3 | "index": [ 4 | "logs*" 5 | ], 6 | "runtime_mappings": { 7 | "repo": { 8 | "type": "keyword", 9 | "script": { 10 | "source": "if (doc.containsKey('repository')) {\r\n emit (doc['repository'].value);\r\n return;\r\n}\r\n\r\ndef fieldname = \"gcp.audit.resource_name\";\r\nif (!doc.containsKey(fieldname)) {\r\n return;\r\n}\r\nif (doc[fieldname].size()>0) {\r\n def field_value = doc[fieldname].value;\r\n if (field_value != null) {\r\n String repo=grok('objects/%%{HOSTNAME:repo}').extract(field_value)?.repo;\r\n if (repo != null) emit(repo); \r\n return;\r\n }\r\n}" 11 | } 12 | } 13 | } 14 | }, 15 | "pivot": { 16 | "group_by": { 17 | "repository": { 18 | "terms": { 19 | "field": "repo" 20 | } 21 | }, 22 | "resource.labels.bucket_name": { 23 | "terms": { 24 | "field": "resource.labels.bucket_name" 25 | } 26 | } 27 | }, 28 | "aggregations": { 29 | "last_access": { 30 | "max": { 31 | "field": "@timestamp" 32 | } 33 | } 34 | } 35 | }, 36 | "description": "Transforms logs data into GCS usage profile", 37 | "dest": { 38 | "index": "gcs-repos" 39 | }, 40 | "sync": { 41 | "time": { 42 | "field": "@timestamp" 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /GoogleCloud/json_templates/es_rule_activation.json: -------------------------------------------------------------------------------- 1 | {"action":"enable", "query": "alert.attributes.tags: \"GCP\""} -------------------------------------------------------------------------------- /GoogleCloud/json_templates/es_vpc_flow_transform.json: -------------------------------------------------------------------------------- 1 | { 2 | "source": { 3 | "index": [ 4 | "logs*" 5 | ], 6 | "query": { 7 | "bool": { 8 | "should": [ 9 | { 10 | "match_phrase": { 11 | "event.dataset": "gcp.vpcflow" 12 | } 13 | } 14 | ], 15 | "minimum_should_match": 1 16 | } 17 | } 18 | }, 19 | "pivot": { 20 | "group_by": { 21 | "gcp.source.vpc.vpc_name": { 22 | "terms": { 23 | "field": "gcp.source.vpc.vpc_name" 24 | } 25 | }, 26 | "gcp.source.vpc.project_id": { 27 | "terms": { 28 | "field": "gcp.source.vpc.project_id" 29 | } 30 | }, 31 | "gcp.source.vpc.subnetwork_name": { 32 | "terms": { 33 | "field": "gcp.source.vpc.subnetwork_name" 34 | } 35 | }, 36 | "source.ip": { 37 | "terms": { 38 | "field": "source.ip" 39 | } 40 | }, 41 | "gcp.source.instance.region": { 42 | "terms": { 43 | "field": "gcp.source.instance.region" 44 | } 45 | }, 46 | "gcp.destination.vpc.vpc_name": { 47 | "terms": { 48 | "field": "gcp.destination.vpc.vpc_name" 49 | } 50 | }, 51 | "gcp.destination.vpc.project_id": { 52 | "terms": { 53 | "field": "gcp.destination.vpc.project_id" 54 | } 55 | }, 56 | "gcp.destination.vpc.subnetwork_name": { 57 | "terms": { 58 | "field": "gcp.destination.vpc.subnetwork_name" 59 | } 60 | }, 61 | "destination.ip": { 62 | "terms": { 63 | "field": "destination.ip" 64 | } 65 | }, 66 | "gcp.destination.instance.region": { 67 | "terms": { 68 | "field": "gcp.destination.instance.region" 69 | } 70 | } 71 | }, 72 | "aggregations": { 73 | "network.bytes.sum": { 74 | "sum": { 75 | "field": "network.bytes" 76 | } 77 | } 78 | } 79 | }, 80 | "dest": { 81 | "index": "gcp_vpc_flow_summary" 82 | }, 83 | "sync": { 84 | "time": { 85 | "field": "@timestamp" 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /GoogleCloud/json_templates/gcp_integration.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gcp", 3 | "namespace": "default", 4 | "description": "Terraformed GCP integration", 5 | "package": { 6 | "name": "gcp", 7 | "title": "Google Cloud Platform", 8 | "version": "2.11.12" 9 | }, 10 | "enabled": true, 11 | "policy_id": "${policy_id}", 12 | "output_id": "", 13 | "vars": { 14 | "project_id": { 15 | "value": "${gcp_project}", 16 | "type": "text" 17 | }, 18 | "credentials_file": { 19 | "type": "text" 20 | }, 21 | "credentials_json": { 22 | "value": ${gcp_credentials_json}, 23 | "type": "text" 24 | } 25 | }, 26 | "inputs": [ 27 | { 28 | "policy_template": "audit", 29 | "type": "gcp-pubsub", 30 | "enabled": %{ if audit_log_topic != ""}true%{ else }false%{ endif }, 31 | "streams": [ 32 | { 33 | "data_stream": { 34 | "type": "logs", 35 | "dataset": "gcp.audit" 36 | }, 37 | "vars": { 38 | "subscription_create": { 39 | "type": "bool", 40 | "value": true 41 | }, 42 | "alternative_host": { 43 | "type": "text" 44 | }, 45 | "topic": { 46 | "type": "text", 47 | "value": "${audit_log_topic}" 48 | }, 49 | "subscription_name": { 50 | "type": "text", 51 | "value": "${audit_log_topic}" 52 | }, 53 | "processors": { 54 | "type": "yaml" 55 | }, 56 | "preserve_original_event": { 57 | "type": "bool", 58 | "value": false 59 | }, 60 | "tags": { 61 | "type": "text", 62 | "value": [ 63 | "forwarded", 64 | "gcp-audit" 65 | ] 66 | } 67 | }, 68 | "enabled": true 69 | } 70 | ] 71 | }, 72 | { 73 | "policy_template": "firewall", 74 | "type": "gcp-pubsub", 75 | "enabled": %{ if firewall_log_topic != ""}true%{ else }false%{ endif }, 76 | "streams": [ 77 | { 78 | "data_stream": { 79 | "type": "logs", 80 | "dataset": "gcp.firewall" 81 | }, 82 | "vars": { 83 | "subscription_create": { 84 | "type": "bool", 85 | "value": true 86 | }, 87 | "alternative_host": { 88 | "type": "text" 89 | }, 90 | "topic": { 91 | "type": "text", 92 | "value": "${firewall_log_topic}" 93 | }, 94 | "subscription_name": { 95 | "type": "text", 96 | "value": "${firewall_log_topic}" 97 | }, 98 | "processors": { 99 | "type": "yaml" 100 | }, 101 | "preserve_original_event": { 102 | "type": "bool", 103 | "value": false 104 | }, 105 | "tags": { 106 | "type": "text", 107 | "value": [ 108 | "forwarded", 109 | "gcp-firewall" 110 | ] 111 | } 112 | }, 113 | "enabled": true 114 | } 115 | ] 116 | }, 117 | { 118 | "policy_template": "vpcflow", 119 | "type": "gcp-pubsub", 120 | "enabled": %{ if vpcflow_log_topic != ""}true%{ else }false%{ endif }, 121 | "streams": [ 122 | { 123 | "data_stream": { 124 | "type": "logs", 125 | "dataset": "gcp.vpcflow" 126 | }, 127 | "vars": { 128 | "subscription_create": { 129 | "type": "bool", 130 | "value": true 131 | }, 132 | "alternative_host": { 133 | "type": "text" 134 | }, 135 | "topic": { 136 | "type": "text", 137 | "value": "${vpcflow_log_topic}" 138 | }, 139 | "subscription_name": { 140 | "type": "text", 141 | "value": "${vpcflow_log_topic}" 142 | }, 143 | "processors": { 144 | "type": "yaml" 145 | }, 146 | "preserve_original_event": { 147 | "type": "bool", 148 | "value": false 149 | }, 150 | "tags": { 151 | "type": "text", 152 | "value": [ 153 | "forwarded", 154 | "gcp-vpcflow" 155 | ] 156 | } 157 | }, 158 | "enabled": true 159 | } 160 | ] 161 | }, 162 | { 163 | "policy_template": "dns", 164 | "type": "gcp-pubsub", 165 | "enabled": %{ if dns_log_topic != ""}true%{ else }false%{ endif }, 166 | "streams": [ 167 | { 168 | "data_stream": { 169 | "type": "logs", 170 | "dataset": "gcp.dns" 171 | }, 172 | "vars": { 173 | "subscription_create": { 174 | "type": "bool", 175 | "value": true 176 | }, 177 | "alternative_host": { 178 | "type": "text" 179 | }, 180 | "topic": { 181 | "type": "text", 182 | "value": "${dns_log_topic}" 183 | }, 184 | "subscription_name": { 185 | "type": "text", 186 | "value": "${dns_log_topic}" 187 | }, 188 | "processors": { 189 | "type": "yaml" 190 | }, 191 | "preserve_original_event": { 192 | "type": "bool", 193 | "value": false 194 | }, 195 | "tags": { 196 | "type": "text", 197 | "value": [ 198 | "forwarded", 199 | "gcp-dns" 200 | ] 201 | } 202 | }, 203 | "enabled": true 204 | } 205 | ] 206 | }, 207 | { 208 | "policy_template": "billing", 209 | "type": "gcp/metrics", 210 | "enabled": false, 211 | "streams": [ 212 | { 213 | "data_stream": { 214 | "type": "metrics", 215 | "dataset": "gcp.billing" 216 | }, 217 | "vars": { 218 | "table_pattern": { 219 | "type": "text", 220 | "value": "gcp_billing_export_v1" 221 | }, 222 | "period": { 223 | "type": "text", 224 | "value": "24h" 225 | }, 226 | "cost_type": { 227 | "type": "text", 228 | "value": "regular" 229 | }, 230 | "dataset_id": { 231 | "type": "text" 232 | } 233 | }, 234 | "enabled": false 235 | } 236 | ] 237 | }, 238 | { 239 | "policy_template": "compute", 240 | "type": "gcp/metrics", 241 | "enabled": true, 242 | "streams": [ 243 | { 244 | "data_stream": { 245 | "type": "metrics", 246 | "dataset": "gcp.compute" 247 | }, 248 | "vars": { 249 | "period": { 250 | "type": "text", 251 | "value": "60s" 252 | }, 253 | "zone": { 254 | "type": "text" 255 | }, 256 | "exclude_labels": { 257 | "type": "bool" 258 | }, 259 | "region": { 260 | "type": "text" 261 | } 262 | }, 263 | "enabled": true 264 | } 265 | ] 266 | }, 267 | { 268 | "policy_template": "firestore", 269 | "type": "gcp/metrics", 270 | "enabled": true, 271 | "streams": [ 272 | { 273 | "data_stream": { 274 | "type": "metrics", 275 | "dataset": "gcp.firestore" 276 | }, 277 | "vars": { 278 | "period": { 279 | "type": "text", 280 | "value": "60s" 281 | }, 282 | "zone": { 283 | "type": "text" 284 | }, 285 | "exclude_labels": { 286 | "type": "bool" 287 | }, 288 | "region": { 289 | "type": "text" 290 | } 291 | }, 292 | "enabled": true 293 | } 294 | ] 295 | }, 296 | { 297 | "policy_template": "loadbalancing", 298 | "type": "gcp/metrics", 299 | "enabled": true, 300 | "streams": [ 301 | { 302 | "data_stream": { 303 | "type": "metrics", 304 | "dataset": "gcp.loadbalancing_metrics" 305 | }, 306 | "vars": { 307 | "period": { 308 | "type": "text", 309 | "value": "60s" 310 | }, 311 | "zone": { 312 | "type": "text" 313 | }, 314 | "exclude_labels": { 315 | "type": "bool" 316 | }, 317 | "region": { 318 | "type": "text" 319 | } 320 | }, 321 | "enabled": true 322 | } 323 | ] 324 | }, 325 | { 326 | "policy_template": "loadbalancing", 327 | "type": "gcp-pubsub", 328 | "enabled": %{ if lb_log_topic != ""}true%{ else }false%{ endif }, 329 | "streams": [ 330 | { 331 | "data_stream": { 332 | "type": "logs", 333 | "dataset": "gcp.loadbalancing_logs" 334 | }, 335 | "vars": { 336 | "subscription_create": { 337 | "type": "bool", 338 | "value": true 339 | }, 340 | "alternative_host": { 341 | "type": "text" 342 | }, 343 | "topic": { 344 | "type": "text", 345 | "value": "${lb_log_topic}" 346 | }, 347 | "subscription_name": { 348 | "type": "text", 349 | "value": "${lb_log_topic}" 350 | }, 351 | "processors": { 352 | "type": "yaml" 353 | }, 354 | "preserve_original_event": { 355 | "type": "bool", 356 | "value": false 357 | }, 358 | "tags": { 359 | "type": "text", 360 | "value": [ 361 | "forwarded", 362 | "gcp-loadbalancing_logs" 363 | ] 364 | } 365 | }, 366 | "enabled": true 367 | } 368 | ] 369 | }, 370 | { 371 | "policy_template": "storage", 372 | "type": "gcp/metrics", 373 | "enabled": true, 374 | "streams": [ 375 | { 376 | "data_stream": { 377 | "type": "metrics", 378 | "dataset": "gcp.storage" 379 | }, 380 | "vars": { 381 | "period": { 382 | "type": "text", 383 | "value": "60s" 384 | }, 385 | "zone": { 386 | "type": "text" 387 | }, 388 | "exclude_labels": { 389 | "type": "bool" 390 | }, 391 | "region": { 392 | "type": "text" 393 | } 394 | }, 395 | "enabled": true 396 | } 397 | ] 398 | }, 399 | { 400 | "policy_template": "gke", 401 | "type": "gcp/metrics", 402 | "enabled": true, 403 | "streams": [ 404 | { 405 | "data_stream": { 406 | "type": "metrics", 407 | "dataset": "gcp.gke" 408 | }, 409 | "vars": { 410 | "period": { 411 | "type": "text", 412 | "value": "60s" 413 | }, 414 | "zone": { 415 | "type": "text" 416 | }, 417 | "exclude_labels": { 418 | "type": "bool" 419 | }, 420 | "region": { 421 | "type": "text" 422 | } 423 | }, 424 | "enabled": true 425 | } 426 | ] 427 | }, 428 | { 429 | "policy_template": "dataproc", 430 | "type": "gcp/metrics", 431 | "enabled": true, 432 | "streams": [ 433 | { 434 | "data_stream": { 435 | "type": "metrics", 436 | "dataset": "gcp.dataproc" 437 | }, 438 | "vars": { 439 | "period": { 440 | "type": "text", 441 | "value": "60s" 442 | }, 443 | "exclude_labels": { 444 | "type": "bool" 445 | }, 446 | "region": { 447 | "type": "text" 448 | } 449 | }, 450 | "enabled": true 451 | } 452 | ] 453 | }, 454 | { 455 | "policy_template": "pubsub", 456 | "type": "gcp/metrics", 457 | "enabled": true, 458 | "streams": [ 459 | { 460 | "data_stream": { 461 | "type": "metrics", 462 | "dataset": "gcp.pubsub" 463 | }, 464 | "vars": { 465 | "period": { 466 | "type": "text", 467 | "value": "60s" 468 | }, 469 | "zone": { 470 | "type": "text" 471 | }, 472 | "exclude_labels": { 473 | "type": "bool" 474 | }, 475 | "region": { 476 | "type": "text" 477 | } 478 | }, 479 | "enabled": true 480 | } 481 | ] 482 | } 483 | ] 484 | } -------------------------------------------------------------------------------- /GoogleCloud/terraform/dataflow.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Create a Dataflow job to read from BigQuery and write to Elastic 3 | # ------------------------------------------------------------- 4 | # resource "google_dataflow_flex_template_job" "read_from_bigquery_to_elasticserach" { 5 | # project = var.google_cloud_project 6 | # provider = google-beta 7 | # name = var.google_cloud_dataflow_job_name 8 | # region = var.google_cloud_region 9 | # container_spec_gcs_path = var.google_cloud_container_spec_gcs_path 10 | # parameters = { 11 | # connectionUrl = ec_deployment.elastic_gc_deployment.elasticsearch[0].cloud_id 12 | # apiKey = data.external.elastic_generate_api_key.result.encoded 13 | # index = var.elastic_index_name 14 | # inputTableSpec = var.google_cloud_inputTableSpec 15 | # maxNumWorkers = var.google_cloud_maxNumWorkers 16 | # maxRetryAttempts = var.google_cloud_maxRetryAttempts 17 | # maxRetryDuration = var.google_cloud_maxRetryDuration 18 | # } 19 | # depends_on = [data.external.elastic_generate_api_key] 20 | # } -------------------------------------------------------------------------------- /GoogleCloud/terraform/ec_deployment.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Deploy Elastic Cloud 3 | # ------------------------------------------------------------- 4 | data "ec_stack" "latest" { 5 | version_regex = "latest" 6 | region = var.elastic_region 7 | } 8 | 9 | resource "ec_deployment" "elastic_deployment" { 10 | name = var.elastic_deployment_name 11 | region = var.elastic_region 12 | version = var.elastic_version == "latest" ? data.ec_stack.latest.version : var.elastic_version 13 | deployment_template_id = var.elastic_deployment_template_id 14 | elasticsearch = { 15 | hot = { 16 | autoscaling = {} 17 | } 18 | 19 | #"remote_cluster" = { 20 | # for_each = { for idx, remote in var.elastic_remotes : idx => remote } 21 | # deployment_id = each.value["id"] 22 | # alias = each.value["alias"] 23 | # } 24 | 25 | } 26 | kibana = {} 27 | integrations_server = {} 28 | } 29 | 30 | output "elastic_cluster_id" { 31 | value = ec_deployment.elastic_deployment.id 32 | } 33 | 34 | output "elastic_cluster_alias" { 35 | value = ec_deployment.elastic_deployment.name 36 | } 37 | 38 | output "elastic_endpoint" { 39 | value = ec_deployment.elastic_deployment.kibana.https_endpoint 40 | } 41 | 42 | output "elastic_cloud_id" { 43 | value = ec_deployment.elastic_deployment.elasticsearch.cloud_id 44 | } 45 | 46 | output "elastic_username" { 47 | value = ec_deployment.elastic_deployment.elasticsearch_username 48 | } 49 | 50 | output "elastic_password" { 51 | value = ec_deployment.elastic_deployment.elasticsearch_password 52 | sensitive=true 53 | } 54 | 55 | # ------------------------------------------------------------- 56 | # Load Policy 57 | # ------------------------------------------------------------- 58 | 59 | data "external" "elastic_create_policy" { 60 | query = { 61 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 62 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 63 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 64 | elastic_json_body = templatefile("${path.module}/../json_templates/default-policy.json", {"policy_name": "GC_${var.google_cloud_project}"}) 65 | } 66 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_create_agent_policy.sh" ] 67 | depends_on = [ec_deployment.elastic_deployment] 68 | } 69 | 70 | data "external" "elastic_add_integration" { 71 | query = { 72 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 73 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 74 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 75 | elastic_json_body = templatefile("${path.module}/../json_templates/gcp_integration.json", 76 | { 77 | "policy_id": data.external.elastic_create_policy.result.id, 78 | "gcp_project": var.google_cloud_project, 79 | "gcp_credentials_json": jsonencode(file(var.google_cloud_service_account_path)), 80 | "audit_log_topic": var.google_pubsub_audit_topic, 81 | "firewall_log_topic": var.google_pubsub_firewall_topic, 82 | "vpcflow_log_topic": var.google_pubsub_vpcflow_topic, 83 | "dns_log_topic": var.google_pubsub_dns_topic, 84 | "lb_log_topic": var.google_pubsub_lb_topic 85 | } 86 | ) 87 | } 88 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 89 | depends_on = [data.external.elastic_create_policy] 90 | } 91 | 92 | # ------------------------------------------------------------- 93 | # Load Rules 94 | # ------------------------------------------------------------- 95 | 96 | data "external" "elastic_load_rules" { 97 | query = { 98 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 99 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 100 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 101 | } 102 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_load_detection_rules.sh" ] 103 | depends_on = [ec_deployment.elastic_deployment] 104 | } 105 | 106 | data "external" "elastic_enable_rules" { 107 | query = { 108 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 109 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 110 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 111 | elastic_json_body = templatefile("${path.module}/../json_templates/es_rule_activation.json",{}) 112 | } 113 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_enable_detection_rules.sh" ] 114 | depends_on = [data.external.elastic_load_rules] 115 | } 116 | 117 | # ------------------------------------------------------------- 118 | # Create and Start transforms 119 | # ------------------------------------------------------------- 120 | 121 | data "external" "elastic_create_transforms" { 122 | query = { 123 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 124 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 125 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 126 | transform_name = "gcs-repo-transform" 127 | elastic_json_body = templatefile("${path.module}/../json_templates/es_repo_transform.json",{}) 128 | } 129 | program = ["sh", "${path.module}/../../lib/elastic_api/es_create_transform.sh" ] 130 | depends_on = [ec_deployment.elastic_deployment] 131 | } 132 | 133 | data "external" "elastic_start_transforms" { 134 | query = { 135 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 136 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 137 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 138 | transform_name = "gcs-repo-transform" 139 | } 140 | program = ["sh", "${path.module}/../../lib/elastic_api/es_start_transform.sh" ] 141 | depends_on = [data.external.elastic_create_transforms] 142 | } 143 | 144 | 145 | ################################################################################ 146 | 147 | data "external" "elastic_create_transform_host_metrics" { 148 | query = { 149 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 150 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 151 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 152 | transform_name = "host-profile-transform" 153 | elastic_json_body = templatefile("${path.module}/../json_templates/es_host_transform.json",{}) 154 | } 155 | program = ["sh", "${path.module}/../../lib/elastic_api/es_create_transform.sh" ] 156 | depends_on = [ec_deployment.elastic_deployment] 157 | } 158 | 159 | data "external" "elastic_start_transform_host_metrics" { 160 | query = { 161 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 162 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 163 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 164 | transform_name = "host-profile-transform" 165 | } 166 | program = ["sh", "${path.module}/../../lib/elastic_api/es_start_transform.sh" ] 167 | depends_on = [data.external.elastic_create_transform_host_metrics] 168 | } 169 | 170 | 171 | 172 | ################################################################################ 173 | 174 | data "external" "elastic_create_transform_vpc_flow" { 175 | query = { 176 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 177 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 178 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 179 | transform_name = "vpc_flow-transform" 180 | elastic_json_body = templatefile("${path.module}/../json_templates/es_vpc_flow_transform.json",{}) 181 | } 182 | program = ["sh", "${path.module}/../../lib/elastic_api/es_create_transform.sh" ] 183 | depends_on = [ec_deployment.elastic_deployment] 184 | } 185 | 186 | data "external" "elastic_start_transform_vpc_flow" { 187 | query = { 188 | elastic_endpoint = ec_deployment.elastic_deployment.elasticsearch.https_endpoint 189 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 190 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 191 | transform_name = "vpc_flow-transform" 192 | } 193 | program = ["sh", "${path.module}/../../lib/elastic_api/es_start_transform.sh" ] 194 | depends_on = [data.external.elastic_create_transform_vpc_flow] 195 | } 196 | 197 | 198 | # ------------------------------------------------------------- 199 | # Load Dashboards 200 | # ------------------------------------------------------------- 201 | 202 | data "external" "elastic_upload_saved_objects" { 203 | query = { 204 | elastic_http_method = "POST" 205 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 206 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 207 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 208 | so_file = "${path.module}/../dashboards/google_cloud_dashboards.ndjson" 209 | } 210 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_upload_saved_objects.sh" ] 211 | depends_on = [ec_deployment.elastic_deployment] 212 | } 213 | 214 | -------------------------------------------------------------------------------- /GoogleCloud/terraform/elastic_agent.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Create Compute VM + Elastic Agent 3 | # ------------------------------------------------------------- 4 | 5 | data "template_file" "install_agent" { 6 | template = file("../../lib/scripts/agent_install.sh") 7 | vars = { 8 | elastic_version = var.elastic_version 9 | elasticsearch_username = ec_deployment.elastic_deployment.elasticsearch_username 10 | elasticsearch_password = ec_deployment.elastic_deployment.elasticsearch_password 11 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 12 | integration_server_endpoint = ec_deployment.elastic_deployment.integrations_server.https_endpoint 13 | policy_id = data.external.elastic_create_policy.result.id 14 | } 15 | } 16 | 17 | resource "google_compute_instance" "vm_instance" { 18 | depends_on = [ec_deployment.elastic_deployment, data.external.elastic_create_policy] ## We want to have the elastic deployment before we install the agent 19 | 20 | name = "elastic-agent" 21 | machine_type = "e2-standard-2" 22 | tags = ["terraformed"] 23 | 24 | 25 | boot_disk { 26 | initialize_params { 27 | image = "debian-cloud/debian-11" 28 | size = 20 29 | } 30 | } 31 | 32 | network_interface { 33 | network = var.google_cloud_network 34 | access_config { 35 | // Ephemeral public IP 36 | } 37 | } 38 | 39 | metadata_startup_script = "${data.template_file.install_agent.rendered}" 40 | } -------------------------------------------------------------------------------- /GoogleCloud/terraform/logging.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Create Audit Log Route to Elastic Agent 3 | # ------------------------------------------------------------- 4 | 5 | 6 | resource "google_pubsub_topic" "audit" { 7 | name = var.google_pubsub_audit_topic 8 | 9 | labels = { 10 | elastic-log = "audit" 11 | } 12 | } 13 | 14 | resource "google_logging_project_sink" "audit" { 15 | name = var.google_pubsub_audit_topic 16 | 17 | # Can export to pubsub, cloud storage, or bigquery 18 | destination = "pubsub.googleapis.com/${google_pubsub_topic.audit.id}" 19 | 20 | filter = var.google_pubsub_audit_filter 21 | 22 | # Use a unique writer (creates a unique service account used for writing) 23 | unique_writer_identity = true 24 | 25 | depends_on = [google_pubsub_topic.audit] 26 | } 27 | 28 | # ------------------------------------------------------------- 29 | # Create Firewall Log Route to Elastic Agent 30 | # ------------------------------------------------------------- 31 | 32 | 33 | resource "google_pubsub_topic" "firewall" { 34 | name = var.google_pubsub_firewall_topic 35 | 36 | labels = { 37 | elastic-log = "firewall" 38 | } 39 | } 40 | 41 | resource "google_logging_project_sink" "firewall" { 42 | name = var.google_pubsub_firewall_topic 43 | 44 | # Can export to pubsub, cloud storage, or bigquery 45 | destination = "pubsub.googleapis.com/${google_pubsub_topic.firewall.id}" 46 | 47 | filter = var.google_pubsub_firewall_filter 48 | 49 | # Use a unique writer (creates a unique service account used for writing) 50 | unique_writer_identity = true 51 | 52 | depends_on = [google_pubsub_topic.firewall] 53 | } 54 | 55 | 56 | # ------------------------------------------------------------- 57 | # Create VPC Flow Log Route to Elastic Agent 58 | # ------------------------------------------------------------- 59 | 60 | 61 | resource "google_pubsub_topic" "vpcflow" { 62 | name = var.google_pubsub_vpcflow_topic 63 | 64 | labels = { 65 | elastic-log = "vpcflow" 66 | } 67 | } 68 | 69 | resource "google_logging_project_sink" "vpcflow" { 70 | name = var.google_pubsub_vpcflow_topic 71 | 72 | # Can export to pubsub, cloud storage, or bigquery 73 | destination = "pubsub.googleapis.com/${google_pubsub_topic.vpcflow.id}" 74 | 75 | filter = var.google_pubsub_vpcflow_filter 76 | 77 | # Use a unique writer (creates a unique service account used for writing) 78 | unique_writer_identity = true 79 | 80 | depends_on = [google_pubsub_topic.vpcflow] 81 | } 82 | 83 | 84 | 85 | # ------------------------------------------------------------- 86 | # Create DNS Log Route to Elastic Agent 87 | # ------------------------------------------------------------- 88 | 89 | 90 | resource "google_pubsub_topic" "dns" { 91 | name = var.google_pubsub_dns_topic 92 | 93 | labels = { 94 | elastic-log = "dns" 95 | } 96 | } 97 | 98 | resource "google_logging_project_sink" "dns" { 99 | name = var.google_pubsub_dns_topic 100 | 101 | # Can export to pubsub, cloud storage, or bigquery 102 | destination = "pubsub.googleapis.com/${google_pubsub_topic.dns.id}" 103 | 104 | filter = var.google_pubsub_dns_filter 105 | 106 | # Use a unique writer (creates a unique service account used for writing) 107 | unique_writer_identity = true 108 | 109 | depends_on = [google_pubsub_topic.dns] 110 | } 111 | 112 | 113 | # ------------------------------------------------------------- 114 | # Create Loadbalancer Log Route to Elastic Agent 115 | # ------------------------------------------------------------- 116 | 117 | 118 | resource "google_pubsub_topic" "lb" { 119 | name = var.google_pubsub_lb_topic 120 | 121 | labels = { 122 | elastic-log = "loadbalancer" 123 | } 124 | } 125 | 126 | resource "google_logging_project_sink" "lb" { 127 | name = var.google_pubsub_lb_topic 128 | 129 | # Can export to pubsub, cloud storage, or bigquery 130 | destination = "pubsub.googleapis.com/${google_pubsub_topic.lb.id}" 131 | 132 | filter = var.google_pubsub_lb_filter 133 | 134 | # Use a unique writer (creates a unique service account used for writing) 135 | unique_writer_identity = true 136 | 137 | depends_on = [google_pubsub_topic.lb] 138 | } 139 | 140 | 141 | # ------------------------------------------------------------- 142 | # Role bindings 143 | # ------------------------------------------------------------- 144 | 145 | resource "google_project_iam_binding" "pubsub_writer_logs" { 146 | project = var.google_cloud_project 147 | role = "roles/pubsub.editor" 148 | 149 | members = [ 150 | google_logging_project_sink.audit.writer_identity, 151 | google_logging_project_sink.firewall.writer_identity, 152 | google_logging_project_sink.vpcflow.writer_identity, 153 | google_logging_project_sink.dns.writer_identity, 154 | google_logging_project_sink.lb.writer_identity, 155 | ] 156 | 157 | depends_on = [ 158 | google_logging_project_sink.audit, 159 | google_logging_project_sink.firewall, 160 | google_logging_project_sink.vpcflow, 161 | google_logging_project_sink.dns, 162 | google_logging_project_sink.lb 163 | ] 164 | } -------------------------------------------------------------------------------- /GoogleCloud/terraform/main.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Terraform provider configuration 3 | # ------------------------------------------------------------- 4 | 5 | terraform { 6 | required_version = ">= 1.0.2" 7 | 8 | required_providers { 9 | ec = { 10 | source = "elastic/ec" 11 | version = ">= 0.4.1" 12 | } 13 | } 14 | } 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /GoogleCloud/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Elastic configuration 3 | # ------------------------------------------------------------- 4 | variable "elastic_version" { 5 | type = string 6 | default = "latest" 7 | } 8 | 9 | variable "elastic_region" { 10 | type = string 11 | default = "gcp-europe-west3" 12 | } 13 | 14 | variable "elastic_deployment_name" { 15 | type = string 16 | default = "Google Cloud Observe and Protect" 17 | } 18 | 19 | variable "elastic_deployment_template_id" { 20 | type = string 21 | default = "gcp-io-optimized-v2" 22 | } 23 | 24 | variable "elastic_remotes" { 25 | type = list( 26 | object({ 27 | id = string 28 | alias = string 29 | }) 30 | ) 31 | default = [] 32 | } 33 | 34 | # ------------------------------------------------------------- 35 | # GCP configuration 36 | # ------------------------------------------------------------- 37 | 38 | variable "google_cloud_project" { 39 | type = string 40 | default = "elastic-pme-team" 41 | } 42 | 43 | variable "google_cloud_region" { 44 | type = string 45 | default = "europe-west3" 46 | } 47 | 48 | variable "google_cloud_service_account_path" { 49 | type = string 50 | } 51 | 52 | variable "google_cloud_network" { 53 | type = string 54 | default = "default" 55 | } 56 | 57 | # ------------------------------------------------------------- 58 | # PubSub configuration 59 | # ------------------------------------------------------------- 60 | 61 | //Audit Logs 62 | variable "google_pubsub_audit_topic" { 63 | type = string 64 | default = "elastic-audit-logs" 65 | } 66 | 67 | variable "google_pubsub_audit_filter" { 68 | type = string 69 | default = "protoPayload.@type=\"type.googleapis.com/google.cloud.audit.AuditLog\"" 70 | } 71 | 72 | //Firewall Logs 73 | variable "google_pubsub_firewall_topic" { 74 | type = string 75 | default = "elastic-firewall-logs" 76 | } 77 | 78 | variable "google_pubsub_firewall_filter" { 79 | type = string 80 | default = "logName:\"compute.googleapis.com%2Ffirewall\"" 81 | } 82 | 83 | //VPC Flow Logs 84 | variable "google_pubsub_vpcflow_topic" { 85 | type = string 86 | default = "elastic-vpcflow-logs" 87 | } 88 | 89 | variable "google_pubsub_vpcflow_filter" { 90 | type = string 91 | default = "log_id(\"compute.googleapis.com/vpc_flows\")" 92 | } 93 | 94 | //DNS Logs 95 | variable "google_pubsub_dns_topic" { 96 | type = string 97 | default = "elastic-dns-logs" 98 | } 99 | 100 | variable "google_pubsub_dns_filter" { 101 | type = string 102 | default = "resource.type=\"dns_query\"" 103 | } 104 | 105 | //Loadbalancer Logs 106 | variable "google_pubsub_lb_topic" { 107 | type = string 108 | default = "elastic-lb-logs" 109 | } 110 | 111 | variable "google_pubsub_lb_filter" { 112 | type = string 113 | default = "resource.type=\"http_load_balancer\"" 114 | } 115 | 116 | # ------------------------------------------------------------- 117 | # BigQuery configuration -- Not used at the moment 118 | # ------------------------------------------------------------- 119 | 120 | variable "google_cloud_container_specs_path" { 121 | type = string 122 | default = "gs://dataflow-templates/latest/flex/BigQuery_to_Elasticsearch" 123 | } 124 | 125 | variable "google_cloud_maxNumWorkers" { 126 | type = number 127 | default = 5 128 | } 129 | 130 | variable "google_cloud_maxRetryAttempts" { 131 | type = string 132 | default = 1 133 | } 134 | 135 | variable "google_cloud_maxRetryDuration" { 136 | type = string 137 | default = 30 138 | } -------------------------------------------------------------------------------- /Kubernetes/EKS/terraform/aws_provider.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.aws_region 3 | access_key = var.aws_access_key 4 | secret_key = var.aws_secret_key 5 | } -------------------------------------------------------------------------------- /Kubernetes/EKS/terraform/ec_deployment.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Deploy Elastic Cloud 3 | # ------------------------------------------------------------- 4 | data "ec_stack" "latest" { 5 | version_regex = "latest" 6 | region = var.elastic_region 7 | } 8 | 9 | resource "ec_deployment" "elastic_deployment" { 10 | name = var.elastic_deployment_name 11 | region = var.elastic_region 12 | version = var.elastic_version == "latest" ? data.ec_stack.latest.version : var.elastic_version 13 | deployment_template_id = var.elastic_deployment_template_id 14 | elasticsearch { 15 | autoscale = "true" 16 | 17 | dynamic "remote_cluster" { 18 | for_each = var.elastic_remotes 19 | content { 20 | deployment_id = remote_cluster.value["id"] 21 | alias = remote_cluster.value["alias"] 22 | } 23 | } 24 | } 25 | kibana {} 26 | integrations_server {} 27 | } 28 | 29 | output "elastic_cluster_id_aws" { 30 | value = ec_deployment.elastic_deployment.id 31 | } 32 | 33 | output "elastic_cluster_alias_aws" { 34 | value = ec_deployment.elastic_deployment.name 35 | } 36 | 37 | output "elastic_endpoint_aws" { 38 | value = ec_deployment.elastic_deployment.kibana.https_endpoint 39 | } 40 | 41 | output "elastic_cloud_id_aws" { 42 | value = ec_deployment.elastic_deployment.elasticsearch.cloud_id 43 | } 44 | 45 | output "elastic_username_aws" { 46 | value = ec_deployment.elastic_deployment.elasticsearch_username 47 | } 48 | 49 | output "elastic_password" { 50 | value = ec_deployment.elastic_deployment.elasticsearch_password 51 | sensitive = true 52 | } 53 | 54 | 55 | # ------------------------------------------------------------- 56 | # Load Policy 57 | # ------------------------------------------------------------- 58 | 59 | data "external" "elastic_create_policy" { 60 | query = { 61 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 62 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 63 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 64 | elastic_json_body = templatefile("${path.module}/../json_templates/default-policy.json", {"policy_name": "EKS"}) 65 | } 66 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_create_agent_policy.sh" ] 67 | depends_on = [ec_deployment.elastic_deployment] 68 | } 69 | 70 | # ------------------------------------------------------------- 71 | # Load Rules 72 | # ------------------------------------------------------------- 73 | 74 | data "external" "elastic_load_rules" { 75 | query = { 76 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 77 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 78 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 79 | } 80 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_load_detection_rules.sh" ] 81 | depends_on = [ec_deployment.elastic_deployment] 82 | } 83 | 84 | # data "external" "elastic_enable_rules" { 85 | # query = { 86 | # kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 87 | # elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 88 | # elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 89 | # elastic_json_body = templatefile("${path.module}/../json_templates/aws_rule_activation.json",{}) 90 | # } 91 | # program = ["sh", "${path.module}/../../lib/elastic_api/kb_enable_detection_rules.sh" ] 92 | # depends_on = [data.external.elastic_load_rules] 93 | # } 94 | 95 | # output "elastic_enable_aws_rules" { 96 | # value = data.external.elastic_enable_rules.result 97 | # depends_on = [ 98 | # data.external.elastic_enable_rules 99 | # ] 100 | # } 101 | 102 | # ------------------------------------------------------------- 103 | # Create and Start transforms 104 | # ------------------------------------------------------------- 105 | 106 | # ------------------------------------------------------------- 107 | # Load Dashboards 108 | # ------------------------------------------------------------- 109 | -------------------------------------------------------------------------------- /Kubernetes/EKS/terraform/eks_agent_manifest.off: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Add Elastic Agent + k8s integration to all existing EKS clusters 3 | # ------------------------------------------------------------- 4 | 5 | data "aws_eks_cluster" "elastic" { 6 | name = var.eks_cluster 7 | } 8 | 9 | data "aws_eks_cluster_auth" "elastic" { 10 | name = var.eks_cluster 11 | } 12 | 13 | provider "kubectl" { 14 | host = data.aws_eks_cluster.elastic.endpoint 15 | cluster_ca_certificate = base64decode(data.aws_eks_cluster.elastic.certificate_authority[0].data) 16 | token = data.aws_eks_cluster_auth.elastic.token 17 | } 18 | 19 | resource "kubectl_manifest" "deploy_agent" { 20 | yaml_body = templatefile("${path.module}/../json_templates/k8s_agent.yaml", 21 | { 22 | "fleet_url": ec_deployment.elastic_deployment.integrations_server.https_endpoint, 23 | "enrollment_token": "xxx" 24 | } 25 | ) 26 | depends_on = [ 27 | ec_deployment.elastic_deployment, 28 | data.external.elastic_add_endpoint_integration, 29 | data.aws_eks_cluster.elastic, 30 | data.aws_eks_cluster_auth.elastic 31 | ] 32 | } -------------------------------------------------------------------------------- /Kubernetes/EKS/terraform/eks_integration.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Load integration policy for Elastic Agent 3 | # ------------------------------------------------------------- 4 | 5 | data "external" "elastic_create_k8s_policy" { 6 | query = { 7 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 8 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 9 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 10 | elastic_json_body = templatefile("${path.module}/../json_templates/default-policy.json", {"policy_name": "k8s"}) 11 | } 12 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_create_agent_policy.sh" ] 13 | depends_on = [ec_deployment.elastic_deployment] 14 | } 15 | 16 | data "external" "elastic_add_k8s_integration" { 17 | query = { 18 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 19 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 20 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 21 | elastic_json_body = templatefile("${path.module}/../json_templates/k8s_integration.json", 22 | { 23 | "policy_id": data.external.elastic_create_k8s_policy.result.id 24 | } 25 | ) 26 | } 27 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 28 | depends_on = [data.external.elastic_create_k8s_policy] 29 | } 30 | 31 | data "external" "elastic_add_cspm_integration" { 32 | query = { 33 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 34 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 35 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 36 | elastic_json_body = templatefile("${path.module}/../json_templates/k8s_cspm_integration.json", 37 | { 38 | "policy_id": data.external.elastic_create_k8s_policy.result.id, 39 | "access_key": var.aws_access_key, 40 | "access_secret": var.aws_secret_key, 41 | } 42 | ) 43 | } 44 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 45 | depends_on = [data.external.elastic_add_k8s_integration] 46 | } 47 | 48 | data "external" "elastic_add_endpoint_integration" { 49 | query = { 50 | kibana_endpoint = ec_deployment.elastic_deployment.kibana.https_endpoint 51 | elastic_username = ec_deployment.elastic_deployment.elasticsearch_username 52 | elastic_password = ec_deployment.elastic_deployment.elasticsearch_password 53 | elastic_json_body = templatefile("${path.module}/../json_templates/k8s_endpoint_integration.json", 54 | { 55 | "policy_id": data.external.elastic_create_k8s_policy.result.id, 56 | } 57 | ) 58 | } 59 | program = ["sh", "${path.module}/../../lib/elastic_api/kb_add_integration_to_policy.sh" ] 60 | depends_on = [data.external.elastic_add_cspm_integration] 61 | } 62 | 63 | # output "elastic_add_endpoint_integration_template" { 64 | # value = templatefile("${path.module}/../json_templates/k8s_endpoint_integration.json", 65 | # { 66 | # "policy_id": data.external.elastic_create_k8s_policy.result.id, 67 | # } 68 | # ) 69 | # } -------------------------------------------------------------------------------- /Kubernetes/EKS/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Elastic configuration 3 | # ------------------------------------------------------------- 4 | variable "elastic_version" { 5 | type = string 6 | default = "latest" 7 | } 8 | 9 | variable "elastic_region" { 10 | type = string 11 | default = "aws-eu-west-2" 12 | } 13 | 14 | variable "elastic_deployment_name" { 15 | type = string 16 | default = "AWS" 17 | } 18 | 19 | variable "elastic_deployment_template_id" { 20 | type = string 21 | default = "aws-general-purpose-arm-v5" 22 | } 23 | 24 | # ------------------------------------------------------------- 25 | # AWS configuration 26 | # ------------------------------------------------------------- 27 | 28 | variable "aws_region" { 29 | type = string 30 | default = "eu-west-1" 31 | } 32 | 33 | variable "aws_access_key" { 34 | type = string 35 | } 36 | 37 | variable "aws_secret_key" { 38 | type = string 39 | } 40 | 41 | variable "eks_cluster" { 42 | type = string 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Monitoring/README.md: -------------------------------------------------------------------------------- 1 | # Create Elastic Cloud Monitoring Clusters 2 | 3 | This example creates one Monitoring Cluster per region. Those clusters can be used to collect the metrics of the Elastic Cloud Elasticsearch Clusters that you are running. 4 | At the moment Elastic needs to collect the monitoring data within the same cluster or within a cluster that is in the same region. Having the monitoring data in a separate cluster is best practice. 5 | In order to get the full overview accross all Elastic Cloud Clusters this script also creates a master cluster that leverages Cross Cluster Search (CCS) to get access to all monitoring data. 6 | All of that happens within the script. 7 | 8 | ## Get started 9 | 10 | You need to create an Elastic Cloud API Key and make it available for terraform using an environment variable. 11 | 12 | In addition to that you need to set the regions where you have clusters deployed within the main.tf script. 13 | 14 | -------------------------------------------------------------------------------- /Monitoring/terraform/maint.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | 4 | required_providers { 5 | ec = { 6 | source = "elastic/ec" 7 | version = "0.5.0" 8 | } 9 | } 10 | } 11 | 12 | provider "ec" { 13 | } 14 | 15 | resource "ec_deployment" "monitoring-clusters" { 16 | for_each = toset( ["us-east-1", "us-west-1"] ) 17 | 18 | name = "${each.key}-monitoring" 19 | 20 | region = each.key 21 | version = "8.5.0" 22 | deployment_template_id = "aws-compute-optimized-v3" 23 | 24 | elasticsearch {} 25 | 26 | } 27 | 28 | resource "ec_deployment" "monitoring-ccs" { 29 | name = "monitoring-ccs" 30 | 31 | region = "aws-us-east-1" 32 | version = "8.5.0" 33 | deployment_template_id = "aws-compute-optimized-v3" 34 | 35 | elasticsearch { 36 | 37 | dynamic "remote_cluster" { 38 | for_each = ec_deployment.monitoring-clusters 39 | 40 | content { 41 | deployment_id = remote_cluster.value.id 42 | alias = remote_cluster.value.name 43 | ref_id = remote_cluster.value.elasticsearch.0.ref_id 44 | } 45 | } 46 | } 47 | 48 | kibana {} 49 | 50 | } -------------------------------------------------------------------------------- /MultiCloud/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .vs/slnx.sqlite 3 | *.tfstate 4 | *.backup 5 | terraform/*.info 6 | terraform/*.tfstate 7 | terraform/.terraform 8 | *.hcl 9 | local_env -------------------------------------------------------------------------------- /MultiCloud/README.md: -------------------------------------------------------------------------------- 1 | # Multi Cloud terraform setup for Elasticsearch 2 | 3 | The project in this repository is creating an Elastic Cloud environment in order to getting started with monitoring and protecting your Cloud Service Providers(CSP) environment in Google, AWS and/or Azure. It is creating all necessary components within the CSPs as well as the in Elastic Cloud using terraform. The whole process will be done in less than 1h. 4 | 5 | You can either install every Cloud Environment separatly or choose the MultiCloud project to install everything at once. By choosing MultiCloud the terraform script will also configure the necessary connection between the clusters in order to do Cross Cluster Search(CSS). Because of that each cluster can live in its own Cloud Provider environment (GCP cluster in GCP, AWS cluser in AWS and so on). This will guarantee a low cost footprint when collecting the relevant data from the providers. But because of CCS every cluster can get queried by one main cluster. 6 | 7 | ## Getting started 8 | 9 | You can decide if you like to install the environment for all Cloud Providers at once or each once independently from each other. No matter what you prefer you need to deploy it within the [MultiCloud](MultiCloud) folder. Before you do that you need to prepare your environment. 10 | 11 | ### Prepare software dependencies 12 | 13 | - [jq](https://stedolan.github.io/jq/download/) 14 | - [terraform](https://www.terraform.io/downloads) 15 | - The Command Line Interface (CLI) tools for each Cloud Provider you wanna use 16 | - [AWS](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) 17 | - [Google Cloud](https://cloud.google.com/sdk/docs/install) 18 | - [Azure](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) 19 | 20 | ### Clone the repository 21 | 22 | ```bash 23 | git clone https://github.com/felix-lessoer/elastic-terraform-examples.git 24 | ``` 25 | 26 | ### Create Elastic Cloud ID following this steps 27 | 28 | [Create EC API key](https://registry.terraform.io/providers/elastic/ec/latest/docs#api-key-authentication-recommended) 29 | 30 | Set env variable for Elastic Cloud: 31 | 32 | ```bash 33 | export EC_API_KEY="[PUT YOUR ELASTIC CLOUD API KEY HERE]" 34 | ``` 35 | 36 | ### Create local env files within the repo 37 | 38 | Target is to have the *aws.json* file, the *azure.json* file and the *gcp.json* file in *MultiCloud/local_env/* to make the terraform commands below working correctly. 39 | ```bash 40 | mkdir local_env 41 | touch aws.json 42 | touch gcp.json 43 | touch azure.json 44 | ``` 45 | 46 | Modify the terraform environment settings to prepare your local env. 47 | 48 | #### For AWS 49 | More AWS configuration remarks you can find in the [AWS](../AWS) folder. 50 | 51 | Minimal config: 52 | ```json 53 | { 54 | "deploy_aws" : true, 55 | "aws_region" : "eu-west-2", 56 | "aws_access_key" : "", 57 | "aws_secret_key" : "" 58 | } 59 | ``` 60 | 61 | List of other optional parameters that can be added to terraform.tfvars.json 62 | | Parameter Name | Default value | Example | Description | 63 | | ------------- | ------------- | ------------- | ------------- | 64 | | elastic_version | latest | 8.4.1 | Used to define the Elastic Search version | 65 | | elastic_region | aws-eu-west-2 | aws-eu-west-2 | Used to set the Elastic Cloud region for the AWS deployment | 66 | | elastic_deployment_name | AWS Observe and Protect | AWS Observe and Protect | Used to define the name for the Elastic deployment | 67 | 68 | #### For Google Cloud 69 | More Google CLoud configuration remarks you can find in the [Google Cloud](../GoogleCloud) folder. 70 | 71 | Minimal config: 72 | ```json 73 | { 74 | "deploy_gc" : true, 75 | "google_cloud_project" : "", 76 | "google_cloud_service_account_path" : "/path/to/service/account/file" 77 | } 78 | ``` 79 | 80 | List of other optional parameters that can be added to terraform.tfvars.json 81 | | Parameter Name | Default value | Example | Description | 82 | | ------------- | ------------- | ------------- | ------------- | 83 | | elastic_version | latest | 8.4.1 | Used to define the Elastic Search version | 84 | | elastic_region | gcp-europe-west3 | gcp-europe-west3 | Used to set the Elastic Cloud region for the Google Cloud deployment | 85 | | elastic_deployment_name | Google Cloud Observe and Protect | Google Cloud Observe and Protect | Used to define the name for the Elastic deployment | 86 | | google_cloud_region | europe-west3 | europe-west3 | Used to change the region where the Google Cloud objects getting installed | 87 | | google_cloud_network | default | my-network | Used to change the network the Elastic Agent VM is installed in. (Network needs to be existent) | 88 | 89 | #### For Azure 90 | More Azure configuration remarks you can find in the [Azure](../Azure) folder. 91 | 92 | Minimal config: 93 | ```json 94 | { 95 | "deploy_azure" : true, 96 | "azure_client_id" : "", 97 | "azure_client_secret" : "", 98 | "azure_tenant_id": "", 99 | "azure_subscription_id": "" 100 | } 101 | ``` 102 | 103 | List of other optional parameters that can be added to terraform.tfvars.json 104 | | Parameter Name | Default value | Example | Description | 105 | | ------------- | ------------- | ------------- | ------------- | 106 | | elastic_version | latest | 8.4.1 | Used to define the Elastic Search version | 107 | | elastic_region | azure-westeurope | azure-westeurope | Used to set the Elastic Cloud region for the Azure deployment | 108 | | elastic_deployment_name | Azure Observe and Protect | Azure cluster | Used to define the name for the Elastic deployment | 109 | | azure_region | West Europe | West Europe | Used to change the region where the Azure objects getting installed | 110 | 111 | 112 | ## Deploy 113 | 114 | For the setup you need to init and apply the terraform configuration in the [Multi Cloud](MultiCloud) root module and start in the terraform folder. Before the apply you need to provide credentials for Elastic Cloud as well as for every Cloud Provider that you want to deploy. Terraform needs access to perform actions in your name. 115 | 116 | After you prepared the settings for each cloud provider you've choosen you should be able to execute the deployment process. 117 | 118 | ### All in one aka Multi Cloud 119 | 120 | If you prefer you install everything at once you need to configure all Cloud Providers. This is the default configuration. 121 | 122 | ### Each example separately 123 | 124 | To install each setup independenly from each other you can disable the creation of the unnecessary clusters also within the [Multi Cloud](MultiCloud) folder. Each module can run on its own. 125 | If you want to add more environments later you just need to change the configuration. 126 | 127 | 128 | List of parameters to de/activate one or more cloud provider environments completly: 129 | | Parameter Name | Default value | Example | Description | 130 | | ------------- | ------------- | ------------- | ------------- | 131 | | deploy_gc | true | false | Used to de/activate the Google Cloud Environment | 132 | | deploy_aws | true | false | Used to de/activate the AWS Environment | 133 | | deploy_azure | true | false | Used to de/activate the Azure Environment | 134 | 135 | ### Run terraform 136 | 137 | #### Initialize within 'terraform' folder in the Multi Cloud module 138 | 139 | ```bash 140 | terraform init 141 | ``` 142 | 143 | #### Check plan to see what will be created by terraform 144 | 145 | ```bash 146 | terraform plan -var-file="../local_env/aws.json" -var-file="../local_env/gcp.json" -var-file="../local_env/azure.json" 147 | ``` 148 | 149 | #### Run with auto-approve will install everything 150 | 151 | First run: 152 | ```bash 153 | terraform apply -var-file="../local_env/aws.json" -var-file="../local_env/gcp.json" -var-file="../local_env/azure.json" -auto-approve 154 | ``` 155 | 156 | After every terraform apply run you get the output that contains all necessary information to access the Elastic Cluster(s). 157 | However as this include sensitive data you need to run in order to see it. 158 | ```bash 159 | terraform output -json 160 | ``` 161 | 162 | The replace part is necessary if you deploy the AWS environment. Without that the Cloud Formation template that is used usually have issues on re apply 163 | ```bash 164 | terraform apply -var-file="../local_env/aws.json" -var-file="../local_env/gcp.json" -var-file="../local_env/azure.json" -replace module.aws_environment[0].aws_serverlessapplicationrepository_cloudformation_stack.esf_cf_stack -auto-approve 165 | ``` 166 | 167 | If you are facing issues while setting up a specific cloud provider, e.g. because of missing credentials for the others you can try to target the apply to a specific module like this: 168 | ```bash 169 | terraform apply -var-file="../local_env/aws.json" -var-file="../local_env/gcp.json" -var-file="../local_env/azure.json" -target module.aws_environment[0] -replace aws_serverlessapplicationrepository_cloudformation_stack.esf_cf_stack -auto-approve 170 | ``` 171 | If this is not working for you, you need to comment out/remove the provider and module blocks within the *modules.tf* file and *_provider.tf* file. 172 | Note: For AWS you need to replace the Serverless Forwarder Application everytime you run terraform. Otherwise the deployment will stuck. 173 | 174 | 175 | 176 | #### Cleanup (Deletes every component that was created by terraform if possible) 177 | 178 | ```bash 179 | terraform destroy -var-file="../local_env/aws.json" -var-file="../local_env/gcp.json" -var-file="../local_env/azure.json" -auto-approve 180 | ``` 181 | -------------------------------------------------------------------------------- /MultiCloud/terraform/aws_provider.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.aws_region 3 | access_key = var.aws_access_key 4 | secret_key = var.aws_secret_key 5 | } -------------------------------------------------------------------------------- /MultiCloud/terraform/aws_variables.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Elastic configuration 3 | # ------------------------------------------------------------- 4 | variable "elastic_aws_region" { 5 | type = string 6 | default = "aws-eu-west-2" 7 | } 8 | 9 | variable "elastic_aws_deployment_name" { 10 | type = string 11 | default = "AWS" 12 | } 13 | 14 | variable "elastic_aws_deployment_template_id" { 15 | type = string 16 | default = "aws-general-purpose-arm-v5" 17 | } 18 | 19 | # ------------------------------------------------------------- 20 | # AWS configuration 21 | # ------------------------------------------------------------- 22 | 23 | variable "aws_region" { 24 | type = string 25 | default = "eu-west-1" 26 | } 27 | 28 | variable "aws_access_key" { 29 | type = string 30 | default = "" 31 | } 32 | 33 | variable "aws_secret_key" { 34 | type = string 35 | default = "" 36 | } 37 | -------------------------------------------------------------------------------- /MultiCloud/terraform/azure_provider.tf: -------------------------------------------------------------------------------- 1 | provider "azurerm" { 2 | features { 3 | key_vault { 4 | purge_soft_delete_on_destroy = true 5 | } 6 | } 7 | 8 | subscription_id = var.azure_subscription_id 9 | client_id = var.azure_client_id 10 | client_secret = var.azure_client_secret 11 | tenant_id = var.azure_tenant_id 12 | } -------------------------------------------------------------------------------- /MultiCloud/terraform/azure_variables.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Elastic configuration 3 | # ------------------------------------------------------------- 4 | variable "elastic_azure_region" { 5 | type = string 6 | default = "azure-westeurope" 7 | } 8 | 9 | variable "elastic_azure_deployment_name" { 10 | type = string 11 | default = "Azure Observe and Protect" 12 | } 13 | 14 | variable "elastic_azure_deployment_template_id" { 15 | type = string 16 | default = "azure-general-purpose" 17 | } 18 | 19 | # ------------------------------------------------------------- 20 | # Azure configuration 21 | # ------------------------------------------------------------- 22 | 23 | variable "azure_region" { 24 | type = string 25 | default = "West Europe" 26 | } 27 | 28 | variable "azure_resource_group" { 29 | type = string 30 | default = "tf-elastic-group" 31 | } 32 | 33 | variable "azure_subscription_id" { 34 | type = string 35 | default = "" 36 | } 37 | 38 | variable "azure_client_id" { 39 | type = string 40 | default = "" 41 | } 42 | 43 | variable "azure_client_secret" { 44 | type = string 45 | default = "" 46 | } 47 | 48 | variable "azure_tenant_id" { 49 | type = string 50 | default = "" 51 | } 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /MultiCloud/terraform/ec_variables.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Elastic configuration for every cluster 3 | # ------------------------------------------------------------- 4 | variable "elastic_version" { 5 | type = string 6 | default = "latest" 7 | } 8 | 9 | variable "elastic_agent_vm_name" { 10 | type = string 11 | default = "elastic-agent" 12 | } -------------------------------------------------------------------------------- /MultiCloud/terraform/gc_provider.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | project = var.google_cloud_project 3 | region = var.google_cloud_region 4 | zone = "${var.google_cloud_region}-a" 5 | credentials = var.google_cloud_service_account_path 6 | } 7 | -------------------------------------------------------------------------------- /MultiCloud/terraform/gc_variables.tf: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------- 2 | # Elastic configuration 3 | # ------------------------------------------------------------- 4 | variable "elastic_gc_region" { 5 | type = string 6 | default = "gcp-europe-west3" 7 | } 8 | 9 | variable "elastic_gc_deployment_name" { 10 | type = string 11 | default = "Google Cloud Observe and Protect" 12 | } 13 | 14 | variable "elastic_gc_deployment_template_id" { 15 | type = string 16 | default = "gcp-io-optimized-v2" 17 | } 18 | 19 | # ------------------------------------------------------------- 20 | # GCP configuration 21 | # ------------------------------------------------------------- 22 | 23 | variable "google_cloud_project" { 24 | type = string 25 | default = "elastic-pme-team" 26 | } 27 | 28 | variable "google_cloud_region" { 29 | type = string 30 | default = "europe-west3" 31 | } 32 | 33 | variable "google_cloud_service_account_path" { 34 | type = string 35 | default = "" 36 | } 37 | 38 | variable "google_cloud_network" { 39 | type = string 40 | default = "default" 41 | } 42 | 43 | # ------------------------------------------------------------- 44 | # PubSub configuration 45 | # ------------------------------------------------------------- 46 | 47 | //Audit Logs 48 | variable "google_pubsub_audit_topic" { 49 | type = string 50 | default = "elastic-audit-logs" 51 | } 52 | 53 | variable "google_pubsub_audit_filter" { 54 | type = string 55 | default = "protoPayload.@type=\"type.googleapis.com/google.cloud.audit.AuditLog\"" 56 | } 57 | 58 | //Firewall Logs 59 | variable "google_pubsub_firewall_topic" { 60 | type = string 61 | default = "elastic-firewall-logs" 62 | } 63 | 64 | variable "google_pubsub_firewall_filter" { 65 | type = string 66 | default = "logName:\"compute.googleapis.com%2Ffirewall\"" 67 | } 68 | 69 | //VPC Flow Logs 70 | variable "google_pubsub_vpcflow_topic" { 71 | type = string 72 | default = "elastic-vpcflow-logs" 73 | } 74 | 75 | variable "google_pubsub_vpcflow_filter" { 76 | type = string 77 | default = "log_id(\"compute.googleapis.com/vpc_flows\")" 78 | } 79 | 80 | //DNS Logs 81 | variable "google_pubsub_dns_topic" { 82 | type = string 83 | default = "elastic-dns-logs" 84 | } 85 | 86 | variable "google_pubsub_dns_filter" { 87 | type = string 88 | default = "resource.type=\"dns_query\"" 89 | } 90 | 91 | //Loadbalancer Logs 92 | variable "google_pubsub_lb_topic" { 93 | type = string 94 | default = "elastic-lb-logs" 95 | } 96 | 97 | variable "google_pubsub_lb_filter" { 98 | type = string 99 | default = "resource.type=\"http_load_balancer\"" 100 | } 101 | 102 | # ------------------------------------------------------------- 103 | # BigQuery configuration -- Not used at the moment 104 | # ------------------------------------------------------------- 105 | 106 | variable "google_cloud_container_spec_gcs_path" { 107 | type = string 108 | default = "gs://dataflow-templates/latest/flex/BigQuery_to_Elasticsearch" 109 | } 110 | 111 | variable "google_cloud_maxNumWorkers" { 112 | type = number 113 | default = 5 114 | } 115 | 116 | variable "google_cloud_maxRetryAttempts" { 117 | type = string 118 | default = 1 119 | } 120 | 121 | variable "google_cloud_maxRetryDuration" { 122 | type = string 123 | default = 30 124 | } -------------------------------------------------------------------------------- /MultiCloud/terraform/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.2" 3 | 4 | required_providers { 5 | ec = { 6 | source = "elastic/ec" 7 | version = ">= 0.4.1" 8 | } 9 | google = { 10 | source = "hashicorp/google" 11 | version = ">= 4.35.0" 12 | } 13 | aws = { 14 | source = "hashicorp/aws" 15 | version = "~> 4.0" 16 | } 17 | kubectl = { 18 | source = "gavinbunney/kubectl" 19 | version = ">= 1.7.0" 20 | } 21 | } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /MultiCloud/terraform/modules.tf: -------------------------------------------------------------------------------- 1 | variable "deploy_aws" { 2 | type = bool 3 | default = true 4 | } 5 | 6 | variable "deploy_gc" { 7 | type = bool 8 | default = true 9 | } 10 | 11 | variable "deploy_azure" { 12 | type = bool 13 | default = true 14 | } 15 | 16 | module "aws_environment" { 17 | source = "../../AWS/terraform" 18 | 19 | elastic_version = var.elastic_version 20 | elastic_region = var.elastic_aws_region 21 | elastic_deployment_name = var.elastic_aws_deployment_name 22 | elastic_deployment_template_id = var.elastic_aws_deployment_template_id 23 | aws_region = var.aws_region 24 | aws_access_key = var.aws_access_key 25 | aws_secret_key = var.aws_secret_key 26 | 27 | ### 28 | # Uncomment the following line to make the AWS cluster the All in One Cluster via CCS 29 | ### 30 | //elastic_remotes = [ 31 | // {id = module.aws_environment[0].elastic_cluster_id, alias = "aws-data"}, 32 | // {id = module.azure_environment[0].elastic_cluster_id, alias = "azure-data"}, 33 | // {id = module.gc_environment[0].elastic_cluster_id, alias = "gcp-data"} 34 | // ] 35 | 36 | count = (var.deploy_aws == true) ? 1 : 0 37 | } 38 | 39 | module "gc_environment" { 40 | source = "../../GoogleCloud/terraform" 41 | 42 | elastic_version = var.elastic_version 43 | elastic_region = var.elastic_gc_region 44 | elastic_deployment_name = var.elastic_gc_deployment_name 45 | elastic_deployment_template_id = var.elastic_gc_deployment_template_id 46 | 47 | google_cloud_project = var.google_cloud_project 48 | google_cloud_region = var.google_cloud_region 49 | google_cloud_service_account_path = var.google_cloud_service_account_path 50 | google_cloud_network = var.google_cloud_network 51 | 52 | ### 53 | # Uncomment the following line to make the Google Cloud cluster the All in One Cluster via CCS 54 | ### 55 | //elastic_remotes = [ 56 | // {id = module.aws_environment[0].elastic_cluster_id, alias = "aws-data"}, 57 | // {id = module.azure_environment[0].elastic_cluster_id, alias = "azure-data"}, 58 | // {id = module.gc_environment[0].elastic_cluster_id, alias = "gcp-data"} 59 | // ] 60 | 61 | count = (var.deploy_gc == true) ? 1 : 0 62 | } 63 | 64 | module "azure_environment" { 65 | source = "../../Azure/terraform" 66 | 67 | elastic_version = var.elastic_version 68 | elastic_region = var.elastic_azure_region 69 | elastic_deployment_name = var.elastic_azure_deployment_name 70 | elastic_deployment_template_id = var.elastic_azure_deployment_template_id 71 | elastic_agent_vm_name = var.elastic_agent_vm_name 72 | 73 | azure_region = var.azure_region 74 | azure_resource_group = var.azure_resource_group 75 | azure_client_id = var.azure_client_id 76 | azure_client_secret = var.azure_client_secret 77 | azure_subscription_id = var.azure_subscription_id 78 | azure_tenant_id = var.azure_tenant_id 79 | 80 | ### 81 | # Uncomment the following line to make the Google Cloud cluster the All in One Cluster via CCS 82 | ### 83 | //elastic_remotes = [ 84 | // {id = module.aws_environment[0].elastic_cluster_id, alias = "aws-data"}, 85 | // {id = module.azure_environment[0].elastic_cluster_id, alias = "azure-data"}, 86 | // {id = module.gc_environment[0].elastic_cluster_id, alias = "gcp-data"} 87 | // ] 88 | 89 | count = (var.deploy_azure == true) ? 1 : 0 90 | } 91 | 92 | output "aws" { 93 | value = module.aws_environment[0] 94 | sensitive = true 95 | } 96 | 97 | output "azure" { 98 | value = module.azure_environment[0] 99 | sensitive = true 100 | } 101 | 102 | output "gc" { 103 | value = module.gc_environment[0] 104 | sensitive = true 105 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Elastic Terraform Examples to build an Multi Cloud Monitoring environment 2 | 3 | The project in this repository is creating an Elastic Cloud environment in order to getting started with monitoring and protecting your Cloud Service Providers(CSP) environment in Google, AWS and/or Azure. It is creating all necessary components within the CSPs as well as the in Elastic Cloud using terraform. The whole process will be done in less than 1h. 4 | 5 | You can either install every Cloud Environment separatly or choose the MultiCloud project to install everything at once. By choosing MultiCloud the terraform script will also configure the necessary connection between the clusters in order to do Cross Cluster Search(CSS). Because of that each cluster can live in its own Cloud Provider environment (GCP cluster in GCP, AWS cluser in AWS and so on). This will guarantee a low cost footprint when collecting the relevant data from the providers. But because of CCS every cluster can get queried by one main cluster. 6 | 7 | ## The AWS environment 8 | 9 | The AWS example is creating an AWS Monitoring and Enhanced Security environment. It creates all necessary AWS Services as well as the Elastic Cloud Cluster for you. The only thing you need to provide is are AWS account credentials that provide the right permissions as well as the Elastic Cloud API Key. It works both: In [Elastic Cloud directly](https://cloud.elastic.co) or via the [AWS Marketplace option for Elastic Cloud](https://ela.st/aws). 10 | 11 | This example will install and configure: 12 | - Elastic Cluster 13 | - AWS EC2 instance with Elastic Agent installed and configured to talk to the Elastic Cluster 14 | - Elastic Agent will be configured to collect available Metric datasets with zero manual configuration 15 | - Elastic SAR app will be used to install the elastic serverless forwarder to collect Logs from S3 and CloudWatch Log Groups 16 | - The Elastic Cluster will be configured with the following additional capabilities 17 | - Preloaded all Elastic Security Detection rules and enabled all AWS related rules 18 | 19 | #### Watch this video to learn more 20 | [![Getting started with Elasticsearch on AWS](https://raw.githubusercontent.com/felix-lessoer/elastic-terraform-examples/main/AWS/AWS_thumbnail.PNG)](https://youtu.be/9PpjxYOOr7c "Getting started with AWS") 21 | 22 | ## The Google Cloud Environment 23 | 24 | The Google Cloud example is creating a Google Cloud Monitoring and Enhanced Security environment. It creates all necessary Google Cloud Services as well as the Elastic Cloud Cluster for you. The only thing you need to provide is an appropriate Google Cloud Service account that has the right permissions and the Elastic Cloud API Key. It works both: In [Elastic Cloud directly](https://cloud.elastic.co) or via the [Google Cloud Marketplace option for Elastic Cloud](https://ela.st/google). 25 | 26 | This example will install and configure: 27 | - Elastic Cluster 28 | - Google Cloud Compute engine with Elastic Agent installed and configured to talk to the Elastic Cluster 29 | - Google Cloud Log routers (Log sinks) with the appropriate filters for Audit, Firewall, VPC Flow, DNS and Loadbalancer Logs. 30 | - Google Cloud PubSub topics to collects the log types above 31 | - Elastic Agent will be configured to collect all the logs and all available Google Cloud Metric datasets with zero manual configuration 32 | - The Elastic Cluster will be configured with the following additional capabilities 33 | - Single pane of glass Google Cloud Dashboard 34 | - Google Cloud Cost optimizer dashboard 35 | - Google Cloud Storage bucket analyzer dashboard 36 | - Elastic transforms to prepare the data for the installed dashboards 37 | - Preloaded all Elastic Security Detection rules and enabled all Google Cloud related rules 38 | 39 | #### Watch this video to learn more 40 | [![Getting started with Elasticsearch on Google Cloud](https://raw.githubusercontent.com/felix-lessoer/elastic-terraform-examples/main/GoogleCloud/Getting-started-with-Google-Cloud-Monitoring-Google-Slides.png)](https://youtu.be/wAIJZmCi6Iw "Getting started with Google Cloud") 41 | 42 | ## The Azure Environment 43 | 44 | The Azure example is enabling an extended view in the monitoring and security data thats created within the Azure platform. 45 | It will create all necessary components like EventHubs within your Azure Account and also configure the Elastic components to collect data from them. 46 | It takes just a few minutes to get it up and running. It works both: In [Elastic Cloud directly](https://cloud.elastic.co) or via the [Azure Marketplace option for Elastic Cloud](https://ela.st/azure). 47 | 48 | This example will install and configure: 49 | - Elastic Cluster 50 | - Azure VM with Elastic Agent installed and configured to talk to the Elastic Cluster 51 | - Azure Diagnostic Settings for Platform and Activity Logs to send it to EventHubs 52 | - Elastic Agent to collect all available Azure Metrics 53 | - The Elastic Cluster will be configured with the following additional capabilities 54 | - Preloaded all Elastic Security Detection rules and enabled all Google Cloud related rules 55 | #### Watch this video to learn more 56 | [![Getting started with Elasticsearch on Azure](https://raw.githubusercontent.com/felix-lessoer/elastic-terraform-examples/main/Azure/AZURE_thumbnail.PNG)](https://youtu.be/SuZyIbFsWcY&list=PLhLSfisesZItKvjRiYt9PGDLYHOjQyzQi "Getting started with Elasticsearch on Azure") 57 | 58 | ## Getting started 59 | 60 | You can decide if you like to install the environment for all Cloud Providers at once or each once independently from each other. No matter what you prefer you need to deploy it within the [MultiCloud](MultiCloud) folder. Before you do that you need to prepare your environment. You will find the comprehensive Getting Started description also within the [MultiCloud](MultiCloud) folder. 61 | 62 | # More Elasticsearch terraform examples 63 | 64 | Other terraform + elastic examples can be found here: 65 | - [Patent Search](https://github.com/MarxDimitri/solution-accelerators/tree/main/patent-search) using Google Cloud BigQuery public dataset 66 | - [AWS Quick Start](https://github.com/aws-ia/terraform-elastic-cloud) 67 | 68 | Kibana Dashboards and other Elastic extensions can be found here 69 | - [Elastic Content Share](https://elastic-content-share.eu/) 70 | - [AWS Cloudformation template](https://elastic-content-share.eu/blog/how-to-create-elastic-cloud-cluster-via-aws-cloud-formation-template/) 71 | 72 | 73 | -------------------------------------------------------------------------------- /lib/elastic_api/es_api_key.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) API_KEY_BODY=\(.api_key_body)"')" 4 | 5 | output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 6 | -H 'Content-Type:application/json' -d "$API_KEY_BODY" \ 7 | ${ELASTIC_ENDPOINT}/_security/api_key | jq '.') 8 | 9 | ENCODED=$( echo $output | jq -r '.encoded' ) 10 | jq -n --arg encoded "$ENCODED" '{"encoded" : $encoded}' 11 | -------------------------------------------------------------------------------- /lib/elastic_api/es_bulk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body) ELASTIC_INDEX_NAME=\(.elastic_index_name)"')" 4 | 5 | output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 6 | -H 'Content-Type:application/json' -d "${ELASTIC_JSON_BODY}" \ 7 | ${ELASTIC_ENDPOINT}/${ELASTIC_INDEX_NAME}/_bulk | jq '.') 8 | 9 | ITEMS=$( echo $output | jq -r '.items' ) 10 | ERROR=$( echo $output | jq -r '.error' ) 11 | jq -n --arg items "$ITEMS" --arg error "$ERROR" '{"items" : $items, "error" : $error}' 12 | -------------------------------------------------------------------------------- /lib/elastic_api/es_create_ilm_policy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 8 | ${ELASTIC_ENDPOINT}/_ilm/policy/patent_search_rollover_policy | jq '.') 9 | 10 | # Return response 11 | ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) 12 | jq -n --arg acknowledged "$ACKNOWLEDGED" '{"acknowledged" : $acknowledged}' 13 | -------------------------------------------------------------------------------- /lib/elastic_api/es_create_index_template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body) ELASTIC_TEMPLATE_NAME=\(.elastic_template_name)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 8 | ${ELASTIC_ENDPOINT}/_index_template/${ELASTIC_TEMPLATE_NAME} | jq '.') 9 | 10 | # Return response 11 | ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) 12 | jq -n --arg acknowledged "$output" '{"acknowledged" : $acknowledged}' 13 | -------------------------------------------------------------------------------- /lib/elastic_api/es_create_ingest_pipeline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body) ELASTIC_PIPELINE_NAME=\(.elastic_pipeline_name)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 8 | ${ELASTIC_ENDPOINT}/_ingest/pipeline/${ELASTIC_PIPELINE_NAME} | jq '.') 9 | 10 | # Return response 11 | ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) 12 | jq -n --arg acknowledged "$output" '{"acknowledged" : $acknowledged}' 13 | -------------------------------------------------------------------------------- /lib/elastic_api/es_create_mapping.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body) ELASTIC_INDEX_NAME=\(.elastic_index_name)"')" 4 | 5 | # Create index 6 | # output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | # ${ELASTIC_ENDPOINT}/${ELASTIC_INDEX_NAME}) 8 | 9 | # Define mapping 10 | output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 11 | -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 12 | ${ELASTIC_ENDPOINT}/${ELASTIC_INDEX_NAME} | jq '.') 13 | 14 | # Return response 15 | ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) 16 | jq -n --arg acknowledged "$ACKNOWLEDGED" '{"acknowledged" : $acknowledged}' 17 | -------------------------------------------------------------------------------- /lib/elastic_api/es_create_transform.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "TRANSFORM_NAME=\(.transform_name) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X PUT -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 8 | ${ELASTIC_ENDPOINT}/_transform/${TRANSFORM_NAME} | jq '.') 9 | 10 | # Return response 11 | ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) 12 | jq -n --arg acknowledged "$ACKNOWLEDGED" '{"acknowledged" : $acknowledged}' 13 | 14 | 15 | -------------------------------------------------------------------------------- /lib/elastic_api/es_start_transform.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "TRANSFORM_NAME=\(.transform_name) ELASTIC_ENDPOINT=\(.elastic_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | ${ELASTIC_ENDPOINT}/_transform/${TRANSFORM_NAME}/_start | jq '.') 8 | 9 | # Return response 10 | ACKNOWLEDGED=$( echo $output | jq -r '.acknowledged' ) 11 | jq -n --arg acknowledged "$ACKNOWLEDGED" '{"acknowledged" : $acknowledged}' 12 | 13 | 14 | -------------------------------------------------------------------------------- /lib/elastic_api/kb_add_integration_to_policy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H "kbn-xsrf: true" -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 8 | ${ELASTIC_ENDPOINT}/api/fleet/package_policies | jq '.') 9 | 10 | # Return response 11 | ID=$( echo $output | jq -r '.item.id' ) 12 | SUCCESS=$( echo $output | jq -r '.success' ) 13 | ERROR=$( echo $output | jq -r '.error' ) 14 | MESSAGE=$( echo $output | jq -r '.message' ) 15 | 16 | jq -n --arg id "$ID" --arg success "$SUCCESS" --arg error "$ERROR" --arg message "$MESSAGE" '{"id": $id, "success" : $success, "error": $error, "message": $message}' -------------------------------------------------------------------------------- /lib/elastic_api/kb_create_agent_policy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H "kbn-xsrf: true" -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 8 | ${ELASTIC_ENDPOINT}/api/fleet/agent_policies?sys_monitoring=true | jq '.') 9 | 10 | # Return response 11 | ID=$( echo $output | jq -r '.item.id' ) 12 | SUCCESS=$( echo $output | jq -r '.success' ) 13 | ERROR=$( echo $output | jq -r '.error' ) 14 | MESSAGE=$( echo $output | jq -r '.message' ) 15 | 16 | if [ "${ERROR}" = "Conflict" ] ;then 17 | ##Regex to extract agent ID from the error message 18 | ID=$( echo $MESSAGE | grep -P "'[0-9a-z\-]+'" -o | grep -P "[0-9a-z\-]+" -o ) 19 | fi 20 | 21 | jq -n --arg id "$ID" --arg success "$SUCCESS" --arg error "$ERROR" --arg message "$MESSAGE" '{"id": $id, "success" : $success, "error": $error, "message": $message}' -------------------------------------------------------------------------------- /lib/elastic_api/kb_enable_detection_rules.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) ELASTIC_JSON_BODY=\(.elastic_json_body)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X POST -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H "kbn-xsrf: true" -H 'Content-Type:application/json' -d "$ELASTIC_JSON_BODY" \ 8 | ${ELASTIC_ENDPOINT}/api/detection_engine/rules/_bulk_action | jq '.') 9 | 10 | # Return response 11 | SUCCESS=$( echo $output | jq -r '.success' ) 12 | 13 | jq -n --arg success "$SUCCESS" '{"success" : $success}' -------------------------------------------------------------------------------- /lib/elastic_api/kb_load_detection_rules.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) SO_FILE=\(.so_file)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X PUT -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H "kbn-xsrf: true" \ 8 | ${ELASTIC_ENDPOINT}/api/detection_engine/rules/prepackaged | jq '.') 9 | 10 | # Return response 11 | RULES=$( echo $output | jq -r '.rules_installed' ) 12 | 13 | jq -n --arg rules "$RULES" '{"rules" : $rules}' -------------------------------------------------------------------------------- /lib/elastic_api/kb_upload_saved_objects.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | eval "$(jq -r '@sh "ELASTIC_HTTP_METHOD=\(.elastic_http_method) ELASTIC_ENDPOINT=\(.kibana_endpoint) ELASTIC_USERNAME=\(.elastic_username) ELASTIC_PASSWORD=\(.elastic_password) SO_FILE=\(.so_file)"')" 4 | 5 | # Define mapping 6 | output=$(curl -s -X ${ELASTIC_HTTP_METHOD} -u "$ELASTIC_USERNAME:$ELASTIC_PASSWORD" \ 7 | -H "kbn-xsrf: true" --form file=@${SO_FILE} \ 8 | ${ELASTIC_ENDPOINT}/api/saved_objects/_import?overwrite=true | jq '.') 9 | 10 | # Return response 11 | SUCCESS=$( echo $output | jq -r '.success' ) 12 | ERROR=$( echo $output | jq -r '.error' ) 13 | MESSAGE=$( echo $output | jq -r '.message' ) 14 | 15 | jq -n --arg success "$SUCCESS" --arg error "$ERROR" --arg message "$MESSAGE" '{"success" : $success, "error": $error, "message": $message}' 16 | #jq -n --arg output "$output" '{"output" : $output}' -------------------------------------------------------------------------------- /lib/scripts/agent_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ####### 4 | ## INIT 5 | ###### 6 | 7 | export ES_VERSION=${elastic_version} 8 | export CLOUD_AUTH=${elasticsearch_username}:${elasticsearch_password} 9 | export KIBANA_URL=${kibana_endpoint} 10 | integration_server_endpoint=${integration_server_endpoint} 11 | export FLEET_URL=$${integration_server_endpoint//apm/fleet} 12 | export HOST_POLICY_ID=${policy_id} 13 | 14 | 15 | echo "deb http://us.archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list 16 | sudo apt-get update 17 | sudo apt-get --assume-yes install jq 18 | 19 | ######### 20 | ## install agent 21 | ######### 22 | 23 | ## get version 24 | ## version=$(curl -XGET -u $CLOUD_AUTH "$${KIBANA_URL}/api/status" -H "kbn-xsrf: true" -H "Content-Type: application/json" | jq -r '.version.number') 25 | echo ES_VERSION=${elastic_version} >> /etc/.env 26 | ## export ES_VERSION=$version 27 | 28 | echo "Command: curl -XGET -u $CLOUD_AUTH \"$${KIBANA_URL}/api/fleet/enrollment_api_keys\"" 29 | 30 | response=$(curl -XGET -u $CLOUD_AUTH "$${KIBANA_URL}/api/fleet/enrollment_api_keys" -H "kbn-xsrf: true" -H "Content-Type: application/json") 31 | echo $response 32 | 33 | echo "using $${HOST_POLICY_ID}" 34 | 35 | endpoint_enroll_key=$( jq -r --arg policy_id "$${HOST_POLICY_ID}" '.list[] | select(.policy_id == $policy_id) | .api_key' <<< "$${response}" ) 36 | export HOST_ENROLL_KEY=$endpoint_enroll_key 37 | echo HOST_ENROLL_KEY=$endpoint_enroll_key >> /etc/.env 38 | 39 | echo "Loading agent" 40 | curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-$ES_VERSION-linux-x86_64.tar.gz 41 | sleep 30 42 | 43 | echo "Unpack agent" 44 | tar xzvf elastic-agent-$ES_VERSION-linux-x86_64.tar.gz 45 | cd elastic-agent-$ES_VERSION-linux-x86_64 46 | 47 | echo "Install agent" 48 | echo "Command: ./elastic-agent install --url=$FLEET_URL --enrollment-token=$HOST_ENROLL_KEY -f" 49 | ./elastic-agent install --url=$FLEET_URL --enrollment-token=$HOST_ENROLL_KEY -f 50 | cd .. 51 | rm elastic-agent-$ES_VERSION-linux-x86_64 -r 52 | rm elastic-agent-$ES_VERSION-linux-x86_64.tar.gz 53 | --------------------------------------------------------------------------------