├── .gitignore ├── presets ├── default_icon.png ├── akamai │ └── waf │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml ├── azure │ └── waf │ │ ├── icon.png │ │ └── README.md ├── okta │ └── syslog │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml ├── aws_sec_lake │ ├── waf │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ ├── route53 │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ ├── s3_data │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ ├── vpc_flow │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ ├── eks_audit │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ ├── security_hub │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ ├── lambda_execution │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ └── cloudtrail_management │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml ├── aws │ └── cloudtrail_iam │ │ ├── icon.png │ │ └── README.md ├── cloudflare │ └── httpreq │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml ├── databricks │ ├── access_audit │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml │ └── query_history │ │ ├── icon.png │ │ ├── README.md │ │ └── preset.yaml └── index.yaml ├── schema ├── index.schema.yaml └── preset.schema.yaml ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | # VS Code 2 | .vscode/ 3 | 4 | # IDEA 5 | .idea 6 | .idea_modules 7 | /.worksheet/ 8 | *.iml 9 | -------------------------------------------------------------------------------- /presets/default_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/default_icon.png -------------------------------------------------------------------------------- /presets/akamai/waf/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/akamai/waf/icon.png -------------------------------------------------------------------------------- /presets/azure/waf/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/azure/waf/icon.png -------------------------------------------------------------------------------- /presets/okta/syslog/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/okta/syslog/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/waf/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/waf/icon.png -------------------------------------------------------------------------------- /presets/aws/cloudtrail_iam/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws/cloudtrail_iam/icon.png -------------------------------------------------------------------------------- /presets/cloudflare/httpreq/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/cloudflare/httpreq/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/route53/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/route53/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/s3_data/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/s3_data/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/vpc_flow/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/vpc_flow/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/eks_audit/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/eks_audit/icon.png -------------------------------------------------------------------------------- /presets/databricks/access_audit/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/databricks/access_audit/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/security_hub/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/security_hub/icon.png -------------------------------------------------------------------------------- /presets/databricks/query_history/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/databricks/query_history/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/lambda_execution/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/lambda_execution/icon.png -------------------------------------------------------------------------------- /presets/aws_sec_lake/cloudtrail_management/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexott/dasl-content-packs/main/presets/aws_sec_lake/cloudtrail_management/icon.png -------------------------------------------------------------------------------- /presets/akamai/waf/README.md: -------------------------------------------------------------------------------- 1 | # Akamai presets 2 | 3 | This preset directory contains Akamai data sources. Current supported data sources: 4 | - http request 5 | 6 | ## Http Request 7 | 8 | Assumed data format: json 9 | 10 | Silver tables: 11 | - akamai_waf_http_activity 12 | 13 | Gold tables: 14 | - http_activity 15 | -------------------------------------------------------------------------------- /presets/cloudflare/httpreq/README.md: -------------------------------------------------------------------------------- 1 | # Cloudflare presets 2 | 3 | This preset directory contains cloudflare data sources. Current supported data sources: 4 | - http request 5 | 6 | ## Http Request 7 | 8 | Assumed data format: json 9 | 10 | Silver tables: 11 | - cloudflare_waf_http_requests 12 | 13 | Gold tables: 14 | - http_activity 15 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/s3_data/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake S3 Data Events presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - S3 DataEvents 6 | 7 | ## S3 DataEvents 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_s3_data_api_activity 13 | 14 | Gold tables: 15 | - api_activity 16 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/eks_audit/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake EKS control plane presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - EKS control plane 6 | 7 | ## EKS control plane 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_eks_audit_api_activity 13 | 14 | Gold tables: 15 | - api_activity 16 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/vpc_flow/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake VPC network flows presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - VPC network flows 6 | 7 | ## VPC network flows 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_vpc_flow_network_activity 13 | 14 | Gold tables: 15 | - network_activity 16 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/lambda_execution/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake Lambda Execution presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - Lambda Execution 6 | 7 | ## Lambda Execution 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_lambda_execution_api_activity 13 | 14 | Gold tables: 15 | - api_activity 16 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/route53/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake Route53 Resolver query log presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - Route53 Resolver query log 6 | 7 | ## Route53 Resolver query log 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_route53_dns_activity 13 | 14 | Gold tables: 15 | - dns_activity 16 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/waf/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake Amazon WAF logs to HTTP Activity presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - Amazon WAF logs to HTTP Activity 6 | 7 | ## Amazon WAF logs to HTTP Activity 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_waf_http_activity 13 | 14 | Gold tables: 15 | - http_activity 16 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/cloudtrail_management/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake CloudTrail Management Event presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - CloudTrail Management 6 | 7 | ## CloudTrail Management Event 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_cloud_trail_mgmt_account_change 13 | - aws_sec_lake_cloud_trail_mgmt_authentication 14 | - aws_sec_lake_cloud_trail_mgmt_api_activity 15 | 16 | Gold tables: 17 | - account_change 18 | - authentication 19 | - api_activity 20 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/security_hub/README.md: -------------------------------------------------------------------------------- 1 | 2 | # AWS Security Lake Security Hub presets 3 | 4 | This preset directory contains AWS Security Lake data sources. Current supported data sources: 5 | - Security Hub 6 | 7 | ## Security Hub 8 | 9 | Assumed data format: parquet 10 | 11 | Silver tables: 12 | - aws_sec_lake_security_hub_findings_vulnerability_finding 13 | - aws_sec_lake_security_hub_findings_compliance_finding 14 | - aws_sec_lake_security_hub_findings_detection_finding 15 | 16 | Gold tables: 17 | - vulnerability_finding 18 | - compliance_finding 19 | - detection_finding 20 | -------------------------------------------------------------------------------- /schema/index.schema.yaml: -------------------------------------------------------------------------------- 1 | $schema: "https://json-schema.org/draft/2020-12/schema" 2 | type: object 3 | properties: 4 | presets: 5 | type: array 6 | items: 7 | type: object 8 | properties: 9 | source: 10 | description: The source (cloud provider, etc.) of the data. 11 | type: string 12 | sourceType: 13 | description: The type of data (e.g. Route53, etc.). This can also be viewed as the service provided by the source. 14 | type: string 15 | required: 16 | - source 17 | - sourceType 18 | required: 19 | - presets 20 | -------------------------------------------------------------------------------- /presets/azure/waf/README.md: -------------------------------------------------------------------------------- 1 | # Akamai presets 2 | 3 | This preset directory contains Azure data sources. Current supported data sources: 4 | - http request 5 | 6 | ## Http Request 7 | 8 | Assumed data format: json 9 | 10 | Silver tables: 11 | - azure_waf_application_gateway_access_log 12 | - azure_waf_application_gateway_firewall_log 13 | - azure_waf_az_fw_application_rule 14 | - azure_waf_az_fw_nat_rule 15 | - azure_waf_az_fw_network_rule 16 | - azure_waf_az_fw_threat_intel 17 | - azure_waf_azure_firewall_application_rule 18 | - azure_waf_azure_firewall_network_rule 19 | 20 | Gold tables: 21 | - http_activity 22 | - network_activity 23 | - security_finding 24 | -------------------------------------------------------------------------------- /presets/index.yaml: -------------------------------------------------------------------------------- 1 | presets: 2 | - source: "akamai" 3 | sourceType: "waf" 4 | - source: "aws_sec_lake" 5 | sourceType: "cloudtrail_management" 6 | - source: "aws_sec_lake" 7 | sourceType: "eks_audit" 8 | - source: "aws_sec_lake" 9 | sourceType: "lambda_execution" 10 | - source: "aws_sec_lake" 11 | sourceType: "route53" 12 | - source: "aws_sec_lake" 13 | sourceType: "s3_data" 14 | - source: "aws_sec_lake" 15 | sourceType: "security_hub" 16 | - source: "aws_sec_lake" 17 | sourceType: "vpc_flow" 18 | - source: "aws_sec_lake" 19 | sourceType: "waf" 20 | - source: "azure" 21 | sourceType: "waf" 22 | - source: "cloudflare" 23 | sourceType: "httpreq" 24 | - source: "okta" 25 | sourceType: "syslog" 26 | - source: "aws" 27 | sourceType: "cloudtrail_iam" -------------------------------------------------------------------------------- /presets/databricks/query_history/README.md: -------------------------------------------------------------------------------- 1 | # Databricks Query History Preset 2 | 3 | This preset contains instructions for processing data found in the Databricks `system.query.history` table within Unity Catalog. All queries executed in a Databricks account are recorded in this table providing deep insight into data interactions. 4 | 5 | 6 | ## Bronze 7 | 8 | A Databricks account's existing `system.query.history` table is used for bronze data. No ingestion is needed for this datasource. 9 | 10 | ## Silver 11 | 12 | The following silver tables are created by this preset: 13 | 14 | - databricks_query_history 15 | 16 | ## Gold 17 | 18 | The following gold OCSF tables are targetted by this preset: 19 | 20 | - api_activity 21 | 22 | ## Further Information 23 | 24 | - [Query history system table reference ](https://docs.databricks.com/aws/en/admin/system-tables/query-history) 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dasl-content-packs 2 | This repository contains preset content pack definitions for processing various data sources, 3 | grouped by source (e.g. AWS) and source type (e.g. Route53 within AWS). These define silver 4 | table pre-transform and transform definitions, and gold table transform definitions. 5 | 6 | ## Layout and Schema 7 | The preset content packs are stored using the pattern: 8 | `presets/{source}/{sourceType}/preset.yaml` 9 | 10 | For example, a preset named "aws_sec_lake_route53" with a source of "aws_sec_lake" and sourceType of 11 | "route53" will be defined in `presets/aws_sec_lake/route53/preset.yaml` 12 | 13 | The schema for a preset is located at [schema/preset.schema.yaml](./schema/preset.schema.yaml) 14 | 15 | All presets must also be listed in [presets/index.yaml](./presets/index.yaml), following the 16 | schema located at [schema/index.schema.yaml](./schema/index.schema.yaml) 17 | -------------------------------------------------------------------------------- /presets/databricks/access_audit/README.md: -------------------------------------------------------------------------------- 1 | # Databricks Audit Table Preset 2 | 3 | This preset contains instructions for processing data found in the Databricks `system.access.audit` table within Unity Catalog. The audit table contains information from a wide variety of sources and so auditing data will flow into a number of gold OCSF tables. 4 | 5 | 6 | ## Bronze 7 | 8 | A Databricks account's existing `system.access.audit` table is used for bronze data. No ingestion is needed for this datasource. 9 | 10 | ## Silver 11 | 12 | The following silver tables are created by this preset: 13 | 14 | - databricks_access_audit 15 | 16 | ## Gold 17 | 18 | The following gold OCSF tables are targetted by this preset: 19 | 20 | - account_change 21 | - api_activity 22 | - authentication 23 | - group_management 24 | 25 | ## Further Information 26 | 27 | - [Audit log reference](https://docs.databricks.com/aws/en/admin/account-settings/audit-logs) 28 | - [Audit log system table reference](https://docs.databricks.com/aws/en/admin/system-tables/audit-logs) -------------------------------------------------------------------------------- /presets/okta/syslog/README.md: -------------------------------------------------------------------------------- 1 | # Okta presets 2 | 3 | This preset contains data processing instructions for the Okta syslog data source with Cribl packaging/metadata. 4 | 5 | ## Exploration & Research 6 | 7 | - [Okta Syslog API EventTypes](https://developer.okta.com/docs/reference/api/event-types/#catalog) 8 | - [CSV File of EventTypes](https://developer.okta.com/docs/okta-event-types.csv) 9 | - [CRIBL Okta Source Setup](https://docs.cribl.io/search/set-up-okta/#provider) 10 | - [List of CRIBL Sources](https://docs.cribl.io/stream/sources/) 11 | - 30 different "up stream" sources - some of which contain multiple products 12 | - Okta is not listed as dedicated source - likely connected via some generic connector 13 | - Seems to contain data from additional 3rd party source 14 | - [ZScaler](https://help.zscaler.com/zia/understanding-nanolog-streaming-service) 15 | - CrowdStrike 16 | - examples from [OCSF](https://github.com/ocsf/examples/tree/main/mappings/markdown/Okta) 17 | 18 | ## Limitations 19 | - currently only supports about 70 event classes present in the sample data 20 | - unsupported event types are being mapped to NULL 21 | - some class-specific OCSF fields are currently not mapped 22 | - uses `tempField` util for the `os` sub-object that is being reused across different objects 23 | 24 | ## Syslog 25 | - `1004` different event types defined by the Okta API 26 | - Assumed data format: jsonL 27 | - gold tables written to 28 | - account_change 29 | - authentication 30 | - authorize_session 31 | - entity_management 32 | - user_access 33 | - group_management 34 | 35 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/waf/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_waf 2 | author: Antimatter 3 | description: "AWS WAF activity logs collected with AWS Security Lake" 4 | title: "AWS SecLake - WAF" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/waf/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "action string,action_id int,activity_id int,activity_name string,category_name string,category_uid int,class_name string,class_uid int,type_name string,type_uid bigint,time bigint,time_dt timestamp,disposition_id int,disposition string,cloud struct,firewall_rule struct,match_location:string,rate_limit:int,type:string,uid:string>,http_request struct>,http_method:string,referrer:string,uid:string,url:struct,user_agent:string,version:string,x_forwarded_for:array>,http_status int,metadata struct,product:struct,name:string,vendor_name:string,version:string>,profiles:array,version:string>,observables array>,severity string,severity_id int,src_endpoint struct,svc_name:string,uid:string>,unmapped map,_rescued_data string" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_waf 14 | filter: "class_uid = 4002" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: category_uid 21 | from: category_uid 22 | - name: category_name 23 | from: category_name 24 | - name: class_uid 25 | from: class_uid 26 | - name: class_name 27 | from: class_name 28 | - name: metadata 29 | from: metadata 30 | - name: severity_id 31 | from: severity_id 32 | - name: severity 33 | from: severity 34 | - name: time 35 | expr: timestamp_millis(time) 36 | - name: time_dt 37 | from: time_dt 38 | - name: type_uid 39 | from: type_uid 40 | - name: type_name 41 | from: type_name 42 | - name: observables 43 | from: observables 44 | - name: unmapped 45 | from: unmapped 46 | - name: action 47 | from: action 48 | - name: cloud 49 | from: cloud 50 | - name: disposition 51 | from: disposition 52 | - name: disposition_id 53 | from: disposition_id 54 | - name: firewall_rule 55 | from: firewall_rule 56 | - name: http_request 57 | from: http_request 58 | - name: http_status 59 | from: http_status 60 | - name: src_endpoint 61 | from: src_endpoint 62 | utils: 63 | unreferencedColumns: 64 | preserve: true 65 | omitColumns: 66 | - time 67 | gold: 68 | - name: http_activity 69 | input: aws_sec_lake_waf 70 | fields: 71 | - name: activity_id 72 | from: activity_id 73 | - name: activity_name 74 | from: activity_name 75 | - name: category_uid 76 | from: category_uid 77 | - name: category_name 78 | from: category_name 79 | - name: class_uid 80 | from: class_uid 81 | - name: class_name 82 | from: class_name 83 | - name: metadata 84 | from: metadata 85 | - name: severity_id 86 | from: severity_id 87 | - name: severity 88 | from: severity 89 | - name: time 90 | from: time 91 | - name: time_dt 92 | from: time_dt 93 | - name: type_uid 94 | from: type_uid 95 | - name: type_name 96 | from: type_name 97 | - name: observables 98 | from: observables 99 | - name: action 100 | from: action 101 | - name: cloud 102 | from: cloud 103 | - name: disposition 104 | from: disposition 105 | - name: disposition_id 106 | from: disposition_id 107 | - name: firewall_rule 108 | from: firewall_rule 109 | - name: http_request 110 | from: http_request 111 | - name: http_status 112 | from: http_status 113 | - name: src_endpoint 114 | from: src_endpoint 115 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/s3_data/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_s3_data 2 | author: Antimatter 3 | description: "AWS S3 DataEvents data collected with AWS Security Lake" 4 | title: "AWS SecLake - S3" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/s3_data/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "metadata struct>,event_code:string,uid:string,profiles:array,version:string>,time bigint,time_dt timestamp,cloud struct,api struct,operation:string,version:string,service:struct,request:struct>,dst_endpoint struct,actor struct,credential_uid:string>,session:struct,invoked_by:string,idp:struct>,http_request struct,src_endpoint struct,session struct,policy struct,resources array>,type:string>>,class_name string,class_uid int,category_name string,category_uid int,severity_id int,severity string,user struct,activity_name string,activity_id int,type_uid bigint,type_name string,status string,is_mfa boolean,unmapped map,accountId bigint,region string,asl_version string,observables array>,eventDay int,_rescued_data string" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_s3_data 14 | filter: "class_uid = 6003" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: category_uid 21 | from: category_uid 22 | - name: category_name 23 | from: category_name 24 | - name: class_uid 25 | from: class_uid 26 | - name: class_name 27 | from: class_name 28 | - name: metadata 29 | from: metadata 30 | - name: severity_id 31 | from: severity_id 32 | - name: severity 33 | from: severity 34 | - name: time 35 | expr: timestamp_millis(time) 36 | - name: time_dt 37 | from: time_dt 38 | - name: type_uid 39 | from: type_uid 40 | - name: type_name 41 | from: type_name 42 | - name: observables 43 | from: observables 44 | - name: status 45 | from: status 46 | - name: unmapped 47 | from: unmapped 48 | - name: api 49 | from: api 50 | - name: cloud 51 | from: cloud 52 | - name: actor 53 | from: actor 54 | - name: policy 55 | from: policy 56 | - name: http_request 57 | from: http_request 58 | - name: resources 59 | from: resources 60 | - name: src_endpoint 61 | from: src_endpoint 62 | utils: 63 | unreferencedColumns: 64 | preserve: true 65 | omitColumns: 66 | - time 67 | gold: 68 | - name: api_activity 69 | input: aws_sec_lake_s3_data 70 | fields: 71 | - name: activity_id 72 | from: activity_id 73 | - name: activity_name 74 | from: activity_name 75 | - name: category_uid 76 | from: category_uid 77 | - name: category_name 78 | from: category_name 79 | - name: class_uid 80 | from: class_uid 81 | - name: class_name 82 | from: class_name 83 | - name: metadata 84 | from: metadata 85 | - name: severity_id 86 | from: severity_id 87 | - name: severity 88 | from: severity 89 | - name: time 90 | from: time 91 | - name: time_dt 92 | from: time_dt 93 | - name: type_uid 94 | from: type_uid 95 | - name: type_name 96 | from: type_name 97 | - name: observables 98 | from: observables 99 | - name: status 100 | from: status 101 | - name: api 102 | from: api 103 | - name: cloud 104 | from: cloud 105 | - name: actor 106 | from: actor 107 | - name: http_request 108 | from: http_request 109 | - name: resources 110 | from: resources 111 | - name: src_endpoint 112 | from: src_endpoint 113 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/eks_audit/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_eks_audit 2 | author: Antimatter 3 | description: "AWS EKS control plane data collected with AWS Security Lake" 4 | title: "AWS SecLake - EKS" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/eks_audit/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "metadata struct>,profiles:array,version:string>,api struct>>>,operation:string,version:string,response:struct>>>,group:struct>,message string,http_request struct,user_agent:string>,actor struct>,type_id:int>,session:struct>,cloud struct,provider:string>,src_endpoint struct>,resources array>,start_time_dt timestamp,time_dt timestamp,time bigint,severity_id int,severity string,class_name string,class_uid int,category_name string,category_uid int,activity_name string,activity_id int,type_name string,type_uid bigint,unmapped map,accountId bigint,region string,asl_version string,observables array>,eventDay int,_rescued_data string" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_eks_audit_api_activity 14 | filter: "class_uid = 6003" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: category_uid 21 | from: category_uid 22 | - name: category_name 23 | from: category_name 24 | - name: class_uid 25 | from: class_uid 26 | - name: class_name 27 | from: class_name 28 | - name: message 29 | from: message 30 | - name: metadata 31 | from: metadata 32 | - name: severity_id 33 | from: severity_id 34 | - name: severity 35 | from: severity 36 | - name: start_time_dt 37 | from: start_time_dt 38 | - name: time 39 | expr: timestamp_millis(time) 40 | - name: time_dt 41 | from: time_dt 42 | - name: type_uid 43 | from: type_uid 44 | - name: type_name 45 | from: type_name 46 | - name: observables 47 | from: observables 48 | - name: unmapped 49 | from: unmapped 50 | - name: api 51 | from: api 52 | - name: cloud 53 | from: cloud 54 | - name: actor 55 | from: actor 56 | - name: http_request 57 | from: http_request 58 | - name: resources 59 | from: resources 60 | - name: src_endpoint 61 | from: src_endpoint 62 | utils: 63 | unreferencedColumns: 64 | preserve: true 65 | omitColumns: 66 | - time 67 | gold: 68 | - name: api_activity 69 | input: aws_sec_lake_eks_audit_api_activity 70 | fields: 71 | - name: activity_id 72 | from: activity_id 73 | - name: activity_name 74 | from: activity_name 75 | - name: category_uid 76 | from: category_uid 77 | - name: category_name 78 | from: category_name 79 | - name: class_uid 80 | from: class_uid 81 | - name: class_name 82 | from: class_name 83 | - name: message 84 | from: message 85 | - name: metadata 86 | from: metadata 87 | - name: severity_id 88 | from: severity_id 89 | - name: severity 90 | from: severity 91 | - name: start_time_dt 92 | from: start_time_dt 93 | - name: time 94 | from: time 95 | - name: time_dt 96 | from: time_dt 97 | - name: type_uid 98 | from: type_uid 99 | - name: type_name 100 | from: type_name 101 | - name: observables 102 | from: observables 103 | - name: api 104 | from: api 105 | - name: cloud 106 | from: cloud 107 | - name: actor 108 | from: actor 109 | - name: http_request 110 | from: http_request 111 | - name: resources 112 | from: resources 113 | - name: src_endpoint 114 | from: src_endpoint 115 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/lambda_execution/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_lambda_execution 2 | author: Antimatter 3 | description: "AWS Lambda Execution data collected with AWS Security Lake" 4 | title: "AWS SecLake - Lambda" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/lambda_execution/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "metadata struct>,event_code:string,uid:string,profiles:array,version:string>,time bigint,time_dt timestamp,cloud struct,api struct,operation:string,version:string,service:struct,request:struct>,dst_endpoint struct,actor struct,credential_uid:string>,session:struct,invoked_by:string,idp:struct>,http_request struct,src_endpoint struct,session struct,policy struct,resources array>,type:string>>,class_name string,class_uid int,category_name string,category_uid int,severity_id int,severity string,user struct,activity_name string,activity_id int,type_uid bigint,type_name string,status string,is_mfa boolean,unmapped map,accountId bigint,region string,asl_version string,observables array>,eventDay int,_rescued_data string" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_lambda_execution_api_activity 14 | filter: "class_uid = 6003" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: category_uid 21 | from: category_uid 22 | - name: category_name 23 | from: category_name 24 | - name: class_uid 25 | from: class_uid 26 | - name: class_name 27 | from: class_name 28 | - name: metadata 29 | from: metadata 30 | - name: severity_id 31 | from: severity_id 32 | - name: severity 33 | from: severity 34 | - name: time 35 | expr: timestamp_millis(time) 36 | - name: time_dt 37 | from: time_dt 38 | - name: type_uid 39 | from: type_uid 40 | - name: type_name 41 | from: type_name 42 | - name: observables 43 | from: observables 44 | - name: status 45 | from: status 46 | - name: unmapped 47 | from: unmapped 48 | - name: api 49 | from: api 50 | - name: cloud 51 | from: cloud 52 | - name: actor 53 | from: actor 54 | - name: policy 55 | from: policy 56 | - name: http_request 57 | from: http_request 58 | - name: resources 59 | from: resources 60 | - name: src_endpoint 61 | from: src_endpoint 62 | utils: 63 | unreferencedColumns: 64 | preserve: true 65 | omitColumns: 66 | - time 67 | gold: 68 | - name: api_activity 69 | input: aws_sec_lake_lambda_execution_api_activity 70 | fields: 71 | - name: activity_id 72 | from: activity_id 73 | - name: activity_name 74 | from: activity_name 75 | - name: category_uid 76 | from: category_uid 77 | - name: category_name 78 | from: category_name 79 | - name: class_uid 80 | from: class_uid 81 | - name: class_name 82 | from: class_name 83 | - name: metadata 84 | from: metadata 85 | - name: severity_id 86 | from: severity_id 87 | - name: severity 88 | from: severity 89 | - name: time 90 | from: time 91 | - name: time_dt 92 | from: time_dt 93 | - name: type_uid 94 | from: type_uid 95 | - name: type_name 96 | from: type_name 97 | - name: observables 98 | from: observables 99 | - name: status 100 | from: status 101 | - name: api 102 | from: api 103 | - name: cloud 104 | from: cloud 105 | - name: actor 106 | from: actor 107 | - name: http_request 108 | from: http_request 109 | - name: resources 110 | from: resources 111 | - name: src_endpoint 112 | from: src_endpoint 113 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/vpc_flow/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_vpc_flow 2 | author: Antimatter 3 | description: "AWS VPC network flows logs collected with AWS Security Lake" 4 | title: "AWS SecLake - VPC" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/vpc_flow/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "metadata struct,vendor_name:string>,profiles:array,version:string>,cloud struct,region:string,zone:string,provider:string>,src_endpoint struct,interface_uid:string,vpc_uid:string,instance_uid:string,subnet_uid:string>,dst_endpoint struct,interface_uid:string,vpc_uid:string,instance_uid:string,subnet_uid:string>,connection_info struct,traffic struct,time bigint,time_dt timestamp,start_time_dt timestamp,end_time_dt timestamp,status_code string,severity_id int,severity string,class_name string,class_uid int,category_name string,category_uid int,activity_name string,activity_id int,action string,action_id int,disposition string,type_uid bigint,type_name string,accountId bigint,region string,asl_version string,unmapped map,observables array>,eventDay int,_rescued_data string" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_vpc_flow 14 | filter: "class_uid = 4001" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: category_uid 21 | from: category_uid 22 | - name: category_name 23 | from: category_name 24 | - name: class_uid 25 | from: class_uid 26 | - name: end_time_dt 27 | from: end_time_dt 28 | - name: class_name 29 | from: class_name 30 | - name: metadata 31 | from: metadata 32 | - name: observables 33 | from: observables 34 | - name: severity_id 35 | from: severity_id 36 | - name: severity 37 | from: severity 38 | - name: start_time_dt 39 | from: start_time_dt 40 | - name: status_code 41 | from: status_code 42 | - name: time 43 | expr: timestamp_millis(time) 44 | - name: time_dt 45 | from: time_dt 46 | - name: type_uid 47 | from: type_uid 48 | - name: type_name 49 | from: type_name 50 | - name: unmapped 51 | from: unmapped 52 | - name: cloud 53 | from: cloud 54 | - name: action 55 | from: action 56 | - name: action_id 57 | from: action_id 58 | - name: disposition 59 | from: disposition 60 | - name: connection_info 61 | from: connection_info 62 | - name: dst_endpoint 63 | from: dst_endpoint 64 | - name: src_endpoint 65 | from: src_endpoint 66 | - name: traffic 67 | from: traffic 68 | utils: 69 | unreferencedColumns: 70 | preserve: true 71 | omitColumns: 72 | - time 73 | gold: 74 | - name: network_activity 75 | input: aws_sec_lake_vpc_flow 76 | fields: 77 | - name: activity_id 78 | from: activity_id 79 | - name: activity_name 80 | from: activity_name 81 | - name: category_uid 82 | from: category_uid 83 | - name: category_name 84 | from: category_name 85 | - name: class_uid 86 | from: class_uid 87 | - name: end_time_dt 88 | from: end_time_dt 89 | - name: class_name 90 | from: class_name 91 | - name: metadata 92 | from: metadata 93 | - name: observables 94 | from: observables 95 | - name: severity_id 96 | from: severity_id 97 | - name: severity 98 | from: severity 99 | - name: start_time_dt 100 | from: start_time_dt 101 | - name: status_code 102 | from: status_code 103 | - name: time 104 | from: time 105 | - name: time_dt 106 | from: time_dt 107 | - name: type_uid 108 | from: type_uid 109 | - name: type_name 110 | from: type_name 111 | - name: cloud 112 | from: cloud 113 | - name: action 114 | from: action 115 | - name: action_id 116 | from: action_id 117 | - name: disposition 118 | from: disposition 119 | - name: connection_info 120 | from: connection_info 121 | - name: dst_endpoint 122 | from: dst_endpoint 123 | - name: src_endpoint 124 | from: src_endpoint 125 | - name: traffic 126 | from: traffic 127 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/route53/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_route53 2 | author: Antimatter 3 | description: "AWS Route53 Resolver query log data collected with AWS Security Lake" 4 | title: "AWS SecLake - Route53" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/route53/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "metadata struct,vendor_name:string>,profiles:array,version:string>,cloud struct,region:string,provider:string>,src_endpoint struct,time bigint,time_dt timestamp,query struct,answers array>,connection_info struct,dst_endpoint struct,firewall_rule struct,severity_id int,severity string,class_name string,class_uid int,category_name string,category_uid int,activity_id int,activity_name string,type_uid bigint,type_name string,rcode_id int,rcode string,disposition string,action string,action_id int,unmapped map,accountId bigint,region string,asl_version string,observables array>,eventDay int,_rescued_data string,actor struct,credential_uid:string>,session:struct,invoked_by:string,idp:struct>" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_route53 14 | filter: "class_uid = 4003" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: actor 21 | from: actor 22 | - name: category_uid 23 | from: category_uid 24 | - name: category_name 25 | from: category_name 26 | - name: class_uid 27 | from: class_uid 28 | - name: class_name 29 | from: class_name 30 | - name: metadata 31 | from: metadata 32 | - name: severity_id 33 | from: severity_id 34 | - name: severity 35 | from: severity 36 | - name: time 37 | expr: timestamp_millis(time) 38 | - name: time_dt 39 | from: time_dt 40 | - name: type_uid 41 | from: type_uid 42 | - name: type_name 43 | from: type_name 44 | - name: observables 45 | from: observables 46 | - name: unmapped 47 | from: unmapped 48 | - name: cloud 49 | from: cloud 50 | - name: src_endpoint 51 | from: src_endpoint 52 | - name: query 53 | from: query 54 | - name: answers 55 | from: answers 56 | - name: connection_info 57 | from: connection_info 58 | - name: dst_endpoint 59 | from: dst_endpoint 60 | - name: firewall_rule 61 | from: firewall_rule 62 | - name: rcode_id 63 | from: rcode_id 64 | - name: rcode 65 | from: rcode 66 | - name: disposition 67 | from: disposition 68 | - name: action 69 | from: action 70 | - name: action_id 71 | from: action_id 72 | - name: accountId 73 | from: accountId 74 | - name: region 75 | from: region 76 | - name: asl_version 77 | from: asl_version 78 | - name: eventDay 79 | from: eventDay 80 | utils: 81 | unreferencedColumns: 82 | preserve: true 83 | omitColumns: 84 | - time 85 | gold: 86 | - name: dns_activity 87 | input: aws_sec_lake_route53 88 | fields: 89 | - name: activity_id 90 | from: activity_id 91 | - name: activity_name 92 | from: activity_name 93 | - name: actor 94 | from: actor 95 | - name: category_uid 96 | from: category_uid 97 | - name: category_name 98 | from: category_name 99 | - name: class_uid 100 | from: class_uid 101 | - name: class_name 102 | from: class_name 103 | - name: metadata 104 | from: metadata 105 | - name: severity_id 106 | from: severity_id 107 | - name: severity 108 | from: severity 109 | - name: time 110 | from: time 111 | - name: time_dt 112 | from: time_dt 113 | - name: type_uid 114 | from: type_uid 115 | - name: type_name 116 | from: type_name 117 | - name: observables 118 | from: observables 119 | - name: action 120 | from: action 121 | - name: action_id 122 | from: action_id 123 | - name: disposition 124 | from: disposition 125 | - name: firewall_rule 126 | from: firewall_rule 127 | - name: connection_info 128 | from: connection_info 129 | - name: dst_endpoint 130 | from: dst_endpoint 131 | - name: src_endpoint 132 | from: src_endpoint 133 | - name: answers 134 | from: answers 135 | - name: query 136 | from: query 137 | - name: rcode 138 | from: rcode 139 | - name: rcode_id 140 | from: rcode_id 141 | -------------------------------------------------------------------------------- /presets/akamai/waf/preset.yaml: -------------------------------------------------------------------------------- 1 | name: akamai_waf 2 | author: Antimatter 3 | description: "A preset for consuming logs generated by Akamai WAF logs" 4 | title: "Akamai - WAF" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/akamai/waf/icon.png" 6 | autoloader: 7 | format: json 8 | cloudFiles: 9 | schemaHints: "attackData struct,format string,geo struct,httpMessage struct,requestId:string,responseHeaders:map,start:string,status:string,tls:string>,serviceID string,type string,version string" 10 | silver: 11 | transform: 12 | - name: akamai_waf_http_activity 13 | fields: 14 | - name: ocsf_activity_id 15 | expr: CASE WHEN LOWER(httpMessage.method) = 'connect' THEN 1 WHEN LOWER(httpMessage.method) = 'delete' THEN 2 WHEN LOWER(httpMessage.method) = 'get' THEN 3 WHEN LOWER(httpMessage.method) = 'head' THEN 4 WHEN LOWER(httpMessage.method) = 'options' THEN 5 WHEN LOWER(httpMessage.method) = 'post' THEN 6 WHEN LOWER(httpMessage.method) = 'put' THEN 7 WHEN LOWER(httpMessage.method) = 'trace' THEN 8 WHEN httpMessage.method IS NULL OR httpMessage.method = '' THEN 0 ELSE 99 END 16 | - name: ocsf_activity_name 17 | expr: CASE WHEN httpMessage.method IS NULL OR httpMessage.method = '' THEN httpMessage.method ELSE CONCAT(UPPER(SUBSTRING(httpMessage.method, 1, 1)), LOWER(SUBSTRING(httpMessage.method, 2))) END 18 | - name: ocsf_category_uid 19 | expr: CAST('4' AS INT) 20 | - name: ocsf_category_name 21 | literal: Network Activity 22 | - name: ocsf_class_uid 23 | expr: CAST('4002' AS INT) 24 | - name: ocsf_class_name 25 | literal: HTTP Activity 26 | - name: ocsf_severity_id 27 | expr: CAST('0' AS INT) 28 | - name: ocsf_severity 29 | literal: Unknown 30 | - name: ocsf_type_uid 31 | expr: CAST((400200 + CASE WHEN LOWER(httpMessage.method) = 'connect' THEN 1 WHEN LOWER(httpMessage.method) = 'delete' THEN 2 WHEN LOWER(httpMessage.method) = 'get' THEN 3 WHEN LOWER(httpMessage.method) = 'head' THEN 4 WHEN LOWER(httpMessage.method) = 'options' THEN 5 WHEN LOWER(httpMessage.method) = 'post' THEN 6 WHEN LOWER(httpMessage.method) = 'put' THEN 7 WHEN LOWER(httpMessage.method) = 'trace' THEN 8 WHEN httpMessage.method IS NULL OR httpMessage.method = '' THEN 0 ELSE 99 END) AS BIGINT) 32 | - name: ocsf_type_name 33 | expr: "CASE WHEN httpMessage.method IS NOT NULL THEN CONCAT('HTTP Activity: ', httpMessage.method) ELSE null END" 34 | - name: time 35 | expr: timestamp_seconds(cast(httpMessage.start AS BIGINT)) 36 | - name: http_status 37 | expr: CAST(httpMessage.status AS int) 38 | - name: ocsf_metadata.uid 39 | expr: attackData.configId 40 | - name: ocsf_metadata.log_provider 41 | from: type 42 | - name: ocsf_metadata.product.name 43 | literal: Kona 44 | - name: ocsf_metadata.product.version 45 | from: version 46 | - name: ocsf_metadata.product.vendor_name 47 | literal: Akamai 48 | - name: ocsf_http_request.length 49 | expr: CAST(httpMessage.bytes AS INT) 50 | - name: ocsf_http_request.http_method 51 | from: httpMessage.Method 52 | - name: ocsf_http_request.url.hostname 53 | from: httpMessage.host 54 | - name: ocsf_http_request.url.path 55 | from: httpMessage.path 56 | - name: ocsf_http_request.url.query_string 57 | from: httpMessage.query 58 | - name: ocsf_http_request.url.port 59 | expr: CAST(httpMessage.port AS INT) 60 | - name: ocsf_http_request.user_agent 61 | from: httpMessage.requestHeaders.`User-Agent` 62 | - name: ocsf_http_request.version 63 | from: httpMessage.protocol 64 | - name: ocsf_http_request.http_headers 65 | expr: "transform( 66 | map_entries( 67 | from_json(to_json(httpMessage.requestHeaders), 'map') 68 | ), 69 | x -> named_struct('name', x.key, 'value', x.value) 70 | )" 71 | - name: ocsf_http_response.code 72 | expr: CAST(httpMessage.status AS int) 73 | - name: ocsf_http_response.length 74 | expr: CAST(httpMessage.bytes AS INT) 75 | - name: ocsf_http_response.http_headers 76 | expr: "transform( 77 | map_entries( 78 | from_json(to_json(httpMessage.responseHeaders), 'map') 79 | ), 80 | x -> named_struct('name', x.key, 'value', x.value) 81 | )" 82 | - name: ocsf_tls.version 83 | from: httpMessage.tls 84 | - name: ocsf_src_endpoint.ip 85 | from: attackData.clientIP 86 | - name: ocsf_src_endpoint.location.city 87 | from: geo.city 88 | - name: ocsf_src_endpoint.location.continent 89 | from: geo.continent 90 | - name: ocsf_src_endpoint.location.country 91 | from: geo.country 92 | - name: ocsf_src_endpoint.location.region 93 | from: geo.regionCode 94 | - name: ocsf_policy.uid 95 | from: attackData.policyId 96 | - name: ocsf_firewall_rule.name 97 | from: attackData.rules 98 | - name: ocsf_firewall_rule.version 99 | from: attackData.ruleVersions 100 | - name: ocsf_firewall_rule.match_details 101 | expr: "ARRAY(attackData.ruleSelectors)" 102 | - name: ocsf_firewall_rule.desc 103 | from: attackData.ruleTags 104 | - name: message 105 | from: attackData.ruleMessages 106 | - name: raw_data 107 | from: attackData.ruleData 108 | - name: action 109 | expr: CASE WHEN attackData.ruleActions = 'deny' THEN 'Denied' WHEN attackData.ruleActions = 'alert' OR attackData.ruleActions = 'monitor' THEN 'Observed' WHEN attackData.ruleActions = 'tarpit' THEN 'Modified' WHEN attackData.ruleActions = 'allow' THEN 'Allowed' ELSE 'Unknown' END 110 | - name: action_id 111 | expr: CASE WHEN attackData.ruleActions = 'deny' THEN 2 WHEN attackData.ruleActions = 'alert' OR attackData.ruleActions = 'monitor' THEN 3 WHEN attackData.ruleActions = 'tarpit' THEN 4 WHEN attackData.ruleActions = 'allow' THEN 1 ELSE 0 END 112 | - name: disposition 113 | expr: CASE WHEN attackData.ruleActions = 'deny' THEN 'Blocked' WHEN attackData.ruleActions = 'alert' THEN 'Alert' WHEN attackData.ruleActions = 'monitor' THEN 'Logged' WHEN attackData.ruleActions = 'tarpit' THEN 'Delayed' WHEN attackData.ruleActions = 'allow' THEN 'Allowed' ELSE 'Unknown' END 114 | - name: disposition_id 115 | expr: CASE WHEN attackData.ruleActions = 'deny' THEN 2 WHEN attackData.ruleActions = 'alert' THEN 19 WHEN attackData.ruleActions = 'monitor' THEN 17 WHEN attackData.ruleActions = 'tarpit' THEN 14 WHEN attackData.ruleActions = 'allow' THEN 1 ELSE 0 END 116 | - name: is_alert 117 | expr: CASE WHEN attackData.ruleActions = 'alert' THEN TRUE ELSE FALSE END 118 | - name: ocsf_enrichments 119 | expr: "array( 120 | named_struct('name', 'clientReputation', 'value', attackData.clientReputation) 121 | )" 122 | - name: ocsf_unmapped 123 | expr: to_json(named_struct('serviceID', serviceID)) 124 | utils: 125 | unreferencedColumns: 126 | preserve: true 127 | gold: 128 | - name: http_activity 129 | input: akamai_waf_http_activity 130 | fields: 131 | - name: activity_id 132 | from: ocsf_activity_id 133 | - name: activity_name 134 | from: ocsf_activity_name 135 | - name: category_uid 136 | from: ocsf_category_uid 137 | - name: category_name 138 | from: ocsf_category_name 139 | - name: class_uid 140 | from: ocsf_class_uid 141 | - name: class_name 142 | from: ocsf_class_name 143 | - name: severity_id 144 | from: ocsf_severity_id 145 | - name: severity 146 | from: ocsf_severity 147 | - name: type_uid 148 | from: ocsf_type_uid 149 | - name: type_name 150 | from: ocsf_type_name 151 | - name: time 152 | from: time 153 | - name: start_time 154 | from: time 155 | - name: action 156 | from: action 157 | - name: action_id 158 | from: action_id 159 | - name: disposition 160 | from: disposition 161 | - name: disposition_id 162 | from: disposition_id 163 | - name: is_alert 164 | from: is_alert 165 | - name: http_status 166 | from: http_status 167 | - name: metadata 168 | from: ocsf_metadata 169 | - name: http_request 170 | from: ocsf_http_request 171 | - name: http_response 172 | from: ocsf_http_response 173 | - name: src_endpoint 174 | from: ocsf_src_endpoint 175 | - name: enrichments 176 | from: ocsf_enrichments 177 | - name: tls 178 | from: ocsf_tls 179 | - name: policy 180 | from: ocsf_policy 181 | - name: firewall_rule 182 | from: ocsf_firewall_rule 183 | - name: message 184 | from: message 185 | - name: raw_data 186 | from: raw_data 187 | - name: unmapped 188 | expr: CAST(ocsf_unmapped AS VARIANT) 189 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/security_hub/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_security_hub 2 | author: Antimatter 3 | description: "AWS Security Hub Findings collected with AWS Security Lake" 4 | title: "AWS SecLake - Security Hub" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/security_hub/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "activity_id int,activity_name string,category_name string,category_uid int,class_name string,class_uid int,cloud struct,provider:string,region:string>,compliance struct,standards:array,status:string>,evidences array>,data:variant,src_endpoint:struct,country:string>>>>,finding_info struct,uid:string>,metadata struct>,log_version:string,processed_time_dt:timestamp,product:struct,name:string,uid:string,vendor_name:string,version:string>,profiles:array,version:string>,observables array>,remediation struct>,resource struct,resources array>,severity string,severity_id int,status string,time bigint,time_dt timestamp,type_name string,type_uid bigint,unmapped map,vulnerabilities array,version:string>>,cve:struct>,epss:struct,modified_time_dt:timestamp,references:array,uid:string>,is_exploit_available:boolean,is_fix_available:boolean,references:array,remediation:struct,vendor_name:string>>,_rescued_data string" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_security_hub_vulnerability_finding 14 | filter: "class_uid = 2002" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: category_uid 21 | from: category_uid 22 | - name: category_name 23 | from: category_name 24 | - name: class_uid 25 | from: class_uid 26 | - name: class_name 27 | from: class_name 28 | - name: metadata 29 | from: metadata 30 | - name: observables 31 | from: observables 32 | - name: severity_id 33 | from: severity_id 34 | - name: severity 35 | from: severity 36 | - name: status 37 | from: status 38 | - name: time 39 | expr: timestamp_millis(time) 40 | - name: time_dt 41 | from: time_dt 42 | - name: type_uid 43 | from: type_uid 44 | - name: type_name 45 | from: type_name 46 | - name: unmapped 47 | from: unmapped 48 | - name: cloud 49 | from: cloud 50 | - name: finding_info 51 | from: finding_info 52 | - name: resource 53 | from: resource 54 | - name: resources 55 | from: resources 56 | - name: vulnerabilities 57 | from: vulnerabilities 58 | utils: 59 | unreferencedColumns: 60 | preserve: true 61 | omitColumns: 62 | - time 63 | - name: aws_sec_lake_security_hub_compliance_finding 64 | filter: "class_uid = 2003" 65 | fields: 66 | - name: activity_id 67 | from: activity_id 68 | - name: activity_name 69 | from: activity_name 70 | - name: category_uid 71 | from: category_uid 72 | - name: category_name 73 | from: category_name 74 | - name: class_uid 75 | from: class_uid 76 | - name: class_name 77 | from: class_name 78 | - name: metadata 79 | from: metadata 80 | - name: observables 81 | from: observables 82 | - name: severity_id 83 | from: severity_id 84 | - name: severity 85 | from: severity 86 | - name: status 87 | from: status 88 | - name: time 89 | expr: timestamp_millis(time) 90 | - name: time_dt 91 | from: time_dt 92 | - name: type_uid 93 | from: type_uid 94 | - name: type_name 95 | from: type_name 96 | - name: unmapped 97 | from: unmapped 98 | - name: cloud 99 | from: cloud 100 | - name: finding_info 101 | from: finding_info 102 | - name: compliance 103 | from: compliance 104 | - name: evidences 105 | from: evidences 106 | - name: remediation 107 | from: remediation 108 | - name: resource 109 | from: resource 110 | - name: resources 111 | from: resources 112 | utils: 113 | unreferencedColumns: 114 | preserve: true 115 | omitColumns: 116 | - time 117 | - name: aws_sec_lake_security_hub_detection_finding 118 | filter: "class_uid = 2004" 119 | fields: 120 | - name: activity_id 121 | from: activity_id 122 | - name: activity_name 123 | from: activity_name 124 | - name: category_uid 125 | from: category_uid 126 | - name: category_name 127 | from: category_name 128 | - name: class_uid 129 | from: class_uid 130 | - name: class_name 131 | from: class_name 132 | - name: metadata 133 | from: metadata 134 | - name: observables 135 | from: observables 136 | - name: severity_id 137 | from: severity_id 138 | - name: severity 139 | from: severity 140 | - name: status 141 | from: status 142 | - name: time 143 | expr: timestamp_millis(time) 144 | - name: time_dt 145 | from: time_dt 146 | - name: type_uid 147 | from: type_uid 148 | - name: type_name 149 | from: type_name 150 | - name: unmapped 151 | from: unmapped 152 | - name: cloud 153 | from: cloud 154 | - name: finding_info 155 | from: finding_info 156 | - name: evidences 157 | from: evidences 158 | - name: remediation 159 | from: remediation 160 | - name: resources 161 | from: resources 162 | - name: vulnerabilities 163 | from: vulnerabilities 164 | utils: 165 | unreferencedColumns: 166 | preserve: true 167 | omitColumns: 168 | - time 169 | gold: 170 | - name: vulnerability_finding 171 | input: aws_sec_lake_security_hub_vulnerability_finding 172 | fields: 173 | - name: activity_id 174 | from: activity_id 175 | - name: activity_name 176 | from: activity_name 177 | - name: category_uid 178 | from: category_uid 179 | - name: category_name 180 | from: category_name 181 | - name: class_uid 182 | from: class_uid 183 | - name: class_name 184 | from: class_name 185 | - name: metadata 186 | from: metadata 187 | - name: observables 188 | from: observables 189 | - name: severity_id 190 | from: severity_id 191 | - name: severity 192 | from: severity 193 | - name: status 194 | from: status 195 | - name: time 196 | from: time 197 | - name: time_dt 198 | from: time_dt 199 | - name: type_uid 200 | from: type_uid 201 | - name: type_name 202 | from: type_name 203 | - name: cloud 204 | from: cloud 205 | - name: finding_info 206 | from: finding_info 207 | - name: resource 208 | from: resource 209 | - name: resources 210 | from: resources 211 | - name: vulnerabilities 212 | from: vulnerabilities 213 | - name: compliance_finding 214 | input: aws_sec_lake_security_hub_compliance_finding 215 | fields: 216 | - name: activity_id 217 | from: activity_id 218 | - name: activity_name 219 | from: activity_name 220 | - name: category_uid 221 | from: category_uid 222 | - name: category_name 223 | from: category_name 224 | - name: class_uid 225 | from: class_uid 226 | - name: class_name 227 | from: class_name 228 | - name: metadata 229 | from: metadata 230 | - name: observables 231 | from: observables 232 | - name: severity_id 233 | from: severity_id 234 | - name: severity 235 | from: severity 236 | - name: status 237 | from: status 238 | - name: time 239 | from: time 240 | - name: time_dt 241 | from: time_dt 242 | - name: type_uid 243 | from: type_uid 244 | - name: type_name 245 | from: type_name 246 | - name: cloud 247 | from: cloud 248 | - name: finding_info 249 | from: finding_info 250 | - name: compliance 251 | from: compliance 252 | - name: evidences 253 | from: evidences 254 | - name: remediation 255 | from: remediation 256 | - name: resource 257 | from: resource 258 | - name: resources 259 | from: resources 260 | - name: detection_finding 261 | input: aws_sec_lake_security_hub_detection_finding 262 | fields: 263 | - name: activity_id 264 | from: activity_id 265 | - name: activity_name 266 | from: activity_name 267 | - name: category_uid 268 | from: category_uid 269 | - name: category_name 270 | from: category_name 271 | - name: class_uid 272 | from: class_uid 273 | - name: class_name 274 | from: class_name 275 | - name: metadata 276 | from: metadata 277 | - name: observables 278 | from: observables 279 | - name: severity_id 280 | from: severity_id 281 | - name: severity 282 | from: severity 283 | - name: status 284 | from: status 285 | - name: time 286 | from: time 287 | - name: time_dt 288 | from: time_dt 289 | - name: type_uid 290 | from: type_uid 291 | - name: type_name 292 | from: type_name 293 | - name: cloud 294 | from: cloud 295 | - name: finding_info 296 | from: finding_info 297 | - name: evidences 298 | from: evidences 299 | - name: remediation 300 | from: remediation 301 | - name: resources 302 | from: resources 303 | - name: vulnerabilities 304 | from: vulnerabilities 305 | -------------------------------------------------------------------------------- /presets/aws_sec_lake/cloudtrail_management/preset.yaml: -------------------------------------------------------------------------------- 1 | name: aws_sec_lake_cloudtrail_management 2 | author: Antimatter 3 | description: "AWS CloudTrail Management Event collected with AWS Security Lake" 4 | title: "AWS SecLake - CloudTrail" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/aws_sec_lake/cloudtrail_management/icon.png" 6 | autoloader: 7 | format: parquet 8 | multiline: true 9 | cloudFiles: 10 | schemaHints: "metadata struct>,event_code:string,uid:string,profiles:array,version:string>,time bigint,time_dt timestamp,cloud struct,api struct,operation:string,version:string,service:struct,request:struct>,dst_endpoint struct,actor struct,credential_uid:string>,session:struct,invoked_by:string,idp:struct>,http_request struct,src_endpoint struct,session struct,policy struct,resources array>,type:string>>,class_name string,class_uid int,category_name string,category_uid int,severity_id int,severity string,user struct,activity_name string,activity_id int,type_uid bigint,type_name string,status string,is_mfa boolean,unmapped map,accountId bigint,region string,asl_version string,observables array>,eventDay int,_rescued_data string" 11 | silver: 12 | transform: 13 | - name: aws_sec_lake_cloudtrail_management_account_change 14 | filter: "class_uid = 3001" 15 | fields: 16 | - name: activity_id 17 | from: activity_id 18 | - name: activity_name 19 | from: activity_name 20 | - name: category_uid 21 | from: category_uid 22 | - name: category_name 23 | from: category_name 24 | - name: class_uid 25 | from: class_uid 26 | - name: class_name 27 | from: class_name 28 | - name: metadata 29 | from: metadata 30 | - name: severity_id 31 | from: severity_id 32 | - name: severity 33 | from: severity 34 | - name: time 35 | expr: timestamp_millis(time) 36 | - name: time_dt 37 | from: time_dt 38 | - name: type_uid 39 | from: type_uid 40 | - name: type_name 41 | from: type_name 42 | - name: observables 43 | from: observables 44 | - name: status 45 | from: status 46 | - name: unmapped 47 | from: unmapped 48 | - name: api 49 | from: api 50 | - name: cloud 51 | from: cloud 52 | - name: actor 53 | from: actor 54 | - name: policy 55 | from: policy 56 | - name: http_request 57 | from: http_request 58 | - name: src_endpoint 59 | from: src_endpoint 60 | - name: user 61 | from: user 62 | utils: 63 | unreferencedColumns: 64 | preserve: true 65 | omitColumns: 66 | - time 67 | - name: aws_sec_lake_cloudtrail_management_authentication 68 | filter: "class_uid = 3002" 69 | fields: 70 | - name: activity_id 71 | from: activity_id 72 | - name: activity_name 73 | from: activity_name 74 | - name: category_uid 75 | from: category_uid 76 | - name: category_name 77 | from: category_name 78 | - name: class_uid 79 | from: class_uid 80 | - name: class_name 81 | from: class_name 82 | - name: metadata 83 | from: metadata 84 | - name: severity_id 85 | from: severity_id 86 | - name: severity 87 | from: severity 88 | - name: time 89 | expr: timestamp_millis(time) 90 | - name: time_dt 91 | from: time_dt 92 | - name: type_uid 93 | from: type_uid 94 | - name: type_name 95 | from: type_name 96 | - name: observables 97 | from: observables 98 | - name: status 99 | from: status 100 | - name: unmapped 101 | from: unmapped 102 | - name: api 103 | from: api 104 | - name: cloud 105 | from: cloud 106 | - name: actor 107 | from: actor 108 | - name: policy 109 | from: policy 110 | - name: http_request 111 | from: http_request 112 | - name: src_endpoint 113 | from: src_endpoint 114 | - name: dst_endpoint 115 | from: dst_endpoint 116 | - name: is_mfa 117 | from: is_mfa 118 | - name: session 119 | from: session 120 | - name: user 121 | from: user 122 | utils: 123 | unreferencedColumns: 124 | preserve: true 125 | omitColumns: 126 | - time 127 | - name: aws_sec_lake_cloudtrail_management_api_activity 128 | filter: "class_uid = 6003" 129 | fields: 130 | - name: activity_id 131 | from: activity_id 132 | - name: activity_name 133 | from: activity_name 134 | - name: category_uid 135 | from: category_uid 136 | - name: category_name 137 | from: category_name 138 | - name: class_uid 139 | from: class_uid 140 | - name: class_name 141 | from: class_name 142 | - name: metadata 143 | from: metadata 144 | - name: severity_id 145 | from: severity_id 146 | - name: severity 147 | from: severity 148 | - name: time 149 | expr: timestamp_millis(time) 150 | - name: time_dt 151 | from: time_dt 152 | - name: type_uid 153 | from: type_uid 154 | - name: type_name 155 | from: type_name 156 | - name: observables 157 | from: observables 158 | - name: status 159 | from: status 160 | - name: unmapped 161 | from: unmapped 162 | - name: api 163 | from: api 164 | - name: cloud 165 | from: cloud 166 | - name: actor 167 | from: actor 168 | - name: policy 169 | from: policy 170 | - name: http_request 171 | from: http_request 172 | - name: resources 173 | from: resources 174 | - name: src_endpoint 175 | from: src_endpoint 176 | utils: 177 | unreferencedColumns: 178 | preserve: true 179 | omitColumns: 180 | - time 181 | gold: 182 | - name: account_change 183 | input: aws_sec_lake_cloudtrail_management_account_change 184 | fields: 185 | - name: activity_id 186 | from: activity_id 187 | - name: activity_name 188 | from: activity_name 189 | - name: category_uid 190 | from: category_uid 191 | - name: category_name 192 | from: category_name 193 | - name: class_uid 194 | from: class_uid 195 | - name: class_name 196 | from: class_name 197 | - name: metadata 198 | from: metadata 199 | - name: severity_id 200 | from: severity_id 201 | - name: severity 202 | from: severity 203 | - name: time 204 | from: time 205 | - name: time_dt 206 | from: time_dt 207 | - name: type_uid 208 | from: type_uid 209 | - name: type_name 210 | from: type_name 211 | - name: observables 212 | from: observables 213 | - name: status 214 | from: status 215 | - name: api 216 | from: api 217 | - name: cloud 218 | from: cloud 219 | - name: actor 220 | from: actor 221 | - name: http_request 222 | from: http_request 223 | - name: src_endpoint 224 | from: src_endpoint 225 | - name: user.uid_alt 226 | expr: (COALESCE(user.uid_alt, actor.user.uid_alt)) 227 | - name: user.uid 228 | expr: (COALESCE(user.uid, actor.user.uid)) 229 | - name: user.name 230 | expr: (COALESCE(user.name, actor.user.name)) 231 | - name: authentication 232 | input: aws_sec_lake_cloudtrail_management_authentication 233 | fields: 234 | - name: activity_id 235 | from: activity_id 236 | - name: activity_name 237 | from: activity_name 238 | - name: category_uid 239 | from: category_uid 240 | - name: category_name 241 | from: category_name 242 | - name: class_uid 243 | from: class_uid 244 | - name: class_name 245 | from: class_name 246 | - name: metadata 247 | from: metadata 248 | - name: severity_id 249 | from: severity_id 250 | - name: severity 251 | from: severity 252 | - name: time 253 | from: time 254 | - name: time_dt 255 | from: time_dt 256 | - name: type_uid 257 | from: type_uid 258 | - name: type_name 259 | from: type_name 260 | - name: observables 261 | from: observables 262 | - name: status 263 | from: status 264 | - name: api 265 | from: api 266 | - name: cloud 267 | from: cloud 268 | - name: actor 269 | from: actor 270 | - name: http_request 271 | from: http_request 272 | - name: src_endpoint 273 | from: src_endpoint 274 | - name: dst_endpoint 275 | from: dst_endpoint 276 | - name: is_mfa 277 | from: is_mfa 278 | - name: session 279 | from: session 280 | - name: user.uid_alt 281 | expr: (COALESCE(user.uid_alt, actor.user.uid_alt)) 282 | - name: user.uid 283 | expr: (COALESCE(user.uid, actor.user.uid)) 284 | - name: user.name 285 | expr: (COALESCE(user.name, actor.user.name)) 286 | - name: api_activity 287 | input: aws_sec_lake_cloudtrail_management_api_activity 288 | fields: 289 | - name: activity_id 290 | from: activity_id 291 | - name: activity_name 292 | from: activity_name 293 | - name: category_uid 294 | from: category_uid 295 | - name: category_name 296 | from: category_name 297 | - name: class_uid 298 | from: class_uid 299 | - name: class_name 300 | from: class_name 301 | - name: metadata 302 | from: metadata 303 | - name: severity_id 304 | from: severity_id 305 | - name: severity 306 | from: severity 307 | - name: time 308 | from: time 309 | - name: time_dt 310 | from: time_dt 311 | - name: type_uid 312 | from: type_uid 313 | - name: type_name 314 | from: type_name 315 | - name: observables 316 | from: observables 317 | - name: status 318 | from: status 319 | - name: api 320 | from: api 321 | - name: cloud 322 | from: cloud 323 | - name: actor 324 | from: actor 325 | - name: http_request 326 | from: http_request 327 | - name: resources 328 | from: resources 329 | - name: src_endpoint 330 | from: src_endpoint 331 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /schema/preset.schema.yaml: -------------------------------------------------------------------------------- 1 | $schema: "https://json-schema.org/draft/2020-12/schema" 2 | type: object 3 | properties: 4 | name: 5 | description: > 6 | The name of the preset. Preset names must be unique and should follow established patterns such as 7 | "${source}_${sourceType}_${optionalInfo}". 8 | type: string 9 | author: 10 | description: > 11 | The person or organization that authored this preset. 12 | type: string 13 | description: 14 | description: > 15 | A description of the preset that will be displayed in the UI. The included content should be a brief and may 16 | include information about the preset's purpose, the data it is designed to work with, and any other relevant 17 | information. 18 | type: string 19 | title: 20 | description: > 21 | A formatted, human-readable title for the preset. This text will be displayed in the UI. 22 | type: string 23 | iconURL: 24 | description: > 25 | A URL to an icon representing the preset. This icon will be displayed in the UI. 26 | type: string 27 | format: uri 28 | autoloader: 29 | description: > 30 | The databricks autoloader configuration for this preset. This configuration will be used to load data from the source 31 | during the bronze stage of the data pipeline. The fields available mirror those found in the DataSourceSpec 32 | except for the location field which is not available here. 33 | type: object 34 | properties: 35 | format: 36 | type: string 37 | description: json | parquet | csv | kafka | txt | cloudFiles 38 | schemaFile: 39 | type: string 40 | description: An optional file containing the schema of the data source. 41 | schema: 42 | type: string 43 | description: An optional string representing the schema of the data source. 44 | multiline: 45 | type: boolean 46 | description: > 47 | A flag indicating whether the JSON records span multiple lines. Only applies if the format is set to 48 | 'json'. 49 | cloudFiles: 50 | type: object 51 | description: "The configuration used by the autoloader if format is 'cloudFiles'." 52 | properties: 53 | schemaHintsFile: 54 | type: string 55 | schemaHints: 56 | type: string 57 | required: 58 | - format 59 | silver: 60 | type: object 61 | properties: 62 | preTransform: 63 | description: > 64 | A list of pretransform definitions that will be used in later stages. Each item will include a name of the 65 | pretransform (used for dataframe variable naming) as well as the FieldSpecs that define the column schema. 66 | Other typical properties such as filter, postFilter, and FieldUtils may be included as well. 67 | type: array 68 | items: 69 | type: object 70 | properties: 71 | name: 72 | description: > 73 | The name of this pretransformation definition. This is used to identify the data transformation 74 | (dataframe) for later use. 75 | type: string 76 | filter: 77 | description: > 78 | A SQL filter to apply to the data at the beginning of a data transformation stage. 79 | type: string 80 | postFilter: 81 | description: > 82 | A SQL filter to apply at the end of a data transformation stage. 83 | type: string 84 | fields: 85 | description: > 86 | A list of FieldSpecs that define the column schema for this pretransformation. Each FieldSpec will 87 | include a name, type, and other properties that define the column schema. 88 | type: array 89 | items: 90 | $ref: '#/components/schemas/FieldSpec' 91 | utils: 92 | description: > 93 | A list of utilities for handling fields, including managing unreferenced fields and extracting fields 94 | from hierarchical or JSON objects. 95 | $ref: '#/components/schemas/FieldUtils' 96 | transform: 97 | description: > 98 | A list of silver transform definitions for data cleaning and processing. Each transformation item will 99 | include the name of the transformation which is used for variable and table naming. Other typical properties 100 | such as filter, postFilter, and FieldUtils may be included as well. 101 | type: array 102 | items: 103 | type: object 104 | properties: 105 | name: 106 | description: > 107 | The name of this silver transform definition. This is used to identify the silver table for writes 108 | as well as the data transformation variable (dataframe) for later use in the related notebook. 109 | type: string 110 | filter: 111 | description: > 112 | A SQL filter to apply to the data at the beginning of a data transformation stage. 113 | type: string 114 | postFilter: 115 | description: > 116 | A SQL filter to apply at the end of a data transformation stage. 117 | type: string 118 | fields: 119 | description: > 120 | A list of FieldSpecs that define the column schema for this data transformation. Each FieldSpec will 121 | include a name, type, and other properties that define the column schema. 122 | type: array 123 | items: 124 | $ref: '#/components/schemas/FieldSpec' 125 | utils: 126 | description: > 127 | A list of utilities for handling fields, including managing unreferenced fields and extracting fields 128 | from hierarchical or JSON objects. 129 | $ref: '#/components/schemas/FieldUtils' 130 | gold: 131 | description: > 132 | The gold transform configuration for this preset. This configuration will be used to transform the silver data 133 | into the gold OCSF tables. Note that you can have duplicate names for array objects which indicates the same 134 | destination table but different source (silver) tables. 135 | type: array 136 | items: 137 | type: object 138 | properties: 139 | name: 140 | description: > 141 | The name of this gold transform definition used to identify the gold table for writes. 142 | type: string 143 | input: 144 | description: > 145 | The silver transform data used as input for this gold table definition. 146 | type: string 147 | filter: 148 | description: > 149 | A SQL filter to apply to the data at the beginning of this stage. 150 | type: string 151 | postFilter: 152 | description: > 153 | A SQL filter to apply at the end of this stage. 154 | type: string 155 | fields: 156 | description: > 157 | A list of FieldSpecs that define the column schema for the gold stage data. Each FieldSpec will 158 | include a name, type, and other properties that define the column schema. 159 | type: array 160 | items: 161 | $ref: '#/components/schemas/FieldSpec' 162 | 163 | components: 164 | required: 165 | - name 166 | - author 167 | - description 168 | - title 169 | - iconURL 170 | schemas: 171 | FieldSpec: 172 | description: > 173 | A FieldSpec object is used to define a field to add to a dataset (dataframe). This field can be derived from an existing field, 174 | an expression, from a literal value, from a join, etc. FieldSpec objects are found wherever the user needs to transform data 175 | including in datasources, custom notebooks, and transforms. 176 | type: object 177 | properties: 178 | name: 179 | type: string 180 | comment: 181 | type: string 182 | description: The comment to apply to the field. 183 | assert: 184 | type: array 185 | description: > 186 | A list of SQL expressions that must evaluate to true for every processed row. 187 | The name of the column can be used in the SQL expression. If the assertion is 188 | false, an operational alert is raised using 'message' for each row. 189 | items: 190 | type: object 191 | properties: 192 | expr: 193 | type: string 194 | message: 195 | description: > 196 | The message to include in the operational alert if the assertion fails. (E.g., "The user email is null"). 197 | type: string 198 | from: 199 | type: string 200 | description: This field obtains its value from the source column of this names. 201 | alias: 202 | type: string 203 | description: This field obtains its value from the destination (transformed) column of this name. 204 | expr: 205 | type: string 206 | description: This field obtains its value from the given SQL expression. 207 | literal: 208 | type: string 209 | description: This field obtains its value from the given literal string. For other data types, use expr. 210 | join: 211 | type: object 212 | properties: 213 | withTable: 214 | type: string 215 | description: The table to join to. 216 | withCSV: 217 | type: object 218 | # we expect to need to add more options here to handle CSVs, hence the object 219 | properties: 220 | path: 221 | type: string 222 | description: The path to the CSV file. 223 | lhs: 224 | type: string 225 | description: The column in the source dataframe to join on. 226 | rhs: 227 | type: string 228 | description: The column in withTable (or withCSV) to join on. 229 | select: 230 | type: string 231 | description: A SQL expression to create the new field from the joined dataset. 232 | FieldUtils: 233 | description: > 234 | Defines utilities for handling fields, including managing unreferenced fields and extracting fields from 235 | hierarchical or JSON objects. 236 | type: object 237 | properties: 238 | unreferencedColumns: 239 | type: object 240 | properties: 241 | preserve: 242 | description: > 243 | Indicates whether columns not referenced in the FieldSpecs should be preserved in the output. This only 244 | applies to the name in the "from" attribute. 245 | type: boolean 246 | embedColumn: 247 | description: > 248 | Specifies a name for a new column to contain all unreferenced fields as a single structured object. 249 | Only applies if preserve is true. 250 | type: string 251 | embedColumnType: 252 | description: > 253 | Specifies the type of structured object that that unreferenced fields will be contained in. 254 | Supported values are: json | struct | variant. Only applies if embedColumn is set. 255 | type: string 256 | omitColumns: 257 | description: > 258 | Lists columns to exclude from the output (either preserved as columns or embedded). Useful for retaining 259 | all columns except the specified ones. 260 | type: array 261 | items: 262 | type: string 263 | duplicatePrefix: 264 | description: > 265 | Adds a specified prefix to resolve ambiguous duplicate field names. This applies only to "preserved" 266 | columns that may be duplicative with something in the field specs. 267 | type: string 268 | jsonExtract: 269 | description: "TODO: this happens before unreferencedColumns" 270 | type: array 271 | items: 272 | type: object 273 | properties: 274 | source: 275 | description: > 276 | The name of the column containing the JSON string from which fields will be extracted. 277 | type: string 278 | omitFields: 279 | description: > 280 | Specifies high-level fields to exclude from extraction. 281 | type: array 282 | items: 283 | type: string 284 | duplicatePrefix: 285 | description: > 286 | Adds a specified prefix to resolve ambiguous duplicate field names generated during extraction. 287 | type: string 288 | embedColumn: 289 | description: > 290 | Specifies a column name to store the extracted JSON object as a structured type. If not specified, 291 | the object is extracted into the top-level structure. 292 | type: string 293 | -------------------------------------------------------------------------------- /presets/databricks/query_history/preset.yaml: -------------------------------------------------------------------------------- 1 | name: databricks_query_history 2 | author: Antimatter 3 | description: "Processes Databricks query logs from the system.query.history table." 4 | title: "Databricks - Query History" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/databricks/query_history/icon.png" 6 | bronze: 7 | skipBronzeLoading: true 8 | silver: 9 | bronzeTables: 10 | - name: system.query.history 11 | 12 | transform: 13 | - name: databricks_query_history 14 | fields: 15 | - name: time 16 | expr: CAST(start_time AS TIMESTAMP) 17 | 18 | - name: ocsf_time_dt 19 | expr: CAST(start_time AS TIMESTAMP) 20 | 21 | - name: ocsf_action_id 22 | expr: | 23 | CASE 24 | WHEN execution_status = 'FINISHED' 25 | OR execution_status = 'CANCELED' THEN 1 26 | ELSE 0 27 | END 28 | 29 | - name: ocsf_action 30 | expr: | 31 | CASE 32 | WHEN execution_status = 'FINISHED' 33 | OR execution_status = 'CANCELED' THEN 'Allowed' 34 | ELSE 'Unknown' 35 | END 36 | 37 | - name: ocsf_activity_id 38 | expr: | 39 | CASE 40 | WHEN statement_type = 'ADD' THEN 1 41 | WHEN statement_type = 'ALTER' THEN 3 42 | WHEN statement_type = 'ANALYZE' THEN 99 43 | WHEN statement_type = 'CACHE' THEN 99 44 | WHEN statement_type = 'COPY' THEN 1 45 | WHEN statement_type = 'CREATE' THEN 1 46 | WHEN statement_type = 'DELETE' THEN 4 47 | WHEN statement_type = 'DESCRIBE' THEN 2 48 | WHEN statement_type = 'DROP' THEN 4 49 | WHEN statement_type = 'GRANT' THEN 1 50 | WHEN statement_type = 'INSERT' THEN 1 51 | WHEN statement_type = 'LIST' THEN 2 52 | WHEN statement_type = 'MERGE' THEN 3 53 | WHEN statement_type = 'OPTIMIZE' THEN 99 54 | WHEN statement_type = 'OTHER' THEN 99 55 | WHEN statement_type = 'REFRESH' THEN 99 56 | WHEN statement_type = 'REPLACE' THEN 3 57 | WHEN statement_type = 'RESET' THEN 99 58 | WHEN statement_type = 'SET' THEN 99 59 | WHEN statement_type = 'SELECT' THEN 2 60 | WHEN statement_type = 'SHOW' THEN 2 61 | WHEN statement_type = 'TRUNCATE' THEN 4 62 | WHEN statement_type = 'UPDATE' THEN 3 63 | WHEN statement_type = 'UNCACHE' THEN 99 64 | WHEN statement_type = 'USE' THEN 99 65 | ELSE 0 66 | END 67 | 68 | - name: ocsf_activity_name 69 | expr: | 70 | CASE 71 | WHEN statement_type = 'ADD' THEN 'Create' 72 | WHEN statement_type = 'ALTER' THEN 'Update' 73 | WHEN statement_type = 'ANALYZE' THEN statement_type 74 | WHEN statement_type = 'CACHE' THEN statement_type 75 | WHEN statement_type = 'COPY' THEN 'Create' 76 | WHEN statement_type = 'CREATE' THEN 'Create' 77 | WHEN statement_type = 'DELETE' THEN 'Delete' 78 | WHEN statement_type = 'DESCRIBE' THEN 'Read' 79 | WHEN statement_type = 'DROP' THEN 'Delete' 80 | WHEN statement_type = 'GRANT' THEN 'Create' 81 | WHEN statement_type = 'INSERT' THEN 'Create' 82 | WHEN statement_type = 'LIST' THEN 'Read' 83 | WHEN statement_type = 'MERGE' THEN 'Update' 84 | WHEN statement_type = 'OPTIMIZE' THEN statement_type 85 | WHEN statement_type = 'OTHER' THEN statement_type 86 | WHEN statement_type = 'REFRESH' THEN statement_type 87 | WHEN statement_type = 'REPLACE' THEN 'Update' 88 | WHEN statement_type = 'RESET' THEN statement_type 89 | WHEN statement_type = 'SET' THEN statement_type 90 | WHEN statement_type = 'SELECT' THEN 'Read' 91 | WHEN statement_type = 'SHOW' THEN 'Read' 92 | WHEN statement_type = 'TRUNCATE' THEN 'Delete' 93 | WHEN statement_type = 'UPDATE' THEN 'Update' 94 | WHEN statement_type = 'UNCACHE' THEN statement_type 95 | WHEN statement_type = 'USE' THEN statement_type 96 | ELSE statement_type 97 | END 98 | 99 | - name: ocsf_actor.user.account.uid 100 | from: account_id 101 | 102 | - name: ocsf_actor.user.name 103 | from: executed_by 104 | 105 | - name: ocsf_actor.user.email_addr 106 | expr: | 107 | CASE 108 | WHEN contains(executed_by, '@') THEN executed_by 109 | ELSE NULL 110 | END 111 | 112 | - name: ocsf_actor.app_name 113 | from: client_application 114 | 115 | - name: ocsf_actor.session.uuid 116 | from: session_id 117 | 118 | - name: ocsf_actor.session.issuer 119 | literal: databricks 120 | 121 | - name: ocsf_actor.session.is_remote 122 | expr: CAST('true' AS BOOLEAN) 123 | 124 | - name: ocsf_api.request.data 125 | expr: CAST(to_json(named_struct('statement_text', statement_text)) AS VARIANT) 126 | 127 | - name: ocsf_api.response.error_message 128 | expr: error_message 129 | 130 | - name: ocsf_cloud.provider 131 | literal: Databricks 132 | 133 | - name: ocsf_cloud.account.uid 134 | from: account_id 135 | 136 | - name: ocsf_cloud.account.type_id 137 | expr: cast('99' as INT) 138 | 139 | - name: ocsf_disposition_id 140 | expr: | 141 | CASE 142 | WHEN execution_status = 'FINISHED' 143 | OR execution_status = 'CANCELED' THEN 1 144 | ELSE 0 145 | END 146 | 147 | - name: ocsf_disposition 148 | expr: | 149 | CASE 150 | WHEN execution_status = 'FINISHED' 151 | OR execution_status = 'CANCELED' THEN 'Allowed' 152 | ELSE 'Unknown' 153 | END 154 | 155 | - name: ocsf_duration 156 | from: CAST(total_duration_ms AS BIGINT) 157 | 158 | - name: ocsf_end_time 159 | expr: CAST(update_time AS TIMESTAMP) 160 | 161 | - name: ocsf_end_time_dt 162 | expr: CAST(update_time AS TIMESTAMP) 163 | 164 | - name: ocsf_metadata.log_name 165 | literal: system.access.audit 166 | 167 | - name: ocsf_metadata.log_provider 168 | literal: Databricks 169 | 170 | - name: ocsf_observables 171 | expr: | 172 | CASE 173 | WHEN query_source IS NOT NULL THEN array( 174 | named_struct( 175 | 'name', 'notebook_id', 176 | 'type_id', 38, 177 | 'type', 'Resource Details Object: notebook_id', 178 | 'value', query_source.notebook_id 179 | ), 180 | named_struct( 181 | 'name', 'job_id', 182 | 'type_id', 38, 183 | 'type', 'Resource Details Object: job_id', 184 | 'value', query_source.job_info.job_id 185 | ), 186 | named_struct( 187 | 'name', 'job_run_id', 188 | 'type_id', 38, 189 | 'type', 'Resource Details Object: job_run_id', 190 | 'value', query_source.job_info.job_run_id 191 | ), 192 | named_struct( 193 | 'name', 'job_task_run_id', 194 | 'type_id', 38, 195 | 'type', 'Resource Details Object: job_task_run_id', 196 | 'value', query_source.job_info.job_task_run_id 197 | ) 198 | ) 199 | ELSE NULL 200 | END 201 | 202 | - name: ocsf_severity_id 203 | expr: CAST('1' AS INT) 204 | 205 | - name: ocsf_severity 206 | literal: Informational 207 | 208 | - name: ocsf_start_time 209 | expr: CAST(start_time AS TIMESTAMP) 210 | 211 | - name: ocsf_start_time_dt 212 | expr: CAST(start_time AS TIMESTAMP) 213 | 214 | - name: ocsf_status 215 | expr: | 216 | CASE 217 | WHEN execution_status = 'FINISHED' THEN 'Success' 218 | WHEN execution_status = 'FAILED' THEN 'Failure' 219 | WHEN execution_status = 'CANCELED' THEN 'Other' 220 | ELSE 'Unknown' 221 | END 222 | 223 | - name: ocsf_status_id 224 | expr: | 225 | CASE 226 | WHEN execution_status = 'FINISHED' THEN 1 227 | WHEN execution_status = 'FAILED' THEN 2 228 | WHEN execution_status = 'CANCELED' THEN 99 229 | ELSE 0 230 | END 231 | 232 | - name: ocsf_unmapped 233 | expr: to_json(named_struct( 234 | 'client_driver', client_driver, 235 | 'compilation_duration_ms', compilation_duration_ms, 236 | 'compute', compute, 237 | 'end_time', end_time, 238 | 'execution_duration_ms', execution_duration_ms, 239 | 'from_result_cache', from_result_cache, 240 | 'pruned_files', pruned_files, 241 | 'produced_rows', produced_rows, 242 | 'query_source', query_source, 243 | 'read_bytes', read_bytes, 244 | 'read_files', read_files, 245 | 'read_io_cache_percent', read_io_cache_percent, 246 | 'read_partitions', read_partitions, 247 | 'read_rows', read_rows, 248 | 'result_fetch_duration_ms', result_fetch_duration_ms, 249 | 'shuffle_read_bytes', shuffle_read_bytes, 250 | 'spilled_local_bytes', spilled_local_bytes, 251 | 'total_task_duration_ms', total_task_duration_ms, 252 | 'waiting_at_capacity_duration_ms', waiting_at_capacity_duration_ms, 253 | 'waiting_for_compute_duration_ms', waiting_for_compute_duration_ms, 254 | 'workspace_id', workspace_id, 255 | 'written_bytes', written_bytes 256 | )) 257 | 258 | utils: 259 | unreferencedColumns: 260 | preserve: true 261 | 262 | gold: 263 | - name: api_activity 264 | input: databricks_query_history 265 | fields: 266 | - name: action 267 | from: ocsf_action 268 | - name: action_id 269 | from: ocsf_action_id 270 | - name: activity_id 271 | from: ocsf_activity_id 272 | - name: activity_name 273 | from: ocsf_activity_name 274 | - name: actor 275 | from: ocsf_actor 276 | - name: api 277 | from: ocsf_api 278 | - name: category_name 279 | literal: Application Activity 280 | - name: category_uid 281 | expr: CAST('6' AS INT) 282 | - name: class_name 283 | literal: API Activity 284 | - name: class_uid 285 | expr: CAST('6003' AS INT) 286 | - name: cloud 287 | from: ocsf_cloud 288 | - name: disposition 289 | from: ocsf_disposition 290 | - name: disposition_id 291 | from: ocsf_disposition_id 292 | - name: duration 293 | from: ocsf_duration 294 | - name: end_time 295 | from: ocsf_end_time 296 | - name: end_time_dt 297 | from: ocsf_end_time_dt 298 | - name: metadata 299 | from: ocsf_metadata 300 | - name: observables 301 | from: ocsf_observables 302 | - name: severity 303 | from: ocsf_severity 304 | - name: severity_id 305 | from: ocsf_severity_id 306 | - name: start_time 307 | from: ocsf_start_time 308 | - name: start_time_dt 309 | from: ocsf_start_time_dt 310 | - name: status 311 | from: ocsf_status 312 | - name: status_id 313 | from: ocsf_status_id 314 | - name: time 315 | from: time 316 | - name: time_dt 317 | from: ocsf_time_dt 318 | - name: type_name 319 | expr: | 320 | CASE 321 | WHEN statement_type = 'ADD' THEN 'API Activity: Create' 322 | WHEN statement_type = 'ALTER' THEN 'API Activity: Update' 323 | WHEN statement_type = 'ANALYZE' THEN 'API Activity: Other' 324 | WHEN statement_type = 'CACHE' THEN 'API Activity: Other' 325 | WHEN statement_type = 'COPY' THEN 'API Activity: Create' 326 | WHEN statement_type = 'CREATE' THEN 'API Activity: Create' 327 | WHEN statement_type = 'DELETE' THEN 'API Activity: Delete' 328 | WHEN statement_type = 'DESCRIBE' THEN 'API Activity: Read' 329 | WHEN statement_type = 'DROP' THEN 'API Activity: Delete' 330 | WHEN statement_type = 'GRANT' THEN 'API Activity: Create' 331 | WHEN statement_type = 'INSERT' THEN 'API Activity: Create' 332 | WHEN statement_type = 'LIST' THEN 'API Activity: Read' 333 | WHEN statement_type = 'MERGE' THEN 'API Activity: Update' 334 | WHEN statement_type = 'OPTIMIZE' THEN 'API Activity: Other' 335 | WHEN statement_type = 'OTHER' THEN 'API Activity: Other' 336 | WHEN statement_type = 'REFRESH' THEN 'API Activity: Other' 337 | WHEN statement_type = 'REPLACE' THEN 'API Activity: Update' 338 | WHEN statement_type = 'RESET' THEN 'API Activity: Other' 339 | WHEN statement_type = 'SET' THEN 'API Activity: Other' 340 | WHEN statement_type = 'SELECT' THEN 'API Activity: Read' 341 | WHEN statement_type = 'SHOW' THEN 'API Activity: Read' 342 | WHEN statement_type = 'TRUNCATE' THEN 'API Activity: Delete' 343 | WHEN statement_type = 'UPDATE' THEN 'API Activity: Update' 344 | WHEN statement_type = 'UNCACHE' THEN 'API Activity: Other' 345 | WHEN statement_type = 'USE' THEN 'API Activity: Other' 346 | ELSE 'API Activity: Other' 347 | END 348 | - name: type_uid 349 | expr: | 350 | CASE 351 | WHEN statement_type = 'ADD' THEN CAST(600301 AS BIGINT) 352 | WHEN statement_type = 'ALTER' THEN CAST(600303 AS BIGINT) 353 | WHEN statement_type = 'ANALYZE' THEN CAST(600399 AS BIGINT) 354 | WHEN statement_type = 'CACHE' THEN CAST(600399 AS BIGINT) 355 | WHEN statement_type = 'COPY' THEN CAST(600301 AS BIGINT) 356 | WHEN statement_type = 'CREATE' THEN CAST(600301 AS BIGINT) 357 | WHEN statement_type = 'DELETE' THEN CAST(600304 AS BIGINT) 358 | WHEN statement_type = 'DESCRIBE' THEN CAST(600302 AS BIGINT) 359 | WHEN statement_type = 'DROP' THEN CAST(600304 AS BIGINT) 360 | WHEN statement_type = 'GRANT' THEN CAST(600301 AS BIGINT) 361 | WHEN statement_type = 'INSERT' THEN CAST(600301 AS BIGINT) 362 | WHEN statement_type = 'LIST' THEN CAST(600302 AS BIGINT) 363 | WHEN statement_type = 'MERGE' THEN CAST(600303 AS BIGINT) 364 | WHEN statement_type = 'OPTIMIZE' THEN CAST(600399 AS BIGINT) 365 | WHEN statement_type = 'OTHER' THEN CAST(600399 AS BIGINT) 366 | WHEN statement_type = 'REFRESH' THEN CAST(600399 AS BIGINT) 367 | WHEN statement_type = 'REPLACE' THEN CAST(600303 AS BIGINT) 368 | WHEN statement_type = 'RESET' THEN CAST(600399 AS BIGINT) 369 | WHEN statement_type = 'SET' THEN CAST(600399 AS BIGINT) 370 | WHEN statement_type = 'SELECT' THEN CAST(600302 AS BIGINT) 371 | WHEN statement_type = 'SHOW' THEN CAST(600302 AS BIGINT) 372 | WHEN statement_type = 'TRUNCATE' THEN CAST(600304 AS BIGINT) 373 | WHEN statement_type = 'UPDATE' THEN CAST(600303 AS BIGINT) 374 | WHEN statement_type = 'UNCACHE' THEN CAST(600399 AS BIGINT) 375 | WHEN statement_type = 'USE' THEN CAST(600399 AS BIGINT) 376 | ELSE CAST(600399 AS BIGINT) 377 | END 378 | - name: unmapped 379 | expr: CAST(ocsf_unmapped AS VARIANT) -------------------------------------------------------------------------------- /presets/cloudflare/httpreq/preset.yaml: -------------------------------------------------------------------------------- 1 | name: cloudflare_httpreq 2 | author: Antimatter 3 | description: "A preset for consuming logs generated by Cloudflare WAF logs" 4 | title: "Cloudflare - HTTP requests" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/cloudflare/httpreq/icon.png" 6 | autoloader: 7 | format: json 8 | cloudFiles: 9 | schemaHints: "CacheCacheStatus string,CacheReserveUsed boolean,CacheResponseBytes int,CacheResponseStatus int,CacheTieredFill boolean,ClientASN int,ClientCountry string,ClientDeviceType string,ClientIP string,ClientIPClass string,ClientMTLSAuthCertFingerprint string,ClientMTLSAuthStatus string,ClientRegionCode string,ClientRequestBytes int,ClientRequestHost string,ClientRequestMethod string,ClientRequestPath string,ClientRequestProtocol string,ClientRequestReferer string,ClientRequestScheme string,ClientRequestSource string,ClientRequestURI string,ClientRequestUserAgent string,ClientSSLCipher string,ClientSSLProtocol string,ClientSrcPort int,ClientTCPRTTMs int,ClientXRequestedWith string,ContentScanObjResults array,ContentScanObjSizes array,ContentScanObjTypes array,EdgeCFConnectingO2O boolean,EdgeColoCode string,EdgeColoID int,EdgeEndTimestamp string,EdgePathingOp string,EdgePathingSrc string,EdgePathingStatus string,EdgeRequestHost string,EdgeResponseBodyBytes int,EdgeResponseBytes int,EdgeResponseCompressionRatio float,EdgeResponseContentType string,EdgeResponseStatus int,EdgeServerIP string,EdgeStartTimestamp string,EdgeTimeToFirstByteMs int,LeakedCredentialCheckResult string,OriginDNSResponseTimeMs bigint,OriginIP string,OriginRequestHeaderSendDurationMs int,OriginResponseBytes int,OriginResponseDurationMs int,OriginResponseHTTPExpires string,OriginResponseHTTPLastModified string,OriginResponseHeaderReceiveDurationMs int,OriginResponseStatus int,OriginResponseTime bigint,OriginSSLProtocol string,OriginTCPHandshakeDurationMs int,OriginTLSHandshakeDurationMs int,ParentRayID string,RayID string,SecurityAction string,SecurityActions array,SecurityRuleDescription string,SecurityRuleID string,SecurityRuleIDs array,SecuritySources array,SmartRouteColoID int,UpperTierColoID int,WAFAttackScore int,WAFFlags string,WAFMatchedVar string,WAFRCEAttackScore int,WAFSQLiAttackScore int,WAFXSSAttackScore int,WorkerCPUTime bigint,WorkerStatus string,WorkerSubrequest boolean,WorkerSubrequestCount int,WorkerWallTimeUs bigint,ZoneName string" 10 | silver: 11 | transform: 12 | - name: cloudflare_waf_http_requests 13 | fields: 14 | - name: ocsf_activity_id 15 | expr: CASE WHEN LOWER(ClientRequestMethod) = 'connect' THEN 1 WHEN LOWER(ClientRequestMethod) = 'delete' THEN 2 WHEN LOWER(ClientRequestMethod) = 'get' THEN 3 WHEN LOWER(ClientRequestMethod) = 'head' THEN 4 WHEN LOWER(ClientRequestMethod) = 'options' THEN 5 WHEN LOWER(ClientRequestMethod) = 'post' THEN 6 WHEN LOWER(ClientRequestMethod) = 'put' THEN 7 WHEN LOWER(ClientRequestMethod) = 'trace' THEN 8 WHEN ClientRequestMethod IS NULL OR ClientRequestMethod = '' THEN 0 ELSE 99 END 16 | - name: ocsf_activity_name 17 | expr: CASE WHEN ClientRequestMethod IS NULL OR ClientRequestMethod = '' THEN ClientRequestMethod ELSE CONCAT(UPPER(SUBSTRING(ClientRequestMethod, 1, 1)), LOWER(SUBSTRING(ClientRequestMethod, 2))) END 18 | - name: ocsf_category_uid 19 | expr: CAST('4' AS INT) 20 | - name: ocsf_category_name 21 | literal: "Network Activity" 22 | - name: ocsf_class_uid 23 | expr: CAST('4002' AS INT) 24 | - name: ocsf_class_name 25 | literal: "HTTP Activity" 26 | - name: time 27 | expr: CAST(EdgeStartTimestamp AS TIMESTAMP) 28 | - name: ocsf_severity_id 29 | expr: CAST('0' AS INT) 30 | - name: ocsf_severity 31 | literal: Unknown 32 | - name: ocsf_type_uid 33 | expr: CAST((400200 + CASE WHEN LOWER(ClientRequestMethod) = 'connect' THEN 1 WHEN LOWER(ClientRequestMethod) = 'delete' THEN 2 WHEN LOWER(ClientRequestMethod) = 'get' THEN 3 WHEN LOWER(ClientRequestMethod) = 'head' THEN 4 WHEN LOWER(ClientRequestMethod) = 'options' THEN 5 WHEN LOWER(ClientRequestMethod) = 'post' THEN 6 WHEN LOWER(ClientRequestMethod) = 'put' THEN 7 WHEN LOWER(ClientRequestMethod) = 'trace' THEN 8 WHEN ClientRequestMethod IS NULL OR ClientRequestMethod = '' THEN 0 ELSE 99 END) AS BIGINT) 34 | - name: ocsf_type_name 35 | expr: "CONCAT('HTTP Activity: ', ClientRequestMethod)" 36 | - name: src_endpoint_type_id 37 | expr: CASE WHEN LOWER(ClientDeviceType) = 'server' THEN 1 WHEN LOWER(ClientDeviceType) = 'desktop' THEN 2 WHEN LOWER(ClientDeviceType) = 'laptop' THEN 3 WHEN LOWER(ClientDeviceType) = 'tablet' THEN 4 WHEN LOWER(ClientDeviceType) = 'mobile' THEN 5 WHEN LOWER(ClientDeviceType) = 'virtual' THEN 6 WHEN LOWER(ClientDeviceType) = 'iot' THEN 7 WHEN LOWER(ClientDeviceType) = 'browser' THEN 8 WHEN LOWER(ClientDeviceType) = 'firewall' THEN 9 WHEN LOWER(ClientDeviceType) = 'switch' THEN 10 WHEN LOWER(ClientDeviceType) = 'hub' THEN 11 WHEN LOWER(ClientDeviceType) = 'router' THEN 12 WHEN LOWER(ClientDeviceType) = 'ids' THEN 13 WHEN LOWER(ClientDeviceType) = 'ips' THEN 14 WHEN LOWER(ClientDeviceType) = 'load balancer' THEN 15 WHEN ClientDeviceType IS NULL OR ClientDeviceType = '' THEN 0 ELSE 99 END 38 | - name: ocsf_metadata.product.name 39 | literal: "Logs" 40 | - name: ocsf_metadata.product.vendor_name 41 | literal: "Cloudflare" 42 | - name: ocsf_metadata.product.feature.name 43 | literal: "HTTP requests" 44 | - name: duration 45 | expr: DATEDIFF(MILLISECOND, CAST(EdgeStartTimestamp AS TIMESTAMP), CAST(EdgeEndTimestamp AS TIMESTAMP)) 46 | - name: ocsf_device.type 47 | expr: CASE WHEN ClientDeviceType IS NULL OR ClientDeviceType = '' THEN ClientDeviceType ELSE CONCAT(UPPER(SUBSTRING(ClientDeviceType, 1, 1)), LOWER(SUBSTRING(ClientDeviceType, 2))) END 48 | - name: ocsf_device.type_id 49 | expr: CASE WHEN ClientDeviceType = 'desktop' THEN 2 WHEN ClientDeviceType = 'mobile' THEN 5 WHEN ClientDeviceType = 'virtual' THEN 6 ELSE 0 END 50 | - name: ocsf_src_endpoint.autonomous_system.number 51 | from: ClientASN 52 | - name: ocsf_src_endpoint.location.country 53 | from: ClientCountry 54 | - name: ocsf_src_endpoint.location.region 55 | from: ClientRegionCode 56 | - name: ocsf_src_endpoint.ip 57 | from: ClientIP 58 | - name: ocsf_src_endpoint.port 59 | from: ClientSrcPort 60 | - name: ocsf_dst_endpoint.ip 61 | from: OriginIP 62 | - name: ocsf_dst_endpoint.hostname 63 | expr: COALESCE(ClientRequestHost, EdgeRequestHost) 64 | - name: ocsf_dst_endpoint.zone 65 | from: ZoneName 66 | - name: ocsf_proxy.ip 67 | from: EdgeServerIP 68 | - name: ocsf_http_request.http_method 69 | from: ClientRequestMethod 70 | - name: ocsf_http_request.referrer 71 | from: ClientRequestReferer 72 | - name: ocsf_http_request.version 73 | from: ClientRequestProtocol 74 | - name: ocsf_http_request.url.path 75 | from: ClientRequestPath 76 | - name: ocsf_http_request.url.scheme 77 | from: ClientRequestScheme 78 | - name: ocsf_http_request.url.url_string 79 | from: ClientRequestURI 80 | - name: ocsf_http_request.user_agent 81 | from: ClientRequestUserAgent 82 | - name: ocsf_http_response.code 83 | expr: CASE WHEN (EdgeServerIP != '') THEN OriginResponseStatus ELSE EdgeResponseStatus END 84 | - name: ocsf_http_response.content_type 85 | from: EdgeResponseContentType 86 | - name: ocsf_http_response.body_length 87 | expr: EdgeResponseBodyBytes 88 | - name: ocsf_http_response.latency 89 | from: EdgeTimeToFirstByteMs 90 | - name: http_status 91 | expr: COALESCE(OriginResponseStatus, EdgeResponseStatus) 92 | - name: ocsf_traffic.bytes 93 | expr: CAST(ClientRequestBytes AS BIGINT) + CAST(CASE WHEN OriginResponseBytes = 0 OR OriginResponseBytes = null THEN EdgeResponseBytes ELSE OriginResponseBytes END AS BIGINT) 94 | - name: ocsf_traffic.bytes_in 95 | expr: CAST(ClientRequestBytes AS BIGINT) 96 | - name: ocsf_traffic.bytes_out 97 | expr: CAST(CASE WHEN OriginResponseBytes = 0 OR OriginResponseBytes = null THEN EdgeResponseBytes ELSE OriginResponseBytes END AS BIGINT) 98 | - name: ocsf_tls.cipher 99 | from: ClientSSLCipher 100 | - name: ocsf_tls.version 101 | expr: CASE WHEN ClientSSLProtocol != 'none' THEN ClientSSLProtocol ELSE OriginSSLProtocol END 102 | - name: action 103 | expr: CASE WHEN array_contains(ARRAY('allow'), SecurityAction) THEN 'Allowed' WHEN array_contains(ARRAY('connectionClose','forceConnectionClose','block'), SecurityAction) THEN 'Denied' WHEN array_contains(ARRAY('skip','bypass','log'), SecurityAction) THEN 'Observed' WHEN array_contains(ARRAY('rewrite','challenge','jschallenge','managedChallenge','managedChallengeNonInteractiveSolved','managedChallengeInteractiveSolved','challengeSolved','jschallengeSolved','challengeBypassed','managedChallengeBypassed','jschallengeBypassed'), SecurityAction) THEN 'Modified' WHEN array_contains(ARRAY('unknown'), SecurityAction) THEN 'Unknown' ELSE 'Other' END 104 | - name: action_id 105 | expr: CASE WHEN array_contains(ARRAY('allow'), SecurityAction) THEN 1 WHEN array_contains(ARRAY('connectionClose','forceConnectionClose','block'), SecurityAction) THEN 2 WHEN array_contains(ARRAY('skip','bypass','log'), SecurityAction) THEN 3 WHEN array_contains(ARRAY('rewrite','challenge','jschallenge','managedChallenge','managedChallengeNonInteractiveSolved','managedChallengeInteractiveSolved','challengeSolved','jschallengeSolved','challengeBypassed','managedChallengeBypassed','jschallengeBypassed'), SecurityAction) THEN 4 WHEN array_contains(ARRAY('unknown'), SecurityAction) THEN 0 ELSE 99 END 106 | - name: disposition 107 | expr: CASE WHEN array_contains(ARRAY('unknown'), SecurityAction) THEN 'Unknown' WHEN array_contains(ARRAY('allow'), SecurityAction) THEN 'Allowed' WHEN array_contains(ARRAY('block'), SecurityAction) THEN 'Blocked' WHEN array_contains(ARRAY('connectionClose', 'forceConnectionClose'), SecurityAction) THEN 'Dropped' WHEN array_contains(ARRAY('challengeSolved', 'jschallengeSolved', 'managedChallengeNonInteractiveSolved', 'managedChallengeInteractiveSolved'), SecurityAction) THEN 'Approved' WHEN array_contains(ARRAY('skip', 'bypass', 'challengeBypassed', 'jschallengeBypassed', 'managedChallengeBypassed'), SecurityAction) THEN 'No Action' WHEN array_contains(ARRAY('log'), SecurityAction) THEN 'Logged' WHEN array_contains(ARRAY('challenge', 'jschallenge', 'managedChallenge'), SecurityAction) THEN 'Challenge' ELSE 'Other' END 108 | - name: disposition_id 109 | expr: CASE WHEN array_contains(ARRAY('unknown'), SecurityAction) THEN 0 WHEN array_contains(ARRAY('allow'), SecurityAction) THEN 1 WHEN array_contains(ARRAY('block'), SecurityAction) THEN 2 WHEN array_contains(ARRAY('connectionClose', 'forceConnectionClose'), SecurityAction) THEN 6 WHEN array_contains(ARRAY('challengeSolved', 'jschallengeSolved', 'managedChallengeNonInteractiveSolved', 'managedChallengeInteractiveSolved'), SecurityAction) THEN 8 WHEN array_contains(ARRAY('skip', 'bypass', 'challengeBypassed', 'jschallengeBypassed', 'managedChallengeBypassed'), SecurityAction) THEN 16 WHEN array_contains(ARRAY('log'), SecurityAction) THEN 17 WHEN array_contains(ARRAY('challenge', 'jschallenge', 'managedChallenge'), SecurityAction) THEN 23 ELSE 99 END 110 | - name: ocsf_firewall_rule.uid 111 | from: SecurityRuleID 112 | - name: ocsf_firewall_rule.desc 113 | from: SecurityRuleDescription 114 | - name: ocsf_firewall_rule.match_location 115 | from: WAFMatchedVar 116 | - name: ocsf_connection_info.uid 117 | from: RayID 118 | - name: risk_score 119 | expr: CASE WHEN WAFRCEAttackScore >= WAFSQLiAttackScore AND WAFRCEAttackScore >= WAFXSSAttackScore AND WAFRCEAttackScore > 0 THEN WAFRCEAttackScore WHEN WAFSQLiAttackScore >= WAFRCEAttackScore AND WAFSQLiAttackScore >= WAFXSSAttackScore AND WAFSQLiAttackScore > 0 THEN WAFSQLiAttackScore WHEN WAFXSSAttackScore >= WAFRCEAttackScore AND WAFXSSAttackScore >= WAFSQLiAttackScore AND WAFXSSAttackScore > 0 THEN WAFXSSAttackScore ELSE 0 END 120 | - name: risk_details 121 | expr: CASE WHEN WAFRCEAttackScore >= WAFSQLiAttackScore AND WAFRCEAttackScore >= WAFXSSAttackScore AND WAFRCEAttackScore > 0 THEN 'Remote code execution.' WHEN WAFSQLiAttackScore >= WAFRCEAttackScore AND WAFSQLiAttackScore >= WAFXSSAttackScore AND WAFSQLiAttackScore > 0 THEN 'SQL injection.' WHEN WAFXSSAttackScore >= WAFRCEAttackScore AND WAFXSSAttackScore >= WAFSQLiAttackScore AND WAFXSSAttackScore > 0 THEN 'Cross-site script.' ELSE 'None' END 122 | - name: ocsf_unmapped 123 | expr: "to_json( 124 | named_struct( 125 | 'CacheCacheStatus', CacheCacheStatus, 126 | 'CacheReserveUsed', CacheReserveUsed, 127 | 'CacheResponseBytes', CacheResponseBytes, 128 | 'CacheResponseStatus', CacheResponseStatus, 129 | 'CacheTieredFill', CacheTieredFill, 130 | 'ClientRequestSource', ClientRequestSource, 131 | 'ClientIPClass', ClientIPClass, 132 | 'ClientMTLSAuthCertFingerprint', ClientMTLSAuthCertFingerprint, 133 | 'ClientMTLSAuthStatus', ClientMTLSAuthStatus, 134 | 'ClientTCPRTTMs', ClientTCPRTTMs, 135 | 'ClientXRequestedWith', ClientXRequestedWith, 136 | 'EdgeCFConnectingO2O', EdgeCFConnectingO2O, 137 | 'EdgeColoCode', EdgeColoCode, 138 | 'EdgeColoID', EdgeColoID, 139 | 'EdgePathingOp', EdgePathingOp, 140 | 'EdgePathingSrc', EdgePathingSrc, 141 | 'EdgePathingStatus', EdgePathingStatus, 142 | 'EdgeResponseCompressionRatio', EdgeResponseCompressionRatio, 143 | 'OriginDNSResponseTimeMs', OriginDNSResponseTimeMs, 144 | 'OriginRequestHeaderSendDurationMs', OriginRequestHeaderSendDurationMs, 145 | 'OriginResponseDurationMs', OriginResponseDurationMs, 146 | 'OriginResponseHTTPExpires', OriginResponseHTTPExpires, 147 | 'OriginResponseHTTPLastModified', OriginResponseHTTPLastModified, 148 | 'OriginResponseHeaderReceiveDurationMs', OriginResponseHeaderReceiveDurationMs, 149 | 'OriginResponseTime', OriginResponseTime, 150 | 'OriginTCPHandshakeDurationMs', OriginTCPHandshakeDurationMs, 151 | 'OriginTLSHandshakeDurationMs', OriginTLSHandshakeDurationMs, 152 | 'SecurityActions', SecurityActions, 153 | 'SecurityRuleIDs', SecurityRuleIDs, 154 | 'SecuritySources', SecuritySources, 155 | 'WAFAttackScore', WAFAttackScore, 156 | 'WAFRCEAttackScore', WAFRCEAttackScore, 157 | 'WAFSQLiAttackScore', WAFSQLiAttackScore, 158 | 'WAFXSSAttackScore', WAFXSSAttackScore, 159 | 'WAFFlags', WAFFlags, 160 | 'LeakedCredentialCheckResult', LeakedCredentialCheckResult, 161 | 'ParentRayID', ParentRayID, 162 | 'ContentScanObjResults', ContentScanObjResults, 163 | 'ContentScanObjSizes', ContentScanObjSizes, 164 | 'ContentScanObjTypes', ContentScanObjTypes, 165 | 'SmartRouteColoID', SmartRouteColoID, 166 | 'UpperTierColoID', UpperTierColoID, 167 | 'WorkerStatus', WorkerStatus, 168 | 'WorkerCPUTime', WorkerCPUTime, 169 | 'WorkerSubrequest', WorkerSubrequest, 170 | 'WorkerSubrequestCount', WorkerSubrequestCount, 171 | 'WorkerWallTimeUs', WorkerWallTimeUs 172 | ) 173 | )" 174 | utils: 175 | unreferencedColumns: 176 | preserve: true 177 | gold: 178 | - name: http_activity 179 | input: cloudflare_waf_http_requests 180 | fields: 181 | - name: activity_id 182 | from: ocsf_activity_id 183 | - name: activity_name 184 | from: ocsf_activity_name 185 | - name: category_uid 186 | from: ocsf_category_uid 187 | - name: category_name 188 | from: ocsf_category_name 189 | - name: class_uid 190 | from: ocsf_class_uid 191 | - name: class_name 192 | from: ocsf_class_name 193 | - name: time 194 | from: time 195 | - name: metadata 196 | from: ocsf_metadata 197 | - name: duration 198 | from: duration 199 | - name: severity_id 200 | from: ocsf_severity_id 201 | - name: severity 202 | from: ocsf_severity 203 | - name: type_uid 204 | from: ocsf_type_uid 205 | - name: type_name 206 | from: ocsf_type_name 207 | - name: device 208 | from: ocsf_device 209 | - name: src_endpoint 210 | from: ocsf_src_endpoint 211 | - name: dst_endpoint 212 | from: ocsf_dst_endpoint 213 | - name: proxy 214 | from: ocsf_proxy 215 | - name: http_request 216 | from: ocsf_http_request 217 | - name: http_response 218 | from: ocsf_http_response 219 | - name: http_status 220 | from: http_status 221 | - name: traffic 222 | from: ocsf_traffic 223 | - name: tls 224 | from: ocsf_tls 225 | - name: action 226 | from: action 227 | - name: action_id 228 | from: action_id 229 | - name: disposition 230 | from: disposition 231 | - name: disposition_id 232 | from: disposition_id 233 | - name: firewall_rule 234 | from: ocsf_firewall_rule 235 | - name: connection_info 236 | from: ocsf_connection_info 237 | - name: risk_score 238 | from: risk_score 239 | - name: risk_details 240 | from: risk_details 241 | - name: unmapped 242 | expr: CAST(ocsf_unmapped AS VARIANT) 243 | -------------------------------------------------------------------------------- /presets/databricks/access_audit/preset.yaml: -------------------------------------------------------------------------------- 1 | name: databricks_access_audit 2 | author: Antimatter 3 | description: "Datasource processing for Databricks access audit logs" 4 | title: "Databricks - Audit Table" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/databricks/access_audit/icon.png" 6 | bronze: 7 | skipBronzeLoading: true 8 | silver: 9 | bronzeTables: 10 | - name: system.access.audit 11 | 12 | transform: 13 | - name: databricks_access_audit 14 | fields: 15 | - name: time 16 | expr: CAST(event_time AS TIMESTAMP) 17 | 18 | - name: ocsf_time_dt 19 | expr: CAST(event_time AS TIMESTAMP) 20 | 21 | - name: ocsf_action_id 22 | expr: | 23 | CASE 24 | WHEN response.status_code >= 200 AND response.status_code < 300 THEN 1 25 | WHEN response.status_code >= 400 AND response.status_code < 500 THEN 2 26 | ELSE 0 27 | END 28 | 29 | - name: ocsf_action 30 | expr: | 31 | CASE 32 | WHEN response.status_code >= 200 AND response.status_code < 300 THEN 'Allowed' 33 | WHEN response.status_code >= 400 AND response.status_code < 500 THEN 'Denied' 34 | ELSE 'Unknown' 35 | END 36 | 37 | - name: action_name 38 | from: action_name 39 | 40 | - name: ocsf_actor.user.account.uid 41 | from: account_id 42 | 43 | - name: ocsf_actor.user.name 44 | from: identity_metadata.run_by 45 | 46 | - name: ocsf_actor.user.email_addr 47 | from: user_identity.email 48 | 49 | - name: ocsf_api.service.name 50 | from: service_name 51 | 52 | - name: ocsf_api.response.code 53 | from: response.status_code 54 | 55 | - name: ocsf_api.response.data 56 | from: CAST(to_json(named_struct('result', response.result)) AS VARIANT) 57 | 58 | - name: ocsf_cloud.provider 59 | literal: Databricks 60 | 61 | - name: ocsf_cloud.account.uid 62 | from: account_id 63 | 64 | - name: ocsf_cloud.account.type_id 65 | expr: cast('99' as INT) 66 | 67 | - name: ocsf_disposition_id 68 | expr: | 69 | CASE 70 | WHEN response.status_code >= 200 AND response.status_code < 300 THEN 1 71 | WHEN response.status_code = 401 THEN 26 72 | WHEN response.status_code >= 400 AND response.status_code < 500 THEN 2 73 | ELSE 0 74 | END 75 | 76 | - name: ocsf_disposition 77 | expr: | 78 | CASE 79 | WHEN response.status_code >= 200 AND response.status_code < 300 THEN 'Allowed' 80 | WHEN response.status_code = 401 THEN 'Unauthorized' 81 | WHEN response.status_code >= 400 AND response.status_code < 500 THEN 'Blocked' 82 | ELSE 'Unknown' 83 | END 84 | 85 | - name: ocsf_dst_endpoint.svc_name 86 | from: service_name 87 | 88 | 89 | - name: ocsf_http_request.args 90 | expr: to_json(request_params) 91 | 92 | - name: ocsf_http_request.uid 93 | from: request_id 94 | 95 | - name: ocsf_http_request.user_agent 96 | from: user_agent 97 | 98 | - name: ocsf_http_response.code 99 | from: response.status_code 100 | 101 | # The Event ID, Code, or Name that the product uses to primarily identify the event. 102 | - name: ocsf_metadata.event_code 103 | from: action_name 104 | 105 | # The event log schema version that specifies the format of the original event. For example 106 | # syslog version or Cisco Log Schema Version. 107 | - name: ocsf_metadata.log_version 108 | from: version 109 | 110 | - name: ocsf_metadata.log_name 111 | literal: system.access.audit 112 | 113 | - name: ocsf_metadata.log_provider 114 | literal: Databricks 115 | 116 | - name: request_params 117 | from: request_params 118 | 119 | - name: ocsf_severity_id 120 | expr: CAST('1' AS INT) 121 | 122 | - name: ocsf_severity 123 | literal: Informational 124 | 125 | - name: ocsf_src_endpoint.ip 126 | from: source_ip_address 127 | 128 | - name: ocsf_status 129 | expr: | 130 | CASE 131 | WHEN response.status_code >= 200 AND response.status_code < 300 THEN 'Success' 132 | WHEN response.status_code >= 400 THEN 'Failure' 133 | ELSE 'Other' 134 | END 135 | 136 | - name: ocsf_status_id 137 | expr: | 138 | CASE 139 | WHEN response.status_code >= 200 AND response.status_code < 300 THEN 1 140 | WHEN response.status_code >= 400 THEN 2 141 | ELSE 99 142 | END 143 | 144 | - name: ocsf_unmapped 145 | expr: to_json(named_struct( 146 | 'workspace_id', workspace_id, 147 | 'request_id', request_id, 148 | 'audit_level', audit_level, 149 | 'identity_metadata', identity_metadata 150 | )) 151 | 152 | utils: 153 | unreferencedColumns: 154 | preserve: true 155 | 156 | gold: 157 | - name: account_change 158 | input: databricks_access_audit 159 | filter: | 160 | action_name = 'activateUser' OR 161 | action_name = 'add' OR 162 | action_name = 'addX509' OR 163 | action_name = 'changePassword' OR 164 | action_name = 'deactivateUser' OR 165 | action_name = 'delete' OR 166 | action_name = 'mfaAddKey' OR 167 | action_name = 'mfaDeleteKey' OR 168 | action_name = 'resetPassword' OR 169 | action_name = 'updateUser' OR 170 | action_name = 'validateEmail' 171 | fields: 172 | - name: time 173 | from: time 174 | 175 | 176 | - name: action_id 177 | from: ocsf_action_id 178 | 179 | - name: activity_id 180 | expr: | 181 | CASE 182 | WHEN 183 | action_name = 'add' OR 184 | action_name = 'addX509' THEN 1 185 | WHEN action_name = 'activateUser' THEN 2 186 | WHEN action_name = 'changePassword' THEN 3 187 | WHEN action_name = 'resetPassword' THEN 4 188 | WHEN action_name = 'deactivateUser' THEN 5 189 | WHEN action_name = 'delete' THEN 6 190 | WHEN action_name = 'mfaAddKey' THEN 10 191 | WHEN action_name = 'mfaDeleteKey' THEN 11 192 | ELSE 99 193 | END 194 | 195 | - name: activity_name 196 | expr: | 197 | CASE 198 | WHEN 199 | action_name = 'add' OR 200 | action_name = 'addX509' THEN 'Create' 201 | WHEN action_name = 'activateUser' THEN 'Enable' 202 | WHEN action_name = 'changePassword' THEN 'Password Change' 203 | WHEN action_name = 'resetPassword' THEN 'Password Reset' 204 | WHEN action_name = 'deactivateUser' THEN 'Disable' 205 | WHEN action_name = 'delete' THEN 'Delete' 206 | WHEN action_name = 'mfaAddKey' THEN 'MFA Factor Enable' 207 | WHEN action_name = 'mfaDeleteKey' THEN 'MFA Factor Disable' 208 | ELSE action_name 209 | END 210 | 211 | - name: actor 212 | from: ocsf_actor 213 | 214 | - name: api 215 | from: ocsf_api 216 | 217 | - name: category_name 218 | literal: Identity & Access Management 219 | 220 | - name: category_uid 221 | expr: CAST('3' AS INT) 222 | 223 | - name: class_name 224 | literal: "Account Change" 225 | 226 | - name: class_uid 227 | expr: CAST('3001' AS INT) 228 | 229 | - name: cloud 230 | from: ocsf_cloud 231 | 232 | - name: disposition_id 233 | from: ocsf_disposition_id 234 | 235 | - name: disposition 236 | from: ocsf_disposition 237 | 238 | - name: http_request 239 | from: ocsf_http_request 240 | 241 | - name: http_response 242 | from: ocsf_http_response 243 | 244 | - name: metadata 245 | from: ocsf_metadata 246 | 247 | - name: raw_data 248 | expr: to_json(request_params) 249 | 250 | - name: severity 251 | from: ocsf_severity 252 | 253 | - name: severity_id 254 | from: ocsf_severity_id 255 | 256 | - name: src_endpoint 257 | from: ocsf_src_endpoint 258 | 259 | - name: status 260 | from: ocsf_status 261 | 262 | - name: status_id 263 | from: ocsf_status_id 264 | 265 | - name: type_name 266 | expr: | 267 | CASE 268 | WHEN 269 | action_name = 'add' OR 270 | action_name = 'addX509' THEN 'Account Change: Create' 271 | WHEN action_name = 'activateUser' THEN 'Account Change: Enable' 272 | WHEN action_name = 'changePassword' THEN 'Password Account Change: Change' 273 | WHEN action_name = 'resetPassword' THEN 'Password Account Change: Reset' 274 | WHEN action_name = 'deactivateUser' THEN 'Account Change: Disable' 275 | WHEN action_name = 'delete' THEN 'Account Change: Delete' 276 | WHEN action_name = 'mfaAddKey' THEN 'Account Change: MFA Factor Enable' 277 | WHEN action_name = 'mfaDeleteKey' THEN 'Account Change: MFA Factor Disable' 278 | ELSE 'Account Change: Other' 279 | END 280 | 281 | - name: type_uid 282 | expr: | 283 | CASE 284 | WHEN 285 | action_name = 'add' OR 286 | action_name = 'addX509' THEN CAST(300101 as BIGINT) 287 | WHEN action_name = 'activateUser' THEN CAST(300102 as BIGINT) 288 | WHEN action_name = 'changePassword' THEN CAST(300103 as BIGINT) 289 | WHEN action_name = 'resetPassword' THEN CAST(300104 as BIGINT) 290 | WHEN action_name = 'deactivateUser' THEN CAST(300105 as BIGINT) 291 | WHEN action_name = 'delete' THEN CAST(300106 as BIGINT) 292 | WHEN action_name = 'mfaAddKey' THEN CAST(300110 as BIGINT) 293 | WHEN action_name = 'mfaDeleteKey' THEN CAST(300111 as BIGINT) 294 | ELSE CAST(300199 as BIGINT) 295 | END 296 | 297 | - name: unmapped 298 | expr: CAST(ocsf_unmapped AS VARIANT) 299 | 300 | - name: group_management 301 | input: databricks_access_audit 302 | filter: | 303 | action_name = 'addPrincipalToGroup' OR 304 | action_name = 'addPrincipalsToGroup' OR 305 | action_name = 'createGroup' OR 306 | action_name = 'getGroupMembers' OR 307 | action_name = 'getGroups' OR 308 | action_name = 'getInheritedGroups' OR 309 | action_name = 'removeGroup' OR 310 | action_name = 'removePrincipalFromGroup' OR 311 | action_name = 'removePrincipalsFromGroup' OR 312 | action_name = 'updateGroup' 313 | 314 | fields: 315 | - name: time 316 | from: time 317 | 318 | - name: action 319 | from: ocsf_action 320 | 321 | - name: action_id 322 | from: ocsf_action_id 323 | 324 | - name: activity_id 325 | expr: | 326 | CASE 327 | WHEN 328 | action_name = 'addPrincipalToGroup' OR 329 | action_name = 'addPrincipalsToGroup' THEN 3 330 | WHEN 331 | action_name = 'removePrincipalFromGroup' OR 332 | action_name = 'removePrincipalsFromGroup' THEN 4 333 | WHEN action_name = 'removeGroup' THEN 5 334 | WHEN action_name = 'createGroup' THEN 6 335 | ELSE 99 336 | END 337 | 338 | - name: activity_name 339 | expr: | 340 | CASE 341 | WHEN 342 | action_name = 'addPrincipalToGroup' OR 343 | action_name = 'addPrincipalsToGroup' THEN 'Add User' 344 | WHEN 345 | action_name = 'removePrincipalFromGroup' OR 346 | action_name = 'removePrincipalsFromGroup' THEN 'Remove User' 347 | WHEN action_name = 'removeGroup' THEN 'Delete' 348 | WHEN action_name = 'createGroup' THEN 'Create' 349 | ELSE action_name 350 | END 351 | 352 | - name: actor 353 | from: ocsf_actor 354 | 355 | - name: api 356 | from: ocsf_api 357 | 358 | - name: category_name 359 | literal: Identity & Access Management 360 | 361 | - name: category_uid 362 | expr: CAST('3' AS INT) 363 | 364 | - name: class_name 365 | literal: Group Management 366 | 367 | - name: class_uid 368 | expr: CAST('3006' AS INT) 369 | 370 | - name: cloud 371 | from: ocsf_cloud 372 | 373 | - name: disposition_id 374 | from: ocsf_disposition_id 375 | 376 | - name: disposition 377 | from: ocsf_disposition 378 | 379 | - name: group.name 380 | from: request_params.targetGroupName 381 | 382 | - name: group.uid 383 | from: request_params.targetGroupId 384 | 385 | 386 | - name: http_request 387 | from: ocsf_http_request 388 | 389 | - name: http_response 390 | from: ocsf_http_response 391 | 392 | - name: metadata 393 | from: ocsf_metadata 394 | 395 | - name: raw_data 396 | expr: to_json(request_params) 397 | 398 | - name: severity 399 | from: ocsf_severity 400 | 401 | - name: severity_id 402 | from: ocsf_severity_id 403 | 404 | - name: src_endpoint 405 | from: ocsf_src_endpoint 406 | 407 | - name: status 408 | from: ocsf_status 409 | 410 | - name: status_id 411 | from: ocsf_status_id 412 | 413 | - name: type_name 414 | expr: | 415 | CASE 416 | WHEN 417 | action_name = 'addPrincipalToGroup' OR 418 | action_name = 'addPrincipalsToGroup' THEN 'Group Management: Add User' 419 | WHEN 420 | action_name = 'removePrincipalFromGroup' OR 421 | action_name = 'removePrincipalsFromGroup' THEN 'Group Management: Remove User' 422 | WHEN action_name = 'removeGroup' THEN 'Group Management: Delete' 423 | WHEN action_name = 'createGroup' THEN 'Group Management: Create' 424 | ELSE 'Group Management: Other' 425 | END 426 | 427 | - name: type_uid 428 | expr: | 429 | CASE 430 | WHEN 431 | action_name = 'addPrincipalToGroup' OR 432 | action_name = 'addPrincipalsToGroup' THEN CAST(300603 as BIGINT) 433 | WHEN 434 | action_name = 'removePrincipalFromGroup' OR 435 | action_name = 'removePrincipalsFromGroup' THEN CAST(300604 as BIGINT) 436 | WHEN action_name = 'removeGroup' THEN CAST(300605 as BIGINT) 437 | WHEN action_name = 'createGroup' THEN CAST(300606 as BIGINT) 438 | ELSE CAST(300699 as BIGINT) 439 | END 440 | 441 | - name: unmapped 442 | expr: CAST(ocsf_unmapped AS VARIANT) 443 | 444 | - name: api_activity 445 | input: databricks_access_audit 446 | fields: 447 | - name: time 448 | from: time 449 | 450 | - name: action_id 451 | from: ocsf_action_id 452 | 453 | - name: activity_id 454 | expr: | 455 | CASE 456 | WHEN LOWER(action_name) LIKE 'create%' THEN 1 457 | 458 | WHEN LOWER(action_name) LIKE 'get%' THEN 2 459 | WHEN LOWER(action_name) LIKE 'list%' THEN 2 460 | 461 | WHEN LOWER(action_name) LIKE 'update%' THEN 3 462 | WHEN LOWER(action_name) LIKE 'change%' THEN 3 463 | WHEN LOWER(action_name) LIKE 'move%' THEN 3 464 | WHEN LOWER(action_name) LIKE 'put%' THEN 3 465 | WHEN LOWER(action_name) LIKE 'rename%' THEN 3 466 | WHEN LOWER(action_name) LIKE 'set%' THEN 3 467 | 468 | WHEN LOWER(action_name) LIKE 'delete%' THEN 4 469 | WHEN LOWER(action_name) LIKE 'remove%' THEN 4 470 | 471 | WHEN action_name IS NULL OR action_name = '' THEN 0 472 | ELSE 99 473 | END 474 | 475 | - name: activity_name 476 | expr: | 477 | CASE 478 | WHEN LOWER(action_name) LIKE 'create%' THEN 'Create' 479 | 480 | WHEN LOWER(action_name) LIKE 'get%' THEN 'Read' 481 | WHEN LOWER(action_name) LIKE 'list%' THEN 'Read' 482 | 483 | WHEN LOWER(action_name) LIKE 'update%' THEN 'Update' 484 | WHEN LOWER(action_name) LIKE 'change%' THEN 'Update' 485 | WHEN LOWER(action_name) LIKE 'move%' THEN 'Update' 486 | WHEN LOWER(action_name) LIKE 'put%' THEN 'Update' 487 | WHEN LOWER(action_name) LIKE 'rename%' THEN 'Update' 488 | WHEN LOWER(action_name) LIKE 'set%' THEN 'Update' 489 | 490 | WHEN LOWER(action_name) LIKE 'delete%' THEN 'Delete' 491 | WHEN LOWER(action_name) LIKE 'remove%' THEN 'Delete' 492 | 493 | WHEN action_name IS NULL OR action_name = '' THEN 'Unknown' 494 | ELSE action_name 495 | END 496 | 497 | - name: actor 498 | from: ocsf_actor 499 | 500 | - name: api 501 | from: ocsf_api 502 | 503 | - name: category_uid 504 | expr: CAST('6' AS INT) 505 | 506 | - name: category_name 507 | literal: Application Activity 508 | 509 | - name: class_uid 510 | expr: CAST('6003' AS INT) 511 | 512 | - name: class_name 513 | literal: "API Activity" 514 | 515 | - name: cloud 516 | from: ocsf_cloud 517 | 518 | - name: disposition_id 519 | from: ocsf_disposition_id 520 | 521 | - name: disposition 522 | from: ocsf_disposition 523 | 524 | - name: dst_endpoint 525 | from: ocsf_dst_endpoint 526 | 527 | - name: http_request 528 | from: ocsf_http_request 529 | 530 | - name: http_response 531 | from: ocsf_http_response 532 | 533 | - name: metadata 534 | from: ocsf_metadata 535 | 536 | - name: severity_id 537 | from: ocsf_severity_id 538 | 539 | - name: severity 540 | from: ocsf_severity 541 | 542 | 543 | - name: src_endpoint 544 | from: ocsf_src_endpoint 545 | 546 | - name: status 547 | from: ocsf_status 548 | 549 | - name: status_id 550 | from: ocsf_status_id 551 | 552 | 553 | - name: type_uid 554 | expr: | 555 | CASE 556 | WHEN LOWER(action_name) LIKE 'create%' THEN CAST(600301 as BIGINT) 557 | 558 | WHEN LOWER(action_name) LIKE 'get%' THEN CAST(600302 as BIGINT) 559 | WHEN LOWER(action_name) LIKE 'list%' THEN CAST(600302 as BIGINT) 560 | 561 | WHEN LOWER(action_name) LIKE 'update%' THEN CAST(600303 as BIGINT) 562 | WHEN LOWER(action_name) LIKE 'change%' THEN CAST(600303 as BIGINT) 563 | WHEN LOWER(action_name) LIKE 'move%' THEN CAST(600303 as BIGINT) 564 | WHEN LOWER(action_name) LIKE 'put%' THEN CAST(600303 as BIGINT) 565 | WHEN LOWER(action_name) LIKE 'rename%' THEN CAST(600303 as BIGINT) 566 | WHEN LOWER(action_name) LIKE 'set%' THEN CAST(600303 as BIGINT) 567 | 568 | WHEN LOWER(action_name) LIKE 'delete%' THEN CAST(600304 as BIGINT) 569 | WHEN LOWER(action_name) LIKE 'remove%' THEN CAST(600304 as BIGINT) 570 | 571 | WHEN action_name IS NULL OR action_name = '' THEN CAST(600300 as BIGINT) 572 | ELSE CAST(600399 as BIGINT) 573 | END 574 | 575 | - name: type_name 576 | expr: | 577 | CASE 578 | WHEN LOWER(action_name) LIKE 'create%' THEN 'API Activity: Create' 579 | 580 | WHEN LOWER(action_name) LIKE 'get%' THEN 'API Activity: Read' 581 | WHEN LOWER(action_name) LIKE 'list%' THEN 'API Activity: Read' 582 | 583 | WHEN LOWER(action_name) LIKE 'update%' THEN 'API Activity: Update' 584 | WHEN LOWER(action_name) LIKE 'change%' THEN 'API Activity: Update' 585 | WHEN LOWER(action_name) LIKE 'move%' THEN 'API Activity: Update' 586 | WHEN LOWER(action_name) LIKE 'put%' THEN 'API Activity: Update' 587 | WHEN LOWER(action_name) LIKE 'rename%' THEN 'API Activity: Update' 588 | WHEN LOWER(action_name) LIKE 'set%' THEN 'API Activity: Update' 589 | 590 | WHEN LOWER(action_name) LIKE 'delete%' THEN 'API Activity: Delete' 591 | WHEN LOWER(action_name) LIKE 'remove%' THEN 'API Activity: Delete' 592 | 593 | WHEN action_name IS NULL OR action_name = '' THEN 'API Activity: Unknown' 594 | ELSE 'API Activity: Other' 595 | END 596 | 597 | - name: unmapped 598 | expr: CAST(ocsf_unmapped AS VARIANT) 599 | 600 | - name: authentication 601 | input: databricks_access_audit 602 | filter: lower(action_name) like '%login' OR lower(action_name) like '%oidc%' OR lower(action_name) like '%authentication%' OR lower(action_name) like '%oauth%' OR lower(action_name) like 'validate%credential' OR lower(action_name) = 'logout' 603 | fields: 604 | - name: time 605 | from: time 606 | 607 | - name: action_id 608 | from: ocsf_action_id 609 | 610 | - name: activity_id 611 | expr: | 612 | CASE 613 | WHEN LOWER(action_name) LIKE '%login' THEN 1 614 | WHEN LOWER(action_name) = 'logout' THEN 2 615 | ELSE 99 616 | END 617 | 618 | - name: activity_name 619 | expr: | 620 | CASE 621 | WHEN LOWER(action_name) LIKE '%login' THEN 'Logon' 622 | WHEN LOWER(action_name) = 'logout' THEN 'Logoff' 623 | ELSE action_name 624 | END 625 | 626 | - name: category_name 627 | literal: Identity & Access Management 628 | 629 | - name: category_uid 630 | expr: CAST('3' AS INT) 631 | 632 | - name: class_name 633 | literal: "Authentication" 634 | 635 | - name: class_uid 636 | expr: CAST('3002' AS INT) 637 | 638 | - name: cloud 639 | from: ocsf_cloud 640 | 641 | - name: disposition_id 642 | from: ocsf_disposition_id 643 | 644 | - name: disposition 645 | from: ocsf_disposition 646 | 647 | - name: dst_endpoint 648 | from: ocsf_dst_endpoint 649 | 650 | - name: http_request 651 | from: ocsf_http_request 652 | 653 | - name: http_response 654 | from: ocsf_http_response 655 | 656 | - name: is_mfa 657 | expr: | 658 | CASE 659 | WHEN action_name = 'mfaLogin' THEN TRUE 660 | 661 | ELSE FALSE 662 | END 663 | 664 | - name: logon_type 665 | expr: | 666 | CASE 667 | WHEN 668 | action_name = 'mintOAuthToken' OR 669 | action_name = 'mintOAuthAuthorizationCode' OR 670 | action_name = 'oidcBrowserLogin' OR 671 | action_name = 'oidcTokenAuthorization' OR 672 | action_name = 'tokenLogin' OR 673 | action_name = 'certLogin' OR 674 | action_name = 'jwtLogin' OR 675 | action_name = 'mfaLogin' OR 676 | action_name = 'samlLogin' OR 677 | action_name = 'validateCredential' OR 678 | action_name = 'validateStorageCredential' OR 679 | action_name = 'accountInHouseOAuthClientAuthentication' OR 680 | action_name = 'workspaceInHouseOAuthClientAuthentication' THEN 'Network' 681 | ELSE 'Unknown' 682 | END 683 | 684 | - name: logon_type_id 685 | expr: | 686 | CASE 687 | WHEN 688 | action_name = 'mintOAuthToken' OR 689 | action_name = 'mintOAuthAuthorizationCode' OR 690 | action_name = 'oidcBrowserLogin' OR 691 | action_name = 'oidcTokenAuthorization' OR 692 | action_name = 'tokenLogin' OR 693 | action_name = 'certLogin' OR 694 | action_name = 'jwtLogin' OR 695 | action_name = 'mfaLogin' OR 696 | action_name = 'samlLogin' OR 697 | action_name = 'validateCredential' OR 698 | action_name = 'validateStorageCredential' OR 699 | action_name = 'accountInHouseOAuthClientAuthentication' OR 700 | action_name = 'workspaceInHouseOAuthClientAuthentication' THEN 3 701 | ELSE 0 702 | END 703 | 704 | - name: metadata 705 | from: ocsf_metadata 706 | 707 | - name: raw_data 708 | expr: to_json(request_params) 709 | 710 | - name: severity_id 711 | from: ocsf_severity_id 712 | 713 | - name: severity 714 | from: ocsf_severity 715 | 716 | - name: src_endpoint 717 | from: ocsf_src_endpoint 718 | 719 | - name: status 720 | from: ocsf_status 721 | 722 | - name: status_id 723 | from: ocsf_status_id 724 | 725 | - name: type_name 726 | expr: | 727 | CASE 728 | WHEN LOWER(action_name) LIKE '%login' THEN 'Authentication: Logon' 729 | WHEN LOWER(action_name) = 'logout' THEN 'Authentication: Logoff' 730 | ELSE 'Authentication: Other' 731 | END 732 | 733 | - name: type_uid 734 | expr: | 735 | CASE 736 | WHEN LOWER(action_name) LIKE '%login' THEN CAST(300201 as BIGINT) 737 | WHEN LOWER(action_name) = 'logout' THEN CAST(300202 as BIGINT) 738 | ELSE CAST(300299 as BIGINT) 739 | END 740 | 741 | - name: unmapped 742 | expr: CAST(ocsf_unmapped AS VARIANT) -------------------------------------------------------------------------------- /presets/aws/cloudtrail_iam/README.md: -------------------------------------------------------------------------------- 1 | # AWS CloudTrail IAM Events 2 | 3 | This preset processes gzipped CloudTrail JSON log files, extracting only IAM-related events while excluding all others. The filtered data is then transformed into structured silver tables, and ultimately into OCSF compliant gold tables. See the sections below for details on how events are categorized and mapped into the silver and gold layers. 4 | 5 | ## Unity Catalog Targets 6 | 7 | ### Silver 8 | 9 | The following silver tables store normalized IAM-related CloudTrail data. Each table corresponds to a specific OCSF class and serves as an intermediate processing layer before populating the gold tables. These tables allow for targeted querying and analysis of IAM operations, broken down by category and use case. 10 | 11 | 12 | | Table Name | Description | 13 | |--------------------------------|-----------------------------------------------------------------------------| 14 | | aws_cloudtrail_account_change | Parsed and normalized CloudTrail events related to IAM account lifecycle actions, such as user creation, deletion, and credential updates. | 15 | | aws_cloudtrail_entity_management | Parsed and normalized CloudTrail events focused on reading, updating, etc., IAM entities. | 16 | | aws_cloudtrail_user_access | Parsed and normalized CloudTrail events related to the management of user permissions/access. | 17 | | aws_cloudtrail_group_management| Parsed and normalized CloudTrail events for group membership operations, including adding or removing users and managing groups. | 18 | | aws_cloudtrail_api_activity | Parsed and normalized CloudTrail events related to IAM CRUD and other actions. | 19 | 20 | 21 | 22 | ### Gold 23 | 24 | The following gold tables contain data that conforms to the [OCSF (Open Cybersecurity Schema Framework)](https://ocsf.io) specification. Each table maps to a specific OCSF class and provides high-quality, schema-aligned datasets for downstream analytics, detection engineering, and threat hunting. 25 | 26 | | Table Name | OCSF Class Name | OCSF Class UID | Description | 27 | |-------------------------|------------------------|----------------|-----------------------------------------------------------------------------| 28 | | account_change | Account Change | 3001 | Events related to modifications of user accounts, such as creation, deletion, or updates to account details. | 29 | | entity_management | Entity Management | 3004 | Events involving the management of entities, such as users, groups, and roles, including their creation and deletion. | 30 | | user_access_management | User Access Management | 3005 | Events related to the management of user access, including permission assignments and policy changes. | 31 | | group_management | Group Management | 3006 | Events concerning the management of groups, including group creation, deletion, and membership changes. | 32 | | api_activity | API Activity | 6003 | Events related to general CRUD (Create, Read, Update, Delete) and other API activities. | 33 | 34 | 35 | ## Event Type Descriptions 36 | 37 | ### Account Change (3001) 38 | 39 | Changes to IAM identities, credentials, or account properties. Canonical activities include: `Create`, `Enable`, `Password Change`, `Password Reset`, `Disable`, `Delete`, `Attach Policy`, `Detach Policy`, `Lock`, `MFA Factor Enable`, `MFA Factor Disable` and `Unlock`. See the [OCSF documentation](https://schema.ocsf.io/1.4.0/classes/account_change?extensions=) for a more detailed description of these values. 40 | 41 | | IAM Event Type | Comment | OCSF Activity Name | 42 | |---------------------------------|---------------------------------------------------|--------------------------| 43 | | ChangePassword | User-initiated password change | Password Change | 44 | | CreateLoginProfile | Creates a password for an IAM user | Create | 45 | | DeleteLoginProfile | Removes IAM user password profile | Delete | 46 | | UpdateLoginProfile | Changes IAM user password or settings | Password Change | 47 | | CreateUser | Adds a new IAM user | Create | 48 | | DeleteUser | Removes an IAM user | Delete | 49 | | UpdateUser | Modifies user name or path | Other | 50 | | CreateVirtualMFADevice | Sets up a virtual MFA device | Other | 51 | | DeactivateMFADevice | Disables MFA for a user | MFA Factor Disable | 52 | | DeleteVirtualMFADevice | Deletes a virtual MFA device | MFA Factor Disable | 53 | | EnableMFADevice | Enables MFA on an IAM user | MFA Factor Enable | 54 | | ResyncMFADevice | Synchronizes MFA token devices | Other | 55 | | UpdateSSHPublicKey | Modifies uploaded SSH key metadata | Other | 56 | | UploadSSHPublicKey | Adds an SSH public key to IAM user | Other | 57 | | DeleteSSHPublicKey | Removes SSH key from IAM user | Other | 58 | | UploadSigningCertificate | Adds an X.509 cert for IAM user | Other | 59 | | DeleteSigningCertificate | Removes X.509 cert from IAM user | Other | 60 | | UpdateSigningCertificate | Updates the status of a signing cert | Other | 61 | | CreateServiceSpecificCredential | Creates credentials for a specific AWS service | Other | 62 | | DeleteServiceSpecificCredential | Deletes service-specific credentials | Other | 63 | | ResetServiceSpecificCredential | Resets password for a service-specific credential | Password Reset | 64 | | UpdateServiceSpecificCredential | Changes status for a service-specific credential | Other | 65 | | CreateAccessKey | Generates a new IAM access key | Other | 66 | | DeleteAccessKey | Deletes an IAM access key | Other | 67 | | UpdateAccessKey | Enables/disables an access key | Other | 68 | | CreateAccountAlias | Creates a friendly alias for the AWS account | Other | 69 | | DeleteAccountAlias | Removes an account alias | Other | 70 | | UpdateAccountPasswordPolicy | Sets account-wide password rules | Other | 71 | | DeleteAccountPasswordPolicy | Removes account-wide password rules | Other | 72 | | AttachRolePolicy | Binds managed policy to role | Attach Policy | 73 | | DetachRolePolicy | Unbinds managed policy from role | Detach Policy | 74 | | PutRolePolicy | Creates inline policy for role | Attach Policy | 75 | | DeleteRolePolicy | Removes inline role policy | Detach Policy | 76 | | PutRolePermissionsBoundary | Adds a role permissions boundary | Attach Policy | 77 | | DeleteRolePermissionsBoundary | Removes role boundary | Detach Policy | 78 | | TagUser | Adds tags to user | Other | 79 | | UntagUser | Removes tags from user | Other | 80 | | TagRole | Adds tags to role | Other | 81 | | UntagRole | Removes tags from role | Other | 82 | 83 | 84 | ### Entity Management (3004) 85 | 86 | Operations or reporting about IAM entities. Canonical activities include: `Create`, `Read`, `Update`, `Delete`, `Move`, `Enroll`, `Unenroll`, `Enable`, `Disable`, `Activate`, `Deactivate`, `Suspend`, `Resume`. 87 | 88 | | IAM Event Type | Comment | OCSF Activity Name | 89 | |-----------------------------------------------|-----------------------------------------------------|--------------------| 90 | | GetGroup | Returns group details | Read | 91 | | GetGroupPolicy | Gets group inline policy | Read | 92 | | GetUser | Returns IAM user details | Read | 93 | | GetUserPolicy | Gets user inline policy | Read | 94 | | GetRole | Returns IAM role details | Read | 95 | | GetRolePolicy | Gets role inline policy | Read | 96 | | GetPolicy | Returns managed policy metadata | Read | 97 | | GetPolicyVersion | Gets a specific version of a managed policy | Read | 98 | | GetInstanceProfile | Returns IAM instance profile details | Read | 99 | | GetLoginProfile | Returns login profile for IAM user | Read | 100 | | GetOpenIDConnectProvider | Returns OIDC provider metadata | Read | 101 | | GetSAMLProvider | Returns SAML provider metadata | Read | 102 | | GetServerCertificate | Returns SSL certificate metadata | Read | 103 | | ListUserTags | Lists tags attached to a user | Read | 104 | | ListGroupsForUser | Groups a user belongs to | Read | 105 | | ListRoleTags | Lists tags on a role | Read | 106 | | ListInstanceProfilesForRole | Profiles associated with a role | Read | 107 | | ListInstanceProfileTags | Tags on an instance profile | Read | 108 | | ListSAMLProviderTags | Tags for a SAML provider | Read | 109 | | ListOpenIDConnectProviderTags | Tags for OIDC provider | Read | 110 | | ListServerCertificateTags | Tags for certs | Read | 111 | | ListMFADevices | Lists MFA devices | Read | 112 | | ListMFADeviceTags | MFA tags | Read | 113 | | ListSigningCertificates | X.509 signing certs | Read | 114 | | ListSSHPublicKeys | SSH keys | Read | 115 | | ListServiceSpecificCredentials | Credentials scoped to AWS services | Read | 116 | | ListPolicyVersions | Versions of a managed policy | Read | 117 | | ListPolicyTags | Tags on policies | Read | 118 | | ListEntitiesForPolicy | Lists entities attached to a policy | Read | 119 | | ListAttachedUserPolicies | Managed policies attached to a user | Read | 120 | | ListAttachedGroupPolicies | Policies attached to group | Read | 121 | | ListAttachedRolePolicies | Policies attached to role | Read | 122 | | GetMFADevice | Retrieves details about an MFA device | Read | 123 | | ListAccessKeys | Lists the access keys for an IAM user | Read | 124 | | ListGroupPolicies | Lists inline policies of a group | Read | 125 | | ListOrganizationsFeatures | Lists enabled org features | Read | 126 | | ListRolePolicies | Lists inline policies attached to a role | Read | 127 | | ListUserPolicies | Lists inline policies attached to a user | Read | 128 | | ListVirtualMFADevices | Lists virtual MFA devices in the account | Read | 129 | | TagMFADevice | Tags MFA device | Update | 130 | | UntagMFADevice | Removes MFA tags | Update | 131 | | TagOpenIDConnectProvider | Tags OIDC provider | Update | 132 | | UntagOpenIDConnectProvider | Untag OIDC provider | Update | 133 | | TagSAMLProvider | Tags SAML provider | Update | 134 | | UntagSAMLProvider | Untag SAML provider | Update | 135 | | TagServerCertificate | Tags server certificate | Update | 136 | | UntagServerCertificate | Untag server certificate | Update | 137 | | SetDefaultPolicyVersion | Changes default policy version | Update | 138 | | TagPolicy | Adds tags to policy | Update | 139 | | UntagPolicy | Removes tags from policy | Update | 140 | | TagInstanceProfile | Adds tags to instance profile | Update | 141 | | UntagInstanceProfile | Removes instance profile tags | Update | 142 | | AddClientIDToOpenIDConnectProvider | Adds a client ID to an existing OIDC provider | Update | 143 | | AddRoleToInstanceProfile | Adds a role to an instance profile | Update | 144 | | RemoveClientIDFromOpenIDConnectProvider | Removes a client ID from an OIDC provider | Update | 145 | | RemoveRoleFromInstanceProfile | Removes a role from an instance profile | Update | 146 | | UpdateAssumeRolePolicy | Updates the trust relationship for a role | Update | 147 | | UpdateOpenIDConnectProviderThumbprint | Updates the thumbprint list for an OIDC provider | Update | 148 | | UpdateRole | Modifies an existing IAM role | Update | 149 | | UpdateRoleDescription | Changes the description of a role | Update | 150 | | UpdateSAMLProvider | Modifies the metadata document for a SAML provider | Update | 151 | | UpdateServerCertificate | Updates the name or path of a server cert | Update | 152 | | CreateInstanceProfile | Creates a new instance profile | Create | 153 | | CreateOpenIDConnectProvider | Creates a new OIDC identity provider | Create | 154 | | CreatePolicy | Creates a managed IAM policy | Create | 155 | | CreatePolicyVersion | Creates a new version of an existing managed policy | Create | 156 | | CreateRole | Creates a new IAM role | Create | 157 | | CreateSAMLProvider | Creates a new SAML identity provider | Create | 158 | | CreateServiceLinkedRole | Creates a service-linked IAM role | Create | 159 | | UploadServerCertificate | Uploads a new server certificate | Create | 160 | | DeleteInstanceProfile | Deletes an existing instance profile | Delete | 161 | | DeleteOpenIDConnectProvider | Deletes an existing OIDC identity provider | Delete | 162 | | DeletePolicy | Deletes a managed IAM policy | Delete | 163 | | DeletePolicyVersion | Deletes a specific version of a managed policy | Delete | 164 | | DeleteRole | Deletes an existing IAM role | Delete | 165 | | DeleteSAMLProvider | Deletes a SAML identity provider | Delete | 166 | | DeleteServerCertificate | Deletes an uploaded server certificate | Delete | 167 | | DeleteServiceLinkedRole | Deletes a service-linked role | Delete | 168 | | DisableOrganizationsRootCredentialsManagement | Disables root credential management for the org | Disable | 169 | | DisableOrganizationsRootSessions | Disables sessions for the org root | Disable | 170 | | EnableOrganizationsRootCredentialsManagement | Enables root credential management for the org | Enable | 171 | | EnableOrganizationsRootSessions | Enables sessions for the org root | Enable | 172 | 173 | 174 | 175 | ### User Access Management (3005) 176 | 177 | Managing permissions and policies attached to IAM users, roles. Canonical activities include: `Assign Privileges`, `Revoke Privileges`. 178 | 179 | | IAM Event Type | Comment | OCSF Activity Name | 180 | |-------------------------------|-----------------------------------------------|----------------------| 181 | | AttachUserPolicy | Binds a managed policy to a user | Assign Privileges | 182 | | DetachUserPolicy | Unbinds policy from a user | Revoke Privileges | 183 | | PutUserPolicy | Creates or updates inline user policy | Assign Privileges | 184 | | DeleteUserPolicy | Deletes an inline user policy | Revoke Privileges | 185 | | PutUserPermissionsBoundary | Sets permissions boundary for user | Assign Privileges | 186 | | DeleteUserPermissionsBoundary | Removes user's permissions boundary | Revoke Privileges | 187 | 188 | 189 | ### Group Management (3006) 190 | 191 | Creation and modification of IAM groups, their memberships and permissions. Canonical activities include: `Assign Privileges`, `Revoke Privileges`, `Add User`, `Remove User`, `Delete`, and `Create`. 192 | 193 | | IAM Event Type | Comment | OCSF Activity Name | 194 | |---------------------|---------------------------------------|----------------------| 195 | | CreateGroup | Creates a new IAM group | Create | 196 | | DeleteGroup | Deletes an IAM group | Delete | 197 | | UpdateGroup | Renames or moves a group | Other | 198 | | AddUserToGroup | Adds user to a group | Add User | 199 | | RemoveUserFromGroup | Removes user from a group | Remove User | 200 | | AttachGroupPolicy | Attaches managed policy to group | Assign Privileges | 201 | | DetachGroupPolicy | Detaches managed policy from group | Revoke Privileges | 202 | | PutGroupPolicy | Creates inline group policy | Assign Privileges | 203 | | DeleteGroupPolicy | Deletes inline group policy | Revoke Privileges | 204 | 205 | 206 | ### API Activity (6003) 207 | 208 | CRUD type actions typically performed through programmatic interfaces, such as AWS SDKs or CLI. Canonical activities include: `Create`, `Read`, `Update` and `Delete`. 209 | 210 | | IAM Event Type | Comment | OCSF Activity Name | 211 | |-------------------------------------------|-------------------------------------------------|--------------------| 212 | | GenerateCredentialReport | Creates CSV report of IAM credentials | Create | 213 | | GenerateOrganizationsAccessReport | Creates report for org-level access analyzer | Create | 214 | | GenerateServiceLastAccessedDetails | Starts analysis of service usage by entity | Create | 215 | | GetCredentialReport | Retrieves most recent credential report | Read | 216 | | GetOrganizationsAccessReport | Fetches an org access analyzer report | Read | 217 | | GetServiceLastAccessedDetails | Gets results of last access analysis | Read | 218 | | GetServiceLastAccessedDetailsWithEntities | Breaks down access by entity | Read | 219 | | GetServiceLinkedRoleDeletionStatus | Checks SL role deletion readiness | Read | 220 | | GetAccountAuthorizationDetails | Returns permission summaries by entity | Read | 221 | | GetAccountSummary | Overview of IAM resources in account | Read | 222 | | GetAccountPasswordPolicy | Returns current password policy | Read | 223 | | GetAccessKeyLastUsed | Shows when access key was last used | Read | 224 | | GetSSHPublicKey | Returns SSH key details | Read | 225 | | GetContextKeysForCustomPolicy | Gets context keys referenced in a custom policy | Read | 226 | | GetContextKeysForPrincipalPolicy | Gets context keys from a principal's policies | Read | 227 | | ListUsers | Lists all IAM users | Read | 228 | | ListGroups | Lists all IAM groups | Read | 229 | | ListRoles | Lists all IAM roles | Read | 230 | | ListInstanceProfiles | Lists instance profiles | Read | 231 | | ListSAMLProviders | Lists SAML identity providers | Read | 232 | | ListOpenIDConnectProviders | OIDC identity providers | Read | 233 | | ListServerCertificates | Lists uploaded SSL certs | Read | 234 | | ListPolicies | Lists all policies | Read | 235 | | ListPoliciesGrantingServiceAccess | Shows what grants service access to user | Read | 236 | | ListAccountAliases | List friendly names for account | Read | 237 | | SetSecurityTokenServicePreferences | Defines global STS session duration prefs | Update | 238 | | SimulateCustomPolicy | Simulates policy evaluation for access test | Other | 239 | | SimulatePrincipalPolicy | Simulates a principal's effective permissions | Other | 240 | 241 | 242 | ## Other Resources 243 | 244 | https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-event-reference-record-contents.html 245 | A complete reference/data dictionary for the CloudTrail events. Note that the reference does not have information for service specific schemas nested within the CloudTrail record. 246 | 247 | https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-events.html#cloudtrail-management-events 248 | A short guide explaining the different CloudTrail event types (i.e., management, data, insight, etc.) 249 | 250 | 251 | https://github.com/aws/aws-sdk-ruby/blob/version-3/apis/iam/2010-05-08/api-2.json 252 | A collection of IAM API event definitions maintained by Amazon in JSON format. -------------------------------------------------------------------------------- /presets/okta/syslog/preset.yaml: -------------------------------------------------------------------------------- 1 | name: okta_syslog 2 | author: Antimatter 3 | description: "Preset for OKTA syslog wrapped by Cribl" 4 | title: "OKTA - Syslog (Preview)" 5 | iconURL: "https://raw.githubusercontent.com/antimatterhq/dasl-content-packs/refs/heads/main/presets/okta/syslog/icon.png" 6 | autoloader: 7 | format: json 8 | cloudFiles: 9 | schemaHints: | 10 | actor struct<`alternateId`:string,`detailEntry`:string,`displayName`:string,`id`:string,`type`:string>, 11 | authenticationContext struct<`authenticationProvider`:string,`authenticationStep`:bigint,`authenticatorContext`:struct<`applicationVersion`:string,`binaryIdentifier`:string,`binaryPath`:string,`bindingMethod`:string,`requestOrigin`:string,`signerDistinguishedName`:string,`signerKeyIdentifier`:string,`userConsent`:string,`validationStatus`:string>,`credentialProvider`:string,`credentialType`:string,`externalSessionId`:string,`interface`:string,`issuer`:struct<`id`:string>,`rootSessionId`:string>, 12 | client struct<`device`:string,`geographicalContext`:struct<`city`:string,`country`:string,`geolocation`:struct<`lat`:double,`lon`:double>,`postalCode`:string,`state`:string>,`id`:string,`ipAddress`:string,`userAgent`:struct<`browser`:string,`os`:string,`rawUserAgent`:string>,`zone`:string>, 13 | debugContext struct<`debugData`:struct<`aaguid`:string,`addedObjects`:string,`appContextName`:string,`appname`:string,`audience`:string,`authCode`:string,`authMethodFirstEnrollment`:string,`authMethodFirstType`:string,`authMethodFirstVerificationTime`:string,`authMethodSecondEnrollment`:string,`authMethodSecondType`:string,`authMethodSecondVerificationTime`:string,`authTime`:string,`authenticationClassRef`:string,`authenticatorMethodChallengeTime`:string,`authnRequestId`:string,`backupEligible`:string,`behaviors`:string,`category`:string,`challengeAuthenticatorsList`:string,`changedAttributes`:string,`clientAuthType`:string,`clientId`:string,`clientSecret`:string,`clientType`:string,`deletedObjects`:string,`detailedmessage`:string,`device`:string,`deviceCategory`:string,`deviceFingerprint`:string,`devicePlatform`:string,`deviceTokenHash`:string,`dtHash`:string,`emailProvider`:string,`emailRequestId`:string,`enrollmentMethodChallengeTime`:string,`expiryTime`:string,`factor`:string,`factorIntent`:string,`grantType`:string,`grantedScopes`:string,`idpType`:string,`importLastToken`:string,`importTrigger`:string,`importType`:string,`initiationType`:string,`ip`:string,`issuedAt`:string,`issuer`:string,`jobId`:string,`jti`:string,`keyTypeUsedForAuthentication`:string,`logOnlySecurityData`:string,`loginResult`:string,`matchedAuthenticatorGroups`:string,`oktaUserAgentExtended`:string,`operationRateLimitScopeType`:string,`operationRateLimitSecondsToReset`:string,`operationRateLimitSubtype`:string,`operationRateLimitThreshold`:string,`operationRateLimitTimeSpan`:string,`operationRateLimitTimeUnit`:string,`operationRateLimitType`:string,`orgId`:string,`origin`:string,`pushOnlyResponseType`:string,`pushWithNumberChallengeResponseType`:string,`redirectUri`:string,`requestId`:string,`requestUri`:string,`requestedScopes`:string,`responseMode`:string,`responseTime`:string,`responseType`:string,`risk`:string,`signOnMode`:string,`state`:string,`subject`:string,`targetEventHookIds`:string,`threatSuspected`:string,`threshold`:string,`timeSpan`:string,`timeUnit`:string,`totalObjects`:string,`totalTime`:string,`traceId`:string,`tunnels`:string,`unchangedObjects`:string,`updatedObjects`:string,`url`:string,`userId`:string>>, 14 | device struct<`device_integrator`:string,`disk_encryption_type`:string,`id`:string,`jailbreak`:boolean,`managed`:boolean,`name`:string,`os_platform`:string,`os_version`:string,`registered`:boolean,`screen_lock_type`:string,`secure_hardware_present`:boolean>, 15 | outcome struct<`reason`:string,`result`:string>, 16 | request struct<`ipChain`:array,`postalCode`:string,`state`:string>,`ip`:string,`source`:string,`version`:string>>>, 17 | securityContext struct<`asNumber`:bigint,`asOrg`:string,`domain`:string,`isProxy`:boolean,`isp`:string>, 18 | target array,`displayName`:string,`id`:string,`type`:string>>, 19 | transaction struct<`detail`:struct<`requestApiTokenClientId`:string,`requestApiTokenId`:string,`rootApiTokenId`:string>,`id`:string,`type`:string> 20 | 21 | silver: 22 | preTransform: 23 | - name: unbox_cribl 24 | fields: 25 | - name: _time 26 | from: _time 27 | - name: actor 28 | from: actor 29 | - name: authenticationContext 30 | from: authenticationContext 31 | - name: client 32 | from: client 33 | - name: debugContext 34 | from: debugContext 35 | - name: device 36 | from: device 37 | - name: displayMessage 38 | from: displayMessage 39 | - name: eventType 40 | from: eventType 41 | - name: host 42 | from: host 43 | - name: legacyEventType 44 | from: legacyEventType 45 | - name: outcome 46 | from: outcome 47 | - name: published 48 | from: published 49 | - name: request 50 | from: request 51 | - name: securityContext 52 | from: securityContext 53 | - name: severity 54 | from: severity 55 | - name: target 56 | from: target 57 | - name: transaction 58 | from: transaction 59 | - name: uuid 60 | from: uuid 61 | - name: version 62 | from: version 63 | - name: _rescued_data 64 | from: _rescued_data 65 | 66 | transform: 67 | - name: okta_syslog 68 | fields: 69 | - name: metadata.product.name 70 | literal: Okta System Log 71 | - name: metadata.product.vendor_name 72 | literal: Okta 73 | - name: metadata.product.version 74 | from: version 75 | - name: metadata.product.uid 76 | from: uuid 77 | - name: metadata.event_code 78 | from: eventType 79 | - name: eventType 80 | from: eventType 81 | - name: ocsf_category_uid 82 | expr: CAST('3' AS INT) 83 | - name: ocsf_category_name 84 | literal: Identity & Access Management 85 | - name: ocsf_class_uid 86 | expr: | 87 | CASE 88 | WHEN LOWER(eventType) = 'app.user_management.grouppush.mapping.okta.users.ignored' THEN CAST('3006' AS INT) 89 | WHEN LOWER(eventType) = 'app.generic.unauth_app_access_attempt' THEN CAST('3002' AS INT) 90 | WHEN LOWER(eventType) = 'app.oauth2.authorize' THEN CAST('3002' AS INT) 91 | WHEN LOWER(eventType) = 'app.oauth2.authorize.code' THEN CAST('3002' AS INT) 92 | WHEN LOWER(eventType) = 'app.oauth2.client.read_client_secret' THEN CAST('3002' AS INT) 93 | WHEN LOWER(eventType) = 'app.oauth2.token.grant' THEN CAST('3002' AS INT) 94 | WHEN LOWER(eventType) = 'app.oauth2.token.grant.access_token' THEN CAST('3002' AS INT) 95 | WHEN LOWER(eventType) = 'app.oauth2.token.grant.id_token' THEN CAST('3002' AS INT) 96 | WHEN LOWER(eventType) = 'app.oauth2.token.grant.refresh_token' THEN CAST('3002' AS INT) 97 | WHEN LOWER(eventType) = 'app.realtimesync.import.details.update_user' THEN CAST('3001' AS INT) 98 | WHEN LOWER(eventType) = 'application.configuration.detect_error' THEN CAST('3004' AS INT) 99 | WHEN LOWER(eventType) = 'application.lifecycle.update' THEN CAST('3004' AS INT) 100 | WHEN LOWER(eventType) = 'application.policy.sign_on.deny_access' THEN CAST('3002' AS INT) 101 | WHEN LOWER(eventType) = 'application.provision.group_push.push_memberships' THEN CAST('3006' AS INT) 102 | WHEN LOWER(eventType) = 'application.provision.user.deactivate' THEN CAST('3001' AS INT) 103 | WHEN LOWER(eventType) = 'application.provision.user.deprovision' THEN CAST('3001' AS INT) 104 | WHEN LOWER(eventType) = 'application.provision.user.push_profile' THEN CAST('3001' AS INT) 105 | WHEN LOWER(eventType) = 'application.user_membership.remove' THEN CAST('3005' AS INT) 106 | WHEN LOWER(eventType) = 'application.user_membership.update' THEN CAST('3005' AS INT) 107 | WHEN LOWER(eventType) = 'core.concurrency.org.limit.violation' THEN CAST('3004' AS INT) 108 | WHEN LOWER(eventType) = 'device.enrollment.create' THEN CAST('3004' AS INT) 109 | WHEN LOWER(eventType) = 'device.lifecycle.activate' THEN CAST('3004' AS INT) 110 | WHEN LOWER(eventType) = 'device.user.add' THEN CAST('3004' AS INT) 111 | WHEN LOWER(eventType) = 'event_hook.delivery' THEN CAST('3004' AS INT) 112 | WHEN LOWER(eventType) = 'group.user_membership.add' THEN CAST('3006' AS INT) 113 | WHEN LOWER(eventType) = 'group.user_membership.remove' THEN CAST('3006' AS INT) 114 | WHEN LOWER(eventType) = 'pki.cert.issue' THEN CAST('3004' AS INT) 115 | WHEN LOWER(eventType) = 'policy.evaluate_sign_on' THEN CAST('3002' AS INT) 116 | WHEN LOWER(eventType) = 'security.request.blocked' THEN CAST('3003' AS INT) 117 | WHEN LOWER(eventType) = 'system.agent.ad.read_ldap' THEN CAST('3004' AS INT) 118 | WHEN LOWER(eventType) = 'system.agent.ad.read_topology' THEN CAST('3004' AS INT) 119 | WHEN LOWER(eventType) = 'system.client.concurrency_rate_limit.notification' THEN CAST('3004' AS INT) 120 | WHEN LOWER(eventType) = 'system.client.rate_limit.notification' THEN CAST('3004' AS INT) 121 | WHEN LOWER(eventType) = 'system.email.delivery' THEN CAST('3004' AS INT) 122 | WHEN LOWER(eventType) = 'system.email.mfa_enroll_notification.sent_message' THEN CAST('3005' AS INT) 123 | WHEN LOWER(eventType) = 'system.email.new_device_notification.sent_message' THEN CAST('3005' AS INT) 124 | WHEN LOWER(eventType) = 'system.import.complete' THEN CAST('3004' AS INT) 125 | WHEN LOWER(eventType) = 'system.import.complete_batch' THEN CAST('3004' AS INT) 126 | WHEN LOWER(eventType) = 'system.import.custom_object.complete' THEN CAST('3004' AS INT) 127 | WHEN LOWER(eventType) = 'system.import.download.complete' THEN CAST('3004' AS INT) 128 | WHEN LOWER(eventType) = 'system.import.download.start' THEN CAST('3004' AS INT) 129 | WHEN LOWER(eventType) = 'system.import.group.complete' THEN CAST('3006' AS INT) 130 | WHEN LOWER(eventType) = 'system.import.group_membership.complete' THEN CAST('3006' AS INT) 131 | WHEN LOWER(eventType) = 'system.import.implicit_deletion.complete' THEN CAST('3004' AS INT) 132 | WHEN LOWER(eventType) = 'system.import.implicit_deletion.start' THEN CAST('3004' AS INT) 133 | WHEN LOWER(eventType) = 'system.import.membership_processing.complete' THEN CAST('3006' AS INT) 134 | WHEN LOWER(eventType) = 'system.import.membership_processing.start' THEN CAST('3006' AS INT) 135 | WHEN LOWER(eventType) = 'system.import.object_creation.complete' THEN CAST('3004' AS INT) 136 | WHEN LOWER(eventType) = 'system.import.object_creation.start' THEN CAST('3004' AS INT) 137 | WHEN LOWER(eventType) = 'system.import.start' THEN CAST('3001' AS INT) 138 | WHEN LOWER(eventType) = 'system.import.user.complete' THEN CAST('3001' AS INT) 139 | WHEN LOWER(eventType) = 'system.import.user.update' THEN CAST('3001' AS INT) 140 | WHEN LOWER(eventType) = 'system.import.user_matching.complete' THEN CAST('3001' AS INT) 141 | WHEN LOWER(eventType) = 'system.import.user_matching.start' THEN CAST('3001' AS INT) 142 | WHEN LOWER(eventType) = 'system.operation.concurrency_limit.violation' THEN CAST('3004' AS INT) 143 | WHEN LOWER(eventType) = 'system.operation.rate_limit.violation' THEN CAST('3004' AS INT) 144 | WHEN LOWER(eventType) = 'system.push.send_factor_verify_push' THEN CAST('3002' AS INT) 145 | WHEN LOWER(eventType) = 'user.account.update_profile' THEN CAST('3001' AS INT) 146 | WHEN LOWER(eventType) = 'user.authentication.auth_via_mfa' THEN CAST('3002' AS INT) 147 | WHEN LOWER(eventType) = 'user.authentication.sso' THEN CAST('3002' AS INT) 148 | WHEN LOWER(eventType) = 'user.authentication.verify' THEN CAST('3002' AS INT) 149 | WHEN LOWER(eventType) = 'user.lifecycle.deactivate' THEN CAST('3001' AS INT) 150 | WHEN LOWER(eventType) = 'user.lifecycle.suspend' THEN CAST('3001' AS INT) 151 | WHEN LOWER(eventType) = 'user.mfa.factor.activate' THEN CAST('3002' AS INT) 152 | WHEN LOWER(eventType) = 'user.mfa.factor.update' THEN CAST('3002' AS INT) 153 | WHEN LOWER(eventType) = 'user.session.access_admin_app' THEN CAST('3002' AS INT) 154 | WHEN LOWER(eventType) = 'user.session.clear' THEN CAST('3003' AS INT) 155 | WHEN LOWER(eventType) = 'user.session.start' THEN CAST('3003' AS INT) 156 | ELSE NULL 157 | END 158 | - name: ocsf_class_name 159 | expr: | 160 | CASE 161 | WHEN LOWER(eventType) = 'app.user_management.grouppush.mapping.okta.users.ignored' THEN 'Group Management' 162 | WHEN LOWER(eventType) = 'app.generic.unauth_app_access_attempt' THEN 'Authentication' 163 | WHEN LOWER(eventType) = 'app.oauth2.authorize' THEN 'Authentication' 164 | WHEN LOWER(eventType) = 'app.oauth2.authorize.code' THEN 'Authentication' 165 | WHEN LOWER(eventType) = 'app.oauth2.client.read_client_secret' THEN 'Authentication' 166 | WHEN LOWER(eventType) = 'app.oauth2.token.grant' THEN 'Authentication' 167 | WHEN LOWER(eventType) = 'app.oauth2.token.grant.access_token' THEN 'Authentication' 168 | WHEN LOWER(eventType) = 'app.oauth2.token.grant.id_token' THEN 'Authentication' 169 | WHEN LOWER(eventType) = 'app.oauth2.token.grant.refresh_token' THEN 'Authentication' 170 | WHEN LOWER(eventType) = 'app.realtimesync.import.details.update_user' THEN 'Account Change' 171 | WHEN LOWER(eventType) = 'application.configuration.detect_error' THEN 'Entity Management' 172 | WHEN LOWER(eventType) = 'application.lifecycle.update' THEN 'Entity Management' 173 | WHEN LOWER(eventType) = 'application.policy.sign_on.deny_access' THEN 'Authentication' 174 | WHEN LOWER(eventType) = 'application.provision.group_push.push_memberships' THEN 'Group Management' 175 | WHEN LOWER(eventType) = 'application.provision.user.deactivate' THEN 'Account Change' 176 | WHEN LOWER(eventType) = 'application.provision.user.deprovision' THEN 'Account Change' 177 | WHEN LOWER(eventType) = 'application.provision.user.push_profile' THEN 'Account Change' 178 | WHEN LOWER(eventType) = 'application.user_membership.remove' THEN 'User Access Management' 179 | WHEN LOWER(eventType) = 'application.user_membership.update' THEN 'User Access Management' 180 | WHEN LOWER(eventType) = 'core.concurrency.org.limit.violation' THEN 'Entity Management' 181 | WHEN LOWER(eventType) = 'device.enrollment.create' THEN 'Entity Management' 182 | WHEN LOWER(eventType) = 'device.lifecycle.activate' THEN 'Entity Management' 183 | WHEN LOWER(eventType) = 'device.user.add' THEN 'Entity Management' 184 | WHEN LOWER(eventType) = 'event_hook.delivery' THEN 'Entity Management' 185 | WHEN LOWER(eventType) = 'group.user_membership.add' THEN 'Group Management' 186 | WHEN LOWER(eventType) = 'group.user_membership.remove' THEN 'Group Management' 187 | WHEN LOWER(eventType) = 'pki.cert.issue' THEN 'Entity Management' 188 | WHEN LOWER(eventType) = 'policy.evaluate_sign_on' THEN 'Authentication' 189 | WHEN LOWER(eventType) = 'security.request.blocked' THEN 'Authorize Session' 190 | WHEN LOWER(eventType) = 'system.agent.ad.read_ldap' THEN 'Entity Management' 191 | WHEN LOWER(eventType) = 'system.agent.ad.read_topology' THEN 'Entity Management' 192 | WHEN LOWER(eventType) = 'system.client.concurrency_rate_limit.notification' THEN 'Entity Management' 193 | WHEN LOWER(eventType) = 'system.client.rate_limit.notification' THEN 'Entity Management' 194 | WHEN LOWER(eventType) = 'system.email.delivery' THEN 'Entity Management' 195 | WHEN LOWER(eventType) = 'system.email.mfa_enroll_notification.sent_message' THEN 'User Access Management' 196 | WHEN LOWER(eventType) = 'system.email.new_device_notification.sent_message' THEN 'User Access Management' 197 | WHEN LOWER(eventType) = 'system.import.complete' THEN 'Entity Management' 198 | WHEN LOWER(eventType) = 'system.import.complete_batch' THEN 'Entity Management' 199 | WHEN LOWER(eventType) = 'system.import.custom_object.complete' THEN 'Entity Management' 200 | WHEN LOWER(eventType) = 'system.import.download.complete' THEN 'Entity Management' 201 | WHEN LOWER(eventType) = 'system.import.download.start' THEN 'Entity Management' 202 | WHEN LOWER(eventType) = 'system.import.group.complete' THEN 'Group Management' 203 | WHEN LOWER(eventType) = 'system.import.group_membership.complete' THEN 'Group Management' 204 | WHEN LOWER(eventType) = 'system.import.implicit_deletion.complete' THEN 'Entity Management' 205 | WHEN LOWER(eventType) = 'system.import.implicit_deletion.start' THEN 'Entity Management' 206 | WHEN LOWER(eventType) = 'system.import.membership_processing.complete' THEN 'Group Management' 207 | WHEN LOWER(eventType) = 'system.import.membership_processing.start' THEN 'Group Management' 208 | WHEN LOWER(eventType) = 'system.import.object_creation.complete' THEN 'Entity Management' 209 | WHEN LOWER(eventType) = 'system.import.object_creation.start' THEN 'Entity Management' 210 | WHEN LOWER(eventType) = 'system.import.start' THEN 'Account Change' 211 | WHEN LOWER(eventType) = 'system.import.user.complete' THEN 'Account Change' 212 | WHEN LOWER(eventType) = 'system.import.user.update' THEN 'Account Change' 213 | WHEN LOWER(eventType) = 'system.import.user_matching.complete' THEN 'Account Change' 214 | WHEN LOWER(eventType) = 'system.import.user_matching.start' THEN 'Account Change' 215 | WHEN LOWER(eventType) = 'system.operation.concurrency_limit.violation' THEN 'Entity Management' 216 | WHEN LOWER(eventType) = 'system.operation.rate_limit.violation' THEN 'Entity Management' 217 | WHEN LOWER(eventType) = 'system.push.send_factor_verify_push' THEN 'Authentication' 218 | WHEN LOWER(eventType) = 'user.account.update_profile' THEN 'Account Change' 219 | WHEN LOWER(eventType) = 'user.authentication.auth_via_mfa' THEN 'Authentication' 220 | WHEN LOWER(eventType) = 'user.authentication.sso' THEN 'Authentication' 221 | WHEN LOWER(eventType) = 'user.authentication.verify' THEN 'Authentication' 222 | WHEN LOWER(eventType) = 'user.lifecycle.deactivate' THEN 'Account Change' 223 | WHEN LOWER(eventType) = 'user.lifecycle.suspend' THEN 'Account Change' 224 | WHEN LOWER(eventType) = 'user.mfa.factor.activate' THEN 'Authentication' 225 | WHEN LOWER(eventType) = 'user.mfa.factor.update' THEN 'Authentication' 226 | WHEN LOWER(eventType) = 'user.session.access_admin_app' THEN 'Authentication' 227 | WHEN LOWER(eventType) = 'user.session.clear' THEN 'Authorize Session' 228 | WHEN LOWER(eventType) = 'user.session.start' THEN 'Authorize Session' 229 | ELSE NULL 230 | END 231 | - name: ocsf_activity_id 232 | expr: | 233 | CASE 234 | WHEN LOWER(eventType) = 'app.user_management.grouppush.mapping.okta.users.ignored' THEN CAST('3' AS INT) 235 | ELSE CAST('99' AS INT) 236 | END 237 | - name: ocsf_activity_name 238 | expr: | 239 | CASE 240 | WHEN LOWER(eventType) = 'app.user_management.grouppush.mapping.okta.users.ignored' THEN 'update' 241 | ELSE 'other' 242 | END 243 | - name: ocsf_actor.user.uid 244 | expr: actor.id 245 | - name: ocsf_actor.user.name 246 | expr: actor.displayName 247 | - name: ocsf_actor.user.email_addr 248 | expr: actor.alternateId 249 | - name: ocsf_actor.user.uid_alt 250 | expr: actor.alternateId 251 | - name: ocsf_actor.session.uid 252 | expr: authenticationContext.externalSessionId 253 | - name: ocsf_actor.session.issuer 254 | expr: authenticationContext.issuer.id 255 | - name: ocsf_actor.user.type 256 | expr: | 257 | CASE 258 | WHEN LOWER(actor.type) = 'user' THEN 'user' 259 | WHEN LOWER(actor.type) = 'publicclientapp' THEN 'other' 260 | WHEN LOWER(actor.type) = 'systemprincipal' THEN 'system' 261 | ELSE 'other' 262 | END 263 | - name: ocsf_actor.user.type_id 264 | expr: | 265 | CASE 266 | WHEN LOWER(actor.type) = 'user' THEN CAST('1' AS INT) 267 | WHEN LOWER(actor.type) = 'publicclientapp' THEN CAST('99' AS INT) 268 | WHEN LOWER(actor.type) = 'systemprincipal' THEN CAST('3' AS INT) 269 | ELSE CAST('99' AS INT) 270 | END 271 | - name: ocsf_status_id 272 | expr: | 273 | CASE 274 | WHEN LOWER(outcome.result) = 'allow' THEN CAST('1' AS INT) 275 | WHEN LOWER(outcome.result) = 'success' THEN CAST('1' AS INT) 276 | WHEN LOWER(outcome.result) = 'deny' THEN CAST('2' AS INT) 277 | WHEN LOWER(outcome.result) = 'failure' THEN CAST('2' AS INT) 278 | ELSE CAST('99' AS INT) 279 | END 280 | - name: ocsf_status 281 | expr: | 282 | CASE 283 | WHEN LOWER(outcome.result) = 'allow' THEN 'success' 284 | WHEN LOWER(outcome.result) = 'success' THEN 'success' 285 | WHEN LOWER(outcome.result) = 'deny' THEN 'failure' 286 | WHEN LOWER(outcome.result) = 'failure' THEN 'failure' 287 | ELSE 'other' 288 | END 289 | - name: ocsf_status_code 290 | expr: LOWER(outcome.result) 291 | - name: ocsf_status_detail 292 | from: outcome.reason 293 | - name: ocsf_time 294 | expr: timestamp_millis(cast(_time AS BIGINT) * 1000) 295 | - name: ocsf_message 296 | from: displayMessage 297 | - name: ocsf_severity 298 | expr: | 299 | CASE 300 | WHEN LOWER(severity) = 'info' THEN 'informational' 301 | WHEN LOWER(severity) = 'warn' THEN 'high' 302 | WHEN LOWER(severity) = 'error' THEN 'fatal' 303 | ELSE 'other' 304 | END 305 | - name: ocsf_severity_id 306 | expr: | 307 | CASE 308 | WHEN LOWER(severity) = 'info' THEN CAST('1' AS INT) 309 | WHEN LOWER(severity) = 'warn' THEN CAST('4' AS INT) 310 | WHEN LOWER(severity) = 'error' THEN CAST('6' AS INT) 311 | ELSE CAST('99' AS INT) 312 | END 313 | 314 | - name: ocsf_device.hostname 315 | from: host 316 | - name: ocsf_device.ip 317 | from: client.ipAddress 318 | - name: ocsf_device.is_trusted 319 | expr: device.secure_hardware_present AND device.jailbreak IS NULL 320 | - name: ocsf_device.name 321 | from: device.name 322 | 323 | - name: ocsf_device.os 324 | from: ocsf_os 325 | - name: ocsf_device.type 326 | expr: | 327 | CASE 328 | WHEN (LOWER(client.device) == 'computer') THEN 'desktop' 329 | WHEN (LOWER(client.device) == 'tablet') THEN 'tablet' 330 | WHEN (LOWER(client.device) == 'mobile') THEN 'mobile' 331 | ELSE 'other' 332 | END 333 | - name: ocsf_device.type_id 334 | expr: | 335 | CASE 336 | WHEN (LOWER(client.device) == 'computer') THEN CAST('2' AS INT) 337 | WHEN (LOWER(client.device) == 'tablet') THEN CAST('4' AS INT) 338 | WHEN (LOWER(client.device) == 'mobile') THEN CAST('5' AS INT) 339 | ELSE CAST('99' AS INT) 340 | END 341 | - name: ocsf_device.uid 342 | from: device.id 343 | 344 | - name: ocsf_http_request.uid 345 | from: debugContext.debugData.requestId 346 | - name: ocsf_http_request.user_agent 347 | from: client.userAgent.rawUserAgent 348 | - name: ocsf_http_request.url.path 349 | from: debugContext.debugData.requestUri 350 | - name: ocsf_http_request.url.query_string 351 | expr: 352 | CASE 353 | WHEN(instr(debugContext.debugData.url, '?') > 0) THEN substring(debugContext.debugData.url, instr(debugContext.debugData.url, '?') + 1) 354 | ELSE NULL 355 | END 356 | - name: ocsf_http_request.url.url_string 357 | from: debugContext.debugData.url 358 | - name: ocsf_src_endpoint.hostname 359 | from: host 360 | - name: ocsf_src_endpoint.ip 361 | from: client.ipAddress 362 | - name: ocsf_src_endpoint.name 363 | expr: | 364 | CASE 365 | WHEN (actor.displayName IS NOT NULL) THEN actor.displayName 366 | WHEN (device.name IS NOT NULL) THEN device.name 367 | ELSE NULL 368 | END 369 | - name: ocsf_src_endpoint.autonomous_system.name 370 | from: securityContext.asOrg 371 | - name: ocsf_src_endpoint.autonomous_system.number 372 | from: securityContext.asNumber 373 | 374 | 375 | - name: ocsf_src_endpoint.os 376 | from: ocsf_os 377 | - name: ocsf_src_endpoint.uid 378 | from: uuid 379 | 380 | - name: ocsf_src_endpoint.type 381 | expr: | 382 | CASE 383 | WHEN (LOWER(client.device) = 'computer') THEN 'desktop' 384 | WHEN (LOWER(client.device) = 'laptop') THEN 'laptop' 385 | WHEN (LOWER(client.device) = 'mobile') THEN 'mobile' 386 | WHEN (LOWER(client.device) = 'tablet') THEN 'tablet' 387 | ELSE 'other' 388 | END 389 | - name: ocsf_src_endpoint.type_id 390 | expr: | 391 | CASE 392 | WHEN (LOWER(client.device) = 'computer') THEN CAST('2' AS INT) 393 | WHEN (LOWER(client.device) = 'laptop') THEN CAST('3' AS INT) 394 | WHEN (LOWER(client.device) = 'tablet') THEN CAST('4' AS INT) 395 | WHEN (LOWER(client.device) = 'mobile') THEN CAST('5' AS INT) 396 | ELSE CAST('99' AS INT) 397 | END 398 | - name: ocsf_src_endpoint.location.city 399 | from: client.geographicalContext.city 400 | - name: ocsf_src_endpoint.location.country 401 | from: client.geographicalContext.country 402 | - name: ocsf_src_endpoint.location.lat 403 | expr: CAST(client.geographicalContext.geolocation.lat AS FLOAT) 404 | - name: ocsf_src_endpoint.location.long 405 | expr: CAST(client.geographicalContext.geolocation.lon AS FLOAT) 406 | - name: ocsf_src_endpoint.location.postal_code 407 | from: client.geographicalContext.postalCode 408 | - name: ocsf_src_endpoint.location.isp 409 | from: securityContext.isp 410 | 411 | 412 | utils: 413 | temporaryFields: 414 | - name: ocsf_os.name 415 | expr: | 416 | CASE 417 | WHEN (device.os_platform IS NULL) THEN client.userAgent.os 418 | ELSE device.os_platform 419 | END 420 | - name: ocsf_os.version 421 | from: device.os_version 422 | - name: ocsf_os.build 423 | from: device.os_version 424 | - name: ocsf_os.type_id 425 | expr: | 426 | CASE 427 | WHEN (LOWER(device.os_platform) LIKE 'android%' OR LOWER(client.userAgent.os) LIKE 'android%') THEN CAST('201' AS INT) 428 | WHEN (LOWER(device.os_platform) LIKE 'linux%' OR LOWER(client.userAgent.os) LIKE 'linux%') THEN CAST('200' AS INT) 429 | WHEN (LOWER(device.os_platform) LIKE 'ubuntu%' OR LOWER(client.userAgent.os) LIKE 'ubuntu%') THEN CAST('200' AS INT) 430 | WHEN (LOWER(device.os_platform) LIKE 'chrome os%' OR LOWER(client.userAgent.os) LIKE 'chrome os%') THEN CAST('200' AS INT) 431 | WHEN (LOWER(device.os_platform) LIKE 'web os%' OR LOWER(client.userAgent.os) LIKE 'web os%') THEN CAST('200' AS INT) 432 | WHEN (LOWER(device.os_platform) LIKE 'mac os%' OR LOWER(client.userAgent.os) LIKE 'mac os%') THEN CAST('300' AS INT) 433 | WHEN (LOWER(device.os_platform) LIKE 'windows%' OR LOWER(client.userAgent.os) LIKE 'windows%') THEN CAST('100' AS INT) 434 | WHEN (LOWER(device.os_platform) LIKE 'ios%' OR LOWER(client.userAgent.os) LIKE 'ios%') THEN CAST('301' AS INT) 435 | ELSE CAST('0' AS INT) 436 | END 437 | - name: ocsf_os.type 438 | expr: | 439 | CASE 440 | WHEN (LOWER(device.os_platform) LIKE 'android%' OR LOWER(client.userAgent.os) LIKE 'android%') THEN 'android' 441 | WHEN (LOWER(device.os_platform) LIKE 'linux%' OR LOWER(client.userAgent.os) LIKE 'linux%') THEN 'windows' 442 | WHEN (LOWER(device.os_platform) LIKE 'ubuntu%' OR LOWER(client.userAgent.os) LIKE 'ubuntu%') THEN 'linux' 443 | WHEN (LOWER(device.os_platform) LIKE 'chrome os%' OR LOWER(client.userAgent.os) LIKE 'chrome os%') THEN 'linux' 444 | WHEN (LOWER(device.os_platform) LIKE 'web os%' OR LOWER(client.userAgent.os) LIKE 'web os%') THEN 'linux' 445 | WHEN (LOWER(device.os_platform) LIKE 'mac os%' OR LOWER(client.userAgent.os) LIKE 'mac os%') THEN 'macos' 446 | WHEN (LOWER(device.os_platform) LIKE 'windows%' OR LOWER(client.userAgent.os) LIKE 'windows%') THEN 'windows' 447 | WHEN (LOWER(device.os_platform) LIKE 'ios%' OR LOWER(client.userAgent.os) LIKE 'ios%') THEN 'ios' 448 | ELSE 'unknown' 449 | END 450 | 451 | unreferencedColumns: 452 | preserve: true 453 | omitColumns: 454 | - _time 455 | 456 | gold: 457 | - name: account_change 458 | input: okta_syslog 459 | filter: ocsf_class_uid = 3001 460 | fields: 461 | - name: type_uid 462 | expr: (ocsf_class_uid * 100) + ocsf_activity_id 463 | - name: category_uid 464 | from: ocsf_category_uid 465 | - name: category_name 466 | from: ocsf_category_name 467 | - name: class_uid 468 | from: ocsf_class_uid 469 | - name: class_name 470 | from: ocsf_class_name 471 | - name: activity_id 472 | from: ocsf_activity_id 473 | - name: activity_name 474 | from: ocsf_activity_name 475 | - name: status 476 | from: ocsf_status 477 | - name: status_id 478 | from: ocsf_status_id 479 | - name: status_code 480 | from: ocsf_status_code 481 | - name: status_detail 482 | from: ocsf_status_detail 483 | - name: time 484 | from: ocsf_time 485 | - name: message 486 | from: ocsf_message 487 | - name: severity 488 | from: ocsf_severity 489 | - name: severity_id 490 | from: ocsf_severity_id 491 | - name: actor 492 | from: ocsf_actor 493 | - name: device 494 | from: ocsf_device 495 | - name: http_request 496 | from: ocsf_http_request 497 | - name: src_endpoint 498 | from: ocsf_src_endpoint 499 | 500 | - name: authentication 501 | input: okta_syslog 502 | filter: ocsf_class_uid = 3002 503 | fields: 504 | - name: type_uid 505 | expr: (ocsf_class_uid * 100) + ocsf_activity_id 506 | - name: category_uid 507 | from: ocsf_category_uid 508 | - name: category_name 509 | from: ocsf_category_name 510 | - name: class_uid 511 | from: ocsf_class_uid 512 | - name: class_name 513 | from: ocsf_class_name 514 | - name: activity_id 515 | from: ocsf_activity_id 516 | - name: activity_name 517 | from: ocsf_activity_name 518 | - name: status 519 | from: ocsf_status 520 | - name: status_id 521 | from: ocsf_status_id 522 | - name: status_code 523 | from: ocsf_status_code 524 | - name: status_detail 525 | from: ocsf_status_detail 526 | - name: time 527 | from: ocsf_time 528 | - name: message 529 | from: ocsf_message 530 | - name: severity 531 | from: ocsf_severity 532 | - name: severity_id 533 | from: ocsf_severity_id 534 | - name: actor 535 | from: ocsf_actor 536 | - name: device 537 | from: ocsf_device 538 | - name: http_request 539 | from: ocsf_http_request 540 | - name: src_endpoint 541 | from: ocsf_src_endpoint 542 | 543 | - name: authorize_session 544 | input: okta_syslog 545 | filter: ocsf_class_uid = 3003 546 | fields: 547 | - name: type_uid 548 | expr: (ocsf_class_uid * 100) + ocsf_activity_id 549 | - name: category_uid 550 | from: ocsf_category_uid 551 | - name: category_name 552 | from: ocsf_category_name 553 | - name: class_uid 554 | from: ocsf_class_uid 555 | - name: class_name 556 | from: ocsf_class_name 557 | - name: activity_id 558 | from: ocsf_activity_id 559 | - name: activity_name 560 | from: ocsf_activity_name 561 | - name: status 562 | from: ocsf_status 563 | - name: status_id 564 | from: ocsf_status_id 565 | - name: status_code 566 | from: ocsf_status_code 567 | - name: status_detail 568 | from: ocsf_status_detail 569 | - name: time 570 | from: ocsf_time 571 | - name: message 572 | from: ocsf_message 573 | - name: severity 574 | from: ocsf_severity 575 | - name: severity_id 576 | from: ocsf_severity_id 577 | - name: actor 578 | from: ocsf_actor 579 | - name: device 580 | from: ocsf_device 581 | - name: http_request 582 | from: ocsf_http_request 583 | - name: src_endpoint 584 | from: ocsf_src_endpoint 585 | - name: dst_endpoint.svc_name 586 | expr: debugContext.debugData.url 587 | 588 | - name: entity_management 589 | input: okta_syslog 590 | filter: ocsf_class_uid = 3004 591 | fields: 592 | - name: type_uid 593 | expr: (ocsf_class_uid * 100) + ocsf_activity_id 594 | - name: category_uid 595 | from: ocsf_category_uid 596 | - name: category_name 597 | from: ocsf_category_name 598 | - name: class_uid 599 | from: ocsf_class_uid 600 | - name: class_name 601 | from: ocsf_class_name 602 | - name: activity_id 603 | from: ocsf_activity_id 604 | - name: activity_name 605 | from: ocsf_activity_name 606 | - name: status 607 | from: ocsf_status 608 | - name: status_id 609 | from: ocsf_status_id 610 | - name: status_code 611 | from: ocsf_status_code 612 | - name: status_detail 613 | from: ocsf_status_detail 614 | - name: time 615 | from: ocsf_time 616 | - name: message 617 | from: ocsf_message 618 | - name: severity 619 | from: ocsf_severity 620 | - name: severity_id 621 | from: ocsf_severity_id 622 | - name: actor 623 | from: ocsf_actor 624 | - name: device 625 | from: ocsf_device 626 | - name: http_request 627 | from: ocsf_http_request 628 | - name: src_endpoint 629 | from: ocsf_src_endpoint 630 | 631 | - name: user_access 632 | input: okta_syslog 633 | filter: ocsf_class_uid = 3005 634 | fields: 635 | - name: type_uid 636 | expr: (ocsf_class_uid * 100) + ocsf_activity_id 637 | - name: category_uid 638 | from: ocsf_category_uid 639 | - name: category_name 640 | from: ocsf_category_name 641 | - name: class_uid 642 | from: ocsf_class_uid 643 | - name: class_name 644 | from: ocsf_class_name 645 | - name: activity_id 646 | from: ocsf_activity_id 647 | - name: activity_name 648 | from: ocsf_activity_name 649 | - name: status 650 | from: ocsf_status 651 | - name: status_id 652 | from: ocsf_status_id 653 | - name: status_code 654 | from: ocsf_status_code 655 | - name: status_detail 656 | from: ocsf_status_detail 657 | - name: time 658 | from: ocsf_time 659 | - name: message 660 | from: ocsf_message 661 | - name: severity 662 | from: ocsf_severity 663 | - name: severity_id 664 | from: ocsf_severity_id 665 | - name: actor 666 | from: ocsf_actor 667 | - name: device 668 | from: ocsf_device 669 | - name: http_request 670 | from: ocsf_http_request 671 | - name: src_endpoint 672 | from: ocsf_src_endpoint 673 | 674 | - name: group_management 675 | input: okta_syslog 676 | filter: ocsf_class_uid = 3006 677 | fields: 678 | - name: type_uid 679 | expr: (ocsf_class_uid * 100) + ocsf_activity_id 680 | - name: category_uid 681 | from: ocsf_category_uid 682 | - name: category_name 683 | from: ocsf_category_name 684 | - name: class_uid 685 | from: ocsf_class_uid 686 | - name: class_name 687 | from: ocsf_class_name 688 | - name: activity_id 689 | from: ocsf_activity_id 690 | - name: activity_name 691 | from: ocsf_activity_name 692 | - name: status 693 | from: ocsf_status 694 | - name: status_id 695 | from: ocsf_status_id 696 | - name: status_code 697 | from: ocsf_status_code 698 | - name: status_detail 699 | from: ocsf_status_detail 700 | - name: time 701 | from: ocsf_time 702 | - name: message 703 | from: ocsf_message 704 | - name: severity 705 | from: ocsf_severity 706 | - name: severity_id 707 | from: ocsf_severity_id 708 | - name: actor 709 | from: ocsf_actor 710 | - name: device 711 | from: ocsf_device 712 | - name: http_request 713 | from: ocsf_http_request 714 | - name: src_endpoint 715 | from: ocsf_src_endpoint 716 | --------------------------------------------------------------------------------