├── LICENSE ├── README.md ├── armis ├── .env.yml ├── README.md ├── armis_client.py ├── main.py ├── main_test.py └── requirements.txt ├── aruba_central ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── azure_eventhub ├── AzureEventhub.zip ├── Azure_eventhub_API_function_app.json ├── README.md ├── azure_eventhub_api_function │ ├── function.json │ ├── main.py │ └── main_test.py ├── azuredeploy_Connector_EventHub_AzureFunctions.json ├── host.json ├── proxies.json └── requirements.txt ├── box_events ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── citrix_auditlogs ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── citrix_sessions ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── common ├── __init__.py ├── auth.py ├── auth_test.py ├── env_constants.py ├── ingest.py ├── ingest_test.py ├── status.py ├── utils.py └── utils_test.py ├── dataminr ├── .env.yml ├── README.md ├── dataminr_client.py ├── main.py ├── main_test.py └── requirements.txt ├── domaintools ├── README.md ├── domaintool_client.py ├── domaintools_env_constants.py ├── fetch_logs.py ├── fetch_logs_test.py ├── images │ ├── adhoc_parameters.png │ ├── chronicle.png │ ├── high_risk_rule.png │ ├── medium_risk_rule.png │ ├── monitoring_list_rule.png │ ├── monitoring_tags_rule.png │ ├── reference_list.png │ └── young_domain_rule.png ├── main.py ├── main_test.py └── requirements.txt ├── duo_activity ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── duo_admin ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── google_cloud_storage ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── misp ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── onelogin_events ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── onelogin_user ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── panw_cortex_xdr ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── proofpoint ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── pubsub ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── slack ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── stix_taxii ├── .env.yml ├── README.md ├── main.py ├── main_test.py ├── requirements.txt ├── taxii_client.py ├── taxii_client_test.py └── test_data │ ├── taxii_v11_collections_response.xml │ ├── taxii_v11_discovery_response.xml │ ├── taxii_v11_discovery_response_without_collection_management.xml │ ├── taxii_v11_indicators_response_empty.xml │ └── taxii_v11_indicators_response_page_1.xml ├── teamcymru_scout ├── README.md ├── fetch_logs.py ├── fetch_logs_test.py ├── images │ ├── adhoc_parameters.png │ ├── chronicle.png │ └── reference_list.png ├── main.py ├── main_test.py ├── requirements.txt ├── teamcymru_scout_client.py ├── teamcymru_scout_client_test.py ├── teamcymru_scout_constants.py └── teamcymru_scout_env_constants.py ├── tenable ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── trend_micro ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt ├── trend_micro_vision ├── .env.yml ├── README.md ├── main.py ├── main_test.py └── requirements.txt └── vectra_xdr ├── README.md ├── constant.py ├── exception.py ├── main.py ├── main_test.py ├── requirements.txt ├── utils.py ├── utils_test.py ├── vectra_client.py └── vectra_client_test.py /README.md: -------------------------------------------------------------------------------- 1 | # Google Security Operations 3p Ingestion Scripts 2 | 3 | ## Deploying the Cloud Function 4 | 5 | ### Setting up the directory 6 | 7 | Create a new directory for the cloud function deployment and add the following 8 | files into that directory: 9 | 10 | 1. *Contents* of the desired platform (i.e. `OneLogin_User`) 11 | 2. `common` directory 12 | 13 | ### Setting the required runtime environment variables 14 | 15 | Edit the .env.yml file to populate all the required environment variables. 16 | Information related to all the environment variables can be found in the 17 | README.md file. 18 | 19 | #### Common runtime environment variables 20 | 21 | Following is the table listing all the Google Security Operations related runtime environment 22 | variables that must be configured for all the ingestion scripts. 23 | 24 | | Variable | Description | Required | Default | Secret | 25 | | ------------------------- | -------------- | -------- | ------- | ------ | 26 | | POLL_INTERVAL | Poll interval | Yes | - | No | 27 | : : for the cloud : : : : 28 | : : function. : : : : 29 | | CHRONICLE_CUSTOMER_ID | Chronicle | Yes | - | No | 30 | : : customer Id. : : : : 31 | | CHRONICLE_REGION | Chronicle | Yes | us | No | 32 | : : region. : : : : 33 | | CHRONICLE_SERVICE_ACCOUNT | Contents of | Yes | - | Yes | 34 | : : the Chronicle : : : : 35 | : : ServiceAccount : : : : 36 | : : JSON file. : : : : 37 | 38 | #### Using secrets 39 | 40 | Environment variables marked as **Secret** must be configured as secrets on 41 | Google Secret Manager. Refer 42 | [this](https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets#create) 43 | page to learn how to create secrets. 44 | 45 | Once the secrets are created on Secret Manager, use the secret's resource name 46 | as the value for environment variables. For example: 47 | 48 | ``` 49 | CHRONICLE_SERVICE_ACCOUNT: projects/{project_id}/secrets/{secret_id}/versions/{version_id} 50 | ``` 51 | 52 | #### Configuring the namespace 53 | 54 | The namespace that the Google Security Operations logs are ingested into can be configured by 55 | setting the `CHRONICLE_NAMESPACE` environment variable. 56 | 57 | ### Deploying the cloud function 58 | 59 | Execute the following command from inside the previously created directory to 60 | deploy the cloud function. 61 | 62 | ``` 63 | gcloud functions deploy --entry-point main --trigger-http --runtime python39 --env-vars-file .env.yml 64 | ``` 65 | 66 | ## Support 67 | 68 | These scripts are provided as examples and are not officially supported. We 69 | welcome feedback on how we can improve them. To submit feedback, go to the 70 | [Chronicle Ingestion Script documentation](https://cloud.google.com/chronicle/docs/ingestion/ingest-using-cloud-functions) 71 | and click "Send Feedback". 72 | 73 | ## Resources 74 | 75 | - [Install the gcloud CLI](https://cloud.google.com/sdk/docs/install) 76 | - [Deploying cloud functions from local machine](https://cloud.google.com/functions/docs/deploying/filesystem) 77 | -------------------------------------------------------------------------------- /armis/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 17 | CHRONICLE_CUSTOMER_ID: 18 | 19 | # Region where the Chronicle instance is located. 20 | CHRONICLE_REGION: us 21 | 22 | # Path of the Google Secret Manager with the version, where the Service Account is stored. 23 | CHRONICLE_SERVICE_ACCOUNT: 24 | 25 | # The namespace that the Chronicle logs are labeled with. 26 | CHRONICLE_NAMESPACE: 27 | 28 | # Time interval in minutes to fetch the data. 29 | # For Example, If the poll interval is 10, then it'll fetch logs after every 10 minutes. 30 | POLL_INTERVAL: "10" 31 | 32 | # Server URL for Armis platform. 33 | ARMIS_SERVER_URL: 34 | 35 | # Path of the Google Secret Manager with the version, where Armis API Secret key is stored. 36 | ARMIS_API_SECRET_KEY: 37 | 38 | # Proxy server URL. 39 | HTTPS_PROXY: 40 | 41 | # Log type to push data into the Chronicle. 42 | CHRONICLE_DATA_TYPE: "ARMIS_ALERTS,ARMIS_ACTIVITIES,ARMIS_DEVICES,ARMIS_VULNERABILITIES" -------------------------------------------------------------------------------- /armis/README.md: -------------------------------------------------------------------------------- 1 | # Armis Chronicle Integration 2 | 3 | This scripts collect the data using API call from the Armis platform for different types of events like alerts, activities, devices, and vulnerabilities. 4 | Furthermore, the collected data will be ingested into Chronicle and parsed by corresponding parsers. 5 | 6 | ### The overall flow of the script 7 | 8 | - Deploying the script to Cloud Function 9 | - Data collection using ingestion script 10 | - Ingest collected data into Chronicle 11 | - Collected data will be parsed through corresponding parsers in Chronicle 12 | 13 | ### Environment Variables 14 | 15 | | Variable | Description | Required | Default | Secret | 16 | | --- | --- | --- | --- | --- | 17 | | CHRONICLE_CUSTOMER_ID | Chronicle customer ID. | Yes | - | No | 18 | | CHRONICLE_REGION | Chronicle region. | Yes | us | No | 19 | | CHRONICLE_SERVICE_ACCOUNT | Contents of the Chronicle ServiceAccount JSON file. | Yes | - | Yes | 20 | | CHRONICLE_NAMESPACE | The namespace that the Chronicle logs are labeled with. | No | - | No | 21 | | POLL_INTERVAL | Frequency interval at which the function executes to get additional log data (in minutes). This duration must be the same as the Cloud Scheduler job interval. | Yes | 10 | No | 22 | | ARMIS_SERVER_URL | Server URL of Armis platform. | Yes | - | No | 23 | | ARMIS_API_SECRET_KEY | Secret key required to authenticate. | Yes | - | Yes | 24 | | HTTPS_PROXY | Proxy server URL. | No | - | No | 25 | | CHRONICLE_DATA_TYPE | Chronicle data type to push data into the Chronicle. | Yes | - | No | 26 | 27 | ### Setting up the directory 28 | 29 | Create a new directory for the cloud function deployment and add the 30 | following files into that directory: 31 | 32 | 1. *Contents* of ingestion script (i.e. `armis`) 33 | 2. `common` directory 34 | 35 | ### Setting the required runtime environment variables 36 | 37 | Edit the .env.yml file to populate all the required environment variables. 38 | Information related to all the environment variables can be found in this file. 39 | 40 | #### Using secrets 41 | 42 | Environment variables marked as **Secret** must be configured as secrets on 43 | Google Secret Manager. Refer [this](https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets#create) 44 | page to learn how to create secrets. 45 | 46 | Once the secrets are created on Secret Manager, use the secret's resource name 47 | as the value for environment variables. For example: 48 | 49 | ``` 50 | CHRONICLE_SERVICE_ACCOUNT: projects/{project_id}/secrets/{secret_id}/versions/{version_id} 51 | ``` 52 | 53 | #### Configuring the namespace 54 | 55 | The namespace that the Chronicle logs are ingested into can be configured by 56 | setting the `CHRONICLE_NAMESPACE` environment variable. 57 | 58 | ### Deploying the cloud function 59 | 60 | Execute the following command from inside the previously created directory to 61 | deploy the cloud function. 62 | 63 | ``` 64 | gcloud functions deploy --gen2 --entry-point main --trigger-http --runtime python39 --env-vars-file .env.yml 65 | ``` 66 | 67 | ### Cloud Function Default Specifications 68 | 69 | | Variable | Default Value | Description | 70 | | --- | --- | --- | 71 | | Memory | 256 MB | Allocated memory for a specific cloud function. | 72 | | Timeout | 60 seconds | Time Interval for the termination of a cloud function. | 73 | | Region | us-central1 | Region for a cloud function. | 74 | | Minimum instances | 0 | Minimum number of instance for a cloud function. | 75 | | Maximum instances | 100 | Maximum number of instances for a cloud function. | 76 | 77 | - The configuration documentation of the above variables can be found here: [link](https://cloud.google.com/functions/docs/configuring) 78 | 79 | ## Steps to fetch the historical data all at once and then continue with the real-time data collection 80 | 81 | - Configure POLL_INTERVAL environment variable in minutes for which the historical data needs to be fetched. 82 | - As the cloud function is configured, the function can be triggered using a scheduler or manually by executing the command in Google Cloud CLI. 83 | 84 | ## Resources 85 | 86 | - [Install the gcloud CLI](https://cloud.google.com/sdk/docs/install) 87 | - [Deploying cloud functions from local machine](https://cloud.google.com/functions/docs/deploying/filesystem) -------------------------------------------------------------------------------- /armis/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 20 | -------------------------------------------------------------------------------- /aruba_central/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 16 | CHRONICLE_CUSTOMER_ID: 17 | 18 | # Region where the Chronicle instance is located. 19 | CHRONICLE_REGION: us 20 | 21 | # Path of the Google Secret Manager with the version, where the Service Account is stored. 22 | CHRONICLE_SERVICE_ACCOUNT: 23 | 24 | # Time interval in minutes to fetch the data. 25 | # For Example, If the poll interval is 5, then it will fetch logs after every 5 minutes. 26 | POLL_INTERVAL: "10" 27 | 28 | # Aruba Central API gateway client ID. 29 | ARUBA_CLIENT_ID: 30 | 31 | # Path of the Google Secret Manager with the version, where Aruba Central client secret is stored. 32 | ARUBA_CLIENT_SECRET_SECRET_PATH: 33 | 34 | # Username of Aruba Central platform. 35 | ARUBA_USERNAME: 36 | 37 | # Path of the Google Secret Manager with the version, where Password of Aruba Central platform is stored. 38 | ARUBA_PASSWORD_SECRET_PATH: 39 | 40 | # Base URL of Aruba Central API gateway. 41 | ARUBA_BASE_URL: 42 | 43 | # Customer ID of Aruba Central platform. 44 | ARUBA_CUSTOMER_ID: 45 | 46 | # The namespace that the Chronicle logs are labeled with. 47 | CHRONICLE_NAMESPACE: -------------------------------------------------------------------------------- /aruba_central/README.md: -------------------------------------------------------------------------------- 1 | # Aruba Central 2 | 3 | This script fetches audit logs from Aruba Central platform and ingests them into Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | Secret | 8 | | --- | ---| --- | --- | --- | 9 | | ARUBA_CLIENT_ID |Aruba Central API gateway client ID. | Yes | - | No | 10 | | ARUBA_CLIENT_SECRET_SECRET_PATH | Aruba Central API gateway client secret. | Yes | - | Yes | 11 | | ARUBA_USERNAME | Username of Aruba Central platform. | Yes | - | No | 12 | | ARUBA_PASSWORD_SECRET_PATH | Password of Aruba Central platform. | Yes | - | Yes | 13 | | ARUBA_BASE_URL | Base URL of Aruba Central API gateway. | Yes | - | No | 14 | | ARUBA_CUSTOMER_ID | Customer ID of Aruba Central platform. | Yes | - | No | 15 | | POLL_INTERVAL | Frequency interval at which the function executes to get additional log data (in minutes). This duration must be the same as the Cloud Scheduler job interval. | Yes | 10 | No | -------------------------------------------------------------------------------- /aruba_central/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch the Audit Trail logs from the Aruba Central platform and ingest into Chronicle.""" 16 | 17 | import datetime 18 | from typing import Any, Dict 19 | 20 | import pycentral.audit_logs 21 | import pycentral.base 22 | import requests 23 | 24 | from common import ingest 25 | from common import status 26 | from common import utils 27 | 28 | 29 | # Environment variable constants. 30 | ENV_ARUBA_CLIENT_ID = "ARUBA_CLIENT_ID" 31 | ENV_ARUBA_CLIENT_SECRET_SECRET_PATH = "ARUBA_CLIENT_SECRET_SECRET_PATH" 32 | ENV_ARUBA_USERNAME = "ARUBA_USERNAME" 33 | ENV_ARUBA_PASSWORD_SECRET_PATH = "ARUBA_PASSWORD_SECRET_PATH" 34 | ENV_ARUBA_BASE_URL = "ARUBA_BASE_URL" 35 | ENV_ARUBA_CUSTOMER_ID = "ARUBA_CUSTOMER_ID" 36 | 37 | # Log type to push data into the Chronicle. 38 | CHRONICLE_DATA_TYPE = "ARUBA_CENTRAL" 39 | 40 | # Date format to be used to parse the date string to the datetime object. 41 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 42 | 43 | 44 | def get_and_ingest_audit_logs(central_info: Dict[str, Any]) -> None: 45 | """Fetch logs from Aruba Central platform and ingest it into Chronicle. 46 | 47 | Args: 48 | central_info (Dict[str, Any]): Parameter dictionary containing 49 | information related Aruba Central and API Gateway for HTTPS 50 | connection. 51 | 52 | Raises: 53 | Exception: When data could not pushed to Chronicle or Error from API 54 | while requesting audit trails. 55 | """ 56 | # Create client object for ArubaCentralBase class. 57 | try: 58 | client = pycentral.base.ArubaCentralBase(central_info=central_info) 59 | # The SDK is logging the error message and exiting the function whenever an 60 | # error occurs in the Aruba Central API. Due to this, explicitly catch 61 | # SystemExit exception and throw HTTPError. 62 | except SystemExit as error: 63 | raise requests.HTTPError( 64 | f"Exception occurred while making API call.\n{error}") from error 65 | except Exception as error: 66 | raise requests.HTTPError( 67 | f"Exception occurred while creating ArubaCentralBase client.\n{error}" 68 | ) from error 69 | 70 | # Create object for Audit class. 71 | audit = pycentral.audit_logs.Audit() 72 | 73 | # Calculate start time based on POLL_INTERVAL. 74 | start_time = utils.get_last_run_at() 75 | print(f"Audit logs will be fetched from {start_time}.") 76 | 77 | # Calculate the epoch time (in seconds) for start time and end time, 78 | # end time will be 'now'. 79 | epoch_start_time = int(start_time.timestamp()) 80 | epoch_end_time = int( 81 | (datetime.datetime.now(datetime.timezone.utc).timestamp())) 82 | 83 | # Initialize the number of page to 0 for the first API call. 84 | page_index = 0 85 | 86 | # Total logs collected from Aruba Central platform. 87 | total_logs = 0 88 | 89 | # API response contains remaining_record key to indicate if next page is 90 | # available or not. In the first iteration, it is set to True. 91 | remaining_records = True 92 | 93 | # Iterate through all the pages until logs are available and ingest data into 94 | # Chronicle. 95 | # Raise HTTPEror if response contains error code other than 200. 96 | while remaining_records: 97 | # Fetch audit trails from the Aruba Central platform. 98 | response = audit.get_traillogs( 99 | client, 100 | offset=page_index, 101 | start_time=epoch_start_time, 102 | end_time=epoch_end_time) 103 | 104 | # If status code is other than 200, raise HTTPError. 105 | if response.get("code") != status.STATUS_OK: 106 | raise requests.HTTPError( 107 | f"Exception occurred while making API call. {response.get('msg')}" 108 | ) 109 | 110 | # Per page response from the API. 111 | per_page_response = response.get("msg", {}) 112 | 113 | audit_logs = per_page_response.get("audit_logs", []) 114 | record_count = per_page_response.get("total", 0) 115 | remaining_records = per_page_response.get("remaining_records", False) 116 | 117 | print(f"{record_count} audit log(s) collected from Aruba Central platform.") 118 | 119 | # Ingest data into the Chronicle. 120 | if audit_logs: 121 | try: 122 | ingest.ingest(audit_logs, CHRONICLE_DATA_TYPE) 123 | except Exception as err: 124 | raise Exception( 125 | "Unable to push the data to the Chronicle. Please check the" 126 | " Chronicle configuration parameters." 127 | ) from err 128 | 129 | total_logs += record_count 130 | page_index += 1 131 | 132 | print(f"Total {total_logs} log(s) ingested successfully into the Chronicle.") 133 | 134 | 135 | def main(request): # pylint: disable=unused-argument 136 | """Entry point for the script. 137 | 138 | Args: 139 | request: Argument to run cloud function. 140 | 141 | Returns: 142 | string: "Ingestion completed." if function execution is successful. 143 | """ 144 | # Fetch the environment variables. 145 | client_id = utils.get_env_var(ENV_ARUBA_CLIENT_ID) 146 | client_secret = utils.get_env_var( 147 | ENV_ARUBA_CLIENT_SECRET_SECRET_PATH, is_secret=True) 148 | customer_id = utils.get_env_var(ENV_ARUBA_CUSTOMER_ID) 149 | username = utils.get_env_var(ENV_ARUBA_USERNAME) 150 | password = utils.get_env_var( 151 | ENV_ARUBA_PASSWORD_SECRET_PATH, is_secret=True) 152 | base_url = utils.get_env_var(ENV_ARUBA_BASE_URL) 153 | 154 | # Create a dictionary of parameters required to pass for creating 155 | # object of ArubaCentralBase class. 156 | central_info = { 157 | "client_id": client_id, 158 | "client_secret": client_secret, 159 | "customer_id": customer_id, 160 | "username": username, 161 | "password": password, 162 | "base_url": base_url, 163 | } 164 | 165 | # Get audit logs from Aruba Central and ingest it into Chronicle. 166 | get_and_ingest_audit_logs(central_info) 167 | 168 | return "Ingestion completed." 169 | -------------------------------------------------------------------------------- /aruba_central/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.28.1 17 | jwt==1.3.1 18 | google-auth==2.15.0 19 | google-cloud-secret-manager==2.13.0 20 | pycentral==0.0.3 -------------------------------------------------------------------------------- /azure_eventhub/AzureEventhub.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/azure_eventhub/AzureEventhub.zip -------------------------------------------------------------------------------- /azure_eventhub/Azure_eventhub_API_function_app.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "Azure EventHub integration with Google Chronicle", 3 | "title": "Azure EventHub integration with Google Chronicle", 4 | "descriptionMarkdown": "The azure function triggers when a new event is encountered in azure event hub and ingests them into Chronicle.", 5 | "additionalRequirementBanner": "The azure function requires Credentials of Chronicle platform to ingest data into the Chronicle.", 6 | "graphQueries": [ 7 | ], 8 | "sampleQueries": [ 9 | ], 10 | "dataTypes": [ 11 | ], 12 | "connectivityCriterias": [ 13 | ], 14 | "availability": { 15 | "status": 1, 16 | "isPreview": true 17 | }, 18 | "permissions": { 19 | "resourceProvider": [ 20 | { 21 | "provider": "Microsoft.OperationalInsights/workspaces", 22 | "permissionsDisplayText": "read and write permissions on the workspace are required.", 23 | "providerDisplayName": "Workspace", 24 | "scope": "Workspace", 25 | "requiredPermissions": { 26 | "write": true, 27 | "read": true, 28 | "delete": true 29 | } 30 | }, 31 | { 32 | "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", 33 | "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", 34 | "providerDisplayName": "Keys", 35 | "scope": "Workspace", 36 | "requiredPermissions": { 37 | "action": true 38 | } 39 | } 40 | ], 41 | "customs": [{ 42 | "name": "Microsoft.Web/sites permissions", 43 | "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." 44 | }, 45 | { 46 | "name": "Chronicle API Credentials/permissions", 47 | "description": "**Chronicle Customer ID**, **Chronicle Region**, **Chronicle Service Account** and **Chronicle Data Type** is required. See the documentation to learn more about API on the `https://cloud.google.com/chronicle/docs/reference/ingestion-api`" 48 | } 49 | ] 50 | }, 51 | "instructionSteps": [{ 52 | "title": "", 53 | "description": ">**NOTE:** This connector uses Azure Functions to connect to the Chronicle Ingestion API and pushes events from Azure Event Hub. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details." 54 | }, 55 | { 56 | "title": "Deploy Azure Function using following deployment steps by Azure Resource Manager (ARM) Template", 57 | "description": "Use this method for automated deployment of the Azure Ingestion Script.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fchronicle%2Fingestion-scripts%2Fmain%2Fazure_eventhub%2Fazuredeploy_Connector_EventHub_AzureFunctions.json)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the below information : \n\t\tFunction Name \n\t\tEvent Hub Name \n\t\tEvent Hub Namespace \n\t\tShared Access Key \n\t\tChronicle Customer ID \n\t\tChronicle Service Account JSON \n\t\tChronicle Region \n\t\tChronicle Data Type \n4. Click **Review + Create ** button. \n5. Click **Create** to deploy." 58 | } 59 | ] 60 | } -------------------------------------------------------------------------------- /azure_eventhub/README.md: -------------------------------------------------------------------------------- 1 | # Azure Event Hub 2 | 3 | This script fetches the Azure EventHub data and ingests it into Chronicle. 4 | 5 | ## Steps to deploy Azure Function 6 | 1. Login into the Microsoft Azure Portal (https://portal.azure.com/). 7 | 2. From the Azure portal search bar, search for **Deploy a custom template** Azure service and select the service from the available options. This step will redirect to the **Custom deployment** page. 8 | 3. Download the ARM template **azuredeploy_Connector_EventHub_AzureFunctions.json** file from the repository. 9 | 4. Click on **Build your own template in the editor** option. This step will open the **Edit template** page. 10 | 5. Click on **Load file** option and upload the ARM template file downloaded at step 3. 11 | 6. Select **Save**. 12 | 7. You see the blade for providing the deployment values. Select the preferred Subscription, Resource Group, Region and provide the Function specific values. Description for all the function-specific parameters are provided in the below table. 13 | 8. Click **Review + Create** button. 14 | 9. Click **Create** to deploy. 15 | 16 | Now the deployed Azure function will trigger for new data in Azure Event Hub and ingest them into Chronicle. 17 | 18 | ## Platform Specific Parameters in Azure Function 19 | 20 | | Parameter | Description | Required | Default | 21 | | --------------------------- | ----------------------------------------- | -------- | ------- | 22 | | Function Name | Function names allow only alphanumeric characters. Special characters are not allowed and length of the name should be less than or equal to 11 characters. | Yes | Chronicle | 23 | | Eventhub Namespace | Namespace of the Eventhub from which data should be collected. | Yes | - | 24 | | Eventhub Name | Name of the Eventhub from which data should be collected. | Yes | - | 25 | | Shared Access Key | Primary key of the Eventhub namespace. | Yes | - | 26 | | Chronicle Customer ID | Customer ID of Google Chronicle. | Yes | - | 27 | | Chronicle Service Account | Provide the Google Chronicle Service Account JSON. | Yes | - | 28 | | Chronicle Region | Specify the Google Chronicle region. | Yes | us | 29 | | Chronicle Data Type | Specify the log type to ingest data into Chronicle. | Yes | - | 30 | 31 | -------------------------------------------------------------------------------- /azure_eventhub/azure_eventhub_api_function/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "scriptFile": "main.py", 3 | "bindings": [ 4 | { 5 | "type": "eventHubTrigger", 6 | "name": "events", 7 | "direction": "in", 8 | "eventHubName": "%EventhubName%", 9 | "connection": "EVENT_HUB_CONNECTION_STRING", 10 | "cardinality": "many", 11 | "consumerGroup": "$Default" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /azure_eventhub/azure_eventhub_api_function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch the logs from Azure Event Hub and ingest them into Chronicle.""" 16 | import json 17 | import logging 18 | from typing import List 19 | 20 | import azure.functions as func 21 | from common import ingest 22 | from common import utils 23 | 24 | # Environment variable constants. 25 | ENV_CHRONICLE_DATA_TYPE = "CHRONICLE_DATA_TYPE" 26 | 27 | 28 | def main(events: List[func.EventHubEvent]) -> None: 29 | """Entrypoint. 30 | 31 | Args: 32 | events: Events from the Azure Event Hub. 33 | """ 34 | # Fetch environment variables. 35 | chronicle_data_type = utils.get_env_var(ENV_CHRONICLE_DATA_TYPE) 36 | events_to_send = [] 37 | 38 | # Iterating over the list of EventHub logs to decode and JSON serialize them. 39 | for event in events: 40 | try: 41 | records = json.loads(event.get_body().decode("utf-8"))["records"] 42 | # Raise error if the event received from the Azure EventHub is not JSON 43 | # serializable. 44 | except json.JSONDecodeError as error: 45 | print("Could not JSON serialize the Azure EventHub log.") 46 | raise RuntimeError( 47 | "The log data from Azure EventHub is not JSON serializable." 48 | ) from error 49 | 50 | # If events are nested in the list form in Eventhub log message. 51 | # Example: {"records": [event1, event2, event3, ...]} 52 | if isinstance(records, list): 53 | events_to_send.extend(records) 54 | else: 55 | events_to_send.append(records) 56 | 57 | events_count = len(events_to_send) 58 | logging.info( 59 | "Parsed %s events from Azure EventHub. Sending events to Chronicle.", 60 | events_count 61 | ) 62 | 63 | try: 64 | # Ingest Azure EventHub logs to Chronicle. 65 | ingest.ingest(events_to_send, chronicle_data_type) 66 | except Exception as error: 67 | raise Exception(f"Unable to push the data to the Chronicle. Error: {error}" # pylint: disable=broad-exception-raised 68 | ) from error 69 | 70 | logging.info( 71 | "Total %s log(s) are successfully ingested to Chronicle.", 72 | events_count, 73 | ) 74 | -------------------------------------------------------------------------------- /azure_eventhub/azure_eventhub_api_function/main_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Unit test case file for main module of AzureEventHub script.""" 16 | import sys 17 | import unittest 18 | from unittest import mock 19 | 20 | INGESTION_SCRIPTS_PATH = "" 21 | SCRIPT_PATH = "" 22 | 23 | # CONSTANTS. 24 | ENV_VARS = ["AZURE_AD"] 25 | 26 | sys.modules[f"{INGESTION_SCRIPTS_PATH}common.ingest"] = mock.MagicMock() 27 | sys.modules[f"{INGESTION_SCRIPTS_PATH}common.utils"] = mock.MagicMock() 28 | 29 | import main 30 | 31 | 32 | class TestEventHubToChronicleIngestion(unittest.TestCase): 33 | """Test cases for Azure EventHub ingestion script.""" 34 | 35 | @mock.patch(f"{SCRIPT_PATH}main.utils") 36 | @mock.patch(f"{SCRIPT_PATH}main.ingest.ingest") 37 | def test_ingestion_successful(self, mock_ingest, mock_utils): 38 | """Test case to verify for successful ingestion of logs.""" 39 | mock_utils.get_env_var.side_effect = ENV_VARS 40 | 41 | events = [mock.MagicMock()] 42 | events[0].get_body.return_value = b'{"records": []}' 43 | main.main(events) 44 | assert mock_ingest.call_count == 1 45 | 46 | @mock.patch(f"{SCRIPT_PATH}main.utils") 47 | def test_json_decode_error(self, mock_utils): 48 | """Test case to verify json loads for failure.""" 49 | mock_utils.get_env_var.side_effect = ENV_VARS 50 | events = [mock.MagicMock()] 51 | events[0].get_body.return_value = b'{"records": [}' 52 | with self.assertRaises(RuntimeError) as error: 53 | main.main(events) 54 | 55 | self.assertEqual( 56 | str(error.exception), 57 | "The log data from Azure EventHub is not JSON serializable.", 58 | ) 59 | 60 | @mock.patch(f"{SCRIPT_PATH}main.utils") 61 | @mock.patch(f"{SCRIPT_PATH}main.ingest.ingest") 62 | def test_list_data_parsing(self, mock_ingest, mock_utils): 63 | """Test case to verify json loads for failure.""" 64 | mock_utils.get_env_var.side_effect = ENV_VARS 65 | events = [mock.MagicMock()] 66 | events[0].get_body.return_value = b'{"records": ["e1", "e2", "e3"]}' 67 | main.main(events) 68 | args, kwargs = mock_ingest.call_args # pylint: disable=unused-variable 69 | 70 | events_expected = ["e1", "e2", "e3"] 71 | assert mock_ingest.call_count == 1 72 | self.assertEqual( 73 | args[0], events_expected 74 | ) 75 | 76 | @mock.patch(f"{SCRIPT_PATH}main.utils") 77 | @mock.patch(f"{SCRIPT_PATH}main.ingest.ingest") 78 | def test_str_data_parsing(self, mock_ingest, mock_utils): 79 | """Test case to verify json loads for failure.""" 80 | mock_utils.get_env_var.side_effect = ENV_VARS 81 | events = [mock.MagicMock()] 82 | events[0].get_body.return_value = b'{"records": "e1"}' 83 | main.main(events) 84 | args, kwargs = mock_ingest.call_args # pylint: disable=unused-variable 85 | 86 | events_expected = ["e1"] 87 | assert mock_ingest.call_count == 1 88 | self.assertEqual( 89 | args[0], events_expected 90 | ) 91 | 92 | @mock.patch(f"{SCRIPT_PATH}main.utils") 93 | @mock.patch(f"{SCRIPT_PATH}main.ingest") 94 | def test_ingest_for_error(self, mock_ingest, mock_utils): 95 | """Test case to verify error is raised for failure in ingest.""" 96 | mock_utils.get_env_var.side_effect = ENV_VARS 97 | mock_ingest.ingest.side_effect = Exception("Custom error for testing") 98 | events = [mock.MagicMock()] 99 | events[0].get_body.return_value = b'{"records": []}' 100 | 101 | with self.assertRaises(Exception) as error: 102 | main.main(events) 103 | 104 | self.assertEqual( 105 | str(error.exception), 106 | "Unable to push the data to the Chronicle. Error: Custom error for" 107 | " testing", 108 | ) 109 | -------------------------------------------------------------------------------- /azure_eventhub/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | }, 11 | "extensionBundle": { 12 | "id": "Microsoft.Azure.Functions.ExtensionBundle", 13 | "version": "[3.*, 4.0.0)" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /azure_eventhub/proxies.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/proxies", 3 | "proxies": {} 4 | } 5 | -------------------------------------------------------------------------------- /azure_eventhub/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # DO NOT include azure-functions-worker in this file 17 | # The Python Worker is managed by Azure Functions platform 18 | # Manually managing azure-functions-worker may cause unexpected issues 19 | 20 | azure-functions 21 | requests==2.28.2 22 | jwt==1.3.1 23 | google-auth==2.16.0 24 | google-cloud-secret-manager==2.15.0 -------------------------------------------------------------------------------- /box_events/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | BOX_CLIENT_ID: 20 | BOX_CLIENT_SECRET: 21 | BOX_SUBJECT_ID: 22 | # Keeping the default value as 5 minutes considering the frequency of updates in Box events. 23 | POLL_INTERVAL: "5" 24 | -------------------------------------------------------------------------------- /box_events/README.md: -------------------------------------------------------------------------------- 1 | # Box Events 2 | 3 | This script pulls events from Box and ingests them into Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | Secret | 8 | | ----------------- | --------------------------------------------------------------- | -------- | ------- | ------ | 9 | | BOX_CLIENT_ID | Client id of box platform (available in box developer console). | Yes | - | No | 10 | | BOX_CLIENT_SECRET | Client secret of box platform. | Yes | - | Yes | 11 | | BOX_SUBJECT_ID | User ID or enterprise ID. | Yes | - | No | 12 | | POLL_INTERVAL | Frequency interval(in minutes) at which the Cloud Function executes. This duration must be same as the cloud scheduler job. | No | 5 | No | 13 | 14 | ## Resources 15 | 16 | - [How to get your API Key](https://support.box.com/hc/en-us/articles/360052055274-Developer-How-to-get-your-API-Key) 17 | -------------------------------------------------------------------------------- /box_events/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch events data from Box API.""" 16 | 17 | import datetime 18 | import requests 19 | 20 | from common import auth 21 | from common import ingest 22 | from common import utils 23 | 24 | # Default page size to fetch events from box. 25 | PAGE_SIZE = 100 26 | 27 | # Box event API endpoint URL. 28 | BOX_EVENTS_URL = "https://api.box.com/2.0/events" 29 | 30 | # Box authentication endpoint URL. 31 | BOX_AUTH_URL = "https://api.box.com/oauth2/token" 32 | 33 | # Log type to push data into Chronicle. 34 | CHRONICLE_DATA_TYPE = "BOX" 35 | 36 | # Environment variables constants. 37 | ENV_BOX_CLIENT_ID = "BOX_CLIENT_ID" 38 | ENV_BOX_CLIENT_SECRET = "BOX_CLIENT_SECRET" 39 | ENV_BOX_SUBJECT_ID = "BOX_SUBJECT_ID" 40 | 41 | BOX_SUBJECT_TYPE = "enterprise" 42 | 43 | 44 | def get_and_ingest_events_from_box( 45 | session: auth.OAuthClientCredentialsAuth) -> None: 46 | """Fetch events from BOX platform and ingest into Chronicle. 47 | 48 | Args: 49 | session (OAuthClientCredentialsAuth): Authorized session for HTTP requests. 50 | 51 | Raises: 52 | TypeError, ValueError: Error when response is not in json format. 53 | """ 54 | # Calculate start time based on POLL_INTERVAL, end time will be 'now'. 55 | # Convert datetime object into the expected format (YYYY-MM-DDTHH:MM:SSSZ). 56 | start_time = utils.get_last_run_at().strftime("%Y-%m-%dT%H:%M:%SZ") 57 | end_time = datetime.datetime.now( 58 | datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 59 | 60 | params = { 61 | "stream_type": "admin_logs", 62 | "limit": PAGE_SIZE, 63 | "created_after": start_time, 64 | "created_before": end_time, 65 | } 66 | 67 | def before_next(request: requests.Request, response: requests.Response): 68 | """Execute function before executing next API call. 69 | 70 | Args: 71 | request (requests.Request): User created request object. 72 | response (requests.Response): Response from HTTP request. 73 | 74 | Returns: 75 | request: Updated request object. 76 | """ 77 | request.params["stream_position"] = response.json().get( 78 | "next_stream_position") 79 | return request 80 | 81 | # Iterate through all the pages if pagination available and ingest data into 82 | # Chronicle. 83 | for response in session.paginate( 84 | "GET", 85 | BOX_EVENTS_URL, 86 | params=params, 87 | has_next=lambda response: response.json().get("chunk_size") != 0, 88 | before_next=before_next, 89 | ): 90 | 91 | try: 92 | box_response = response.json() 93 | except (TypeError, ValueError) as error: 94 | print( 95 | "ERROR: Unexpected data format received while collecting Box events") 96 | raise error 97 | 98 | data_list = box_response.get("entries", []) 99 | print(f"Retrieved {len(data_list)} Box events from the last API call.") 100 | 101 | # Ingest data into the Chronicle. 102 | if data_list: 103 | ingest.ingest(data_list, CHRONICLE_DATA_TYPE) 104 | 105 | 106 | def main(request): # pylint: disable=unused-argument 107 | """Entrypoint. 108 | 109 | Args: 110 | request: Request to execute the cloud function. 111 | 112 | Returns: 113 | string: "Ingestion completed." 114 | """ 115 | # Fetching values from the environment variables. 116 | client_id = utils.get_env_var(ENV_BOX_CLIENT_ID) 117 | client_secret = utils.get_env_var(ENV_BOX_CLIENT_SECRET, is_secret=True) 118 | box_subject_id = utils.get_env_var(ENV_BOX_SUBJECT_ID) 119 | 120 | def before_request(request): 121 | request.data["box_subject_type"] = BOX_SUBJECT_TYPE 122 | request.data["box_subject_id"] = box_subject_id 123 | return request 124 | 125 | # Create a new Box session. 126 | session = auth.OAuthClientCredentialsAuth( 127 | BOX_AUTH_URL, 128 | client_id, 129 | client_secret, 130 | before_request=before_request, 131 | ) 132 | 133 | get_and_ingest_events_from_box(session) 134 | 135 | return "Ingestion completed." 136 | -------------------------------------------------------------------------------- /box_events/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /citrix_auditlogs/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | CITRIX_CLIENT_ID: 20 | CITRIX_CLIENT_SECRET: 21 | CITRIX_CUSTOMER_ID: 22 | # Keeping the defult value as 30 minutes considering the frequency interval of Citrix audit logs. 23 | POLL_INTERVAL: "30" 24 | URL_DOMAIN: 25 | -------------------------------------------------------------------------------- /citrix_auditlogs/README.md: -------------------------------------------------------------------------------- 1 | # Citrix Audit Logs 2 | 3 | This script collects audit logs from citrix and ingests them into Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | Secret | 8 | | --------------------------- | ----------------------------------------- | -------- | ------- | ------ | 9 | | CITRIX_CLIENT_ID | Client ID of Citrix platform. | Yes | - | No | 10 | | CITRIX_CLIENT_SECRET | Client Secret of Citrix platform. | Yes | - | Yes | 11 | | CITRIX_CUSTOMER_ID | ID of the customer. | Yes | - | No | 12 | | POLL_INTERVAL | Frequency interval(in minutes) at which the Cloud Function executes. This duration must be same as the cloud scheduler job. | No | 30 | No | 13 | | URL_DOMAIN | Citrix Cloud Endpoint. | Yes | - | No | 14 | -------------------------------------------------------------------------------- /citrix_auditlogs/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /citrix_sessions/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | CITRIX_CLIENT_ID: 20 | CITRIX_CLIENT_SECRET: 21 | CITRIX_CUSTOMER_ID: 22 | # Keeping the default value as 30 minutes considering the frequency of updates in session metadata. 23 | POLL_INTERVAL: "30" 24 | URL_DOMAIN: -------------------------------------------------------------------------------- /citrix_sessions/README.md: -------------------------------------------------------------------------------- 1 | # Citrix Sessions 2 | 3 | This script collects citrix metadata which helps in user traffic monitoring on 4 | citrix environments. 5 | 6 | ## Platform Specific Environment Variables 7 | 8 | | Variable | Description | Required | Default | Secret | 9 | | --------------------------- | ----------------------------------------- | -------- | ------- | ------ | 10 | | CITRIX_CLIENT_ID | Client ID of Citrix platform. | Yes | - | No | 11 | | CITRIX_CLIENT_SECRET | Client Secret of Citrix platform. | Yes | - | Yes | 12 | | CITRIX_CUSTOMER_ID | ID of the customer. | Yes | - | No | 13 | | POLL_INTERVAL | Frequency interval(in minutes) at which the Cloud Function executes. This duration must be same as the cloud scheduler job. | No | 30 | No | 14 | | URL_DOMAIN | Citrix Cloud Endpoint. | Yes | - | No | 15 | -------------------------------------------------------------------------------- /citrix_sessions/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Common module for Chronicle ingestion scripts.""" 16 | 17 | -------------------------------------------------------------------------------- /common/env_constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Common environment constants to be used across the project.""" 16 | 17 | ENV_POLL_INTERVAL = "POLL_INTERVAL" 18 | ENV_CHRONICLE_CUSTOMER_ID = "CHRONICLE_CUSTOMER_ID" 19 | ENV_CHRONICLE_REGION = "CHRONICLE_REGION" 20 | ENV_CHRONICLE_SERVICE_ACCOUNT = "CHRONICLE_SERVICE_ACCOUNT" 21 | ENV_CHRONICLE_NAMESPACE = "CHRONICLE_NAMESPACE" 22 | ENV_CHRONICLE_DATA_TYPE = "CHRONICLE_DATA_TYPE" 23 | ENV_GCP_BUCKET_NAME = "GCP_BUCKET_NAME" 24 | ENV_REDIS_HOST = "REDIS_HOST" 25 | ENV_REDIS_PORT = "REDIS_PORT" 26 | -------------------------------------------------------------------------------- /common/status.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """HTTP status constants to be used across the project.""" 16 | import http 17 | 18 | STATUS_OK = http.HTTPStatus.OK.value # For status code 200. 19 | STATUS_BAD_REQUEST = http.HTTPStatus.BAD_REQUEST.value # For status code 400. 20 | STATUS_UNAUTHORIZED = http.HTTPStatus.UNAUTHORIZED.value # For status code 401. 21 | STATUS_FORBIDDEN = http.HTTPStatus.FORBIDDEN.value # For status code 403. 22 | STATUS_NOT_FOUND = http.HTTPStatus.NOT_FOUND.value # For status code 404. 23 | # For status code 429. 24 | STATUS_TOO_MANY_REQUESTS = http.HTTPStatus.TOO_MANY_REQUESTS.value 25 | # For status code 500. 26 | STATUS_INTERNAL_SERVER_ERROR = http.HTTPStatus.INTERNAL_SERVER_ERROR.value 27 | -------------------------------------------------------------------------------- /common/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Utility functions required for ingestion scripts.""" 16 | 17 | import datetime 18 | import json 19 | import os 20 | from typing import Dict, Any 21 | 22 | from google.cloud import secretmanager 23 | 24 | from common import env_constants 25 | 26 | 27 | def get_env_var( 28 | name: str, 29 | required: bool = True, 30 | default: Any = None, 31 | is_secret: bool = False, 32 | ) -> Any: 33 | """Gets an environment variable. 34 | 35 | Args: 36 | name (str): Name of the environment variable. 37 | required (Optional[bool]): Script will exit with RuntimeError if this is 38 | True and variable is not set. Defaults to True. 39 | default (Optional[Any]): Default value to return in case the env variable is 40 | not set. Defaults to None. 41 | is_secret (bool): Script will get data from Google Cloud Secret Manager in 42 | case it is set to true. 43 | 44 | Returns: 45 | Any: Value of the environment variable. 46 | 47 | Raises: 48 | RuntimeError: Raises when required name is not in environment variable. 49 | """ 50 | if name not in os.environ and required: 51 | raise RuntimeError(f"Environment variable {name} is required.") 52 | if is_secret: 53 | return get_value_from_secret_manager(os.environ[name]) 54 | if name not in os.environ or (name in os.environ and 55 | not os.environ[name].strip()): 56 | return default 57 | return os.environ[name] 58 | 59 | 60 | def get_last_run_at() -> datetime.datetime: 61 | """Calculates the start time for data collection based on POLL_INTERVAL environment variable. 62 | 63 | If the POLL_INTERVAL environment variable is not set, then the function will 64 | return the start time as the last 5 minutes from the current time. 65 | 66 | Returns: 67 | datetime.datetime: Start time for data collection. 68 | 69 | Raises: 70 | RuntimeError: If the value of the POLL_INTERVAL is negative or zero. 71 | """ 72 | try: 73 | # If the POLL_INTERVAL is not passed, the default value will considered as 74 | # last 5 minutes from the current time. 75 | poll_interval = get_env_var( 76 | env_constants.ENV_POLL_INTERVAL, required=False, default=5) 77 | 78 | if int(poll_interval) <= 0: 79 | raise ValueError 80 | 81 | return datetime.datetime.now( 82 | datetime.timezone.utc) - datetime.timedelta(minutes=int(poll_interval)) 83 | except ValueError as error: 84 | raise RuntimeError( 85 | "Invalid value provided for the POLL_INTERVAL environment variable. A " 86 | "POLL_INTERVAL should be a non-zero positive integer value.") from error 87 | 88 | 89 | def get_value_from_secret_manager(resource_path: str) -> str: 90 | """Retrieve the value of the secret from the Google Cloud Secret Manager. 91 | 92 | Args: 93 | resource_path (str): Path of the secret with version included. Ex.: 94 | "projects//secrets//versions/1", 95 | "projects//secrets//versions/latest" 96 | 97 | Returns: 98 | str: Payload for secret. 99 | """ 100 | # Create the Secret Manager client. 101 | client = secretmanager.SecretManagerServiceClient() 102 | 103 | # Access the secret version. 104 | response = client.access_secret_version(name=resource_path) 105 | return response.payload.data.decode("UTF-8") 106 | 107 | 108 | def load_service_account(service_account: str, 109 | product_name: str) -> Dict[str, Any]: 110 | """Load a service account string to the dictionary. 111 | 112 | Args: 113 | service_account (str): Service account string. 114 | product_name (str): The name of the product whose service_account string 115 | is serialized. 116 | 117 | Returns: 118 | service_account_dict (Dict): Parsed service account dictionary from 119 | given string. 120 | 121 | Raises: 122 | RuntimeError: If the provided service account string is not JSON 123 | serializable. 124 | """ 125 | try: 126 | return json.loads(service_account) 127 | except json.JSONDecodeError as error: 128 | print("Could not load the service account string.") 129 | raise RuntimeError( 130 | f"Invalid Service Account JSON provided for {product_name}.") from error 131 | 132 | 133 | def cloud_logging(message: str, severity: str = "INFO") -> None: 134 | """Function for logging in google cloud function. 135 | 136 | Args: 137 | message (str): The message to log 138 | severity (str): severity of the message. Defaults to "INFO". 139 | """ 140 | print(json.dumps({"severity": severity, "message": message})) 141 | -------------------------------------------------------------------------------- /common/utils_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Unit test file for utils.py file.""" 16 | 17 | import datetime 18 | import unittest 19 | from unittest import mock 20 | 21 | from common import utils 22 | 23 | # Path to common framework. 24 | INGESTION_SCRIPTS_PATH = ( 25 | "common" 26 | ) 27 | 28 | 29 | class TestUtilsFromCommon(unittest.TestCase): 30 | """Unit test class for utils.""" 31 | 32 | def test_get_env_var_runtime_error(self): 33 | """Test case to verify that the RuntimeError is raised when the name not found in the environment variable and is_required is set to True. 34 | 35 | Asserts: 36 | RuntimeError is thrown if the requirement environment variable doesn't 37 | exist. 38 | """ 39 | self.assertRaises(RuntimeError, utils.get_env_var, "test", required=True) 40 | 41 | @mock.patch.dict("{}.utils.os.environ".format(INGESTION_SCRIPTS_PATH), 42 | {"poll_interval": "10"}) 43 | def test_get_env_var_success(self): 44 | """Test case to verify that the correct value is returned for the poll_interval if it exists in the environment variable. 45 | 46 | Asserts: 47 | get_env_var() returns the expected value of existing environment variable. 48 | """ 49 | self.assertEqual(utils.get_env_var("poll_interval"), "10") 50 | 51 | def test_get_env_var_default_value(self): 52 | """Test case to verify that the defualt value is returned for the variable which does not exist in the environment variables. 53 | 54 | Asserts: 55 | get_env_var() returns the default value set for the optional environment 56 | variable if the variable is not found. 57 | """ 58 | self.assertEqual( 59 | utils.get_env_var("poll_interval", required=False, default=10), 10) 60 | 61 | @mock.patch( 62 | "{}.utils.get_value_from_secret_manager".format(INGESTION_SCRIPTS_PATH)) 63 | @mock.patch.dict("{}.utils.os.environ".format(INGESTION_SCRIPTS_PATH), 64 | {"POLL_INTERVAL": "10"}) 65 | def test_get_env_var_secret_value(self, mocked_get_value_from_secret_manager): 66 | """Test case to verify that the correct value is returned when is_secret is set to True. 67 | 68 | Args: 69 | mocked_get_value_from_secret_manager (mock.Mock): Mocked object of 70 | SecretManager() class. 71 | 72 | Asserts: 73 | get_env_var() method leverages the SecretManager() class to access the 74 | value of environment variable stored in the Google Secret Manager. 75 | """ 76 | mocked_get_value_from_secret_manager.return_value = "10" 77 | self.assertEqual( 78 | utils.get_env_var("POLL_INTERVAL", is_secret=True), "10") 79 | 80 | @mock.patch.dict("{}.utils.os.environ".format(INGESTION_SCRIPTS_PATH), 81 | {"POLL_INTERVAL": "-10"}) 82 | def test_get_last_run_at_invalid(self): 83 | """Test case to verify that the RuntimeError is raised when the poll_interval value is zero or negative. 84 | 85 | Asserts: 86 | get_env_var() method raises RuntimeError for negative values of 87 | POLL_INTERVAL environment variable. 88 | """ 89 | self.assertRaises(RuntimeError, utils.get_last_run_at) 90 | 91 | @mock.patch("{}.utils.datetime".format(INGESTION_SCRIPTS_PATH)) 92 | @mock.patch.dict("{}.utils.os.environ".format(INGESTION_SCRIPTS_PATH), 93 | {"POLL_INTERVAL": "10"}) 94 | def test_get_last_run_at_success(self, mocked_datetime): 95 | """Test case to verify that the get_last_run_at returns the valid datetime object. 96 | 97 | Args: 98 | mocked_datetime (mock.Mock): Mocked object of datetime module. 99 | 100 | Asserts: 101 | get_last_run_at() method returns a datetime object based on POLL_INTERVAL 102 | environment variable. 103 | """ 104 | mocked_datetime.datetime.now.return_value = datetime.datetime( 105 | 2022, 1, 1, 6, 30, 00) 106 | mocked_datetime.timedelta.return_value = datetime.timedelta(seconds=600) 107 | self.assertEqual(utils.get_last_run_at(), 108 | datetime.datetime(2022, 1, 1, 6, 20, 00)) 109 | 110 | @mock.patch("{}.utils.datetime".format(INGESTION_SCRIPTS_PATH)) 111 | @mock.patch( 112 | "{}.utils.get_env_var".format(INGESTION_SCRIPTS_PATH), return_value=10) 113 | def test_get_last_run_at_for_get_env_var(self, mocked_get_env_var, 114 | mocked_datetime): 115 | """Test case to verify that the get_last_run_at calls the get_env_var with valid arguments. 116 | 117 | Args: 118 | mocked_get_env_var (int): Mocked return value of get_env_var() 119 | mocked_datetime (mock.Mock): Mocked object of datetime module. 120 | 121 | Asserts: 122 | get_last_run_at() method returns a datetime object based on POLL_INTERVAL 123 | environment variable. 124 | get_env_var() method is called with valid parameters. 125 | """ 126 | mocked_datetime.datetime.now.return_value = datetime.datetime( 127 | 2022, 1, 1, 6, 30, 00) 128 | mocked_datetime.timedelta.return_value = datetime.timedelta(seconds=600) 129 | self.assertEqual(utils.get_last_run_at(), 130 | datetime.datetime(2022, 1, 1, 6, 20, 00)) 131 | self.assertEqual(mocked_get_env_var.mock_calls[0], 132 | mock.call("POLL_INTERVAL", required=False, default=5)) 133 | -------------------------------------------------------------------------------- /dataminr/.env.yml: -------------------------------------------------------------------------------- 1 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 2 | CHRONICLE_CUSTOMER_ID: 3 | 4 | # Region where the Chronicle instance is located. 5 | CHRONICLE_REGION: us 6 | 7 | # Path of the Google Secret Manager with the version, where the Service Account is stored. 8 | CHRONICLE_SERVICE_ACCOUNT: 9 | 10 | # The namespace that the Chronicle logs are labeled with. 11 | CHRONICLE_NAMESPACE: 12 | 13 | # Dataminr Client ID from the Dataminr Platform 14 | DATAMINR_CLIENT_ID: 15 | 16 | # Path of the Google Secret Manager with the version, where Dataminr API Client Secret stored. 17 | DATAMINR_CLIENT_SECRET: 18 | 19 | # Limit numbers of alerts to be fetch in one Dataminr API call. 20 | DATAMINR_ALERT_LIMIT: "40" 21 | 22 | # Fetch Alert for provided Dataminr list names. 23 | DATAMINR_WATCHLIST_NAMES: 24 | 25 | # Fetch Alert based on search text. 26 | # This parameter only applicable if Watch list names parameter is not provided or provided with empty value. 27 | DATAMINR_ALERT_QUERY: 28 | 29 | # GCP bucket name where Dataminr checkpoint would be save. 30 | GCP_BUCKET_NAME: 31 | 32 | # Proxy server URL. 33 | HTTPS_PROXY: -------------------------------------------------------------------------------- /dataminr/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.31.0 2 | jwt==1.3.1 3 | google-auth==2.6.0 4 | google-cloud-storage==2.10.0 5 | google-cloud-secret-manager==2.10.0 6 | -------------------------------------------------------------------------------- /domaintools/domaintool_client.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """DomainTools Client class to get the enriched domains information.""" 16 | 17 | from typing import List 18 | 19 | import domaintools 20 | import requests 21 | 22 | from common import utils 23 | 24 | ERROR_MSG = "Error: {}" 25 | 26 | 27 | class DomainToolClient: 28 | """DomainToolClient class which will enrich the domains.""" 29 | 30 | def __init__(self, domaintool_user: str, domaintool_password: str) -> None: 31 | self.domaintool_user = domaintool_user 32 | self.domaintool_password = domaintool_password 33 | self.api = self.generate_api() 34 | 35 | def enrich(self, queued_domains: List[str]): 36 | """Enrich the domains and return them. 37 | 38 | Args: 39 | queued_domains (List): The domains to enrich from DomainTools 40 | 41 | Raises: 42 | NotAuthorizedException: When request in unauthorized 43 | ServiceUnavailableException: When the Query limits are exhausted 44 | requests.exceptions.ProxyError: Unable to connect to Proxy 45 | requests.exceptions.SSLError: Problem in SSL configuration 46 | Exception: Any other Exception raised 47 | Returns: 48 | List: List of enriched domains 49 | """ 50 | 51 | try: 52 | response = self.api.iris_enrich(*list(queued_domains)).response() 53 | return response 54 | except domaintools.exceptions.NotAuthorizedException as e: 55 | utils.cloud_logging( 56 | "The credentials provided for DomainTools are invalid.", 57 | severity="ERROR", 58 | ) 59 | raise e 60 | except domaintools.exceptions.ServiceUnavailableException as e: 61 | raise e 62 | except requests.exceptions.ProxyError as e: 63 | utils.cloud_logging(ERROR_MSG.format(e), severity="ERROR") 64 | except requests.exceptions.SSLError as e: 65 | utils.cloud_logging(ERROR_MSG.format(e), severity="ERROR") 66 | except Exception as e: # pylint: disable=broad-except 67 | utils.cloud_logging(ERROR_MSG.format(e), severity="ERROR") 68 | raise e 69 | 70 | def generate_api(self): 71 | """Generate the API Object for DomainTools. 72 | 73 | Returns: 74 | Any: Object of Generated API 75 | """ 76 | return domaintools.API( 77 | self.domaintool_user, 78 | self.domaintool_password 79 | ) 80 | -------------------------------------------------------------------------------- /domaintools/domaintools_env_constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """DomainTools Environment variable constants.""" 16 | 17 | ENV_DOMAINTOOLS_API_USERNAME = "DOMAINTOOLS_API_USERNAME" 18 | ENV_DOMAINTOOLS_API_KEY = "DOMAINTOOLS_API_KEY" 19 | ENV_DNSDB_API_KEY = "DNSDB_API_KEY" 20 | ENV_LOG_TYPE_FILE_PATH = "LOG_TYPE_FILE_PATH" 21 | ENV_PROVISIONAL_TTL = "PROVISIONAL_TTL" 22 | ENV_NON_PROVISIONAL_TTL = "NON_PROVISIONAL_TTL" 23 | ENV_ALLOW_LIST = "ALLOW_LIST" 24 | ENV_MONITORING_LIST = "MONITORING_LIST" 25 | ENV_MONITORING_TAGS = "MONITORING_TAGS" 26 | ENV_BULK_ENRICHMENT = "BULK_ENRICHMENT" 27 | ENV_FETCH_SUBDOMAINS_FOR_MAX_DOMAINS = "FETCH_SUBDOMAINS_FOR_MAX_DOMAINS" 28 | CHRONICLE_DATA_TYPE = "DOMAINTOOLS_THREATINTEL" 29 | DNSDB_URL = "https://api.dnsdb.info/dnsdb/v2/lookup/rrset/name/*.{}/NS?limit=50&time_last_after=-21600" 30 | ERROR_MSG = "Unable to fetch reference list. Error: {}" 31 | TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" 32 | -------------------------------------------------------------------------------- /domaintools/images/adhoc_parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/adhoc_parameters.png -------------------------------------------------------------------------------- /domaintools/images/chronicle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/chronicle.png -------------------------------------------------------------------------------- /domaintools/images/high_risk_rule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/high_risk_rule.png -------------------------------------------------------------------------------- /domaintools/images/medium_risk_rule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/medium_risk_rule.png -------------------------------------------------------------------------------- /domaintools/images/monitoring_list_rule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/monitoring_list_rule.png -------------------------------------------------------------------------------- /domaintools/images/monitoring_tags_rule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/monitoring_tags_rule.png -------------------------------------------------------------------------------- /domaintools/images/reference_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/reference_list.png -------------------------------------------------------------------------------- /domaintools/images/young_domain_rule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/domaintools/images/young_domain_rule.png -------------------------------------------------------------------------------- /domaintools/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.31.0 2 | jwt==1.3.1 3 | google-auth==2.6.0 4 | google-cloud-secret-manager==2.10.0 5 | domaintools_api==1.0.1 6 | google-api-python-client==2.97.0 7 | google-cloud-storage==2.10.0 8 | redis==5.0.0 9 | tldextract==5.0.1 -------------------------------------------------------------------------------- /duo_activity/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 17 | CHRONICLE_CUSTOMER_ID: 18 | 19 | # Region where the Chronicle instance is located. 20 | CHRONICLE_REGION: us 21 | 22 | # Path of the Google Secret Manager with the version, where the Service Account is stored. 23 | CHRONICLE_SERVICE_ACCOUNT: 24 | 25 | # The namespace that the Chronicle logs are labeled with. 26 | CHRONICLE_NAMESPACE: 27 | 28 | # The URL of Duo Security API 29 | BACKSTORY_API_V1_URL: 30 | 31 | # The path of file where the checkpoint timestamp of last ingested log is stored 32 | CHECKPOINT_FILE_PATH: 33 | 34 | # The duration for which the logs are fetched 35 | LOG_FETCH_DURATION: 36 | 37 | # The DUO secret key required to fetch logs from the DUO API 38 | DUO_SECRET_KEY: 39 | 40 | # The DUO integration key required to fetch logs from the DUO API 41 | DUO_INTEGRATION_KEY: 42 | -------------------------------------------------------------------------------- /duo_activity/README.md: -------------------------------------------------------------------------------- 1 | # Duo Activity 2 | 3 | This script is for fetching the logs API calls from DUO platform. 4 | Furthermore, the collected data will be ingested into Chronicle and parsed by corresponding parsers. 5 | 6 | ### The overall flow of the script: 7 | - Deploying the script to Cloud Function 8 | - Data collection using ingestion script 9 | - Ingest collected data into Chronicle 10 | - Collected data will be parsed through corresponding parsers in Chronicle 11 | 12 | ### Pre-Requisites 13 | - Chronicle console and Chronicle service account. 14 | - Duo Activity API credentials (API url, API key) 15 | - GCP Project with the below required permissions: 16 | - GCP user and project service account should have Owner permissions 17 | - GCP Services 18 | - Cloud function 19 | - Secret Manager 20 | - Cloud Scheduler 21 | 22 | ### Environment Variables 23 | 24 | | Variable | Description | Required | Default | Secret | 25 | | --- | --- | --- | --- | --- | 26 | | CHRONICLE_CUSTOMER_ID | Chronicle customer Id. | Yes | - | No | 27 | | CHRONICLE_REGION | Chronicle region. | Yes | us | No | 28 | | SERVICE_ACCOUNT_FILE | Path of the Google Secret Manager with the version, where the Service Account is stored. | Yes | - | Yes | 29 | | CHRONICLE_NAMESPACE | The namespace that the Chronicle logs are labeled with. | No | - | No | 30 | | BACKSTORY_API_V1_URL | Duo Activity API URL | Yes | - | No | 31 | | DUO_SECRET_KEY | Duo secret key required to authenticate. | Yes | - | Yes | 32 | | DUO_INTEGRATION_KEY | Duo integration key required to authenticate. | Yes | - | Yes | 33 | | LOG_FETCH_DURATION | The total duration for which the logs are fetched in one API call | No | 1 | No | 34 | | CHECKPOINT_FILE_PATH | The path of file where checkpoint timestamp information is stored | No | checkpoint.json | No | 35 | 36 | ### Setting up the directory 37 | Create a zip file of the cloud function with the contents of the following files: 38 | 39 | 1. *Content*s of the ingestion script (i.e. `duo_activity`) 40 | 2. `common` directory 41 | 42 | ### Setting the required runtime environment variables 43 | 44 | Edit the .env.yml file to populate all the required environment variables. 45 | Information related to all the environment variables can be found in the 46 | README.md file. 47 | 48 | #### Using secrets 49 | 50 | Environment variables marked as **Secret** must be configured as secrets on 51 | Google Secret Manager. Refer [this](https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets#create) 52 | page to learn how to create secrets. 53 | 54 | Once the secrets are created on Secret Manager, use the secret's resource name 55 | as the value for environment variables. For example: 56 | 57 | ``` 58 | CHRONICLE_SERVICE_ACCOUNT: projects/{project_id}/secrets/{secret_id}/versions/{version_id} 59 | ``` 60 | 61 | #### Configuring the namespace 62 | 63 | The namespace that the Chronicle logs are ingested into can be configured by 64 | setting the `CHRONICLE_NAMESPACE` environment variable. 65 | 66 | ### Deploying the cloud function 67 | 68 | The directory containing duo_activity ingestion script should be uploaded as a ZIP file in the Source code field in the Google Cloud Console. 69 | Refer [this](https://cloud.google.com/functions/docs/console-quickstart) 70 | page to learn how to create and deploy a cloud function. 71 | 72 | -------------------------------------------------------------------------------- /duo_activity/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.3 2 | jwt==1.3.1 3 | google-auth==2.30.0 4 | google-cloud-secret-manager==2.20.0 5 | google-api-python-client==2.134.0 6 | google-cloud-storage==2.17.0 7 | redis==5.0.8 8 | tldextract==5.0.1 9 | -------------------------------------------------------------------------------- /duo_admin/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | POLL_INTERVAL: 20 | DUO_API_DETAILS: -------------------------------------------------------------------------------- /duo_admin/README.md: -------------------------------------------------------------------------------- 1 | # Duo Admin 2 | 3 | This script is for fetching the logs from DUO platform and ingesting to Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | | Variable | Description | Required | Default | Secret | 7 | | ------------------------- | -------------------------------------------------------------------------------------------- | -------- | ------- | ------ | 8 | | DUO_API_DETAILS | Content of DUO account JSON file. | Yes | - | Yes | 9 | | POLL_INTERVAL | Fetch within the last x amount of time, where x can be defined in minutes (for example : 30) | No | 10 | No | 10 | -------------------------------------------------------------------------------- /duo_admin/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch logs from DUO.""" 16 | 17 | import datetime 18 | import json 19 | 20 | import duo_client 21 | 22 | from common import env_constants 23 | from common import ingest 24 | from common import utils 25 | 26 | # Log type to push data into Chronicle. 27 | CHRONICLE_DATA_TYPE = "DUO_ADMIN" 28 | 29 | # Environment variables constants. 30 | ENV_DUO_API_DETAILS = "DUO_API_DETAILS" 31 | 32 | # Default initialization of environment variable. 33 | DUO_API_IKEY = None 34 | DUO_API_SKEY = None 35 | DUO_API_HOSTNAME = None 36 | POLL_INTERVAL = None 37 | 38 | # Response consists of maximum 1000 log entries. 39 | PAGE_SIZE = 1000 40 | 41 | # Date format for the API. 42 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 43 | 44 | 45 | def get_last_timestamp(duo_logs) -> int: 46 | """Get the last timestamp to retrieve the next set of Duo Admin logs. 47 | 48 | Args: 49 | duo_logs (list): List of Duo Admin logs. 50 | 51 | Returns: 52 | max_timestamp (int): Latest timestamp retrieved from the Duo Admin logs. 53 | """ 54 | max_timestamp = 0 55 | for log in duo_logs: 56 | max_timestamp = max(max_timestamp, int(log["timestamp"])) 57 | 58 | return max_timestamp 59 | 60 | 61 | def get_and_ingest_logs(): 62 | """Fetch logs from Duo client and ingest into Chronicle.""" 63 | # Calculating start time based on POLL_INTERVAL. 64 | start_time = utils.get_last_run_at() 65 | 66 | # Creating a human readable format of the start_time to print in the 67 | # logs for debugging purposes. 68 | start_time_str = start_time.strftime(DATE_FORMAT) 69 | 70 | # The API requires the time in the epoch. So, calculating total 71 | # seconds from the January 1970 and converting it into seconds. 72 | start_time = int((start_time - datetime.datetime( 73 | 1970, 1, 1, tzinfo=datetime.timezone.utc)).total_seconds()) 74 | 75 | print( 76 | f"Retrieving the last {POLL_INTERVAL} mins of logs since {start_time_str}" 77 | ) 78 | 79 | log_count = PAGE_SIZE 80 | 81 | # Creating session with Duo client. 82 | admin_api = duo_client.Admin( 83 | ikey=DUO_API_IKEY, skey=DUO_API_SKEY, host=DUO_API_HOSTNAME) 84 | 85 | # Iterate through all the pages if pagination available and ingest data into 86 | # Chronicle. 87 | # If log_count is less than 1000, no need to check for next entries. 88 | while log_count == PAGE_SIZE: 89 | data_list = [] 90 | 91 | # Getting data from Duo platform. 92 | logs = admin_api.get_administrator_log(mintime=start_time) 93 | log_count = len(logs) 94 | 95 | # No need to ingest logs for empty response. 96 | if log_count == 0: 97 | break 98 | 99 | data_list.extend(iter(logs)) 100 | print(f"Retrieved {log_count} Duo admin logs from the last API call.") 101 | 102 | # Fetching the maximum timestamp from the collected logs for the next API 103 | # call. 104 | if log_count == PAGE_SIZE: 105 | start_time = get_last_timestamp(logs) + 1 106 | # Human readable format of the start time. 107 | start_time_str = (datetime.datetime.fromtimestamp( 108 | start_time, tz=datetime.timezone.utc)).strftime(DATE_FORMAT) 109 | 110 | print(f"Next page records to be collected from {start_time_str}.") 111 | 112 | # Ingest data into the Chronicle. 113 | ingest.ingest(data_list, CHRONICLE_DATA_TYPE) 114 | 115 | 116 | def main(req) -> str: # pylint: disable=unused-argument 117 | """Entrypoint. 118 | 119 | Args: 120 | req: Request to execute the cloud function. 121 | 122 | Returns: 123 | string: "Ingestion completed." 124 | """ 125 | global DUO_API_IKEY 126 | global DUO_API_SKEY 127 | global DUO_API_HOSTNAME 128 | global POLL_INTERVAL 129 | 130 | # Interval should match what you have configured in Cloud Scheduler. 131 | POLL_INTERVAL = utils.get_env_var( 132 | env_constants.ENV_POLL_INTERVAL, required=False, default=10) 133 | 134 | # Duo Admin API integration key. 135 | DUO_API_IKEY = json.loads( 136 | utils.get_env_var(ENV_DUO_API_DETAILS, is_secret=True))["ikey"] 137 | 138 | # Duo Admin API secret key. 139 | DUO_API_SKEY = json.loads( 140 | utils.get_env_var(ENV_DUO_API_DETAILS, is_secret=True))["skey"] 141 | 142 | # Duo Admin API hostname. 143 | DUO_API_HOSTNAME = json.loads( 144 | utils.get_env_var(ENV_DUO_API_DETAILS, is_secret=True))["api_host"] 145 | 146 | # Fetch and ingest Duo admin logs into Chronicle. 147 | get_and_ingest_logs() 148 | 149 | return "Ingestion completed." 150 | -------------------------------------------------------------------------------- /duo_admin/main_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Unit test case file for duo client.""" 16 | import datetime 17 | import sys 18 | import time 19 | 20 | import unittest 21 | from unittest import mock 22 | 23 | INGESTION_SCRIPTS_PATH = "" 24 | SCRIPT_PATH = "" 25 | 26 | sys.modules["{}common.ingest".format(INGESTION_SCRIPTS_PATH)] = mock.Mock() 27 | 28 | import main 29 | 30 | 31 | def mock_get_env_var(*args, **unused_kwargs): 32 | """Mock and return env variable values. 33 | 34 | Args: 35 | *args (list[Any]): Any number of positional arguments. 36 | **unused_kwargs (list[Any]): Any number of keyword arguments. 37 | 38 | Returns: 39 | Value of POLL_INTERVAL environment variable as 10. 40 | Dummy values of DUO_API_DETAILS environment variable. 41 | Dummy value as "test" for any other environment variable. 42 | """ 43 | if args[0] == "POLL_INTERVAL": 44 | return 10 45 | elif args[0] == "DUO_API_DETAILS": 46 | return '{"ikey": "", "skey": "", "api_host": ""}' 47 | else: 48 | return "test" 49 | 50 | 51 | @mock.patch( 52 | f"{SCRIPT_PATH}main.utils.get_env_var", 53 | side_effect=mock_get_env_var) 54 | @mock.patch(f"{SCRIPT_PATH}main.ingest.ingest") 55 | @mock.patch(f"{SCRIPT_PATH}main.duo_client.Admin") 56 | class TestDuoAdminIngestion(unittest.TestCase): 57 | """Test cases to verify Duo Admin ingestion script.""" 58 | 59 | def test_no_logs_to_ingest(self, mocked_duo_admin, mocked_ingest, 60 | unused_mocked_get_env_var): 61 | """Test case to ensure that we break the loop when there are no logs to ingest. 62 | 63 | Args: 64 | mocked_duo_admin (mock.Mock): Mocked object of duo_client.admmin module. 65 | mocked_ingest (mock.Mock): Mocked object of ingest method. 66 | unused_mocked_get_env_var (mock.Mock): Mocked object of get_env_var 67 | method. 68 | 69 | Asserts: 70 | Validates that ingest() method is not called if no records are returned 71 | from the Duo Admin API. 72 | """ 73 | mock_duo_client = mock.Mock() 74 | mock_duo_client.get_administrator_log.return_value = [] 75 | mocked_duo_admin.return_value = mock_duo_client 76 | 77 | main.main(req="") 78 | 79 | self.assertEqual(mocked_ingest.call_count, 0) 80 | 81 | @mock.patch(f"{SCRIPT_PATH}main.utils.datetime") 82 | def test_log_retrieve_time(self, mocked_script_datetime, mocked_duo_admin, 83 | unused_mocked_ingest, unused_mocked_get_env_var): 84 | """Test case to verify the log retrieve time is as expected. 85 | 86 | Args: 87 | mocked_script_datetime (mock.Mock): Mocked object of datetime module 88 | imported in ingestion script. 89 | mocked_duo_admin (mock.Mock): Mocked object of duo_client.admmin module. 90 | unused_mocked_ingest (mock.Mock): Mocked object of ingest method. 91 | unused_mocked_get_env_var (mock.Mock): Mocked object of get_env_var 92 | method. 93 | 94 | Asserts: 95 | Validates the start date from which data collection will start for Duo 96 | Admin logs. By default, the start date will be (now - 10 minutes). 97 | """ 98 | now_date = datetime.datetime( 99 | 2022, 1, 1, 10, 15, 15, 234566, tzinfo=datetime.timezone.utc) 100 | mocked_script_datetime.datetime.now.return_value = now_date 101 | mocked_script_datetime.timedelta.side_effect = datetime.timedelta 102 | 103 | mock_duo_client = mock.Mock() 104 | mock_duo_client.get_administrator_log.return_value = [] 105 | mocked_duo_admin.return_value = mock_duo_client 106 | 107 | main.main(req="") 108 | 109 | _, kwargs = mock_duo_client.get_administrator_log.call_args 110 | # (2022-01-01 10:15:15) - 10 minutes = (2022-01-01 10:05:15) 111 | expected_log_start_time = 1641031515 112 | self.assertEqual(kwargs.get("mintime"), expected_log_start_time) 113 | 114 | def test_pagination(self, mocked_duo_admin, mocked_ingest, 115 | unused_mocked_get_env_var): 116 | """Test case to verify we fetch next page records when the log count is 1000. 117 | 118 | Args: 119 | mocked_duo_admin (mock.Mock): Mocked object of duo_client.admmin module. 120 | mocked_ingest (mock.Mock): Mocked object of ingest method. 121 | unused_mocked_get_env_var (mock.Mock): Mocked object of get_env_var 122 | method. 123 | 124 | Asserts: 125 | Validates that the ingest() method is called twice if number of records 126 | are more than 1000. 127 | Validates the number of records being sent to the ingest() method during 128 | the execution. 129 | """ 130 | dummy_logs = [ 131 | [{ 132 | "id": i, "timestamp": int(time.time() - i) 133 | } for i in range(1001, 1, -1)], # 1st page, 1000 logs 134 | [{ 135 | "id": i, "timestamp": int(time.time() - i) 136 | } for i in range(1545, 1001, -1)] # 2nd page, 544 logs 137 | ] 138 | mock_duo_client = mock.Mock() 139 | mock_duo_client.get_administrator_log.side_effect = dummy_logs 140 | mocked_duo_admin.return_value = mock_duo_client 141 | 142 | main.main(req="") 143 | 144 | actual_calls = mocked_ingest.mock_calls 145 | expected_calls = [ 146 | mock.call(dummy_logs[0], "DUO_ADMIN"), # Call ingest with 1000 logs 147 | mock.call(dummy_logs[1], "DUO_ADMIN") # Call ingest with 544 logs 148 | ] 149 | self.assertEqual(mocked_ingest.call_count, 2) 150 | self.assertEqual(actual_calls, expected_calls) 151 | 152 | def test_get_max_timestamp( 153 | self, 154 | mocked_duo_admin, mocked_ingest, # pylint: disable=unused-argument 155 | unused_mocked_get_env_var): 156 | """Test case to verify if the maximum timestamp is identified from the logs. 157 | 158 | Args: 159 | mocked_duo_admin (mock.Mock): Mocked object of duo_client.admmin module. 160 | mocked_ingest (mock.Mock): Mocked object of ingest method. 161 | unused_mocked_get_env_var (mock.Mock): Mocked object of get_env_var 162 | method. 163 | 164 | Asserts: 165 | Record with maximum timestamp is returned by the get_max_timestamp() 166 | method. 167 | """ 168 | dummy_logs = [{"id": dummy_val, "timestamp": dummy_val 169 | } for dummy_val in range(0, 1000)] 170 | 171 | actual_latest_timestamp = main.get_last_timestamp(dummy_logs) 172 | expected_latest_timestamp = 999 173 | 174 | self.assertEqual(actual_latest_timestamp, expected_latest_timestamp) 175 | -------------------------------------------------------------------------------- /duo_admin/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | duo_client==4.3.2 17 | requests==2.27.1 18 | jwt==1.3.1 19 | google-auth==2.6.0 20 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /google_cloud_storage/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 17 | CHRONICLE_CUSTOMER_ID: 18 | 19 | # Region where the Chronicle instance is located. 20 | CHRONICLE_REGION: us 21 | 22 | # Path of the Google Secret Manager with the version, where the Service Account is stored. 23 | CHRONICLE_SERVICE_ACCOUNT: 24 | 25 | # Name of the GCS Buckets from which to fetch the data. 26 | # Multiple bucket names can be provided, comma seperated. Example: buck1,buck2. 27 | GCS_BUCKET_NAME: 28 | 29 | # Path to the Google Secret Manager with the version, where the Service Account is stored. 30 | # This will be used to authenticate with the GCP to collect the logs from the Cloud Storage. 31 | GCP_SERVICE_ACCOUNT_SECRET_PATH: 32 | 33 | # Time interval in minutes to fetch the data. Ex. If poll interval is 5, 34 | # then it will fetch logs after every 5 minutes. 35 | POLL_INTERVAL: "60" 36 | 37 | # Log type to push data into the Chronicle platform. 38 | CHRONICLE_DATA_TYPE: 39 | 40 | # The namespace that the Chronicle logs are labeled with. 41 | CHRONICLE_NAMESPACE: 42 | -------------------------------------------------------------------------------- /google_cloud_storage/README.md: -------------------------------------------------------------------------------- 1 | # Google Cloud Storage 2 | 3 | This script retrieves the system logs of the compute engine from the google cloud bucket and ingest them in Chronicle. 4 | ## Platform Specific Environment Variables 5 | 6 | | Variable | Description | Required | Default | Secret | 7 | |---|---|---|---|---| 8 | | POLL_INTERVAL | Frequency interval at which the function executes to get additional log data (in minutes). This duration must be the same as the Cloud Scheduler job interval. | No | 60 | No | 9 | | GCS_BUCKET_NAME | Name of the GCS Bucket from which to fetch the data. | Yes | - | No | 10 | | GCP_SERVICE_ACCOUNT_SECRET_PATH | Path to the secret in Secret Manager that stores the GCP Service Account JSON file. | Yes | - | Yes | 11 | | CHRONICLE_DATA_TYPE | Log type to push data into the Chronicle platform. | Yes | - | No | 12 | -------------------------------------------------------------------------------- /google_cloud_storage/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch the logs stored in the Google Cloud Storage Bucket and ingest into Chronicle.""" 16 | import datetime 17 | import json 18 | from typing import Any 19 | 20 | from google.cloud import exceptions 21 | from google.cloud import storage 22 | 23 | from common import ingest 24 | from common import utils 25 | 26 | # Date format to be used to parse the date string to the datetime object. 27 | DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f%z" 28 | 29 | # Encoding system for Unicode. 30 | UTF_8 = "utf-8" 31 | 32 | # Product name. 33 | PRODUCT_NAME = "google cloud platform" 34 | 35 | # Environment variable constants. 36 | ENV_GCP_SERVICE_ACCOUNT_SECRET_PATH = "GCP_SERVICE_ACCOUNT_SECRET_PATH" 37 | ENV_CHRONICLE_DATA_TYPE = "CHRONICLE_DATA_TYPE" 38 | ENV_GCS_BUCKET_NAME = "GCS_BUCKET_NAME" 39 | 40 | 41 | def get_and_ingest_logs(storage_client: storage.Client, bucket_names: list[Any], 42 | start_time: datetime.datetime, 43 | chronicle_data_type: str) -> None: 44 | """Get logs from the GCP Storage Bucket and ingest them into Chronicle. 45 | 46 | Args: 47 | storage_client (storage): GCS Storage Client object to be used. 48 | bucket_names (list): List of bucket names to read the logs from. 49 | start_time (datetime): Time from which to start fetching the logs. 50 | chronicle_data_type (str): Log type to push data into the Chronicle 51 | platform. 52 | 53 | Raises: 54 | Exception: Raised error for unexpected behavior. 55 | """ 56 | print( 57 | "Retrieving blobs which are created after" 58 | f" {start_time.strftime(DATE_FORMAT)}." 59 | ) 60 | 61 | no_of_logs = 0 62 | 63 | # Iterate over the GCS buckets to fetch the logs. 64 | for bucket_name in bucket_names: 65 | try: 66 | print( 67 | "Creating a storage bucket object with the provided bucket name:" 68 | f" {bucket_name}." 69 | ) 70 | bucket_object = storage_client.get_bucket(bucket_name) 71 | except exceptions.NotFound as error: 72 | raise RuntimeError( 73 | f"The specified bucket '{bucket_name}' does not exist.") from error 74 | except Exception as error: 75 | raise RuntimeError( 76 | f"Error occurred while creating the object for '{bucket_name}'" 77 | ) from error 78 | 79 | # Filter the blobs which are created after start_time. 80 | for blob in bucket_object.list_blobs(): 81 | blob_created_time = blob.time_created 82 | 83 | # If the blob is created after the start_time_obj, then only we'll fetch 84 | # the data from it. 85 | if blob_created_time >= start_time: 86 | try: 87 | blob_data = json.loads(blob.download_as_text(encoding=UTF_8)) 88 | 89 | # If the blob is not in JSON string, then we split the content 90 | # before parsing it. 91 | except json.JSONDecodeError: 92 | try: 93 | blob_str_data = blob.download_as_text( 94 | encoding=UTF_8).split("\n")[:-1] 95 | blob_data = [json.loads(data) for data in blob_str_data] 96 | except json.JSONDecodeError as error: 97 | print(f"Could not load the log data from blob {blob.name}.") 98 | raise RuntimeError( 99 | f"The log data from {blob.name} is not JSON serializable." 100 | ) from error 101 | 102 | # Ingest blob data into Chronicle. 103 | try: 104 | ingest.ingest(blob_data, chronicle_data_type) 105 | except Exception as error: 106 | raise Exception( 107 | "Unable to push the data to the Chronicle.") from error 108 | 109 | no_of_logs += len(blob_data) 110 | 111 | if not no_of_logs: 112 | print("No newer logs found for the given bucket.") 113 | else: 114 | print(f"Total {no_of_logs} log(s) are successfully ingested to Chronicle.") 115 | 116 | 117 | # Requests is a user input dictionary passed while running the cloud function. 118 | # The script does not use these params. 119 | def main(request) -> str: # pylint: disable=unused-argument 120 | """Entrypoint. 121 | 122 | Args: 123 | request: Request to execute the cloud function. 124 | 125 | Returns: 126 | string: "Ingestion completed". 127 | """ 128 | # Fetching the environment variables. 129 | gcp_service_account = utils.get_env_var( 130 | ENV_GCP_SERVICE_ACCOUNT_SECRET_PATH, is_secret=True) 131 | chronicle_data_type = utils.get_env_var(ENV_CHRONICLE_DATA_TYPE) 132 | 133 | gcs_bucket_name = utils.get_env_var(ENV_GCS_BUCKET_NAME) 134 | bucket_names = [ 135 | bucket.lower().strip() for bucket in gcs_bucket_name.strip().split(",") 136 | ] 137 | 138 | start_time = utils.get_last_run_at() 139 | 140 | # Load GCP service account JSON. 141 | gcp_service_account_dict = utils.load_service_account( 142 | gcp_service_account, PRODUCT_NAME) 143 | 144 | # Creating storage client based on provided GCP service account JSON. 145 | try: 146 | print("Creating a storage client from GCP service account.") 147 | storage_client = storage.Client.from_service_account_info( 148 | gcp_service_account_dict) 149 | except ValueError as error: 150 | raise RuntimeError( 151 | "Invalid Google Cloud Service Account provided.") from error 152 | except Exception as error: 153 | raise RuntimeError( 154 | "Error occurred while creating the GCS Client.") from error 155 | 156 | # Fetch and ingest the logs into the Chronicle. 157 | get_and_ingest_logs(storage_client, bucket_names, start_time, 158 | chronicle_data_type) 159 | 160 | return "Ingestion completed." 161 | -------------------------------------------------------------------------------- /google_cloud_storage/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | requests==2.28.1 18 | jwt==1.3.1 19 | google-auth==2.15.0 20 | google-cloud==0.34.0 21 | google-cloud-core==2.3.2 22 | google-cloud-secret-manager==2.13.0 23 | google-cloud-storage==2.7.0 -------------------------------------------------------------------------------- /misp/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | API_KEY: 20 | TARGET_SERVER: 21 | # Keeping the default value as 5 minutes considering the frequency of data. 22 | POLL_INTERVAL: "5" 23 | ORG_NAME: -------------------------------------------------------------------------------- /misp/README.md: -------------------------------------------------------------------------------- 1 | # Malware Information Sharing Platform 2 | 3 | This script is for fetching the logs from MISP platform and ingesting to Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | | Variable | Description | Required | Default | Secret | 7 | | ------------------------- | -------------------------------------------------------------------------------------------- | -------- | ------- | ------ | 8 | | TARGET_SERVER | Your IP address, getting after creating MISP instance. | Yes | - | No | 9 | | API_KEY | Pass API key's secret manager path to authentication. | Yes | - | Yes | 10 | | POLL_INTERVAL | Frequency interval(in minutes) at which the Cloud Function executes. This duration must be same as the cloud scheduler job. | No | 5 | No | 11 | | ORG_NAME | Organization name for filtering events. | Yes | - | No | 12 | -------------------------------------------------------------------------------- /misp/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch data from MISP API.""" 16 | 17 | from typing import Optional 18 | 19 | import requests 20 | 21 | from common import env_constants 22 | from common import ingest 23 | from common import status 24 | from common import utils 25 | 26 | 27 | # Environment variable constants. 28 | ENV_API_KEY = "API_KEY" 29 | ENV_TARGET_SERVER = "TARGET_SERVER" 30 | ENV_ORG_NAME = "ORG_NAME" 31 | 32 | # List of unwanted keys in event json. 33 | KEYS_TO_REMOVE = [ 34 | "Event", 35 | "Tag", 36 | "EventReport", 37 | "Object", 38 | "Galaxy", 39 | "RelatedEvent", 40 | "ShadowAttribute", 41 | "Orgc", 42 | "Org", 43 | "Feed", 44 | ] 45 | 46 | # Log type to push data into Chronicle. 47 | CHRONICLE_DATA_TYPE = "MISP_IOC" 48 | 49 | 50 | def get_and_ingest_events(api_key: str, 51 | target_server: str, 52 | start_time: str, 53 | org_name: Optional[str] = None): 54 | """Get logs from 3p resources. 55 | 56 | Args: 57 | api_key(str): key for authentication. 58 | target_server(str): 3p resource ip address. 59 | start_time(str): add time interval in minutes. 60 | org_name(Optional[str]): organization name to filter data. 61 | """ 62 | headers = { 63 | "Authorization": api_key, 64 | "Accept": "application/json", 65 | "Content-Type": "application/json", 66 | } 67 | 68 | params = { 69 | # Timestamp represents start time i.e., 70 | # start_time minutes before current time. 71 | "timestamp": f"{start_time}m", 72 | } 73 | print(f"Retrieving event data from last {start_time}m.") 74 | 75 | # If organization name provided, update params. 76 | if org_name is not None: 77 | params["org_name"] = org_name 78 | 79 | data_list = [] 80 | response_events = None 81 | try: 82 | url = f"https://{target_server}/events/restSearch" 83 | req = requests.post(url, json=params, headers=headers) 84 | 85 | response_events = req.json() 86 | 87 | if req.status_code != status.STATUS_OK: 88 | print(f"HTTP Error: {req.status_code}, Reason: {response_events}") 89 | 90 | req.raise_for_status() 91 | 92 | # Iterate through all the events and ingest data into Chronicle. 93 | for data in response_events.get("response", []): 94 | event_json = data.get("Event", {}) 95 | 96 | # Remove unwanted key-value and append the 97 | # updated dictionary to data_list. 98 | updated_dict = { 99 | key: event_json.get(key) 100 | for key in event_json 101 | if key not in KEYS_TO_REMOVE 102 | } 103 | data_list.append(updated_dict) 104 | 105 | print(f"Retrieved {len(data_list)} MISP events from the last API call.") 106 | 107 | # Ingest the logs to the Chronicle. 108 | ingest.ingest(data_list, CHRONICLE_DATA_TYPE) 109 | 110 | except Exception as error: 111 | print( 112 | "ERROR: Unexpected error occured while fetching events from the MISP" 113 | " API." 114 | ) 115 | raise error 116 | 117 | 118 | def main(req): # pylint: disable=unused-argument 119 | """Entrypoint. 120 | 121 | Args: 122 | req: Request to execute the cloud function. 123 | 124 | Returns: 125 | string: "Ingestion completed." 126 | """ 127 | 128 | api_key = utils.get_env_var(ENV_API_KEY, is_secret=True) 129 | target_server = utils.get_env_var(ENV_TARGET_SERVER) 130 | poll_interval = utils.get_env_var( 131 | env_constants.ENV_POLL_INTERVAL, required=False, default=5) 132 | org_name = utils.get_env_var(ENV_ORG_NAME) 133 | 134 | get_and_ingest_events(api_key, target_server, poll_interval, org_name) 135 | 136 | return "Ingestion completed." 137 | -------------------------------------------------------------------------------- /misp/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /onelogin_events/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | CLIENT_ID: 20 | CLIENT_SECRET: 21 | TOKEN_ENDPOINT: https://api.us.onelogin.com/auth/oauth2/v2/token 22 | # Keeping the default value as 5 minutes considering the frequency of OneLogin events. 23 | POLL_INTERVAL: "5" -------------------------------------------------------------------------------- /onelogin_events/README.md: -------------------------------------------------------------------------------- 1 | # OneLogin Events 2 | 3 | This script pulls events from OneLogin and ingests them into Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | Secret | 8 | | -------------- | --------------------------------------------------------------- | -------- | ------- | ------ | 9 | | CLIENT_ID | Client ID of OneLogin platform | Yes | - | No | 10 | | CLIENT_SECRET | Client Secret of OneLogin platform. | Yes | - | Yes | 11 | | POLL_INTERVAL | Frequency interval(in minutes) at which the Cloud Function executes. This duration must be same as the cloud scheduler job. | No | 5 | No | 12 | | TOKEN_ENDPOINT | URL for token request. | No | https://api.us.onelogin.com/auth/oauth2/v2/token | No | 13 | 14 | -------------------------------------------------------------------------------- /onelogin_events/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch events from the Onelogin platform.""" 16 | 17 | import datetime 18 | 19 | from common import auth 20 | from common import ingest 21 | from common import status 22 | from common import utils 23 | 24 | # API URL for OneLogin Events. 25 | ONELOGIN_EVENTS_URL = "https://api.us.onelogin.com/api/1/events" 26 | 27 | # OneLogin Authentication URL. 28 | ONELOGIN_AUTH_URL = "https://api.us.onelogin.com/auth/oauth2/v2/token" 29 | 30 | # Log type to push data into Chronicle. 31 | CHRONICLE_DATA_TYPE = "ONELOGIN_SSO" 32 | 33 | # Date format to be used in API. 34 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" 35 | 36 | # Environment variable constants. 37 | ENV_CLIENT_ID = "CLIENT_ID" 38 | ENV_CLIENT_SECRET = "CLIENT_SECRET" 39 | ENV_TOKEN_ENDPOINT = "TOKEN_ENDPOINT" 40 | 41 | 42 | def get_and_ingest_events(http_session: auth.OAuthClientCredentialsAuth): 43 | """Get events from the OneLogin platform and ingest into Chronicle. 44 | 45 | Args: 46 | http_session (auth.OAuthClientCredentialsAuth): Session object to get 47 | events from. 48 | 49 | Raises: 50 | TypeError, ValueError: Error when response is not in json format. 51 | """ 52 | # Calculate start time based on POLL_INTERVAL, end time will be 'now'. 53 | # Convert datetime object into the expected format (YYYY-MM-DDTHH:MM:SSSZ). 54 | start_time = utils.get_last_run_at().strftime(DATE_FORMAT)[:-3] + "Z" 55 | end_time = datetime.datetime.now( 56 | datetime.timezone.utc).strftime(DATE_FORMAT)[:-3] + "Z" 57 | 58 | next_url = (f"{ONELOGIN_EVENTS_URL}?since={start_time}&until={end_time}") 59 | 60 | # Iterate through all the pages if pagination available and ingest data 61 | # into Chronicle. 62 | while next_url is not None: 63 | events_url = next_url 64 | 65 | # Get the response from the OneLogin API. 66 | request_events = http_session.get(events_url) 67 | 68 | try: 69 | response_events = request_events.json() 70 | except (TypeError, ValueError) as error: 71 | print( 72 | "ERROR: Unexpected data format received while collecting OneLogin" 73 | " events." 74 | ) 75 | raise error 76 | 77 | # If REST API status code is not 200. 78 | if request_events.status_code != status.STATUS_OK: 79 | print(f"HTTP Error: {request_events.status_code}," 80 | " Reason: {resp_json}.") 81 | 82 | request_events.raise_for_status() 83 | 84 | data_list = response_events.get("data", []) 85 | print( 86 | f"Retrieved {len(data_list)} OneLogin events from the last" 87 | " API call." 88 | ) 89 | 90 | # Ingest data into Chronicle. 91 | if data_list: 92 | ingest.ingest(response_events["data"], CHRONICLE_DATA_TYPE) 93 | 94 | # Prepare the URL to fetch the next page for OneLogin events. 95 | next_url = response_events.get("pagination", {}).get("next_link") 96 | 97 | 98 | def main(request): # pylint: disable=unused-argument 99 | """Entrypoint. 100 | 101 | Args: 102 | request: Request to execute the cloud function. 103 | 104 | Returns: 105 | string: "Ingestion completed." 106 | """ 107 | 108 | # Fetching values from the environment variables. 109 | token_endpoint = utils.get_env_var( 110 | ENV_TOKEN_ENDPOINT, 111 | required=False, 112 | default=ONELOGIN_AUTH_URL, 113 | ) 114 | client_id = utils.get_env_var(ENV_CLIENT_ID) 115 | client_secret = utils.get_env_var(ENV_CLIENT_SECRET, is_secret=True) 116 | 117 | # Creating the session object. 118 | session = auth.OAuthClientCredentialsAuth(token_endpoint, client_id, 119 | client_secret) 120 | 121 | # Fetch and ingest the events from OneLogin platform into Chronicle. 122 | get_and_ingest_events(session) 123 | 124 | return "Ingestion completed." 125 | -------------------------------------------------------------------------------- /onelogin_events/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /onelogin_user/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | CLIENT_ID: 20 | CLIENT_SECRET: 21 | TOKEN_ENDPOINT: https://api.us.onelogin.com/auth/oauth2/v2/token 22 | # Keeping the default value as 30 minutes considering the freqeuency of OneLogin Users data. 23 | POLL_INTERVAL: "30" -------------------------------------------------------------------------------- /onelogin_user/README.md: -------------------------------------------------------------------------------- 1 | # OneLogin Users 2 | 3 | This script pulls events from OneLogin and ingests them into Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | 8 | | -------------- | --------------------------------------------------------------- | -------- | ------- | 9 | | CLIENT_ID | Client ID of OneLogin platform | Yes | - | No | 10 | | CLIENT_SECRET | Client Secret of OneLogin platform | Yes | - | Yes | 11 | | POLL_INTERVAL | Frequency interval(in minutes) at which the Cloud Function executes. This duration must be same as the cloud scheduler job. | No | 30 | No | 12 | | TOKEN_ENDPOINT | URL for token request. | No | https://api.us.onelogin.com/auth/oauth2/v2/token | No | 13 | -------------------------------------------------------------------------------- /onelogin_user/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch user data from Onelogin Users API.""" 16 | 17 | import datetime 18 | 19 | from common import auth 20 | from common import ingest 21 | from common import status 22 | from common import utils 23 | 24 | # API URL for OneLogin Users. 25 | ONELOGIN_USERS_URL = "https://api.us.onelogin.com/api/1/users" 26 | 27 | # Onelogin authentication endpoint URL. 28 | ONELOGIN_AUTH_URL = "https://api.us.onelogin.com/auth/oauth2/v2/token" 29 | 30 | # Log type to push data in Chronicle. 31 | CHRONICLE_DATA_TYPE = "ONELOGIN_USER_CONTEXT" 32 | 33 | # Date format to be used in API. 34 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" 35 | 36 | # Environment variable constants. 37 | ENV_CLIENT_ID = "CLIENT_ID" 38 | ENV_CLIENT_SECRET = "CLIENT_SECRET" 39 | ENV_TOKEN_ENDPOINT = "TOKEN_ENDPOINT" 40 | 41 | 42 | def get_and_ingest_users(http_session: auth.OAuthClientCredentialsAuth) -> None: 43 | """Get user data from the OneLogin platform and ingest into Chronicle. 44 | 45 | Args: 46 | http_session (OAuthClientCredentialsAuth): Session object to get users from. 47 | 48 | Raises: 49 | TypeError, ValueError: Error when response is not in json format. 50 | """ 51 | # Calculate start time based on POLL_INTERVAL, end time will be 'now'. 52 | # Convert datetime object into the expected format (YYYY-MM-DDTHH:MM:SSSZ). 53 | start_time = utils.get_last_run_at().strftime(DATE_FORMAT)[:-3] + "Z" 54 | end_time = datetime.datetime.now( 55 | datetime.timezone.utc).strftime(DATE_FORMAT)[:-3] + "Z" 56 | 57 | next_url = (f"{ONELOGIN_USERS_URL}?since={start_time}&until={end_time}") 58 | 59 | # Iterate through all the pages if pagination available and ingest data 60 | # into Chronicle. 61 | while next_url is not None: 62 | users_url = next_url 63 | 64 | # Get the response from the OneLogin API. 65 | request_users = http_session.get(users_url) 66 | 67 | try: 68 | response_users = request_users.json() 69 | except (ValueError, TypeError) as error: 70 | print( 71 | "ERROR: Unexpected data format received while collecting OneLogin" 72 | " users." 73 | ) 74 | raise error 75 | 76 | # If REST API status code is not 200. 77 | if request_users.status_code != status.STATUS_OK: 78 | print(f"HTTP Error: {request_users.status_code}," 79 | " Reason: {resp_json}.") 80 | 81 | request_users.raise_for_status() 82 | 83 | data_list = response_users.get("data", []) 84 | print( 85 | f"Retrieved {len(data_list)} OneLogin users data from the last" 86 | " API call." 87 | ) 88 | 89 | # Ingest data into Chronicle. 90 | if data_list: 91 | ingest.ingest(response_users["data"], CHRONICLE_DATA_TYPE) 92 | 93 | # Prepare the URL to fetch the next page for OneLogin users. 94 | next_url = response_users.get("pagination", {}).get("next_link") 95 | 96 | 97 | def main(request) -> str: # pylint: disable=unused-argument 98 | """Entrypoint. 99 | 100 | Args: 101 | request: Request to execute the cloud function. 102 | 103 | Returns: 104 | string: "Ingestion completed." 105 | """ 106 | # Fetching values from the environment variables. 107 | token_endpoint = utils.get_env_var( 108 | ENV_TOKEN_ENDPOINT, 109 | required=False, 110 | default=ONELOGIN_AUTH_URL, 111 | ) 112 | client_id = utils.get_env_var(ENV_CLIENT_ID) 113 | client_secret = utils.get_env_var(ENV_CLIENT_SECRET, is_secret=True) 114 | 115 | # Creating the session object. 116 | session = auth.OAuthClientCredentialsAuth(token_endpoint, client_id, 117 | client_secret) 118 | 119 | # Fetch and ingest the user data from OneLogin platform into Chronicle. 120 | get_and_ingest_users(session) 121 | 122 | return "Ingestion completed." 123 | -------------------------------------------------------------------------------- /onelogin_user/main_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Unit test file for OneLogin user.""" 16 | 17 | import datetime 18 | import sys 19 | 20 | 21 | import unittest 22 | from unittest import mock 23 | 24 | from google.auth.transport import requests 25 | import requests as req 26 | 27 | INGESTION_SCRIPTS_PATH = "" 28 | SCRIPT_PATH = "" 29 | 30 | # Mock the chronicle library 31 | sys.modules['{}common.ingest'.format( 32 | INGESTION_SCRIPTS_PATH)] = mock.MagicMock() 33 | 34 | import main 35 | 36 | 37 | def mock_get_env_var(*args, **kwargs): # pylint: disable=unused-argument 38 | """Mock and return env variable values.""" 39 | if args[0] == 'POLL_INTERVAL': 40 | return 10 41 | else: 42 | return 'test' 43 | 44 | 45 | @mock.patch( 46 | '{}main.utils.get_env_var'.format( 47 | SCRIPT_PATH), side_effect=mock_get_env_var) 48 | @mock.patch('{}main.auth.OAuthClientCredentialsAuth'.format(SCRIPT_PATH)) 49 | @mock.patch('{}main.ingest.ingest'.format(SCRIPT_PATH)) 50 | class TestGetUsersFromOneLogin(unittest.TestCase): 51 | """Unit test case class for OneLogin User.""" 52 | 53 | def test_empty_response(self, mock_ingest, unused_mock_oauth, 54 | unused_mock_env_var): 55 | """Test that the `get_users` function ingest empty list without any error in case of no data from API it self. 56 | """ 57 | mocked_session = mock.Mock() 58 | mocked_get = mock.Mock() 59 | mocked_get.json.return_value = { 60 | 'data': [], 61 | 'pagination': { 62 | 'next_link': None 63 | } 64 | } 65 | mocked_session.get.return_value = mocked_get 66 | main.get_and_ingest_users(mocked_session) 67 | 68 | self.assertEqual(mock_ingest.call_count, 0) 69 | 70 | def test_multiple_page_response(self, mock_ingest, unused_mock_oauth, 71 | unused_mock_env_var): 72 | """Test that the `get_users` function ingest list of events without any error in case of multiple pages from API. 73 | """ 74 | mocked_session = mock.Mock() 75 | mocked_get = mock.Mock() 76 | mocked_get.json.side_effect = [{ 77 | 'data': [{ 78 | 'id': 1 79 | }], 80 | 'pagination': { 81 | 'next_link': 'next_link_1' 82 | } 83 | }, { 84 | 'data': [{ 85 | 'id': 2 86 | }], 87 | 'pagination': { 88 | 'next_link': None 89 | } 90 | }] 91 | 92 | mocked_session.get.return_value = mocked_get 93 | main.get_and_ingest_users(mocked_session) 94 | 95 | actual_calls = mock_ingest.mock_calls 96 | expected_calls = [ 97 | mock.call([{ 98 | 'id': 1 99 | }], 'ONELOGIN_USER_CONTEXT'), 100 | mock.call([{ 101 | 'id': 2 102 | }], 'ONELOGIN_USER_CONTEXT') 103 | ] 104 | self.assertEqual(actual_calls, expected_calls) 105 | self.assertEqual(mock_ingest.call_count, 2) 106 | 107 | @mock.patch('{}main.get_and_ingest_users'.format(SCRIPT_PATH)) 108 | def test_http_error(self, unused_mock_ingest, unused_mock_oauth, 109 | unused_mock_env_var, mock_events): 110 | """Test that `main` function raises exception if API returns error.""" 111 | mock_events.side_effect = requests.requests.exceptions.HTTPError() 112 | 113 | with self.assertRaises(requests.requests.exceptions.HTTPError): 114 | main.main(request='') 115 | 116 | @mock.patch('builtins.print') 117 | def test_value_error(self, mocked_print, unused_mock_ingest, 118 | unused_mock_oauth, unused_mock_env_var): 119 | """Test case to verify that we raise ValueError when we get invalid JSON response. 120 | """ 121 | mocked_session = mock.Mock() 122 | mocked_response = req.Response() 123 | mocked_response.status_code = 200 124 | mocked_response.data = None 125 | 126 | mocked_session.get.return_value = mocked_response 127 | with self.assertRaises(ValueError): 128 | main.get_and_ingest_users(mocked_session) 129 | 130 | mocked_print.assert_called_with( 131 | 'ERROR: Unexpected data format received while collecting OneLogin' 132 | ' users.' 133 | ) 134 | 135 | @mock.patch('builtins.print') 136 | def test_type_error(self, mocked_print, unused_mock_ingest, unused_mock_oauth, 137 | unused_mock_env_var): 138 | """Test case to verify that we raise TypeError when we get JSON response in unexpected format. 139 | """ 140 | mocked_session = mock.Mock() 141 | mocked_response = req.Response() 142 | mocked_response.status_code = 200 143 | mocked_response._content = 12 # pylint: disable=protected-access 144 | mocked_session.get.return_value = mocked_response 145 | 146 | with self.assertRaises(TypeError): 147 | main.get_and_ingest_users(mocked_session) 148 | 149 | mocked_print.assert_called_with( 150 | 'ERROR: Unexpected data format received while collecting OneLogin' 151 | ' users.' 152 | ) 153 | 154 | @mock.patch(f'{SCRIPT_PATH}main.utils.datetime') 155 | @mock.patch(f'{SCRIPT_PATH}main.datetime') 156 | def test_log_retrieve_time(self, mocked_utils_datetime 157 | , mocked_script_datetime, unused_mock_ingest, 158 | unused_mock_oauth, unused_mock_env_var): 159 | """Test case to verify that log retrieve time is passed as expected.""" 160 | now_date = datetime.datetime( 161 | 2022, 1, 1, 10, 15, 15, 234566, tzinfo=datetime.timezone.utc) 162 | mocked_script_datetime.datetime.now.return_value = now_date 163 | mocked_script_datetime.timedelta.side_effect = datetime.timedelta 164 | mocked_utils_datetime.datetime.now.return_value = now_date 165 | mocked_utils_datetime.timedelta.side_effect = datetime.timedelta 166 | 167 | mocked_session = mock.Mock() 168 | mocked_get = mock.Mock() 169 | mocked_get.json.return_value = { 170 | 'data': [], 171 | 'pagination': { 172 | 'next_link': None 173 | } 174 | } 175 | mocked_session.get.return_value = mocked_get 176 | 177 | main.get_and_ingest_users(mocked_session) 178 | 179 | args, _ = mocked_session.get.call_args 180 | expected_start_date = '2022-01-01T10:05:15.234Z' # (now - 10 minutes) 181 | expected_end_date = '2022-01-01T10:15:15.234Z' 182 | expected_url = f'https://api.us.onelogin.com/api/1/users?since={expected_start_date}&until={expected_end_date}' 183 | self.assertEqual(args[0], expected_url) 184 | -------------------------------------------------------------------------------- /onelogin_user/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /panw_cortex_xdr/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 17 | CHRONICLE_CUSTOMER_ID: 18 | 19 | # Region where the Chronicle instance is located. 20 | CHRONICLE_REGION: us 21 | 22 | # Path of the Google Secret Manager with the version, where the Service Account is stored. 23 | CHRONICLE_SERVICE_ACCOUNT: 24 | 25 | # Name of the GCS Buckets from which to fetch the data. 26 | # Multiple bucket names can be provided, comma seperated. Example: buck1,buck2. 27 | GCS_BUCKET_NAME: 28 | 29 | # Path to the Google Secret Manager with the version, where the Service Account is stored. 30 | # This will be used to authenticate with the GCP to collect the logs from the Cloud Storage. 31 | GCP_SERVICE_ACCOUNT_SECRET_PATH: 32 | 33 | # Time interval in minutes to fetch the data. Ex. If poll interval is 5, 34 | # then it will fetch logs after every 5 minutes. 35 | POLL_INTERVAL: "60" 36 | 37 | # Log type to push data into the Chronicle platform. 38 | CHRONICLE_DATA_TYPE: 39 | 40 | # The namespace that the Chronicle logs are labeled with. 41 | CHRONICLE_NAMESPACE: 42 | -------------------------------------------------------------------------------- /panw_cortex_xdr/README.md: -------------------------------------------------------------------------------- 1 | # Palo Alto Cortex XDR 2 | 3 | This script retrieves the Palo Alto Cortex XDR exported logs from the google cloud bucket and ingest them in Chronicle. 4 | The Palo Alto Cortex XDR exported log file is a gzipped multiple lines of JSON. 5 | 6 | ## Platform Specific Environment Variables 7 | 8 | | Variable | Description | Required | Default | Secret | 9 | |---|---|---|---|---| 10 | | POLL_INTERVAL | Frequency interval at which the function executes to get additional log data (in minutes). This duration must be the same as the Cloud Scheduler job interval. | No | 60 | No | 11 | | GCS_BUCKET_NAME | Name of the GCS Bucket from which to fetch the data. | Yes | - | No | 12 | | GCP_SERVICE_ACCOUNT_SECRET_PATH | Path to the secret in Secret Manager that stores the GCP Service Account JSON file. | Yes | - | Yes | 13 | | CHRONICLE_DATA_TYPE | Log type to push data into the Chronicle platform. | Yes | - | No | 14 | -------------------------------------------------------------------------------- /panw_cortex_xdr/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch the logs stored in the Google Cloud Storage Bucket and ingest into Chronicle.""" 16 | import datetime 17 | import gzip 18 | import json 19 | from typing import Any 20 | 21 | from google.cloud import exceptions 22 | from google.cloud import storage 23 | 24 | from common import ingest 25 | from common import utils 26 | 27 | # Date format to be used to parse the date string to the datetime object. 28 | DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f%z" 29 | 30 | # Encoding system for Unicode. 31 | UTF_8 = "utf-8" 32 | 33 | # Product name. 34 | PRODUCT_NAME = "google cloud platform" 35 | 36 | # Environment variable constants. 37 | ENV_GCP_SERVICE_ACCOUNT_SECRET_PATH = "GCP_SERVICE_ACCOUNT_SECRET_PATH" 38 | ENV_CHRONICLE_DATA_TYPE = "CHRONICLE_DATA_TYPE" 39 | ENV_GCS_BUCKET_NAME = "GCS_BUCKET_NAME" 40 | 41 | 42 | def get_gzip_blob_data(blob: storage.Blob) -> list[str]: 43 | """Download and process Palo Alto Network Cortex XDR log file. 44 | 45 | The log file is a gzipped multiple lines of JSON, unzip and parse the log 46 | file and return the list of JSON str. 47 | 48 | Args: 49 | blob: storage.Blob: Palo Alto Network Cortex XDR log file. 50 | 51 | Returns: 52 | list[str]: list of JSON string. 53 | 54 | Raises: 55 | Exception: Raised error for unexpected behavior. 56 | """ 57 | 58 | try: 59 | decompressed = gzip.decompress(blob.download_as_bytes()) 60 | file_content = decompressed.decode("utf-8").split("\n") 61 | # the last item is empty after the split 62 | return [json.loads(data) for data in file_content if data] 63 | except Exception as error: 64 | raise RuntimeError( 65 | f"Could not load the log data from blob {blob.name}." 66 | ) from error 67 | 68 | 69 | def get_and_ingest_logs(storage_client: storage.Client, bucket_names: list[Any], 70 | start_time: datetime.datetime, 71 | chronicle_data_type: str) -> None: 72 | """Get logs from the GCP Storage Bucket and ingest them into Chronicle. 73 | 74 | Args: 75 | storage_client (storage): GCS Storage Client object to be used. 76 | bucket_names (list): List of bucket names to read the logs from. 77 | start_time (datetime): Time from which to start fetching the logs. 78 | chronicle_data_type (str): Log type to push data into the Chronicle 79 | platform. 80 | 81 | Raises: 82 | Exception: Raised error for unexpected behavior. 83 | """ 84 | print( 85 | "Retrieving blobs which are created after" 86 | f" {start_time.strftime(DATE_FORMAT)}." 87 | ) 88 | 89 | no_of_logs = 0 90 | 91 | # Iterate over the GCS buckets to fetch the logs. 92 | for bucket_name in bucket_names: 93 | try: 94 | print( 95 | "Creating a storage bucket object with the provided bucket name:" 96 | f" {bucket_name}." 97 | ) 98 | bucket_object = storage_client.get_bucket(bucket_name) 99 | except exceptions.NotFound as error: 100 | raise RuntimeError( 101 | f"The specified bucket '{bucket_name}' does not exist.") from error 102 | except Exception as error: 103 | raise RuntimeError( 104 | f"Error occurred while creating the object for '{bucket_name}'" 105 | ) from error 106 | 107 | # Filter the blobs which are created after start_time. 108 | for blob in bucket_object.list_blobs(): 109 | blob_created_time = blob.time_created 110 | 111 | # If the blob is created after the start_time_obj, then only we'll fetch 112 | # the data from it. 113 | if blob_created_time >= start_time: 114 | blob_data = get_gzip_blob_data(blob) 115 | if not blob_data: 116 | continue 117 | 118 | # Ingest blob data into Chronicle. 119 | try: 120 | ingest.ingest(blob_data, chronicle_data_type) 121 | except Exception as error: 122 | raise RuntimeError( 123 | "Unable to push the data to the Chronicle.") from error 124 | 125 | no_of_logs += len(blob_data) 126 | 127 | if not no_of_logs: 128 | print("No newer logs found for the given bucket.") 129 | else: 130 | print(f"Total {no_of_logs} log(s) are successfully ingested to Chronicle.") 131 | 132 | 133 | # Requests is a user input dictionary passed while running the cloud function. 134 | # The script does not use these params. 135 | def main(request) -> str: # pylint: disable=unused-argument 136 | """Entrypoint. 137 | 138 | Args: 139 | request: Request to execute the cloud function. 140 | 141 | Returns: 142 | string: "Ingestion completed". 143 | """ 144 | # Fetching the environment variables. 145 | gcp_service_account = utils.get_env_var( 146 | ENV_GCP_SERVICE_ACCOUNT_SECRET_PATH, is_secret=True) 147 | chronicle_data_type = utils.get_env_var(ENV_CHRONICLE_DATA_TYPE) 148 | 149 | gcs_bucket_name = utils.get_env_var(ENV_GCS_BUCKET_NAME) 150 | bucket_names = [ 151 | bucket.lower().strip() for bucket in gcs_bucket_name.strip().split(",") 152 | ] 153 | 154 | start_time = utils.get_last_run_at() 155 | 156 | # Load GCP service account JSON. 157 | gcp_service_account_dict = utils.load_service_account( 158 | gcp_service_account, PRODUCT_NAME) 159 | 160 | # Creating storage client based on provided GCP service account JSON. 161 | try: 162 | print("Creating a storage client from GCP service account.") 163 | storage_client = storage.Client.from_service_account_info( 164 | gcp_service_account_dict) 165 | except ValueError as error: 166 | raise RuntimeError( 167 | "Invalid Google Cloud Service Account provided.") from error 168 | except Exception as error: 169 | raise RuntimeError( 170 | "Error occurred while creating the GCS Client.") from error 171 | 172 | # Fetch and ingest the logs into the Chronicle. 173 | get_and_ingest_logs(storage_client, bucket_names, start_time, 174 | chronicle_data_type) 175 | 176 | return "Ingestion completed." 177 | -------------------------------------------------------------------------------- /panw_cortex_xdr/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | requests==2.28.1 18 | jwt==1.3.1 19 | google-auth==2.15.0 20 | google-cloud==0.34.0 21 | google-cloud-core==2.3.2 22 | google-cloud-secret-manager==2.13.0 23 | google-cloud-storage==2.7.0 -------------------------------------------------------------------------------- /proofpoint/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 16 | CHRONICLE_CUSTOMER_ID: 17 | 18 | # Region where the Chronicle instance is located. 19 | CHRONICLE_REGION: us 20 | 21 | # Path of the Google Secret Manager with the version, where the Chronicle Service Account is stored. 22 | CHRONICLE_SERVICE_ACCOUNT: 23 | 24 | # Log type according to the service to push data into the Chronicle platform. 25 | CHRONICLE_DATA_TYPE: 26 | 27 | # The namespace that the Chronicle logs are labeled with. 28 | CHRONICLE_NAMESPACE: 29 | 30 | # Base URL of the Proofpoint API Gateway. 31 | PROOFPOINT_SERVER_URL: 32 | 33 | # Service principle of Proofpoint client. 34 | PROOFPOINT_SERVICE_PRINCIPLE: 35 | 36 | # Path of the Google Secret Manager with the version, where Proofpoint secret is stored. 37 | PROOFPOINT_SECRET: 38 | 39 | # A number indicating how many days the data should be retrieved for. Accepted values are 14, 30 and 90. 40 | PROOFPOINT_RETRIEVAL_RANGE: "30" 41 | -------------------------------------------------------------------------------- /proofpoint/README.md: -------------------------------------------------------------------------------- 1 | # Proofpoint VAP 2 | 3 | This script fetches the users who are mostly attacked in a particular organization within a given time period and ingest them into the Chronicle platform. 4 | 5 | ## Platform Specific Environment Variables 6 | | Variable | Description | Required | Default | Secret | 7 | |---|---|---|---|---| 8 | | CHRONICLE_DATA_TYPE | Log type to push data into the Chronicle platform. | Yes | - | No | 9 | | PROOFPOINT_SERVER_URL | Base URL of Proofpoint Server API gateway. | Yes | - | No | 10 | | PROOFPOINT_SERVICE_PRINCIPLE | Service Principle of Proofpoint platform. | Yes | - | No | 11 | | PROOFPOINT_SECRET | Path of the Google Secret Manager with the version, where the Password of Proofpoint platform is stored. | Yes | - | Yes | 12 | | PROOFPOINT_RETRIEVAL_RANGE | Number indicating from how many days the data should be retrieved. Accepted values are 14, 30 and 90. | No | 30 | No | 13 | 14 | ## Note 15 | - Proofpoint script retrieval range (PROOFPOINT_RETRIEVAL_RANGE) can be set as 14 days, 30 days or 90 days. This acts as the start date to fetch the data. 16 | - The user can configure cloud scheduler to run the cloud function after every 6 hours to fetch the data of the last 30 days. 17 | - The Proofpoint People API records are updated once in every 24 hours, hence running the script every 6 hours will enable the user to fetch and ingest updated data efficiently with minimum data duplication. 18 | - The Proofpoint API has no provision to fetch only the updated records in the given time range hence data duplication is a possible scenario. -------------------------------------------------------------------------------- /proofpoint/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | requests==2.28.1 17 | jwt==1.3.1 18 | google-auth==2.15.0 19 | google-cloud-secret-manager==2.14.0 20 | -------------------------------------------------------------------------------- /pubsub/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: -------------------------------------------------------------------------------- /pubsub/README.md: -------------------------------------------------------------------------------- 1 | ## DESCRIPTION 2 | --- 3 | 4 | **This script is for fetching message information from the PUBSUB Subscriptions and ingesting to Chronicle.** 5 | 6 | ## PREREQUISITE 7 | --- 8 | This Cloud Function expects the data from PUBSUB in the JSON format. If the message is not received in the expected format, the function would skip the message and continue the data collection. 9 |
Since there can be multiple subscriptions, the user can deploy the Cloud Function once, and configure multiple Cloud Schedulers to collect messages from respective subscriptions.
10 | 11 | 12 | **List of Environment variables:** 13 |
Below details need to be provided in the Body section of Cloud Scheduler to allow the ingestion script for data collection.
NOTE: The details need to be provided in the JSON format only.
14 | 15 | | Variable | Description | Required | Default | Secret | 16 | | ------------------- | -------------------------------------------------------- | -------- | ------- | ------ | 17 | | PROJECT_ID | PUBSUB project ID. | Yes | - | No | 18 | | SUBSCRIPTION_ID | PUBSUB Subscription ID. | Yes | - | No | 19 | | CHRONICLE_DATA_TYPE | Log type to be provided while pushing data to Chronicle. | Yes | - | No | 20 | -------------------------------------------------------------------------------- /pubsub/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch logs from PUBSUB.""" 16 | 17 | from concurrent import futures 18 | import json 19 | import sys 20 | from typing import Any, Dict, List, Union 21 | 22 | from google.cloud import pubsub_v1 23 | 24 | from common import env_constants 25 | from common import ingest 26 | 27 | # Default initialization of variable. 28 | PAYLOAD_SIZE = None 29 | PAYLOAD = None 30 | CHRONICLE_DATA_TYPE = None 31 | 32 | # The threshold to use for ingesting the data to the Chronicle. 33 | PAYLOAD_THRESHOLD = 500000 34 | 35 | # Default timeout to wait for subscriber to send a message. 36 | DEFAULT_TIMEOUT = 5 37 | 38 | 39 | # Generate package to sent to Chronicle. 40 | def build_and_ingest_payload(log: Union[Dict[Any, Any], List[Any]]) -> str: 41 | """Build payload from logs fetched from PUBSUB and ingest it to Chronicle. 42 | 43 | Args: 44 | log: Logs to be ingested in the Chronicle. 45 | 46 | Returns: 47 | str: OK if ingestion successful. 48 | """ 49 | global PAYLOAD_SIZE, PAYLOAD 50 | 51 | if PAYLOAD_SIZE == 0: 52 | # Build a new object. 53 | PAYLOAD = [] 54 | log = json.dumps(log) 55 | PAYLOAD.append(log) 56 | PAYLOAD_SIZE = PAYLOAD_SIZE + (sys.getsizeof(json.dumps(PAYLOAD))) 57 | else: 58 | log = json.dumps(log) 59 | logsize = sys.getsizeof(log) 60 | # Send when the payload hits a certain size. 61 | if PAYLOAD_SIZE + logsize > PAYLOAD_THRESHOLD: 62 | # Ingest collected payload data. 63 | ingest.ingest(PAYLOAD, CHRONICLE_DATA_TYPE) 64 | # Reset payload. 65 | PAYLOAD_SIZE = 0 66 | PAYLOAD = [] 67 | # Append the event. 68 | PAYLOAD.append(log) 69 | PAYLOAD_SIZE = PAYLOAD_SIZE + (sys.getsizeof(log)) 70 | 71 | return "OK" 72 | 73 | 74 | def main(req): # pylint: disable=unused-argument 75 | """Entrypoint. 76 | 77 | Args: 78 | req: Request to execute the cloud function. 79 | 80 | Returns: 81 | string: "Ingestion completed." 82 | """ 83 | global PAYLOAD_SIZE, PAYLOAD, CHRONICLE_DATA_TYPE 84 | PAYLOAD_SIZE = 0 85 | PAYLOAD = [] 86 | 87 | # Expecting values during cloud schedule trigger. 88 | request_json = req.get_json(silent=True) 89 | 90 | if request_json: 91 | project_id = request_json.get("PROJECT_ID", "") 92 | subscription_id = request_json.get("SUBSCRIPTION_ID", "") 93 | CHRONICLE_DATA_TYPE = request_json.get( 94 | env_constants.ENV_CHRONICLE_DATA_TYPE) 95 | else: 96 | print("Did not get configuration parameters from request body.") 97 | 98 | subscriber = pubsub_v1.SubscriberClient() 99 | subscription_path = subscriber.subscription_path(project_id, subscription_id) 100 | 101 | def get_and_ingest_messages( 102 | message: pubsub_v1.subscriber.message.Message) -> None: 103 | """Get message from the subscription. 104 | 105 | Args: 106 | message: Message received from subscription. 107 | 108 | Raises: 109 | ValueError, TypeError: Error when received message is not in json format. 110 | """ 111 | print(f"Received {message.data!r}.") 112 | message.ack() 113 | data = (message.data).decode("utf-8") 114 | try: 115 | data = json.loads(data) 116 | except (ValueError, TypeError) as error: 117 | print("ERROR: Unexpected data format received " 118 | "while collecting message details from subscription") 119 | raise error 120 | 121 | build_and_ingest_payload(data) 122 | 123 | future = subscriber.subscribe( 124 | subscription_path, callback=get_and_ingest_messages) 125 | 126 | with subscriber: 127 | try: 128 | future.result(timeout=DEFAULT_TIMEOUT) 129 | except futures.TimeoutError: 130 | future.cancel() # Trigger the shutdown. 131 | future.result() # Block until the shutdown is complete. 132 | 133 | if PAYLOAD_SIZE > 0: 134 | ingest.ingest(PAYLOAD, CHRONICLE_DATA_TYPE) 135 | 136 | return "Ingestion completed." 137 | -------------------------------------------------------------------------------- /pubsub/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | google-cloud-pubsub 17 | requests==2.27.1 18 | jwt==1.3.1 19 | google-auth==2.6.0 20 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /slack/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | CHRONICLE_CUSTOMER_ID: 16 | CHRONICLE_REGION: us 17 | CHRONICLE_SERVICE_ACCOUNT: 18 | CHRONICLE_NAMESPACE: 19 | # Keeping the default value as 5 minutes considering the frequency of Slack audit logs. 20 | POLL_INTERVAL: "5" 21 | SLACK_ADMIN_TOKEN: -------------------------------------------------------------------------------- /slack/README.md: -------------------------------------------------------------------------------- 1 | # Slack 2 | 3 | This script is for fetching the audit logs from SLACK platform and ingesting to Chronicle. 4 | 5 | ## Platform Specific Environment Variables 6 | | Variable | Description | Required | Default | Secret | 7 | | --- | --- | --- | --- | --- | 8 | | SLACK_ADMIN_TOKEN | Authentication token. | Yes | - | Yes | 9 | | POLL_INTERVAL | Frequency interval(in minutes) at which the Cloud Function executes. This duration must be same as the cloud scheduler job. | No | 5 | No | 10 | -------------------------------------------------------------------------------- /slack/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch audit logs from Slack environment.""" 16 | 17 | import datetime 18 | import requests 19 | 20 | from common import ingest 21 | from common import status 22 | from common import utils 23 | 24 | # Log type to push data into Chronicle. 25 | CHRONICLE_DATA_TYPE = "SLACK_AUDIT" 26 | 27 | # Slack logs API endpoint URL. 28 | SLACK_LOGS_URL = "https://api.slack.com/audit/v1/logs" 29 | 30 | # Environment variable constants. 31 | ENV_SLACK_ADMIN_TOKEN = "SLACK_ADMIN_TOKEN" 32 | 33 | # Default initialization of variable. 34 | SLACK_ADMIN_TOKEN = None 35 | 36 | # Date format to be used in the API. 37 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 38 | 39 | 40 | def get_and_ingest_audit_logs() -> None: 41 | """Fetch audit logs from Slack API, process it and ingest into Chronicle. 42 | 43 | Raises: 44 | TypeError, ValueError: Error when response is not in json format. 45 | """ 46 | # Calculating start_time based on the provided poll interval, it will be a 47 | # datetime object. 48 | start_time = utils.get_last_run_at() 49 | 50 | # Creating a human readable format of the start_time to print in the 51 | # logs for debugging purposes. 52 | start_time_str = start_time.strftime(DATE_FORMAT) 53 | 54 | # The API requires the time in the epoch. So, calculating total 55 | # seconds from the January 1970 and converting it into seconds. 56 | start_time = int((start_time - datetime.datetime( 57 | 1970, 1, 1, tzinfo=datetime.timezone.utc)).total_seconds()) 58 | 59 | print(f"Retrieving the Slack audit logs since: {start_time_str}") 60 | print("Processing logs...") 61 | 62 | url = f"{SLACK_LOGS_URL}?oldest={start_time}" 63 | headers = { 64 | "Accept": "application/json", 65 | "Content-Type": "application/json", 66 | "Authorization": f"Bearer {SLACK_ADMIN_TOKEN}", 67 | } 68 | 69 | # Iterate through all the pages if pagination available and ingest data 70 | # into Chronicle. 71 | while True: 72 | data_list = [] 73 | 74 | print(f"Processing set of results with start time: {start_time}") 75 | 76 | resp = requests.get(url=url, headers=headers) 77 | 78 | try: 79 | response = resp.json() 80 | except (TypeError, ValueError) as error: 81 | print( 82 | "ERROR: Unexpected data format received while collecting audit logs") 83 | raise error 84 | 85 | if resp.status_code != status.STATUS_OK: 86 | print(f"HTTP Error: {resp.status_code}, Reason: {response}") 87 | 88 | resp.raise_for_status() 89 | 90 | log_count = len(response.get("entries", [])) 91 | 92 | print(f"Retrieved {log_count} audit logs from the API call") 93 | 94 | # No need to ingest logs for empty response. 95 | if log_count == 0: 96 | break 97 | 98 | data_list.extend(iter(response["entries"])) 99 | print(f"Retrieved {len(data_list)} Slack audit logs from the last" 100 | " API call.") 101 | 102 | # Ingest data into Chronicle. 103 | ingest.ingest(data_list, CHRONICLE_DATA_TYPE) 104 | 105 | next_cursor = response["response_metadata"]["next_cursor"] 106 | # Update the url if next cursor is available. 107 | if next_cursor: 108 | url = f"{SLACK_LOGS_URL}?oldest={start_time}&cursor={next_cursor}" 109 | print( 110 | f"More records expected.. (processed {log_count} records)") 111 | else: 112 | print("Logs processed successfully.") 113 | break 114 | 115 | 116 | def main(req) -> str: # pylint: disable=unused-argument 117 | """Entrypoint. 118 | 119 | Args: 120 | req: Request to execute the cloud function. 121 | 122 | Returns: 123 | string: "Ingestion completed." 124 | """ 125 | global SLACK_ADMIN_TOKEN 126 | 127 | # Slack admin token. 128 | SLACK_ADMIN_TOKEN = utils.get_env_var( 129 | ENV_SLACK_ADMIN_TOKEN, is_secret=True) 130 | 131 | # Method to fetch audit logs and ingest to chronicle. 132 | get_and_ingest_audit_logs() 133 | 134 | return "Ingestion completed." 135 | -------------------------------------------------------------------------------- /slack/main_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Unit tests for the 'main' module.""" 16 | 17 | import datetime 18 | import sys 19 | 20 | import unittest 21 | from unittest import mock 22 | import requests 23 | 24 | INGESTION_SCRIPTS_PATH = "" 25 | SCRIPT_PATH = "" 26 | 27 | sys.modules["{}common.ingest".format(INGESTION_SCRIPTS_PATH)] = mock.Mock() 28 | 29 | import main 30 | 31 | 32 | def mock_get_env_var(*args, **unused_kwargs): 33 | """Mock and return env variable values.""" 34 | if args[0] == "POLL_INTERVAL": 35 | return 10 36 | else: 37 | return "test" 38 | 39 | 40 | # Mock data. 41 | _test_entities = [{ 42 | "entries": [{ 43 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i1" 44 | }, { 45 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i2" 46 | }], 47 | "response_metadata": { 48 | "next_cursor": "dXNlcjpVMEc5V0ZYTlo=" 49 | } 50 | }, { 51 | "entries": [{ 52 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i3" 53 | }, { 54 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i4" 55 | }], 56 | "response_metadata": { 57 | "next_cursor": "" 58 | } 59 | }] 60 | 61 | 62 | def get_mock_response(): 63 | """Return a mock response.""" 64 | response = mock.Mock() 65 | response.raise_for_status = mock.Mock() 66 | response.status_code = 200 67 | return response 68 | 69 | 70 | @mock.patch( 71 | "{}main.utils.get_env_var".format(SCRIPT_PATH), 72 | side_effect=mock_get_env_var) 73 | @mock.patch("{}main.ingest.ingest".format(SCRIPT_PATH)) 74 | @mock.patch("{}main.requests.get".format(SCRIPT_PATH)) 75 | class TestSlackIngestion(unittest.TestCase): 76 | """Test cases to verify Slack ingestion script.""" 77 | 78 | @mock.patch("builtins.print") 79 | def test_http_error(self, mocked_print, mocked_get, unused_mocked_ingest, 80 | unused_mocked_get_env_var): 81 | """Test case to ensure that we raise errors when status code other than 2XX is encountered.""" 82 | response = get_mock_response() 83 | response.raise_for_status.side_effect = requests.HTTPError() 84 | response.status_code = 400 85 | response.json.return_value = { 86 | "code": 87 | "access_denied", 88 | "description": 89 | "You do not have sufficient permissions to view this resource." 90 | } 91 | mocked_get.return_value = response 92 | 93 | with self.assertRaises(requests.HTTPError): 94 | main.main(req="") 95 | 96 | mocked_print.assert_called_with( 97 | "HTTP Error: 400, Reason: {'code': 'access_denied', 'description': 'You do not have sufficient permissions to view this resource.'}" 98 | ) 99 | 100 | @mock.patch("builtins.print") 101 | def test_value_error(self, mocked_print, mocked_get, unused_mocked_ingest, 102 | unused_mocked_get_env_var): 103 | """Test case to ensure that we raise error when we encounter ValueError from JSON response.""" 104 | response = get_mock_response() 105 | response.json.side_effect = ValueError 106 | mocked_get.return_value = response 107 | 108 | with self.assertRaises(ValueError): 109 | main.main(req="") 110 | 111 | mocked_print.assert_called_with( 112 | "ERROR: Unexpected data format received while collecting audit logs") 113 | 114 | def test_no_logs_to_ingest(self, mocked_get, mocked_ingest, 115 | unused_mocked_get_env_var): 116 | """Test case to ensure that we break the loop when there are no logs to ingest.""" 117 | response = get_mock_response() 118 | response.json.return_value = {"entries": []} 119 | mocked_get.return_value = response 120 | 121 | main.main(req="") 122 | 123 | self.assertEqual(mocked_ingest.call_count, 0) 124 | 125 | @mock.patch("{}main.utils.datetime".format(SCRIPT_PATH)) 126 | def test_log_retrieve_time(self, mocked_utils_datetime, 127 | mocked_get, unused_mocked_ingest, 128 | unused_mocked_get_env_var): 129 | """Test case to verify the log retrieve time is as expected.""" 130 | now_date = datetime.datetime( 131 | 2022, 1, 1, 10, 15, 15, 234566, tzinfo=datetime.timezone.utc) 132 | mocked_utils_datetime.datetime.now.return_value = now_date 133 | mocked_utils_datetime.timedelta.side_effect = datetime.timedelta 134 | 135 | response = get_mock_response() 136 | response.json.return_value = {"entries": []} 137 | mocked_get.return_value = response 138 | 139 | main.main(req="") 140 | 141 | _, kwargs = mocked_get.call_args 142 | 143 | # (2022-01-01 10:15:15) - 5 minutes = (2022-01-01 10:05:15) 144 | expected_log_retrieve_time = 1641031515 145 | self.assertEqual( 146 | kwargs.get("url"), 147 | f"https://api.slack.com/audit/v1/logs?oldest={expected_log_retrieve_time}" 148 | ) 149 | 150 | def test_pagination(self, mocked_get, mocked_ingest, 151 | unused_mocked_get_env_var): 152 | """Test case to verify we fetch next page records when the API response contains next cursor.""" 153 | response = get_mock_response() 154 | response.json.side_effect = _test_entities 155 | mocked_get.return_value = response 156 | 157 | main.main(req="") 158 | 159 | actual_calls = mocked_ingest.mock_calls 160 | expected_calls = [ 161 | mock.call([{ 162 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i1" 163 | }, { 164 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i2" 165 | }], "SLACK_AUDIT"), # Call ingest with 1st page logs 166 | mock.call([{ 167 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i3" 168 | }, { 169 | "id": "0123a45b-6c7d-8900-e12f-3456789gh0i4" 170 | }], "SLACK_AUDIT") # Call ingest with 2nd page logs 171 | ] 172 | self.assertEqual(mocked_ingest.call_count, 2) 173 | self.assertEqual(actual_calls, expected_calls) 174 | -------------------------------------------------------------------------------- /slack/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.27.1 17 | jwt==1.3.1 18 | google-auth==2.6.0 19 | google-cloud-secret-manager==2.10.0 -------------------------------------------------------------------------------- /stix_taxii/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 17 | CHRONICLE_CUSTOMER_ID: 18 | 19 | # Region where the Chronicle instance is located. 20 | CHRONICLE_REGION: us 21 | 22 | # Path of the Google Secret Manager with the version, where the Service Account is stored. 23 | CHRONICLE_SERVICE_ACCOUNT: 24 | 25 | # The Discovery url of the TAXII Server. 26 | TAXII_DISCOVERY_URL: 27 | 28 | # The Username to be used for authentication. 29 | TAXII_USERNAME: 30 | 31 | # Path of the Google Secret Manager with the version, where the Password for TAXII Server is stored. 32 | TAXII_PASSWORD_SECRET_PATH: 33 | 34 | # The STIX/TAXII Version to be used. Supported versions are 1.1, 2.0, 2.1. 35 | TAXII_VERSION: 36 | 37 | # Comma separated collection names from which to collect indicators. 38 | # If not provided, will fetch from all available collections. 39 | TAXII_COLLECTION_NAMES: 40 | 41 | # Time interval in minutes to fetch the data. 42 | # Ex. If poll interval is 60, then it'll fetch indicators after every 60 minutes. 43 | POLL_INTERVAL: "60" -------------------------------------------------------------------------------- /stix_taxii/README.md: -------------------------------------------------------------------------------- 1 | # STIX/TAXII Feed 2 | 3 | This script pulls indicators from STIX/TAXII server and ingests them into Chronicle. 4 | 5 | ## List of Environment Variables 6 | | Variable | Description | Required | Default | Secret | 7 | | -------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | ------ | 8 | | POLL_INTERVAL | Frequency interval (in minutes) at which the Cloud Function executes. This duration must be same as the Cloud Scheduler job.| No | 60 | No | 9 | | TAXII_VERSION | The STIX/TAXII version to use. Possible options are 1.1, 2.0, 2.1 | Yes | - | No | 10 | | TAXII_DISCOVERY_URL | Discovery URL of TAXII server. | Yes | - | No | 11 | | TAXII_COLLECTION_NAMES | Collections (CSV) from which to fetch the data. Leave empty to fetch data from all of the collections. | No | - | No | 12 | | TAXII_USERNAME | Username required for authentication if any. | No | - | No | 13 | | TAXII_PASSWORD_SECRET_PATH | Password required for authentication if any. | No | - | Yes | 14 | -------------------------------------------------------------------------------- /stix_taxii/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch indicators from the STIX/TAXII Server and ingest into Chronicle.""" 16 | 17 | from common import ingest 18 | from common import utils 19 | import taxii_client 20 | 21 | # Environment variable constants. 22 | ENV_TAXII_DISCOVERY_URL = "TAXII_DISCOVERY_URL" 23 | ENV_TAXII_USERNAME = "TAXII_USERNAME" 24 | ENV_TAXII_PASSWORD_SECRET_PATH = "TAXII_PASSWORD_SECRET_PATH" 25 | ENV_TAXII_VERSION = "TAXII_VERSION" 26 | ENV_TAXII_COLLECTION_NAMES = "TAXII_COLLECTION_NAMES" 27 | 28 | # Log type to push data into Chronicle. 29 | CHRONICLE_DATA_TYPE = "STIX" 30 | 31 | 32 | def get_and_ingest_indicators(client: taxii_client.TAXIIClient) -> None: 33 | """Get indicators from STIX/TAXII server and ingest them into Chronicle. 34 | 35 | Args: 36 | client (taxii_client.TAXIIClient): TAXII Client to be used for collecting 37 | indicators. 38 | 39 | Raises: 40 | Exception: If any error occurred while fetching and ingesting the 41 | indicators. 42 | """ 43 | # Calculate the start time based on the POLL_INTERVAL environment variable. 44 | start_time = utils.get_last_run_at() 45 | 46 | # Convert the datetime object to STIX compliant datetime string. 47 | # Expected format is (YYYY-MM-DDTHH:MM:SSS.SSSZ). 48 | start_time = taxii_client.convert_date_to_stix_format(start_time) 49 | 50 | # Pull indicators from the TAXII server. 51 | try: 52 | fetched_indicators = client.pull_indicators(start_time) 53 | except Exception as error: 54 | raise Exception( 55 | "Failure occurred while fetching the indicators from the STIX/TAXII " 56 | "server." 57 | ) from error 58 | 59 | # Ingest data into Chronicle. 60 | try: 61 | ingest.ingest(fetched_indicators, CHRONICLE_DATA_TYPE) 62 | except Exception as error: 63 | raise Exception( 64 | "Failure occurred while ingesting the indicators into Chronicle." 65 | ) from error 66 | 67 | 68 | def main(req) -> str: # pylint: disable=unused-argument 69 | """Entrypoint. 70 | 71 | Args: 72 | req: Request to execute the cloud function. 73 | 74 | Returns: 75 | string: "Ingestion completed." 76 | """ 77 | # Fetch the environment variables. 78 | discovery_url = utils.get_env_var(ENV_TAXII_DISCOVERY_URL) 79 | username = utils.get_env_var(ENV_TAXII_USERNAME, required=False) 80 | password = utils.get_env_var( 81 | ENV_TAXII_PASSWORD_SECRET_PATH, is_secret=True, required=False) 82 | # Possible values of TAXII version are 1.1, 2.0 or 2.1. 83 | taxii_version = utils.get_env_var(ENV_TAXII_VERSION) 84 | # Provide specific collection names from which the indicators should be 85 | # collected. These collection names are specific to the STIX/TAXII server. 86 | # By default, the indicators will be collected from all the collections. 87 | collection_names = utils.get_env_var( 88 | ENV_TAXII_COLLECTION_NAMES, required=False) 89 | 90 | # Create a Taxii client based on the provided parameters. 91 | client = taxii_client.TAXIIClient( 92 | discovery_url=discovery_url, 93 | username=username, 94 | password=password, 95 | taxii_version=taxii_version, 96 | collection_names=collection_names) 97 | 98 | # Fetch and ingest the indicators from STIX/TAXII server into Chronicle. 99 | get_and_ingest_indicators(client) 100 | 101 | return "Ingestion completed." 102 | -------------------------------------------------------------------------------- /stix_taxii/main_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Unit test case file for main module of STIX/TAXII ingestion script.""" 16 | 17 | import datetime 18 | import sys 19 | 20 | import unittest 21 | from unittest import mock 22 | 23 | INGESTION_SCRIPTS_PATH = "" 24 | SCRIPT_PATH = "" 25 | 26 | sys.modules["{}common.ingest".format(INGESTION_SCRIPTS_PATH)] = mock.Mock() 27 | 28 | import main 29 | import taxii_client 30 | 31 | # Test value for poll interval. 32 | TEST_POLL_INTERVAL = 15 33 | 34 | 35 | def get_mock_response() -> mock.Mock: 36 | """Return a mock response. 37 | 38 | Returns: 39 | mock.Mock: Mock response. 40 | """ 41 | response = mock.Mock() 42 | response.raise_for_status = mock.Mock() 43 | response.status_code = 200 44 | return response 45 | 46 | 47 | class TestStixTaxiiIngestion(unittest.TestCase): 48 | """Test cases for the main function.""" 49 | 50 | @mock.patch( 51 | f"{SCRIPT_PATH}main.taxii_client." 52 | "convert_date_to_stix_format") 53 | @mock.patch( 54 | f"{SCRIPT_PATH}main.taxii_client.TAXIIClient") 55 | @mock.patch(f"{SCRIPT_PATH}main.utils.get_env_var") 56 | @mock.patch(f"{INGESTION_SCRIPTS_PATH}common.auth.requests.Session.send") 57 | @mock.patch(f"{SCRIPT_PATH}main.ingest.ingest") 58 | def test_main_success(self, mocked_ingest, mocked_send, *unused_args): 59 | """Test case to verify that the ingest function should be called once when the TAXII Server returns valid response. 60 | """ 61 | mock_response_1 = get_mock_response() 62 | mock_response_1.json.return_value = {"access_token": "test_access_token"} 63 | 64 | mock_response_2 = get_mock_response() 65 | mock_response_2.json.return_value = {"entries": [], "chunk_size": 0} 66 | 67 | mocked_send.side_effect = [mock_response_1, mock_response_2] 68 | 69 | main.main(req="") 70 | 71 | self.assertEqual(mocked_ingest.call_count, 1) 72 | 73 | def test_convert_date_to_stix_format(self): 74 | """Test case to verify that the convert_date_to_stix format returns the valid date string when provided a valid datetime object. 75 | """ 76 | dt_object = datetime.datetime(2022, 1, 1, 5, 45, 58, 564783) 77 | dt_object = dt_object.replace(tzinfo=datetime.timezone.utc) 78 | 79 | self.assertEqual( 80 | taxii_client.convert_date_to_stix_format(dt_object), 81 | "2022-01-01T05:45:58.564783Z") 82 | -------------------------------------------------------------------------------- /stix_taxii/requirements.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | cabby==0.1.23 16 | stix==1.2.0.11 17 | taxii2-client==2.3.0 18 | requests==2.28.1 19 | jwt==1.3.1 20 | google-auth==2.6.0 21 | google-cloud-secret-manager==2.10.0 22 | requests-mock==1.10.0 -------------------------------------------------------------------------------- /stix_taxii/test_data/taxii_v11_collections_response.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | test collection 4 | 5 | 6 | 7 | urn:taxii.mitre.org:protocol:https:1.0 8 | https://dummy.com/taxii/poll/ 9 | urn:taxii.mitre.org:message:xml:1.1 10 | 11 | 12 | urn:taxii.mitre.org:protocol:https:1.0 13 | https://dummy.com/taxii/inbox/ 14 | urn:taxii.mitre.org:message:xml:1.1 15 | 16 | 17 | 18 | test2 collection 19 | 20 | 21 | 22 | urn:taxii.mitre.org:protocol:https:1.0 23 | https://dummy.com/taxii/poll/ 24 | urn:taxii.mitre.org:message:xml:1.1 25 | 26 | 27 | -------------------------------------------------------------------------------- /stix_taxii/test_data/taxii_v11_discovery_response.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | urn:taxii.mitre.org:protocol:https:1.0 4 | https://dummy.com/taxii/ 5 | urn:taxii.mitre.org:message:xml:1.1 6 | Test Server Discovery Service 7 | 8 | 9 | urn:taxii.mitre.org:protocol:https:1.0 10 | https://dummy.com/taxii/collection/ 11 | urn:taxii.mitre.org:message:xml:1.1 12 | Test TAXII Server Collections 13 | 14 | 15 | urn:taxii.mitre.org:protocol:https:1.0 16 | https://dummy.com/taxii/poll/ 17 | urn:taxii.mitre.org:message:xml:1.1 18 | Test poll service 19 | 20 | -------------------------------------------------------------------------------- /stix_taxii/test_data/taxii_v11_discovery_response_without_collection_management.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | urn:taxii.mitre.org:protocol:https:1.0 4 | https://dummy.com/taxii/ 5 | urn:taxii.mitre.org:message:xml:1.1 6 | Test Server Discovery Service 7 | 8 | 9 | urn:taxii.mitre.org:protocol:https:1.0 10 | https://dummy.com/taxii/poll/ 11 | urn:taxii.mitre.org:message:xml:1.1 12 | Test poll service 13 | 14 | -------------------------------------------------------------------------------- /stix_taxii/test_data/taxii_v11_indicators_response_empty.xml: -------------------------------------------------------------------------------- 1 | 2 | 2022-11-14T00:00:00+00:00 3 | 0 4 | 5 | -------------------------------------------------------------------------------- /stix_taxii/test_data/taxii_v11_indicators_response_page_1.xml: -------------------------------------------------------------------------------- 1 | 2 | 2022-11-01T00:00:00+00:00 3 | 2022-11-02T00:00:00+00:00 4 | 1 5 | 6 | 7 | 8 | 9 | 10 | 11 | mal_domain: test.com 12 | malicious-activity 13 | mal_domain 14 | test description 15 | 16 | 2022-09-12T15:50:21.286000+00:00 17 | 9999-12-31T00:00:00+00:00 18 | 19 | 20 | High 21 | 22 | 23 | 24 | mal_domain: demo.com 25 | malicious-activity 26 | mal_domain 27 | test description 2 28 | 29 | 2022-09-12T15:50:21.048000+00:00 30 | 9999-12-31T00:00:00+00:00 31 | 32 | 33 | High 34 | 35 | 36 | 37 | 38 | 39 | 2022-11-01T15:16:14.021157+00:00 40 | 41 | 42 | -------------------------------------------------------------------------------- /teamcymru_scout/images/adhoc_parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/teamcymru_scout/images/adhoc_parameters.png -------------------------------------------------------------------------------- /teamcymru_scout/images/chronicle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/teamcymru_scout/images/chronicle.png -------------------------------------------------------------------------------- /teamcymru_scout/images/reference_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chronicle/ingestion-scripts/7d2dea0d5b02ef1be806008e70d5a286e8ebdf09/teamcymru_scout/images/reference_list.png -------------------------------------------------------------------------------- /teamcymru_scout/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.3 2 | jwt==1.3.1 3 | google-auth==2.30.0 4 | google-cloud-secret-manager==2.20.0 5 | google-api-python-client==2.134.0 6 | google-cloud-storage==2.17.0 7 | redis==5.0.8 8 | tldextract==5.0.1 9 | ipaddress==1.0.23 10 | -------------------------------------------------------------------------------- /teamcymru_scout/teamcymru_scout_constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # pylint: disable=line-too-long 16 | """Team Cymru Scout Constants.""" 17 | 18 | PROTOCOL = "https://" 19 | VERIFY_SSL = True 20 | IPS_CHUNKSIZE = 10 21 | SIZE_THRESHOLD_BYTES = 950000 22 | 23 | 24 | class Endpoints: 25 | """Team Cymru Scout Endpoints.""" 26 | 27 | CYMRU_SERVER_ADDRESS = "scout.cymru.com/api/scout" 28 | USAGE = "/usage" 29 | IP_FOUNDATION = "/ip/foundation" 30 | IP_DETAILS = "/ip/{ip}/details" 31 | DOMAIN_DETAILS = "/search" 32 | 33 | 34 | class Rest: 35 | """Team Cymru Scout Rest Constants.""" 36 | 37 | STATUS_FORCELIST = list(range(500, 600)) + [ 38 | 429, 39 | ] 40 | REQUEST_TIMEOUT = 300 41 | MAX_RETRIES = 3 42 | BACKOFF_FACTOR = 60 43 | 44 | 45 | DOMAIN_REGEX = r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{1,}$" 46 | IPV4_REGEX = r"^(?:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$" 47 | IPV6_REGEX = ( 48 | r"^(?:(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}|" 49 | r"(?:[A-Fa-f0-9]{1,4}:){1,7}:|" 50 | r"(?:[A-Fa-f0-9]{1,4}:){1,6}:[A-Fa-f0-9]{1,4}|" 51 | r"(?:[A-Fa-f0-9]{1,4}:){1,5}(:[A-Fa-f0-9]{1,4}){1,2}|" 52 | r"(?:[A-Fa-f0-9]{1,4}:){1,4}(:[A-Fa-f0-9]{1,4}){1,3}|" 53 | r"(?:[A-Fa-f0-9]{1,4}:){1,3}(:[A-Fa-f0-9]{1,4}){1,4}|" 54 | r"(?:[A-Fa-f0-9]{1,4}:){1,2}(:[A-Fa-f0-9]{1,4}){1,5}|" 55 | r"[A-Fa-f0-9]{1,4}:(:[A-Fa-f0-9]{1,4}){1,6}|" 56 | r":((:[A-Fa-f0-9]{1,4}){1,7}|:)|" 57 | r"fe80:(:[A-Fa-f0-9]{0,4}){0,4}%[0-9a-zA-Z]{1,}|" 58 | r"::(ffff(:0{1,4}){0,1}:){0,1}" 59 | r"((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}" 60 | r"(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])|" 61 | r"(?:[A-Fa-f0-9]{1,4}:){1,4}:" 62 | r"((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}" 63 | r"(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9]))$" 64 | ) 65 | -------------------------------------------------------------------------------- /teamcymru_scout/teamcymru_scout_env_constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Team Cymru Scout env constants.""" 16 | 17 | ENV_TEAMCYMRU_SCOUT_AUTH_TYPE = "TEAMCYMRU_SCOUT_AUTH_TYPE" 18 | ENV_TEAMCYMRU_SCOUT_ACCOUNT_NAME = "TEAMCYMRU_SCOUT_ACCOUNT_NAME" 19 | ENV_TEAMCYMRU_SCOUT_API_USERNAME = "TEAMCYMRU_SCOUT_API_USERNAME" 20 | ENV_TEAMCYMRU_SCOUT_API_PASSWORD = "TEAMCYMRU_SCOUT_API_PASSWORD" 21 | ENV_TEAMCYMRU_SCOUT_API_KEY = "TEAMCYMRU_SCOUT_API_KEY" 22 | ENV_IP_ENRICHMENT_LIST = "IP_ENRICHMENT_LIST" 23 | ENV_DOMAIN_SEARCH_LIST = "DOMAIN_SEARCH_LIST" 24 | ENV_LIVE_INVESTIGATION_LIST = "LIVE_INVESTIGATION_LIST" 25 | CHRONICLE_DATA_TYPE = "TEAM_CYMRU_SCOUT_THREATINTEL" 26 | ENV_IP_ENRICHMENT_SIZE = "IP_ENRICHMENT_SIZE" 27 | ENV_FORCE_IP_ENRICHMENT_DETAIL = "FORCE_IP_ENRICHMENT_DETAIL" 28 | ENV_IP_ENRICHMENT_TAGS = "IP_ENRICHMENT_TAGS" 29 | ENV_LOG_TYPE_FILE_PATH = "LOG_TYPE_FILE_PATH" 30 | ENV_PROVISIONAL_TTL = "PROVISIONAL_TTL" 31 | -------------------------------------------------------------------------------- /tenable/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 17 | CHRONICLE_CUSTOMER_ID: 18 | 19 | # Region where the Chronicle instance is located. 20 | CHRONICLE_REGION: us 21 | 22 | # Path of the Google Secret Manager with the version, where the Chronicle Service Account is stored. 23 | CHRONICLE_SERVICE_ACCOUNT: 24 | 25 | # The Tenable access key to be used for authentication. 26 | TENABLE_ACCESS_KEY: 27 | 28 | # Path of the Google Secret Manager with the version, where the secret key for Tenable server is 29 | # stored. 30 | TENABLE_SECRET_KEY_PATH: 31 | 32 | # Type of data to fetch from Tenable. Supported data types are ASSETS and VULNERABILITIES. 33 | TENABLE_DATA_TYPE: ASSETS, VULNERABILITIES 34 | 35 | # The state of vulnerabilities to fetch from Tenable. Supported states are OPEN, REOPENED, and FIXED. 36 | TENABLE_VULNERABILITY: OPEN, REOPENED 37 | 38 | # Time interval in minutes to fetch the data. 39 | # For example, if poll interval is 60, then it will fetch data at every 60 minutes. 40 | POLL_INTERVAL: "360" 41 | 42 | #The namespace that the Chronicle logs are labeled with. 43 | CHRONICLE_NAMESPACE: 44 | -------------------------------------------------------------------------------- /tenable/README.md: -------------------------------------------------------------------------------- 1 | # Tenable IO 2 | 3 | This script retrieves the Assets and Vulnerabilities from Tenable.io, and ingest them into the Chronicle platform. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | Secret | 8 | |-------------------------|-----------------------------------------------|----------|-------------------------|--------| 9 | | POLL_INTERVAL | Frequency interval (in minutes) at which the Cloud Function executes. This duration must be same as the Cloud Scheduler job. | No | 360 | No | 10 | | TENABLE_ACCESS_KEY | The access key to be used for authentication. | Yes | - | No | 11 | | TENABLE_SECRET_KEY_PATH | Path of the Google Secret Manager with the version, where the password for Tenable server is stored. | Yes | - | Yes | 12 | | TENABLE_DATA_TYPE | Type of data to fetch from Tenable. Supported data types are ASSETS and VULNERABILITIES. | No | ASSETS, VULNERABILITIES | No | 13 | | TENABLE_VULNERABILITY | The state of vulnerabilities to fetch from Tenable. Supported states are OPEN, REOPENED, and FIXED. | No | OPEN, REOPENED | No | 14 | -------------------------------------------------------------------------------- /tenable/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | pyTenable==1.4.11 17 | requests==2.28.1 18 | jwt==1.3.1 19 | google-auth==2.15.0 20 | google-cloud-secret-manager==2.13.0 -------------------------------------------------------------------------------- /trend_micro/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 16 | CHRONICLE_CUSTOMER_ID: 17 | 18 | # Region where the Chronicle instance is located. 19 | CHRONICLE_REGION: us 20 | 21 | # Path of the Google Secret Manager with the version, where the Chronicle Service Account is stored. 22 | CHRONICLE_SERVICE_ACCOUNT: 23 | 24 | # Service URL of the Trend Micro Cloud App service. 25 | TREND_MICRO_SERVICE_URL: 26 | 27 | # Path of the Google Secret Manager with the version, where the authentication token for Trend Micro Server is stored. 28 | TREND_MICRO_AUTHENTICATION_TOKEN: 29 | 30 | # The name of the protected service, whose logs to retrieve. Supports comma-separated values. 31 | # By default, it will fetch logs for all types of services. 32 | # Possible values: exchange, sharepoint, onedrive, dropbox, box, googledrive, gmail, 33 | # teams, exchangeserver, salesforce_sandbox, salesforce_production, teams_chat 34 | TREND_MICRO_SERVICE: "exchange, sharepoint, onedrive, dropbox, box, googledrive, gmail, teams, exchangeserver, salesforce_sandbox, salesforce_production, teams_chat" 35 | 36 | # The type of the security event, whose logs to retrieve. Supports comma-separated values. 37 | # By default, it will fetch logs for all types of security events. 38 | # Possible values: securityrisk, virtualanalyzer, ransomware, dlp 39 | TREND_MICRO_EVENT: "securityrisk, virtualanalyzer, ransomware, dlp" 40 | 41 | # Log type according to the service to push data into the Chronicle platform. 42 | CHRONICLE_DATA_TYPE: 43 | 44 | # The namespace that the Chronicle logs are labeled with. 45 | CHRONICLE_NAMESPACE: 46 | 47 | # Time interval in minutes to fetch the data. 48 | # For example, if the poll interval is 5, then it will fetch events after every 5 minutes. 49 | POLL_INTERVAL: "10" 50 | -------------------------------------------------------------------------------- /trend_micro/README.md: -------------------------------------------------------------------------------- 1 | # Trend Micro 2 | 3 | This script retrieves the security logs from Trend Micro platform, and ingests them into the Chronicle platform. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | Secret | 8 | |---|---|---|---|---| 9 | | POLL_INTERVAL | Frequency interval at which the function executes to get additional log data (in minutes). This duration must be the same as the Cloud Scheduler job interval. | No | 10 | No | 10 | | CHRONICLE_DATA_TYPE | Log type according to the service to push data into the Chronicle platform. | Yes | - | No | 11 | | TREND_MICRO_AUTHENTICATION_TOKEN | Path of the Google Secret Manager with the version, where the authentication token for Trend Micro Server is stored. | Yes | - | Yes | 12 | | TREND_MICRO_SERVICE_URL | Service URL of the Cloud App Security service. | Yes | - | No | 13 | | TREND_MICRO_SERVICE | The name of the protected service, whose logs to retrieve. Supports comma-separated values. Possible values: exchange, sharepoint, onedrive, dropbox, box, googledrive, gmail, teams, exchangeserver, salesforce_sandbox, salesforce_production, teams_chat. | No | exchange, sharepoint, onedrive, dropbox, box, googledrive, gmail, teams, exchangeserver, salesforce_sandbox, salesforce_production, teams_chat | No | 14 | | TREND_MICRO_EVENT | The type of the security event, whose logs to retrieve. Supports comma-separated values. Possible values: securityrisk, virtualanalyzer, ransomware, dlp. | No | securityrisk, virtualanalyzer, ransomware, dlp | No | 15 | -------------------------------------------------------------------------------- /trend_micro/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | requests==2.28.1 17 | jwt==1.3.1 18 | google-auth==2.15.0 19 | google-cloud-secret-manager==2.13.0 -------------------------------------------------------------------------------- /trend_micro_vision/.env.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Chronicle Customer ID to be used for pushing the logs to the Chronicle. 16 | CHRONICLE_CUSTOMER_ID: 17 | 18 | # Region where the Chronicle instance is located. 19 | CHRONICLE_REGION: us 20 | 21 | # Path of the Google Secret Manager with the version, where the Chronicle Service Account is stored. 22 | CHRONICLE_SERVICE_ACCOUNT: 23 | 24 | # Path of the Google Secret Manager with the version, where the authentication token for Trend Micro 25 | # Vision One Server is stored. 26 | TREND_MICRO_AUTHENTICATION_TOKEN: 27 | 28 | # Trend Micro Vision One region where the service endpoint is located. 29 | # For example: api.in.xdr.trendmicro.com 30 | TREND_MICRO_DOMAIN: 31 | 32 | # Type of data to ingest in Chronicle. 33 | # Possible values: AUDIT_LOGS, ALERTS 34 | TREND_MICRO_DATA_TYPE: "AUDIT_LOGS, ALERTS" 35 | 36 | # The namespace that the Chronicle logs are labeled with. 37 | CHRONICLE_NAMESPACE: 38 | 39 | # Time interval in minutes to fetch the data. 40 | # For example, if the poll interval is 5, then it will fetch events after every 5 minutes. 41 | POLL_INTERVAL: "10" -------------------------------------------------------------------------------- /trend_micro_vision/README.md: -------------------------------------------------------------------------------- 1 | # Trend Micro Vision One 2 | 3 | This script retrieves the audit logs from Trend Micro Vision One platform, and ingests them into the Chronicle platform. 4 | 5 | ## Platform Specific Environment Variables 6 | 7 | | Variable | Description | Required | Default | Secret | 8 | |---|---|---|---|---| 9 | | POLL_INTERVAL | Frequency interval at which the function executes to get additional log data (in minutes). This duration must be the same as the Cloud Scheduler job interval. | No | 10 | No | 10 | | TREND_MICRO_AUTHENTICATION_TOKEN | Path of the Google Secret Manager with the version, where the authentication token for Trend Micro Vision One Server is stored. | Yes | - | Yes | 11 | | TREND_MICRO_DOMAIN | Trend Micro Vision One region where the service endpoint is located. For example: api.in.xdr.trendmicro.com | Yes | - | No | 12 | | TREND_MICRO_DATA_TYPE | Type of data to ingest in Chronicle. Possible Values: AUDIT_LOGS, ALERTS. | No | AUDIT_LOGS, ALERTS | No | 13 | -------------------------------------------------------------------------------- /trend_micro_vision/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Fetch audit logs from the Trend Micro Vision One and ingest into Chronicle.""" 16 | 17 | import requests 18 | 19 | from common import ingest 20 | from common import status 21 | from common import utils 22 | 23 | # The date format to be used for converting python datetime object to 24 | # human-readable string. 25 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 26 | 27 | # Possible data types. 28 | VALID_DATA_TYPES = ["audit_logs", "alerts"] 29 | 30 | TREND_MICRO_AUDIT_LOGS_DATA_TYPE = "audit_logs" 31 | TREND_MICRO_ALERTS_DATA_TYPE = "alerts" 32 | 33 | # Log type to push data into Chronicle. 34 | CHRONICLE_DATA_TYPE = "TRENDMICRO_VISION_ONE" 35 | 36 | # By default, the script will collect data of audit logs and alerts. 37 | DEFAULT_TREND_MICRO_DATA_TYPE = ( 38 | f"{TREND_MICRO_AUDIT_LOGS_DATA_TYPE}, {TREND_MICRO_ALERTS_DATA_TYPE}" 39 | ) 40 | 41 | # Environment variable constants. 42 | ENV_TREND_MICRO_AUTHENTICATION_TOKEN = "TREND_MICRO_AUTHENTICATION_TOKEN" 43 | ENV_TREND_MICRO_DOMAIN = "TREND_MICRO_DOMAIN" 44 | ENV_TREND_MICRO_DATA_TYPE = "TREND_MICRO_DATA_TYPE" 45 | 46 | 47 | def get_and_ingest_vision_one_logs( 48 | authentication_token: str, domain: str, data_type: str 49 | ) -> None: 50 | """Fetch audit logs/alerts from Trend Micro Vision One platform and ingest them into Chronicle. 51 | 52 | Args: 53 | authentication_token (str): Authentication token used to authenticate with 54 | the API. 55 | domain (str): Region where the service endpoint is located. 56 | data_type (str): Type of data to fetch and ingest into Chronicle. 57 | 58 | Raises: 59 | RuntimeError: If any error occurred while fetching and ingesting audit 60 | logs/alerts. 61 | """ 62 | 63 | # Calculate the start time based on the POLL_INTERVAL environment variable. 64 | start_time = utils.get_last_run_at().strftime(DATE_FORMAT) 65 | log_count = 0 66 | 67 | headers = {"Authorization": "Bearer " + authentication_token} 68 | 69 | # Set URL based on data types. 70 | if data_type == TREND_MICRO_AUDIT_LOGS_DATA_TYPE: # URL for audit_logs. 71 | url = f"https://{domain}/v3.0/audit/logs?startDateTime={start_time}&labels=all&top=200" 72 | else: # URL for alerts. 73 | url = f"https://{domain}/v3.0/workbench/alerts?startDateTime={start_time}" 74 | 75 | print(f"Retrieving {data_type} added after {start_time}.") 76 | 77 | # Get the audit logs/alerts from the Trend Micro Vision One until nextLink is 78 | # present in response. 79 | while True: 80 | response = requests.get(url, headers=headers) 81 | response_status = response.status_code 82 | 83 | # Retrieve the json response. 84 | try: 85 | json_response = response.json() 86 | except (ValueError, TypeError) as error: 87 | raise ValueError( 88 | f"Unexpected data format received while collecting {data_type} from" 89 | " Trend Micro Vision One." 90 | ) from error 91 | 92 | # If the response status code is other than 200, then raise the error. 93 | if response_status != status.STATUS_OK: 94 | error_message = json_response.get("message", json_response) 95 | raise RuntimeError( 96 | f"Failed to get {data_type} from Trend Micro Vision One with status" 97 | f" code {response_status}. Error message: {error_message}." 98 | ) 99 | 100 | # Ingest Trend Micro Vision One audit logs/alerts to the Chronicle platform. 101 | data_list = json_response.get("items", []) 102 | 103 | if data_list: 104 | log_count += len(data_list) 105 | try: 106 | ingest.ingest(data_list, CHRONICLE_DATA_TYPE) 107 | except Exception as error: 108 | raise RuntimeError( 109 | f"Unable to push Trend Micro Vision One {data_type} into Chronicle:" 110 | f" {error}." 111 | ) from error 112 | 113 | # Update the URL for the next page, if the nextLink is present. 114 | if json_response.get("nextLink"): 115 | url = json_response["nextLink"] 116 | else: 117 | break 118 | 119 | if log_count: 120 | print(f"Successfully ingested {log_count} log(s) into Chronicle.") 121 | else: 122 | print(f"No new {data_type} found in the given time range.") 123 | 124 | 125 | # Request is a user input dictionary passed while running the cloud function. 126 | # The script does not use these parameters. 127 | def main(request) -> str: # pylint: disable=unused-argument 128 | """Entrypoint. 129 | 130 | Args: 131 | request: Request to execute the cloud function. 132 | Returns: 133 | str: "Ingestion completed". 134 | """ 135 | # Fetch the environment variables. 136 | authentication_token = utils.get_env_var( 137 | ENV_TREND_MICRO_AUTHENTICATION_TOKEN, is_secret=True) 138 | domain = utils.get_env_var(ENV_TREND_MICRO_DOMAIN) 139 | trend_micro_data_type = utils.get_env_var( 140 | ENV_TREND_MICRO_DATA_TYPE, 141 | required=False, 142 | default=DEFAULT_TREND_MICRO_DATA_TYPE, 143 | ) 144 | 145 | # Create a list of data type from CSV string. 146 | data_type = [ 147 | data.lower().strip() for data in trend_micro_data_type.strip().split(",") 148 | ] 149 | 150 | # Get audit logs from Trend Micro Vision One and ingest it into Chronicle. 151 | if TREND_MICRO_AUDIT_LOGS_DATA_TYPE in data_type: 152 | get_and_ingest_vision_one_logs( 153 | authentication_token, domain, TREND_MICRO_AUDIT_LOGS_DATA_TYPE 154 | ) 155 | 156 | # Get alerts from Trend Micro Vision One and ingest it into Chronicle. 157 | if TREND_MICRO_ALERTS_DATA_TYPE in data_type: 158 | get_and_ingest_vision_one_logs( 159 | authentication_token, domain, TREND_MICRO_ALERTS_DATA_TYPE 160 | ) 161 | 162 | return "Ingestion completed." 163 | -------------------------------------------------------------------------------- /trend_micro_vision/requirements.txt: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | requests==2.28.1 17 | jwt==1.3.1 18 | google-auth==2.15.0 19 | google-cloud-secret-manager==2.13.0 -------------------------------------------------------------------------------- /vectra_xdr/constant.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Ventra XDR Constants.""" 16 | 17 | # ENVIRONMENT VARIABLES CONSTANTS 18 | ENV_VAR_INCLUDE_SCORE_DECREASES = "INCLUDE_SCORE_DECREASES" 19 | ENV_VAR_VLANS = "VLANS" 20 | ENV_VAR_INCLUDE_INFO_CATEGORY = "INCLUDE_INFO_CATEGORY" 21 | ENV_VAR_INCLUDE_TRIAGED = "INCLUDE_TRIAGED" 22 | ENV_VECTRA_BASE_URL = "VECTRA_PORTAL_URL" 23 | ENV_GCP_BUCKET_NAME = "GCP_BUCKET_NAME" 24 | ENV_HISTORICAL = "HISTORICAL" 25 | ENV_CLIENT_ID_SECRECT_NAME = "CLIENT_ID" 26 | ENV_CLIENT_SECRET_SECRET_NAME = "SECRET_KEY" 27 | ENV_GCP_PROJECT_NUMBER = "GCP_PROJECT_NUMBER" 28 | 29 | ENV_VAR_LOCKDOWN = "ENABLE_LOCKDOWN" 30 | ENV_VAR_AUDIT = "ENABLE_AUDIT" 31 | ENV_VAR_HEALTH = "ENABLE_HEALTH" 32 | ENV_VAR_DETECTION = "ENABLE_DETECTION" 33 | ENV_VAR_SCORING = "ENABLE_SCORING" 34 | 35 | # API ENDPOINTS 36 | API_VERSION = "api/v3.4" 37 | VECTRA_ACCESS_TOKEN_ENDPOINT = "oauth2/token" 38 | VECTRA_LOCKDOWN_ENDPOINT = API_VERSION + "/lockdown" 39 | VECTRA_AUDIT_ENDPOINT = API_VERSION + "/events/audits" 40 | VECTRA_HEALTH_ENDPOINT = API_VERSION + "/health" 41 | VECTRA_SCORING_ENDPOINT = API_VERSION + "/events/entity_scoring" 42 | VECTRA_DETECTION_ENDPOINT = API_VERSION + "/events/detections" 43 | VECTRA_APP_VERSION = "1.0.0" 44 | VECTRA_APP_USER_AGENT = "vectra-rux-csiem-" + VECTRA_APP_VERSION 45 | 46 | HEADERS = {"Accept": "application/json", "User-Agent": VECTRA_APP_USER_AGENT} 47 | ERRORS = { 48 | "RATE_LIMIT_EXCEEDED": "Rate limit exceeded. Please wait and try again.", 49 | "REFRESH_TOKEN_EXPIRE_MESSAGE": ( 50 | "Please try reauthenticating using API client credentials" 51 | ), 52 | } 53 | 54 | # OTHERS 55 | RETRY_COUNT = 3 56 | RETRY_COUNT_TOKEN = 1 57 | DEFAULT_REQUEST_TIMEOUT = 60 58 | WAIT_TIME_FOR_RETRY = 30 59 | MAX_EVENT_LIMIT = 100 60 | TIME_STAMP_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # 2024-06-30T01:29:13Z 61 | HOST_TYPE = "host" 62 | ACCOUNT_TYPE = "account" 63 | NEXT_CHECKPOINT = "next_checkpoint" 64 | REMAINING_COUNT = "remaining_count" 65 | CHRONICLE_DATA_TYPE = "VECTRA_XDR" 66 | METHOD_INTERVAL = 60 67 | GCP_BUCKET_FILE_NAME = "checkpoint.json" 68 | VECTRA_API_TOKEN_SECRET_NAME = "vectra_api_token" 69 | 70 | # Default Values 71 | DEFAULT_VALUES = { 72 | ENV_VAR_INCLUDE_SCORE_DECREASES: "false", 73 | ENV_VAR_VLANS: "false", 74 | ENV_VAR_INCLUDE_INFO_CATEGORY: "true", 75 | ENV_VAR_INCLUDE_TRIAGED: "false", 76 | ENV_VAR_LOCKDOWN: "true", 77 | ENV_VAR_AUDIT: "true", 78 | ENV_VAR_HEALTH: "true", 79 | ENV_VAR_DETECTION: "true", 80 | ENV_VAR_SCORING: "true", 81 | ENV_HISTORICAL: "false", 82 | } 83 | -------------------------------------------------------------------------------- /vectra_xdr/exception.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # pylint: disable=g-bad-exception-name 16 | 17 | """Custom exceptions for Vectra XDR ingestion script.""" 18 | 19 | 20 | class VectraException(Exception): 21 | """Base exception for Vectra XDR ingestion script.""" 22 | 23 | 24 | class UnauthorizeException(VectraException): 25 | """Unauthorized user.""" 26 | 27 | 28 | class RefreshTokenException(VectraException): 29 | """Exception if refresh token is expired or invalid.""" 30 | 31 | 32 | class RateLimitException(VectraException): 33 | """Exception for rate limit.""" 34 | 35 | 36 | class InternalSeverError(VectraException): 37 | """Internal Server Error.""" 38 | 39 | 40 | class BadRequestException(VectraException): 41 | """Exception for bad request.""" 42 | -------------------------------------------------------------------------------- /vectra_xdr/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | """Main script for Vectra XDR ingestion.""" 16 | 17 | from common import utils 18 | import constant 19 | import utils as vectra_utils 20 | import vectra_client 21 | 22 | 23 | def main(request): # pylint: disable=unused-argument 24 | try: 25 | secret_manager_client = vectra_utils.SecretManagerClient() 26 | client_id = secret_manager_client.get_secrets( 27 | secret_name=vectra_utils.get_environment_variable( 28 | constant.ENV_CLIENT_ID_SECRECT_NAME, 29 | is_required=True, 30 | is_secret=True, 31 | ), 32 | secret_format_is_json_type=False, 33 | ) 34 | client_secrets = secret_manager_client.get_secrets( 35 | secret_name=vectra_utils.get_environment_variable( 36 | constant.ENV_CLIENT_SECRET_SECRET_NAME, 37 | is_required=True, 38 | is_secret=True, 39 | ), 40 | secret_format_is_json_type=False, 41 | ) 42 | base_url = vectra_utils.get_environment_variable( 43 | constant.ENV_VECTRA_BASE_URL, is_required=True 44 | ) 45 | bucket_name = vectra_utils.get_environment_variable( 46 | constant.ENV_GCP_BUCKET_NAME, is_required=True 47 | ) 48 | 49 | vectra_client_instance = vectra_client.VectraClient( 50 | client_id=client_id, 51 | client_secret=client_secrets, 52 | base_url=base_url, 53 | bucket_name=bucket_name, 54 | secret_manager_client=secret_manager_client, 55 | ) 56 | 57 | methods_map = { 58 | constant.ENV_VAR_DETECTION: ( 59 | vectra_client_instance.get_and_ingest_detection_events 60 | ), 61 | constant.ENV_VAR_SCORING: ( 62 | vectra_client_instance.get_and_ingest_entity_scoring_events 63 | ), 64 | constant.ENV_VAR_LOCKDOWN: ( 65 | vectra_client_instance.get_and_ingest_lockdown_events 66 | ), 67 | constant.ENV_VAR_AUDIT: ( 68 | vectra_client_instance.get_and_ingest_audit_events 69 | ), 70 | constant.ENV_VAR_HEALTH: ( 71 | vectra_client_instance.get_and_ingest_health_events 72 | ), 73 | } 74 | 75 | # Get enabled methods from environment variables 76 | enabled_methods = [] 77 | for method_name, method_func in methods_map.items(): 78 | if vectra_utils.get_environment_variable(method_name) == "true": 79 | enabled_methods.append(method_func) 80 | 81 | # Check if there are no methods to run 82 | if not enabled_methods: 83 | utils.cloud_logging( 84 | "No methods enabled. Please set proper environment variables.", 85 | severity="ERROR", 86 | ) 87 | return "No methods enabled. Please set proper environment variables.", 400 88 | 89 | utils.cloud_logging( 90 | "Enabled methods:" 91 | f" {', '.join([method.__name__ for method in enabled_methods])}", 92 | ) 93 | # Run enabled methods with intervals 94 | try: 95 | vectra_utils.run_methods_with_intervals(enabled_methods) 96 | utils.cloud_logging("Methods executed successfully.") 97 | return "data ingestion completed" 98 | except Exception as e: # pylint: disable=broad-except 99 | utils.cloud_logging( 100 | "Unknown exception occurred while executing methods parallelly." 101 | f" Error message: {str(e)}", 102 | severity="ERROR", 103 | ) 104 | return f"Error executing methods: {str(e)}" 105 | 106 | except Exception as e: # pylint: disable=broad-except 107 | utils.cloud_logging( 108 | "Unknown exception occurred while retrieving the environment" 109 | f" credentials. Error message: {e}", 110 | severity="ERROR", 111 | ) 112 | return f"Error initializing: {str(e)}", 500 113 | -------------------------------------------------------------------------------- /vectra_xdr/requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.32.3 2 | google-cloud-secret-manager==2.22.0 3 | google-cloud-storage==2.19.0 4 | google-auth==2.30.0 5 | --------------------------------------------------------------------------------