├── LICENSE ├── README.md ├── add_cloud_props_azure └── add_cloud_props_azure.py ├── add_cloud_props_lambda └── add_cloud_props_lambda.py ├── add_observations └── add_observations.py ├── bash_rx360_auth └── bash_rx360_auth.sh ├── change_dashboard_owner └── change_dashboard_owner.py ├── cloudformation_traffic_mirror └── cloudformation_traffic_mirror.yml ├── create_backup └── create_backup.py ├── create_custom_devices ├── create_custom_devices.py └── device_list.csv ├── create_device_groups ├── create_device_groups.py └── device_group_list.csv ├── deploy_kubernetes_daemon ├── Dockerfile ├── init.sh └── rpcapd_daemon.yaml ├── extract_device_list └── extract_device_list.py ├── extract_files └── extract_files.py ├── extract_metrics └── extract_metrics.py ├── f5_irules ├── extrahop_shared_secret_export_multiple ├── extrahop_shared_secret_export_single └── extrahop_shared_secret_proto ├── lambda_traffic_mirror └── lambda_traffic_mirror.py ├── migrate_detection_hiding ├── migrate_detection_hiding.py └── migrate_detection_hiding_enterprise.py ├── migrate_saml ├── create_local_user_groups.py ├── create_saml_accounts.py ├── delete_remote_users.py ├── remote_to_saml.csv ├── retrieve_local_user_groups.py ├── retrieve_remote_users.py ├── retrieve_sharing.py └── transfer_sharing.py ├── ml_api_logger └── ml_api_logger.go ├── py_rx360_auth └── py_rx360_auth.py ├── query_records_explore └── query_records_explore.py ├── query_records_third_party └── query_records_third_party.py ├── rollback_firmware ├── rollback_firmware.py └── systems.csv ├── search_device └── search_device.py ├── self-managed-sensor-rx360-connect ├── self-managed-sensor-rx360-connect.py └── sensors.csv ├── specify_custom_make_model ├── custom_config.csv └── specify_custom_make_model.py ├── specify_high_value ├── ip_list.csv └── specify_high_value.py ├── sunburst ├── .gitignore ├── README.md ├── sunburst_detect.py └── threats.json ├── tag_device ├── ip_list.csv └── tag_device.py ├── update_network_localities ├── create_network_localities.py ├── example_input.csv ├── example_output.csv ├── retrieve_network_localities.py └── util.py ├── upgrade_system ├── systems.csv └── upgrade_system.py ├── upgrade_system_cloud ├── systems.csv └── upgrade_system_cloud.py ├── upgrade_system_url ├── systems.csv └── upgrade_system_url.py ├── upload_ids_rules ├── ids.csv └── upload_ids_rules.py └── upload_stix ├── systems.csv ├── upload_stix.py └── upload_stix_rx360.py /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2020, ExtraHop Networks 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ExtraHop code examples 2 | 3 | This repository includes example scripts that configure and automate interactions with ExtraHop systems through the REST and Trigger APIs. These scripts demonstrate functionality that can help developers write their own integrations and tools. These scripts are not intended to be deployed in production environments. 4 | 5 | ## Scripts 6 | 7 | This repository contains the following example scripts. 8 | 9 | ### add-cloud-props-azure 10 | 11 | This Python script imports Azure device properties into the ExtraHop system. The script assigns cloud device properties to every device discovered by the ExtraHop system with a MAC address that belongs to an Azure VM network interface. 12 | 13 | For more information, see [Add device cloud instance properties through the REST API](https://docs.extrahop.com/current/rest-add-cloud-prop/). 14 | 15 | ### add-cloud-props-lambda 16 | 17 | This Python script imports AWS EC2 instance properties into the ExtraHop system. The script maps network interfaces of EC2 instances to devices discovered on the ExtraHop system by MAC address. 18 | 19 | For more information, see [Add device cloud instance properties through the REST API](https://docs.extrahop.com/current/rest-add-cloud-prop/). 20 | 21 | ### add-observations 22 | 23 | This Python script creates associations on the ExtraHop system based on a CSV log file from OpenVPN. 24 | 25 | For more information, see [Add observations through the REST API](https://docs.extrahop.com/current/rest-add-observation/) 26 | 27 | ### bash-rx360-auth 28 | 29 | This Bash script generates a temporary API access token with the cURL command and then authenticates two simple requests with the token that retrieve devices and device groups from the Reveal(x) 360 REST API. 30 | 31 | For more information, see [Generate a REST API token](https://docs.extrahop.com/current/rx360-rest-api/#generate-a-rest-api-token) 32 | 33 | ### change-dashboard-owner 34 | 35 | This Python script searches for all dashboards owned by a given user account and changes the owner to a new user account. 36 | 37 | For more information, see [Change a dashboard owner through the REST API](https://docs.extrahop.com/current/rest-change-dashboard-ownership/) 38 | 39 | ### cloudformation-traffic-mirror 40 | 41 | This AWS CloudFormation template automatically mirrors traffic from EC2 instances to your ExtraHop sensors. 42 | 43 | For more information, see [Automate AWS Traffic Mirroring with CloudFormation](https://docs.extrahop.com/current/lambda-traffic-mirror/). 44 | 45 | ### create-backup 46 | 47 | This Python script creates backups of ExtraHop system customizations, such as bundles, triggers, dashboards, and users through the REST API. The script creates the backups on the ExtraHop system and then downloads each backup to the local machine. 48 | 49 | For more information, see [Back up the ExtraHop system through the REST API](https://docs.extrahop.com/current/rest-backup/). 50 | 51 | ### create-custom-device 52 | 53 | This Python script creates custom devices by reading criteria from a CSV file. 54 | 55 | For more information, see [Create custom devices through the REST API](https://docs.extrahop.com/current/rest-create-custom-devices/) 56 | 57 | ### create-device-groups 58 | 59 | This Python script creates device groups through the REST API. The script creates each device group by reading a list of IP addresses and CIDR blocks from a CSV file. 60 | 61 | For more information, see [Create a device group through the REST API](https://docs.extrahop.com/current/rest-create-device-group/). 62 | 63 | ### deploy-kubernetes-daemon 64 | 65 | This directory contains files that configure a Kubernetes DaemonSet that forwards packets from pods to the ExtraHop system. 66 | 67 | For more information, see [Configure packet forwarding for Kubernetes pods](https://docs.extrahop.com/current/configure-rpcap-kubernetes). 68 | 69 | ### extract-device-list 70 | 71 | This Python script extracts the device list, including all device metadata, and writes the list to a CSV file. 72 | 73 | For more information, see [Extract the device list through the REST API](https://docs.extrahop.com/current/rest-extract-devices/). 74 | 75 | ### extract-files 76 | 77 | This Python script extracts files from packets that match a specified query. 78 | 79 | For more information, see [Extract files from packets through the REST API](https://docs.extrahop.com/current/rest-extract-files/) 80 | 81 | ### extract-metrics 82 | 83 | This Python script extracts the total count of HTTP responses a server with an ID of 1298 sent over five minute time intervals and then writes the values to a csv file. 84 | 85 | For more information, see [Extract metrics through the REST API](https://docs.extrahop.com/current/rest-extract-metrics/). 86 | 87 | ### f5-irules 88 | 89 | For more information about these iRules, see [Session key forwarding from an F5 LTM](https://docs.extrahop.com/current/customers/deploy-eda-f5ltm) 90 | 91 | ### lambda-traffic-mirror 92 | 93 | This Python script automatically mirrors traffic from EC2 instances to your ExtraHop sensors. This script has been deprecated and is replaced by the traffic mirroring CloudFormation template. 94 | 95 | For more information about this script, see [Automate traffic mirroring with AWS Lambda](https://docs.extrahop.com/9.5/lambda-traffic-mirror/). For information about the CloudFormation template, see [Automate AWS Traffic Mirroring with CloudFormation](https://docs.extrahop.com/current/lambda-traffic-mirror/). 96 | 97 | ### migrate-detection-hiding 98 | 99 | The Python scripts in this directory migrate detection hiding rules. The migrate-detection-hiding.py script migrates tuning rules from an ECA VM to Reveal(x) 360. The migrate-detection-hiding-enterprise.py script migrates tuning rules from an ECA VM to another ECA VM. 100 | 101 | For more information, see [Migrate detection hiding rules](rest-migrate-detection-rules). 102 | 103 | ### migrate-saml 104 | 105 | The Python scripts in this directory migrate user customizations from remote users to SAML through the REST API. 106 | 107 | For more information, see [Migrate to SAML from LDAP through the REST API](https://docs.extrahop.com/current/migrate-saml-rest/). 108 | 109 | ### ml-api-logger 110 | 111 | This Go script recieves and writes logs sent from an ExtraHop sensor or console. The logs contain a record of all API interactions between the sensor or console and the ExtraHop Machine Learning Service. 112 | 113 | For more information, see [Export logs for Machine Learning Service API interactions](https://docs.extrahop.com/current/export-ml-logs) 114 | 115 | ### py-rx360-auth 116 | 117 | This Python script generates a temporary API access token and then authenticates two simple requests with the token that retrieve devices and device groups from the Reveal(x) 360 REST API. 118 | 119 | For more information, see [Generate a REST API token](https://docs.extrahop.com/current/rx360-rest-api/#generate-a-rest-api-token). 120 | 121 | ### query-records-explore 122 | 123 | This Python script retrieves records from an Explore appliance. 124 | 125 | For more information, see [Query for records through the REST API](https://docs.extrahop.com/current/rest-query-records/). 126 | 127 | ### query-records-third-party 128 | 129 | This Python script retrieves records from a third-party and cloud recordstores. 130 | 131 | For more information, see [Query for records through the REST API](https://docs.extrahop.com/current/rest-query-records/). 132 | 133 | ### rollback-firmware 134 | 135 | This Python script rolls back firmware on multiple ExtraHop systems by reading URLs and API keys from a CSV file. Rolling back the firmware on an appliance resets the datastore and removes all metrics. 136 | 137 | For more information, see [Roll back firmware through the REST API](https://docs.extrahop.com/current/rest-rollback/) 138 | 139 | ### search-device 140 | 141 | This Python script searches for a list of devices by IP address. The script then outputs the ExtraHop discovery ID for each IP address. 142 | 143 | For more information, see [Search for a device through the REST API](https://docs.extrahop.com/current/rest-search-for-device/). 144 | 145 | ### self-managed-sensor-rx360-connect 146 | 147 | This Python script connects a list of sensors to Reveal(x) 360 through the REST API. The script reads a list of sensor URLs and Reveal(x) 360 tokens from a CSV file. You must [generate the Reveal(x) 360 tokens](https://docs.extrahop.com/current/configure-ccp/#generate-a-token) before running the script. 148 | 149 | For more information, see [Connect to Reveal(x) 360 from self-managed sensors through the REST API](https://docs.extrahop.com/current/rest-connect-ccp/). 150 | 151 | ### specify-custom-make-model 152 | 153 | This Python script reads custom makes and models from a CSV file and adds them to devices with specified IP addresses. 154 | 155 | For more information, see [Specify custom device makes and models through the REST API](https://docs.extrahop.com/current/rest-specify-custom-make-model/). 156 | 157 | ### specify-high-value 158 | 159 | This Python script reads IP addresses from a CSV file and specifies all devices with those IP addresses as high value. 160 | 161 | For more information, see [Specify devices as high value through the REST API](https://docs.extrahop.com/current/rest-specify-high-value/). 162 | 163 | ### sunburst 164 | 165 | The Python scripts and JSON file in this directory search the ExtraHop system for indicators of the SUNBURST backdoor attack through the REST API. The SUNBURST trojan is dormant for long periods of time and might only occasionally contact external resources. To search for the large number of suspicious hostnames and IP addresses over a long period of time, we recommend that you query metrics through the REST API. 166 | 167 | ### tag-device 168 | 169 | This Python script creates a device tag and then assigns the tag to all devices with the IP addresses specified in a CSV file. 170 | 171 | For more information, see [Tag a device through the REST API](https://docs.extrahop.com/current/rest-tag-device/). 172 | 173 | ### update-network-localities 174 | 175 | The Python scripts in this directory help consolidate and add descriptive names to network localities that were created before upgrading to ExtraHop firmware version 9.0. 176 | 177 | For more information, see [Update network localities](https://docs.extrahop.com/current/update-network-localities/) 178 | 179 | ### upgrade-system 180 | 181 | This Python script upgrades multiple ExtraHop systems by reading URLs, API keys, and firmware file paths from a CSV file. 182 | 183 | For more information, see [Upgrade ExtraHop firmware through the REST API](https://docs.extrahop.com/8.6/rest-upgrade-firmware/). 184 | 185 | ### upgrade-system-cloud 186 | 187 | This Python script downloads firmware from ExtraHop Cloud Services and upgrades multiple ExtraHop systems by reading URLs and API keys from a CSV file. This script is compatible with ExtraHop systems running firmware version 8.7 or later. 188 | 189 | For more information, see [Upgrade ExtraHop firmware through the REST API with ExtraHop Cloud Services](https://docs.extrahop.com/current/rest-upgrade-cloud/). 190 | 191 | ### upgrade-system-url 192 | 193 | This Python script downloads firmware images and upgrades multiple ExtraHop systems by reading URLs and API keys from a CSV file. Firmware is downloaded from a specified URL. This script is compatible with ExtraHop systems running firmware version 8.7 or later. 194 | 195 | For more information, see [Upgrade ExtraHop firmware through the REST API](https://docs.extrahop.com/current/rest-upgrade-firmware/). 196 | 197 | ### upload-ids-rules 198 | 199 | This Python script uploads a set of curated IDS rules from the ExtraHop Customer Portal to consoles and sensors. 200 | 201 | For more information, see [Upload IDS rules to the ExtraHop system through the REST API](https://docs.extrahop.com/current/rest-upload-ids-rules/) 202 | 203 | ### upload-stix 204 | 205 | This Python script uploads all STIX files in a given directory to a list of ExtraHop systems 206 | 207 | For more information, see [Upload STIX files through the REST API](https://docs.extrahop.com/current/rest-upload-stix/). 208 | 209 | ## Related resources 210 | 211 | * [ExtraHop API documentation](https://docs.extrahop.com/current/api/) 212 | * [Trigger API Reference](https://docs.extrahop.com/current/extrahop-trigger-api/) 213 | * [REST API Guide](https://docs.extrahop.com/current/rest-api-guide/) 214 | * [ExtraHop product documentation](https://docs.extrahop.com/current/) 215 | * [ExtraHop corporate site](https://www.extrahop.com/) 216 | -------------------------------------------------------------------------------- /add_cloud_props_azure/add_cloud_props_azure.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | 9 | from azure.mgmt.compute import ComputeManagementClient 10 | from azure.mgmt.network import NetworkManagementClient 11 | from azure.common.credentials import ServicePrincipalCredentials 12 | import os 13 | import json 14 | import sys 15 | import requests 16 | from urllib.parse import urlunparse 17 | 18 | # The IP address or hostname of the ExtraHop system. 19 | HOST = "extrahop.example.com" 20 | # The ExtraHop API key. 21 | APIKEY = "123456789abcdefghijklmnop" 22 | 23 | subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") 24 | credentials = ServicePrincipalCredentials( 25 | client_id=os.environ["AZURE_CLIENT_ID"], 26 | secret=os.environ["AZURE_CLIENT_SECRET"], 27 | tenant=os.environ["AZURE_TENANT_ID"], 28 | ) 29 | compute_client = ComputeManagementClient(credentials, subscription_id) 30 | network_client = NetworkManagementClient(credentials, subscription_id) 31 | region = "eastus" 32 | 33 | 34 | result_list_pub = compute_client.virtual_machine_images.list_publishers(region,) 35 | 36 | 37 | def addEHids(mac_addresses, mapping): 38 | """ 39 | Method that retrieves ExtraHop device IDs and maps them to Azure network interfaces. 40 | 41 | Parameters: 42 | mac_addresses (list): List of Azure network interface MAC addresses 43 | mapping (list): List of dictionaries containing Azure network interface metadata 44 | 45 | Returns: 46 | to_do (list): List of network interfaces that have been mapped to ExtraHop device IDs 47 | not_found (list): List of network interface MAC addresses that were not found on the ExtraHop system 48 | """ 49 | url = urlunparse(("https", HOST, "/api/v1/devices/search", "", "", "")) 50 | headers = {"Authorization": "ExtraHop apikey=%s" % APIKEY} 51 | rules = [] 52 | for macaddr in mac_addresses: 53 | rules.append({"field": "macaddr", "operand": macaddr, "operator": "="}) 54 | search = {"filter": {"operator": "or", "rules": rules}} 55 | r = requests.post( 56 | url, headers=headers, data=json.dumps(search) 57 | ) 58 | if r.status_code != 200: 59 | print("Error! Unable to retrieve devices from ExtraHop.") 60 | print(r.text) 61 | sys.exit() 62 | mac_id_map = {} 63 | for device in r.json(): 64 | macaddr = device["macaddr"] 65 | if device["is_l3"] == True: 66 | continue 67 | if macaddr in mac_id_map: 68 | mac_id_map[macaddr].append(device["id"]) 69 | else: 70 | mac_id_map[macaddr] = [device["id"]] 71 | return_list = [] 72 | for instance in mapping: 73 | aws_mac = instance["macaddr"] 74 | if aws_mac in mac_id_map: 75 | instance["id"] = mac_id_map[aws_mac] 76 | instance.pop("macaddr") 77 | return_list.append(instance) 78 | else: 79 | print( 80 | "MAC address: " 81 | + instance["macaddr"] 82 | + " not found on the ExtraHop system" 83 | ) 84 | return return_list 85 | 86 | 87 | def updateMeta(device, dev_id): 88 | """ 89 | Method that adds cloud properties to devices in the ExtraHop system. 90 | 91 | Parameters: 92 | device (dict): The cloud properties of the device 93 | dev_id (str): The ID of the device 94 | 95 | Returns: 96 | bool: Indicates whether the request was successful 97 | """ 98 | url = urlunparse( 99 | ("https", HOST, f"/api/v1/devices/{str(dev_id)}", "", "", "") 100 | ) 101 | headers = {"Authorization": "ExtraHop apikey=%s" % APIKEY} 102 | r = requests.patch(url, headers=headers, data=json.dumps(device)) 103 | if r.status_code != 204: 104 | return False 105 | else: 106 | return True 107 | 108 | 109 | az_map = [] 110 | for vm in compute_client.virtual_machines.list_all(): 111 | vm_name = vm.name 112 | vm_id = vm.vm_id 113 | vm_size = vm.hardware_profile.vm_size 114 | subscription_id = vm.id.split("/")[2] 115 | for interface in vm.network_profile.network_interfaces: 116 | name = interface.id.split("/")[-1] 117 | subnet = interface.id.split("/")[4] 118 | nic_group = interface.id.split("/")[-5] 119 | nic = network_client.network_interfaces.get(nic_group, name) 120 | macaddr = nic.mac_address.replace("-", ":") 121 | vnet_name = nic.ip_configurations[0].subnet.id.split("/")[-3] 122 | az_map.append( 123 | { 124 | "macaddr": macaddr, 125 | "cloud_instance_name": vm_name, 126 | "cloud_instance_type": vm_size, 127 | "cloud_instance_id": vm_id, 128 | "cloud_account": subscription_id, 129 | "vpc_id": vnet_name, 130 | } 131 | ) 132 | 133 | mac_addresses = [x["macaddr"] for x in az_map] 134 | az_map = addEHids(mac_addresses, az_map) 135 | 136 | updates = [] 137 | failed = [] 138 | for device in az_map: 139 | ids = device.pop("id") 140 | for i in ids: 141 | success = updateMeta(device, i) 142 | if success: 143 | updates.append("Device ID: " + str(i)) 144 | else: 145 | failed.append("Device ID: " + str(i)) 146 | 147 | if updates: 148 | print("Updated cloud properties for the following devices:") 149 | for u in updates: 150 | print(" " + u) 151 | if failed: 152 | print("Failed to update cloud properties for the following devices:") 153 | for f in failed: 154 | print(" " + f) 155 | -------------------------------------------------------------------------------- /add_cloud_props_lambda/add_cloud_props_lambda.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import boto3 9 | import json 10 | import requests 11 | import os 12 | import sys 13 | from botocore.config import Config 14 | from urllib.parse import urlunparse 15 | 16 | # The private IP address or hostname of the EC2 instance of the ExtraHop system. 17 | HOST = "extrahop.example.com" 18 | # The ExtraHop API key. 19 | APIKEY = "123456789abcdefghijklmnop" 20 | 21 | 22 | def addEHids(mac_addresses, mapping): 23 | """ 24 | Method that retrieves ExtraHop device IDs and maps them to EC2 network interfaces. 25 | 26 | Parameters: 27 | mac_addresses (list): List of EC2 network interface MAC addresses 28 | mapping (list): List of dictionaries containing EC2 network interface metadata 29 | 30 | Returns: 31 | to_do (list): List of network interfaces that have been mapped to ExtraHop device IDs 32 | not_found (list): List of network interface MAC addresses that were not found on the ExtraHop system 33 | """ 34 | url = urlunparse(("https", HOST, "/api/v1/devices/search", "", "", "")) 35 | headers = {"Authorization": "ExtraHop apikey=%s" % APIKEY} 36 | rules = [] 37 | for macaddr in mac_addresses: 38 | rules.append({"field": "macaddr", "operand": macaddr, "operator": "="}) 39 | search = {"filter": {"operator": "or", "rules": rules}} 40 | r = requests.post( 41 | url, headers=headers, data=json.dumps(search) 42 | ) 43 | if r.status_code != 200: 44 | print("Error! Unable to retrieve devices from ExtraHop.") 45 | print(r.text) 46 | sys.exit() 47 | mac_id_map = {} 48 | for device in r.json(): 49 | macaddr = device["macaddr"].lower() 50 | if device["is_l3"] == True: 51 | continue 52 | if macaddr in mac_id_map: 53 | mac_id_map[macaddr].append(device["id"]) 54 | else: 55 | mac_id_map[macaddr] = [device["id"]] 56 | to_do = [] 57 | not_found = [] 58 | for instance in mapping: 59 | aws_mac = instance["macaddr"] 60 | if aws_mac in mac_id_map: 61 | instance["id"] = mac_id_map[aws_mac] 62 | instance.pop("macaddr") 63 | to_do.append(instance) 64 | else: 65 | not_found.append(aws_mac) 66 | return to_do, not_found 67 | 68 | 69 | def updateMeta(device, dev_id): 70 | """ 71 | Method that adds cloud properties to devices in the ExtraHop system. 72 | 73 | Parameters: 74 | device (dict): The cloud properties of the device 75 | dev_id (str): The ID of the device 76 | 77 | Returns: 78 | bool: Indicates whether the request was successful 79 | """ 80 | url = urlunparse( 81 | ("https", HOST, f"/api/v1/devices/{str(dev_id)}", "", "", "") 82 | ) 83 | headers = {"Authorization": "ExtraHop apikey=%s" % APIKEY} 84 | r = requests.patch(url, headers=headers, data=json.dumps(device)) 85 | if r.status_code != 204: 86 | return False 87 | else: 88 | return True 89 | 90 | 91 | def lambda_handler(event, context): 92 | aws_map = [] 93 | print("start") 94 | ec2 = boto3.client( 95 | "ec2", 96 | config=Config( 97 | connect_timeout=10, 98 | read_timeout=10, 99 | retries={"total_max_attempts": 50}, 100 | ), 101 | ) 102 | response = ec2.describe_instances() 103 | reservations = response["Reservations"] 104 | for reservation in reservations: 105 | for instance in reservation["Instances"]: 106 | instance_id = instance["InstanceId"] 107 | instance_type = instance["InstanceType"] 108 | instance_name = "" 109 | for tag in instance["Tags"]: 110 | if tag["Key"] == "Name": 111 | instance_name = tag["Value"] 112 | for interface in instance["NetworkInterfaces"]: 113 | instance_owner = interface["OwnerId"] 114 | vpc_id = interface["VpcId"] 115 | macaddr = interface["MacAddress"] 116 | aws_map.append( 117 | { 118 | "macaddr": macaddr, 119 | "cloud_instance_id": instance_id, 120 | "cloud_instance_type": instance_type, 121 | "cloud_instance_name": instance_name, 122 | "cloud_account": instance_owner, 123 | "vpc_id": vpc_id, 124 | } 125 | ) 126 | mac_addresses = [x["macaddr"] for x in aws_map] 127 | aws_map, not_found = addEHids(mac_addresses, aws_map) 128 | updates = [] 129 | failed = [] 130 | for device in aws_map: 131 | ids = device.pop("id") 132 | for i in ids: 133 | success = updateMeta(device, i) 134 | if success: 135 | updates.append(str(i)) 136 | else: 137 | failed.append(str(i)) 138 | results = { 139 | "updated_device_ids": updates, 140 | "update_failed_device_ids": failed, 141 | "macaddr_not_found_on_eh": not_found, 142 | } 143 | return {"statusCode": 200, "body": json.dumps(results)} 144 | -------------------------------------------------------------------------------- /add_observations/add_observations.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import csv 10 | import time 11 | import requests 12 | from urllib.parse import urlunparse 13 | 14 | # The IP address or hostname of the ExtraHop system. 15 | HOST = "extrahop.example.com" 16 | # The API key. 17 | API_KEY = "123456789abcdefghijklmnop" 18 | # The name of the CSV file. 19 | CSV_FILE = "log.csv" 20 | # The source of the observations. 21 | SOURCE = "OpenVPN" 22 | 23 | # The name of the column in the CSV file that specifies the IP addresses of the VPN clients on your internal network. 24 | ASSOCIATED_IPADDR = "Real IP" 25 | # The name of the column in the CSV file that specifies the external IP addresses assigned to the users on the public internet. 26 | IPADDR = "VPN IP" 27 | # The name of the column in the CSV file that specifies the time that the observation was created by the source. 28 | TIMESTAMP = "Start Time" 29 | 30 | 31 | def readCSV(associated_ipaddr, ipaddr, timestamp): 32 | """ 33 | Method that extracts observations from a CSV file. 34 | 35 | Parameters: 36 | associated_ipaddr (str): The name of the column that specifies IP addresses of VPN clients 37 | ipaddr (str): The name of the column that specifies IP addresses of users on the public internet 38 | timestamp (str): The name of the column that specifies the observation creation time 39 | 40 | Returns: 41 | observations (list): A list of observation dictionaries 42 | """ 43 | observations = [] 44 | with open(CSV_FILE, "rt", encoding="ascii") as f: 45 | reader = csv.reader(f) 46 | header = next(reader, None) 47 | for row in reader: 48 | observations.append( 49 | { 50 | "associated_ipaddr": row[header.index(associated_ipaddr)], 51 | "ipaddr": row[header.index(ipaddr)], 52 | "timestamp": translateTime(row[header.index(timestamp)]), 53 | } 54 | ) 55 | return observations 56 | 57 | 58 | def translateTime(t): 59 | """ 60 | Method that translates a formatted timestamp into epoch time. 61 | 62 | Parameters: 63 | t (str): The formatted timestamp 64 | 65 | Returns: 66 | str: The epoch time 67 | """ 68 | pattern = "%m/%d/%y %H:%M:%S" 69 | return int(time.mktime(time.strptime(t, pattern))) 70 | 71 | 72 | def makeObservations(observations): 73 | """ 74 | Method that sends observations to the ExtraHop system 75 | 76 | Parameters: 77 | observations (list): A list of observation dictionaries 78 | 79 | """ 80 | url = urlunparse(("https", HOST, "/api/v1/observations/associatedipaddrs", "", "", "")) 81 | headers = { 82 | "Content-Type": "application/json", 83 | "Accept": "application/json", 84 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 85 | } 86 | data = {"observations": observations, "source": SOURCE} 87 | r = requests.post(url, headers=headers, data=json.dumps(data)) 88 | if r.status_code == 202: 89 | print(r.text) 90 | else: 91 | print("Observation upload failed") 92 | print(r.text) 93 | 94 | 95 | observations = readCSV(ASSOCIATED_IPADDR, IPADDR, TIMESTAMP) 96 | makeObservations(observations) 97 | -------------------------------------------------------------------------------- /bash_rx360_auth/bash_rx360_auth.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | # The hostname of the Reveal(x) 360 API. This hostname is displayed in Reveal(x) 9 | # 360 on the API Access page under API Endpoint. The hostname does not 10 | # include the /oauth/token. 11 | HOST="https://example.api.com" 12 | # The ID of the REST API credentials. 13 | ID="abcdefg123456789" 14 | # The secret of the REST API credentials. 15 | SECRET="123456789abcdefg987654321abcdefg" 16 | 17 | # Note: Remove the --wrap=0 option if running on a Mac machine. The option is required 18 | # only on Linux machines. 19 | AUTH=$(printf "$ID:$SECRET" | base64 --wrap=0) 20 | 21 | ACCESS_TOKEN=$(curl -s \ 22 | -H "Authorization: Basic ${AUTH}" \ 23 | -H "Content-Type: application/x-www-form-urlencoded" \ 24 | --request POST \ 25 | ${HOST}/oauth2/token \ 26 | -d "grant_type=client_credentials" \ 27 | | jq -r '.access_token') 28 | 29 | curl -v -X POST -H "Authorization: Bearer ${ACCESS_TOKEN}" --header "Content-Type: application/json" -d "{ \"active_from\": 1, \"active_until\": 0, \"limit\": 100}" ${HOST}/api/v1/devices/search 30 | curl -v -H "Authorization: Bearer ${ACCESS_TOKEN}" ${HOST}/api/v1/devicegroups 31 | -------------------------------------------------------------------------------- /change_dashboard_owner/change_dashboard_owner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | from urllib.parse import urlunparse 11 | 12 | # The IP address or hostname of the ExtraHop system. 13 | HOST = "extrahop.example.com" 14 | # The API key. 15 | API_KEY = "123456789abcdefghijklmnop" 16 | # The username of the current dashboard owner. 17 | CURRENT = "marksmith" 18 | # The username of the new dashboard owner. 19 | NEW = "paulanderson" 20 | 21 | 22 | def getDashboards(): 23 | """ 24 | Method that retrieves all dashboards from an ExtraHop system. 25 | 26 | Returns: 27 | list: The dashboards on the ExtraHop system 28 | """ 29 | url = urlunparse(("https", HOST, "/api/v1/dashboards", "", "", "")) 30 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 31 | r = requests.get(url, headers=headers) 32 | if r.status_code == 200: 33 | return r.json() 34 | else: 35 | print("Failed to retrieve dashboards") 36 | print(r.text) 37 | print(status_code) 38 | 39 | 40 | def changeOwner(dashboard): 41 | """ 42 | Method that changes the owner of a dashboard. 43 | 44 | Parameters: 45 | dashboard (dict): The dashboard dictionary 46 | """ 47 | url = urlunparse( 48 | ("https", HOST, f"/api/v1/dashboards/{dashboard['id']}", "", "", "") 49 | ) 50 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 51 | data = {"owner": NEW} 52 | r = requests.patch(url, headers=headers, json=data) 53 | if r.status_code == 204: 54 | print(f"Switching owner of {dashboard['name']} from {CURRENT} to {NEW}") 55 | else: 56 | print(f"Failed to update owner of dashboard {dashboard['name']}") 57 | print(r.text) 58 | print(status_code) 59 | 60 | 61 | def main(): 62 | dashboards = getDashboards() 63 | for dashboard in dashboards: 64 | if dashboard["owner"] == CURRENT: 65 | changeOwner(dashboard) 66 | 67 | 68 | if __name__ == "__main__": 69 | main() 70 | -------------------------------------------------------------------------------- /cloudformation_traffic_mirror/cloudformation_traffic_mirror.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion : '2010-09-09' 2 | Description: VPC Traffic Mirror Manager 3 | Parameters: 4 | TagMirror: 5 | Description: Specify the Tag for Traffic Mirror orchestration 6 | Type: String 7 | Default: Mirror 8 | 9 | 10 | Resources: 11 | TrafficMirrorManagerLambda: 12 | Type: AWS::Lambda::Function 13 | Properties: 14 | Description: This function automates the creation and management of VPC Traffic Mirror Sessions 15 | FunctionName: ExtraHop-Traffic-Mirror-Manager 16 | PackageType: Zip 17 | Code: 18 | S3Bucket: !Join [ '-', [ 'extrahop-onboarding', !Ref AWS::Region ] ] 19 | S3Key: public/traffic-mirror-manager.zip 20 | Runtime: python3.10 21 | Handler: traffic-mirror-manager.lambda_handler 22 | Role: !GetAtt LambdaTMMRole.Arn 23 | Timeout: 30 24 | Environment: 25 | Variables: 26 | TAGKEY: !Ref TagMirror 27 | TMMTagRule: 28 | Type: AWS::Events::Rule 29 | Properties: 30 | Name: ExtraHop-Tag-Rule 31 | Description: > 32 | Trigger for Tag Events for ExtraHop Traffic Mirror Manager 33 | EventPattern: 34 | source: 35 | - aws.ec2 36 | detail-type: 37 | - AWS API Call via CloudTrail 38 | detail: 39 | eventName: 40 | - CreateTags 41 | - DeleteTags 42 | eventSource: 43 | - ec2.amazonaws.com 44 | requestParameters: 45 | tagSet: 46 | items: 47 | key: 48 | - !Ref TagMirror 49 | State: ENABLED 50 | Targets: 51 | - 52 | Arn: !GetAtt TrafficMirrorManagerLambda.Arn 53 | Id: ExtraHop-Traffic-Mirror-Manager 54 | TMMTagLambdaPerms: 55 | Type: AWS::Lambda::Permission 56 | DependsOn: TrafficMirrorManagerLambda 57 | Properties: 58 | FunctionName: !Ref TrafficMirrorManagerLambda 59 | Action: lambda:InvokeFunction 60 | Principal: events.amazonaws.com 61 | SourceArn: !GetAtt TMMTagRule.Arn 62 | 63 | TMMExtraRule: 64 | Type: AWS::Events::Rule 65 | Properties: 66 | Name: ExtraHop-Extras-Rule 67 | Description: > 68 | Trigger for EC2/Delete Events for ExtraHop Traffic Mirror Manager 69 | EventPattern: 70 | source: 71 | - aws.ec2 72 | detail-type: 73 | - AWS API Call via CloudTrail 74 | detail: 75 | eventName: 76 | - RunInstances 77 | - DeleteTrafficMirrorSession 78 | eventSource: 79 | - ec2.amazonaws.com 80 | State: ENABLED 81 | Targets: 82 | - Arn: !GetAtt TrafficMirrorManagerLambda.Arn 83 | Id: ExtraHop-Traffic-Mirror-Manager 84 | TMMExtraLambdaPerms: 85 | Type: AWS::Lambda::Permission 86 | DependsOn: TrafficMirrorManagerLambda 87 | Properties: 88 | FunctionName: !Ref TrafficMirrorManagerLambda 89 | Action: lambda:InvokeFunction 90 | Principal: events.amazonaws.com 91 | SourceArn: !GetAtt TMMExtraRule.Arn 92 | LambdaTMMRole: 93 | Type: AWS::IAM::Role 94 | Properties: 95 | RoleName: !Join [ '-', [ 'ExtraHop-Traffic-Mirror-Manager', !Ref AWS::Region ] ] 96 | Description: Lambda Execution Role for ExtraHop Traffic Mirror Manager 97 | Path: / 98 | AssumeRolePolicyDocument: 99 | Version: "2012-10-17" 100 | Statement: 101 | Sid: RoleTrustPolicy 102 | Effect: Allow 103 | Principal: 104 | Service: lambda.amazonaws.com 105 | Action: sts:AssumeRole 106 | Policies: 107 | - PolicyName: PermissionsPolicy 108 | PolicyDocument: 109 | Version: "2012-10-17" 110 | Statement: 111 | - Sid: EC2Actions 112 | Effect: Allow 113 | Action: 114 | - 'ec2:DescribeInstances' 115 | - 'ec2:DescribeNetworkInterfaces' 116 | - 'ec2:DescribeTrafficMirrorTargets' 117 | - 'ec2:DescribeTrafficMirrorFilters' 118 | - 'ec2:CreateTrafficMirrorSession' 119 | - 'ec2:DeleteTrafficMirrorSession' 120 | - 'ec2:DescribeTrafficMirrorSessions' 121 | - 'ec2:DescribeTags' 122 | - 'ec2:CreateTags' 123 | - 'ec2:ModifyTrafficMirrorSession' 124 | Resource: "*" 125 | - Sid: CloudwatchLogsPublish1 126 | Effect: Allow 127 | Action: 128 | - logs:CreateLogGroup 129 | Resource: 130 | - !Join [':', ['arn', !Ref AWS::Partition, 'logs', !Ref AWS::Region, !Ref AWS::AccountId, '*'] ] 131 | - Sid: CloudwatchLogsPublish2 132 | Effect: Allow 133 | Action: 134 | - logs:CreateLogStream 135 | - logs:PutLogEvents 136 | Resource: 137 | - !Join [':', ['arn', !Ref AWS::Partition, 'logs', !Ref AWS::Region, !Ref AWS::AccountId, 'log-group', '/aws/lambda/ExtraHop-Traffic-Mirror-Manager', '*'] ] 138 | -------------------------------------------------------------------------------- /create_backup/create_backup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2020 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import sys 11 | from urllib.parse import urlunparse 12 | 13 | HOST = "extrahop.example.com" 14 | API_KEY = "123456789abcdefghijklmnop" 15 | BACKUP_NAME = "mybackup" 16 | 17 | 18 | def createBackup(BACKUP_NAME): 19 | url = urlunparse(("https", HOST, "/api/v1/customizations", "", "", "")) 20 | headers = { 21 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 22 | "Content-Type": "application/json", 23 | } 24 | data = {"name": BACKUP_NAME} 25 | r = requests.post(url, headers=headers, data=json.dumps(data)) 26 | if r.status_code == 201: 27 | return r.headers["Location"].split("/")[-1] 28 | else: 29 | print("Unable to create backup") 30 | print(r.text) 31 | print(r.status_code) 32 | sys.exit() 33 | 34 | 35 | def getIdName(backup_id): 36 | url = urlunparse(("https", HOST, f"/api/v1/customizations/{backup_id}", "", "", "")) 37 | headers = { 38 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 39 | "Content-Type": "application/json", 40 | } 41 | r = requests.get(url, headers=headers) 42 | if r.status_code == 200: 43 | return json.loads(r.text)[".meta"]["name"] 44 | else: 45 | print("Unable to retrieve backup ID") 46 | print(r.text) 47 | print(r.status_code) 48 | sys.exit() 49 | 50 | 51 | def downloadBackup(backup_id): 52 | url = urlunparse( 53 | ( 54 | "https", 55 | HOST, 56 | f"/api/v1/customizations/{str(backup_id)}/download", 57 | "", 58 | "", 59 | "", 60 | ) 61 | ) 62 | headers = { 63 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 64 | "accept": "application/exbk", 65 | } 66 | r = requests.post(url, headers=headers) 67 | if r.status_code == 200: 68 | return r.content 69 | else: 70 | print("Unable to download backup") 71 | print(r.status_code) 72 | print(r.text) 73 | sys.exit() 74 | 75 | 76 | def writeBackup(backup, BACKUP_NAME): 77 | new_name = BACKUP_NAME.replace(":", "") 78 | filepath = new_name + ".exbk" 79 | with open(filepath, "wb") as b: 80 | b.write(bytes(backup)) 81 | print("Success! Backup file name:") 82 | print(filepath) 83 | 84 | 85 | backup_id = createBackup(BACKUP_NAME) 86 | new_backup_name = getIdName(backup_id) 87 | backup = downloadBackup(backup_id) 88 | writeBackup(backup, new_backup_name) 89 | -------------------------------------------------------------------------------- /create_custom_devices/create_custom_devices.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | from urllib.parse import urlunparse 11 | import csv 12 | import os.path 13 | 14 | # The IP address or hostname of the ExtraHop system. 15 | HOST = "extrahop.example.com" 16 | # The API key. 17 | API_KEY = "123456789abcdefghijklmnop" 18 | # The path of the CSV file relative to the location of the script file. 19 | CSV_FILE = "device_list.csv" 20 | 21 | 22 | def readCSV(): 23 | """ 24 | Method that reads custom device criteria from a CSV file. 25 | 26 | Returns: 27 | devices (list): A list of device dictionaries 28 | """ 29 | devices = [] 30 | with open(CSV_FILE, "rt", encoding="ascii") as f: 31 | reader = csv.reader(f) 32 | for row in reader: 33 | device = {} 34 | ips = [] 35 | device["name"] = row.pop(0) 36 | device["extrahop_id"] = row.pop(0) 37 | device["description"] = row.pop(0) 38 | for ip in row: 39 | ips.append({"ipaddr": ip}) 40 | device["criteria"] = ips 41 | devices.append(device) 42 | return devices 43 | 44 | 45 | def createDevice(device): 46 | """ 47 | Method that creates a custom device. 48 | 49 | Parameters: 50 | device(dict): A device dictionary 51 | 52 | Returns: 53 | dev_id(string): The ID of the custom device 54 | """ 55 | headers = { 56 | "Content-Type": "application/json", 57 | "Accept": "application/json", 58 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 59 | } 60 | url = urlunparse(("https", HOST, "/api/v1/customdevices", "", "", "")) 61 | r = requests.post(url, headers=headers, json=device) 62 | if r.status_code == 201: 63 | print(f"Created custom device: {device['name']}") 64 | return dev_id 65 | else: 66 | print(f"Failed to create custom device: {device['name']}") 67 | print(r.text) 68 | print(r.status_code) 69 | 70 | 71 | def main(): 72 | devices = readCSV() 73 | for device in devices: 74 | dev_id = createDevice(device) 75 | 76 | 77 | if __name__ == "__main__": 78 | main() 79 | -------------------------------------------------------------------------------- /create_custom_devices/device_list.csv: -------------------------------------------------------------------------------- 1 | France,francehq,The location of our office in France,192.168.0.103,192.168.0.105,192.168.0.101 2 | Holland,hollandhq,The location of our office in Holland,192.168.0.102 3 | California,californiahq,The location of our office in California,192.168.0.104,192.168.0.103 4 | -------------------------------------------------------------------------------- /create_device_groups/create_device_groups.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2020 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import csv 11 | import os.path 12 | from urllib.parse import urlunparse 13 | import base64 14 | import sys 15 | 16 | # The IP address or hostname of the ExtraHop appliance or Reveal(x) 360 API 17 | HOST = "extrahop.example.com" 18 | 19 | # For Reveal(x) 360 authentication 20 | # The ID of the REST API credentials. 21 | ID = "abcdefg123456789" 22 | # The secret of the REST API credentials. 23 | SECRET = "123456789abcdefg987654321abcdefg" 24 | # A global variable for the temporary API access token (leave blank) 25 | TOKEN = "" 26 | 27 | # For self-managed appliance authentication 28 | # The API key. 29 | API_KEY = "123456789abcdefghijklmnop" 30 | 31 | CSV_FILE = "device_group_list.csv" 32 | 33 | 34 | def getToken(): 35 | """ 36 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 37 | 38 | Returns: 39 | str: A temporary API access token 40 | """ 41 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 42 | headers = { 43 | "Authorization": "Basic " + auth, 44 | "Content-Type": "application/x-www-form-urlencoded", 45 | } 46 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 47 | r = requests.post( 48 | url, 49 | headers=headers, 50 | data="grant_type=client_credentials", 51 | ) 52 | try: 53 | return r.json()["access_token"] 54 | except: 55 | print(r.text) 56 | print(r.status_code) 57 | print("Error retrieveing token from Reveal(x) 360") 58 | sys.exit() 59 | 60 | 61 | def getAuthHeader(force_token_gen=False): 62 | """ 63 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 64 | token. For self-managed appliances, adds an API key. 65 | 66 | Parameters: 67 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 68 | 69 | Returns: 70 | header (str): The value for the header key in the headers dictionary 71 | """ 72 | global TOKEN 73 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 74 | return f"ExtraHop apikey={API_KEY}" 75 | else: 76 | if TOKEN == "" or force_token_gen == True: 77 | TOKEN = getToken() 78 | return f"Bearer {TOKEN}" 79 | 80 | 81 | def readCSV(): 82 | devices = [] 83 | with open(CSV_FILE, "rt", encoding="ascii") as f: 84 | reader = csv.reader(f) 85 | for row in reader: 86 | device = {} 87 | device["name"] = row.pop(0) 88 | device["description"] = row.pop(0) 89 | rules = [] 90 | for ip in row: 91 | rules.append( 92 | {"field": "ipaddr", "operand": ip, "operator": "="} 93 | ) 94 | device["filter"] = {"rules": rules, "operator": "or"} 95 | devices.append(device) 96 | return devices 97 | 98 | 99 | def createDevice(device): 100 | url = urlunparse(("https", HOST, "/api/v1/devicegroups", "", "", "")) 101 | headers = { 102 | "Content-Type": "application/json", 103 | "Accept": "application/json", 104 | "Authorization": getAuthHeader(), 105 | } 106 | r = requests.post(url, headers=headers, data=json.dumps(device)) 107 | if r.status_code != 201: 108 | print("Could not create device group: " + device["name"]) 109 | print(r.status_code) 110 | print(r.json()) 111 | else: 112 | print("Created custom device group: " + device["name"]) 113 | 114 | 115 | devices = readCSV() 116 | for device in devices: 117 | createDevice(device) 118 | -------------------------------------------------------------------------------- /create_device_groups/device_group_list.csv: -------------------------------------------------------------------------------- 1 | Paris,The location of our office in France,10.10.1.255,10.10.10.135,192.168.0.0/26 2 | Hamburg,The location of our office in Germany,192.168.0.0/24 3 | San Francisco,The location of our office in California,192.168.0.104,192.168.0.103 4 | -------------------------------------------------------------------------------- /deploy_kubernetes_daemon/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | # docker build -t rpcapd --build-arg RPCAPD_DEB_ARCHIVE=rpcapd_9.3.7_amd64.22.04.deb . 4 | 5 | RUN apt-get update && \ 6 | apt-get install -y iproute2 7 | 8 | ARG RPCAPD_DEB_ARCHIVE 9 | 10 | ENV INTERFACE any-eth 11 | ENV EXTRAHOP_SENSOR_IP eda 12 | 13 | COPY init.sh . 14 | COPY ${RPCAPD_DEB_ARCHIVE} . 15 | 16 | RUN dpkg --unpack ${RPCAPD_DEB_ARCHIVE} && \ 17 | rm ${RPCAPD_DEB_ARCHIVE} && \ 18 | chmod +x init.sh 19 | 20 | ENTRYPOINT ["./init.sh"] 21 | -------------------------------------------------------------------------------- /deploy_kubernetes_daemon/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eu 4 | 5 | exec 6>&1 6 | exec > rpcapd.ini 7 | 8 | OLDIFS=$IFS 9 | IFS=',' 10 | for s in ${EXTRAHOP_SENSOR_IP} 11 | do 12 | echo "ActiveClient = ${s},${RPCAPD_TARGET_PORT}" 13 | done 14 | servicesubnet=" " 15 | for s in ${SVCNET} 16 | do 17 | servicesubnet="${servicesubnet}and (not ip net ${s}) " 18 | done 19 | IFS=$OLDIFS 20 | echo "NullAuthPermit = YES" 21 | exec 1>&6 6>&- 22 | 23 | # This script finds the subnets of the other nodes and pods in the cluster. 24 | # The script then builds a BPF that sends traffic to the ExtraHop sensor 25 | # only once. 26 | 27 | main_dev=$(ip route list | grep default | grep -o 'dev.*' | awk '{print $2}') 28 | if [ "${main_dev}" = "" ] 29 | then 30 | echo "Could not look up main network device, output:" 31 | ip route list 32 | exit 255 33 | fi 34 | 35 | # Get a list of netmasks for the default device, separated with a comma so compose_cluster_rule can parse correctly 36 | subnets=$(ip route show dev "${main_dev}" scope link | grep / | cut -d' ' -f1 | paste -s -d ',') 37 | if [ "${subnets}" = "" ] 38 | then 39 | echo "Could not lookup primary subnet, routing table for ${main_dev}:" 40 | ip route show dev "${main_dev}" 41 | exit 255 42 | fi 43 | 44 | # The PODNET variable is a subnet that is a superset of all pod subnets on all nodes 45 | if [ "${PODNET}" != "" ] 46 | then 47 | subnets="${subnets},${PODNET}" 48 | fi 49 | 50 | # Arguments: subnet, direction 51 | # Returns: rule 52 | compose_cluster_rule() { 53 | rule='( not (' 54 | subnets_number=0 55 | OLDIFS=$IFS 56 | IFS=',' 57 | for subnet in $1 58 | do 59 | if [ $subnets_number -eq 0 ] 60 | then 61 | subnets_number=1 62 | else 63 | rule="${rule} or" 64 | fi 65 | rule="${rule} ip ${2} net ${subnet}" 66 | done 67 | IFS=$OLDIFS 68 | rule="${rule} ) )" 69 | echo "${rule}" 70 | } 71 | subnetrule=$(compose_cluster_rule "${subnets}" "src or dst") 72 | 73 | # Arguments: direction 74 | # Returns: rule 75 | compose_local_rule() { 76 | direction='' 77 | addrcmp='' 78 | portcmp='' 79 | if [ "$1" = "src" ] 80 | then 81 | direction="inbound" 82 | addrcmp="(ip[12:4] >= ip[16:4])" 83 | portcmp="<" 84 | else 85 | direction="outbound" 86 | # For outbound, the lower address is chosen because 87 | # that means we should always match the IP broadcasts, which will 88 | # use the highest address in their subnet. 89 | addrcmp="(ip[12:4] <= ip[16:4])" 90 | portcmp=">" 91 | fi 92 | 93 | # Some packets do not have a port because they are fragments or they are not TCP. 94 | noportrule="((ip[6:2] & 0x1fff != 0) or ((not tcp)) and ${addrcmp})" 95 | 96 | tcprule="(tcp and ((tcp[0:2] ${portcmp} tcp[2:2]) or ((tcp[0:2] = tcp[2:2]) and ${addrcmp})))" 97 | 98 | rule="(${direction} and (${noportrule} or ${tcprule}))" 99 | 100 | echo "${rule}" 101 | } 102 | 103 | # The local rule specifies that only the side with the higher port 104 | # number should forward the packet if the traffic is within 105 | # the Kubernetes cluster. If the higher port is unknown, the side 106 | # with the higher IP address should forward the packet. 107 | localsrcrule=$(compose_local_rule "src") 108 | localdstrule=$(compose_local_rule "dst") 109 | 110 | bpf_rules="not ip or (not ip proto 4) ${servicesubnet} and (${subnetrule} or ${localsrcrule} or ${localdstrule})" 111 | 112 | echo /opt/extrahop/sbin/rpcapd -v -f rpcapd.ini -i "${INTERFACE}" -D -F "${bpf_rules}" 113 | /opt/extrahop/sbin/rpcapd -v -f rpcapd.ini -i "${INTERFACE}" -D -F "${bpf_rules}" 114 | -------------------------------------------------------------------------------- /deploy_kubernetes_daemon/rpcapd_daemon.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: extrahop 5 | --- 6 | apiVersion: apps/v1 7 | kind: DaemonSet 8 | metadata: 9 | name: extrahop-rpcapd 10 | namespace: extrahop 11 | labels: 12 | component: extrahop-rpcapd 13 | spec: 14 | selector: 15 | matchLabels: 16 | component: extrahop-rpcapd 17 | template: 18 | metadata: 19 | labels: 20 | component: extrahop-rpcapd 21 | spec: 22 | affinity: 23 | nodeAffinity: 24 | requiredDuringSchedulingIgnoredDuringExecution: 25 | nodeSelectorTerms: 26 | - matchExpressions: 27 | - key: extrahop/norpcapd 28 | operator: DoesNotExist 29 | hostNetwork: true 30 | hostPID: true 31 | containers: 32 | - name: rpcapd 33 | image: EXAMPLE-REGISTRY/rpcapd:latest 34 | securityContext: 35 | privileged: true 36 | env: 37 | - name: EXTRAHOP_SENSOR_IP 38 | value: 10.10.10.10 39 | - name: RPCAPD_TARGET_PORT 40 | value: "2003" 41 | - name: PODNET 42 | value: 10.10.11.0/24 43 | - name: SVCNET 44 | value: 10.10.12.0/24 45 | tolerations: 46 | - key: node-role.kubernetes.io/master 47 | operator: Exists 48 | -------------------------------------------------------------------------------- /extract_device_list/extract_device_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import json 10 | import csv 11 | import datetime 12 | import sys 13 | from urllib.parse import urlunparse 14 | import base64 15 | 16 | # The IP address or hostname of the ExtraHop appliance or Reveal(x) 360 API 17 | HOST = "extrahop.example.com" 18 | 19 | # For Reveal(x) 360 authentication 20 | # The ID of the REST API credentials. 21 | ID = "abcdefg123456789" 22 | # The secret of the REST API credentials. 23 | SECRET = "123456789abcdefg987654321abcdefg" 24 | # A global variable for the temporary API access token (leave blank) 25 | TOKEN = "" 26 | 27 | # For self-managed appliance authentication 28 | # The API key. 29 | API_KEY = "123456789abcdefghijklmnop" 30 | 31 | # The file that output will be written to. 32 | FILENAME = "devices.csv" 33 | # The maximum number of devices to retrieve with each GET request. 34 | LIMIT = 1000 35 | # Determines whether L2 parent devices are retrieved. 36 | SAVEL2 = False 37 | # Retrieves only devices that are currently under advanced analysis. 38 | ADVANCED_ONLY = False 39 | # Retrieves only devices that are considered high value. 40 | HIGH_VALUE_ONLY = False 41 | 42 | 43 | def getToken(): 44 | """ 45 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 46 | 47 | Returns: 48 | str: A temporary API access token 49 | """ 50 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 51 | headers = { 52 | "Authorization": "Basic " + auth, 53 | "Content-Type": "application/x-www-form-urlencoded", 54 | } 55 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 56 | r = requests.post( 57 | url, 58 | headers=headers, 59 | data="grant_type=client_credentials", 60 | ) 61 | try: 62 | return r.json()["access_token"] 63 | except: 64 | print(r.text) 65 | print(r.status_code) 66 | print("Error retrieveing token from Reveal(x) 360") 67 | sys.exit() 68 | 69 | 70 | def getAuthHeader(force_token_gen=False): 71 | """ 72 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 73 | token. For self-managed appliances, adds an API key. 74 | 75 | Parameters: 76 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 77 | 78 | Returns: 79 | header (str): The value for the header key in the headers dictionary 80 | """ 81 | global TOKEN 82 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 83 | return f"ExtraHop apikey={API_KEY}" 84 | else: 85 | if TOKEN == "" or force_token_gen == True: 86 | TOKEN = getToken() 87 | return f"Bearer {TOKEN}" 88 | 89 | 90 | def getAllDevices(): 91 | """ 92 | Method that retrieves all devices from the ExtraHop system. 93 | 94 | Returns: 95 | device_list(list): A list of all devices on the system 96 | """ 97 | continue_search = True 98 | offset = 0 99 | device_list = [] 100 | while continue_search: 101 | new_devices = getDevices(LIMIT, offset) 102 | device_list += new_devices 103 | if len(new_devices) > 0: 104 | print(f"Retrieved {str(len(device_list))} devices") 105 | offset += LIMIT 106 | else: 107 | continue_search = False 108 | return device_list 109 | 110 | 111 | def getDevices(limit, offset): 112 | """ 113 | Method that retrieves a set of devices from the ExtraHop system. 114 | 115 | Parameters: 116 | limit (int): The maximum number of devices to retrieve 117 | offset (int): The number of device results to skip 118 | 119 | Returns: 120 | devices (list): A list of device dictionaries 121 | """ 122 | url = urlunparse( 123 | ( 124 | "https", 125 | HOST, 126 | f"/api/v1/devices/search", 127 | "", 128 | "", 129 | "", 130 | ) 131 | ) 132 | headers = { 133 | "Authorization": getAuthHeader(), 134 | "Accept": "application/json", 135 | } 136 | data = { 137 | "limit": limit, 138 | "offset": offset 139 | } 140 | r = requests.post(url, headers=headers, json=data) 141 | if r.status_code == 200: 142 | return r.json() 143 | else: 144 | print("Error retrieving Device list") 145 | print(r.status_code) 146 | print(r.text) 147 | sys.exit() 148 | 149 | 150 | def saveToCSV(devices): 151 | """ 152 | Method that saves a list of device dictionaries to a CSV file. 153 | 154 | Parameters: 155 | devices (list): The device dictionaries 156 | 157 | Returns: 158 | saved (list): A list of devices that were saved to the CSV file 159 | skipped (list): A list of devices that were not saved to the CSV file 160 | """ 161 | print(f"Saving devices to CSV file") 162 | with open(FILENAME, "w") as csvfile: 163 | csvwriter = csv.writer(csvfile, dialect="excel") 164 | csvwriter.writerow(list(devices[0].keys())) 165 | saved = [] 166 | skipped = [] 167 | for device in devices: 168 | if ADVANCED_ONLY == False or ( 169 | ADVANCED_ONLY == True and device["analysis"] == "advanced" 170 | ): 171 | if HIGH_VALUE_ONLY == False or ( 172 | HIGH_VALUE_ONLY == True and device["critical"] == True 173 | ): 174 | if device["is_l3"] | SAVEL2: 175 | saved.append(device) 176 | device["mod_time"] = datetime.datetime.fromtimestamp( 177 | device["mod_time"] / 1000.0 178 | ) 179 | device[ 180 | "user_mod_time" 181 | ] = datetime.datetime.fromtimestamp( 182 | device["user_mod_time"] / 1000.0 183 | ) 184 | device[ 185 | "discover_time" 186 | ] = datetime.datetime.fromtimestamp( 187 | device["discover_time"] / 1000.0 188 | ) 189 | csvwriter.writerow(list(device.values())) 190 | else: 191 | skipped.append(device) 192 | else: 193 | skipped.append(device) 194 | else: 195 | skipped.append(device) 196 | return saved, skipped 197 | 198 | 199 | def main(): 200 | devices = getAllDevices() 201 | if devices: 202 | saved, skipped = saveToCSV(devices) 203 | print( 204 | f"Saved {str(len(saved))} devices to {FILENAME}.", 205 | ) 206 | print(f"Skipped {str(len(skipped))} devices.") 207 | 208 | 209 | if __name__ == "__main__": 210 | main() 211 | -------------------------------------------------------------------------------- /extract_files/extract_files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2024 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import json 10 | import csv 11 | import datetime 12 | import sys 13 | from urllib.parse import urlunparse 14 | import base64 15 | 16 | # The IP address or hostname of the ExtraHop appliance or RevealX 360 API 17 | HOST = "extrahop.example.com" 18 | 19 | # For RevealX 360 authentication 20 | # The ID of the REST API credentials. 21 | ID = "abcdefg123456789" 22 | # The secret of the REST API credentials. 23 | SECRET = "123456789abcdefg987654321abcdefg" 24 | # A global variable for the temporary API access token (leave blank) 25 | TOKEN = "" 26 | 27 | # For self-managed appliance authentication 28 | # The API key. 29 | API_KEY = "123456789abcdefghijklmnop" 30 | 31 | # The parameters of the packet search 32 | SEARCH = {"from": "-30m", "output": "extract", "ip1": "10.10.10.10"} 33 | 34 | 35 | def getToken(): 36 | """ 37 | Method that generates and retrieves a temporary API access token for RevealX 360 authentication. 38 | 39 | Returns: 40 | str: A temporary API access token 41 | """ 42 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 43 | headers = { 44 | "Authorization": "Basic " + auth, 45 | "Content-Type": "application/x-www-form-urlencoded", 46 | } 47 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 48 | r = requests.post( 49 | url, 50 | headers=headers, 51 | data="grant_type=client_credentials", 52 | ) 53 | try: 54 | return r.json()["access_token"] 55 | except: 56 | print(r.text) 57 | print(r.status_code) 58 | print("Error retrieveing token from RevealX 360") 59 | sys.exit() 60 | 61 | 62 | def getAuthHeader(force_token_gen=False): 63 | """ 64 | Method that adds an authorization header for a request. For RevealX 360, adds a temporary access 65 | token. For self-managed appliances, adds an API key. 66 | 67 | Parameters: 68 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 69 | 70 | Returns: 71 | header (str): The value for the header key in the headers dictionary 72 | """ 73 | global TOKEN 74 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 75 | return f"ExtraHop apikey={API_KEY}" 76 | else: 77 | if TOKEN == "" or force_token_gen == True: 78 | TOKEN = getToken() 79 | return f"Bearer {TOKEN}" 80 | 81 | 82 | def fileSearch(search): 83 | """ 84 | Method that extracts files from packets that match the specified search parameters. 85 | 86 | Parameters: 87 | search(dict): A dictionary that specifies the packet search parameters. 88 | 89 | Returns: 90 | device_list(list): A list of all devices on the system 91 | """ 92 | url = urlunparse( 93 | ( 94 | "https", 95 | HOST, 96 | f"/api/v1/packets/search", 97 | "", 98 | "", 99 | "", 100 | ) 101 | ) 102 | headers = { 103 | "Authorization": getAuthHeader(), 104 | "Accept": "application/json", 105 | } 106 | r = requests.post(url, headers=headers, json=search) 107 | if r.status_code == 200: 108 | return r.content 109 | else: 110 | print("Unable to retrieve files") 111 | print(r.status_code) 112 | print(r.text) 113 | sys.exit() 114 | 115 | 116 | def main(): 117 | output_filename = "extracted_files.zip" 118 | extracted_files = fileSearch(SEARCH) 119 | with open(output_filename, "wb") as file: 120 | file.write(extracted_files) 121 | print(f"Wrote extracted files to {output_filename}.") 122 | 123 | 124 | if __name__ == "__main__": 125 | main() 126 | -------------------------------------------------------------------------------- /extract_metrics/extract_metrics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import csv 10 | import time 11 | import requests 12 | from urllib.parse import urlunparse 13 | import base64 14 | import sys 15 | 16 | # The IP address or hostname of the ExtraHop appliance or Reveal(x) 360 API 17 | HOST = "extrahop.example.com" 18 | 19 | # For Reveal(x) 360 authentication 20 | # The ID of the REST API credentials. 21 | ID = "abcdefg123456789" 22 | # The secret of the REST API credentials. 23 | SECRET = "123456789abcdefg987654321abcdefg" 24 | # A global variable for the temporary API access token (leave blank) 25 | TOKEN = "" 26 | 27 | # For self-managed appliance authentication 28 | # The API key. 29 | API_KEY = "123456789abcdefghijklmnop" 30 | 31 | # The filepath of the CSV file to save metrics to 32 | FILENAME = "output.csv" 33 | 34 | 35 | def getToken(): 36 | """ 37 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 38 | 39 | Returns: 40 | str: A temporary API access token 41 | """ 42 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 43 | headers = { 44 | "Authorization": "Basic " + auth, 45 | "Content-Type": "application/x-www-form-urlencoded", 46 | } 47 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 48 | r = requests.post( 49 | url, 50 | headers=headers, 51 | data="grant_type=client_credentials", 52 | ) 53 | try: 54 | return r.json()["access_token"] 55 | except: 56 | print(r.text) 57 | print(r.status_code) 58 | print("Error retrieveing token from Reveal(x) 360") 59 | sys.exit() 60 | 61 | 62 | def getAuthHeader(force_token_gen=False): 63 | """ 64 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 65 | token. For self-managed appliances, adds an API key. 66 | 67 | Parameters: 68 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 69 | 70 | Returns: 71 | header (str): The value for the header key in the headers dictionary 72 | """ 73 | global TOKEN 74 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 75 | return f"ExtraHop apikey={API_KEY}" 76 | else: 77 | if TOKEN == "" or force_token_gen == True: 78 | TOKEN = getToken() 79 | return f"Bearer {TOKEN}" 80 | 81 | 82 | def getMetrics(object_type, metric_category, name, object_ids, cycle): 83 | """ 84 | Method that retrieves metrics from the ExtraHop system 85 | 86 | Parameters: 87 | object_type (str): The type of object to retrieve metrics for 88 | metric_category (str): The category of object to retrieve metrics for 89 | name (str): The name of the metric to retrieve 90 | object_ids (list): A list of numeric IDs that identify the objects to retrieve metrics for 91 | cycle (str): The aggregation period for metrics 92 | 93 | Returns: 94 | metrics (list): A list of metric objects 95 | """ 96 | data = { 97 | "object_type": object_type, 98 | "metric_category": metric_category, 99 | "metric_specs": [{"name": name}], 100 | "object_ids": object_ids, 101 | "cycle": cycle, 102 | } 103 | headers = {"Authorization": getAuthHeader()} 104 | url = urlunparse(("https", HOST, "/api/v1/metrics", "", "", "")) 105 | r = requests.post(url, headers=headers, json=data) 106 | if r.status_code == 200: 107 | j = r.json() 108 | print( 109 | f'Extracted {str(len(j["stats"]))} metrics from {str(j["from"])} until {str(j["until"])}' 110 | ) 111 | return j["stats"] 112 | else: 113 | print("Failed to retrieve metrics") 114 | print(r.status_code) 115 | print(r.text) 116 | 117 | 118 | def saveMetrics(metrics, filename): 119 | """ 120 | Method that saves metrics to a CSV file. 121 | 122 | Parameters: 123 | metrics (list): The list of metric objects 124 | filename (str): The filename of the CSV file 125 | """ 126 | with open(filename, "w") as csvfile: 127 | csvwriter = csv.writer(csvfile, dialect="excel") 128 | headers = [] 129 | for header in metrics[0]: 130 | headers.append(header) 131 | csvwriter.writerow(headers) 132 | for metric in metrics: 133 | metric["time"] = time.strftime( 134 | "%Y-%m-%d %H:%M:%S", time.localtime(metric["time"] / 1000) 135 | ) 136 | metric["values"] = str(metric["values"][0]) 137 | csvwriter.writerow(list(metric.values())) 138 | 139 | 140 | def main(): 141 | metrics = getMetrics("device", "http_server", "rsp", [1907], "1hr") 142 | saveMetrics(metrics, FILENAME) 143 | 144 | 145 | if __name__ == "__main__": 146 | main() 147 | -------------------------------------------------------------------------------- /f5_irules/extrahop_shared_secret_export_multiple: -------------------------------------------------------------------------------- 1 | when RULE_INIT { 2 | # Here, you must define the name of the sideband virtual server to send secrets 3 | set static::edas { 4 | extrahop_shared_secret_sideband_eda01 5 | extrahop_shared_secret_sideband_eda02 6 | } 7 | } 8 | 9 | when CLIENTSSL_HANDSHAKE { 10 | if { [ catch { 11 | if { [SSL::cipher version] eq "TLSv1.3" } { 12 | call handleMultiSecret 13 | } 14 | else { 15 | call handleSecret 16 | } 17 | } err ] } { 18 | log local0. [concat "Error:" $err] 19 | return 20 | } 21 | } 22 | 23 | when SERVERSSL_HANDSHAKE { 24 | if { [ catch { 25 | if { [SSL::cipher version] eq "TLSv1.3" } { 26 | call handleMultiSecret 27 | } 28 | else { 29 | call handleSecret 30 | } 31 | } err ] } { 32 | log local0. [concat "Error:" $err] 33 | return 34 | } 35 | } 36 | 37 | proc handleMultiSecret {} { 38 | set client_rand [SSL::clientrandom] 39 | 40 | set client_hs [SSL::tls13_secret client hs] 41 | set secret_message [binary format H* "f4[format %X [expr { ([string length $client_hs] / 2) + 33 } ]]0000[set client_rand][set client_hs]"] 42 | call sendSecret $secret_message 43 | 44 | set server_hs [SSL::tls13_secret server hs] 45 | set secret_message [binary format H* "f4[format %X [expr { ([string length $server_hs] / 2) + 33 } ]]0001[set client_rand][set server_hs]"] 46 | call sendSecret $secret_message 47 | 48 | set client_app [SSL::tls13_secret client app] 49 | set secret_message [binary format H* "f4[format %X [expr { ([string length $client_app] / 2) + 33 } ]]0002[set client_rand][set client_app]"] 50 | call sendSecret $secret_message 51 | 52 | set server_app [SSL::tls13_secret server app] 53 | set secret_message [binary format H* "f4[format %X [expr { ([string length $server_app] / 2) + 33 } ]]0003[set client_rand][set server_app]"] 54 | call sendSecret $secret_message 55 | 56 | } 57 | 58 | proc handleSecret {} { 59 | set client_rand [SSL::clientrandom] 60 | set secret [SSL::sessionsecret] 61 | 62 | set secret_message [binary format H* "f35000$client_rand$secret"] 63 | call sendSecret $secret_message 64 | } 65 | 66 | proc sendSecret { secret_message } { 67 | 68 | set length [string length $secret_message] 69 | # Verify that the message is the appropriate length ( 70 | if { $length != 83 && $length != 68 && $length != 84}{ 71 | return 72 | } 73 | 74 | set cmp_unit [TMM::cmp_unit] 75 | 76 | foreach eda $static::edas { 77 | 78 | # Session table key 79 | set key "${cmp_unit}_conn_${eda}" ; 80 | # Session table data for $key for this dest_addr 81 | set conn [session lookup dest_addr $key] ; 82 | 83 | if { $conn eq "" }{ 84 | set conn [connect -timeout 1000 -idle 300 -status conn_status $eda] 85 | if { $conn_status ne "connected" }{ 86 | return 87 | } 88 | session add dest_addr $key "$conn" 300 89 | } 90 | else { 91 | # Attempt sideband connection re-use 92 | set conn_info [connect info -status $conn] 93 | set conn_state [lindex [lindex $conn_info 0] 0] 94 | if { $conn_state ne "connected" }{ 95 | set conn [connect -timeout 1000 -idle 300 -status conn_status $eda] 96 | if { $conn_status ne "connected" }{ 97 | return 98 | } 99 | session add dest_addr $key "$conn" 300 100 | } 101 | } 102 | 103 | # Send secret message 104 | set send_bytes [send -timeout 1000 -status send_status $conn $secret_message] 105 | recv -timeout 1 $conn 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /f5_irules/extrahop_shared_secret_export_single: -------------------------------------------------------------------------------- 1 | # Updated 1/27/21 2 | # 3 | when RULE_INIT { 4 | # Here, you must define the name of the sideband virtual server to send secrets 5 | set static::virtual_server "extrahop_shared_secret_sideband" 6 | } 7 | 8 | when CLIENTSSL_HANDSHAKE { 9 | if { [ catch { 10 | if { [SSL::cipher version] eq "TLSv1.3" } { 11 | call handleMultiSecret 12 | } 13 | else { 14 | call handleSecret 15 | } 16 | } err ] } { 17 | log local0. [concat "Error:" $err] 18 | return 19 | } 20 | } 21 | 22 | when SERVERSSL_HANDSHAKE { 23 | if { [ catch { 24 | if { [SSL::cipher version] eq "TLSv1.3" } { 25 | call handleMultiSecret 26 | } 27 | else { 28 | call handleSecret 29 | } 30 | } err ] } { 31 | log local0. [concat "Error:" $err] 32 | return 33 | } 34 | } 35 | 36 | proc handleMultiSecret {} { 37 | set client_rand [SSL::clientrandom] 38 | 39 | set client_hs [SSL::tls13_secret client hs] 40 | set secret_message [binary format H* "f4[format %X [expr { ([string length $client_hs] / 2) + 33 } ]]0000[set client_rand][set client_hs]"] 41 | call sendSecret $secret_message 42 | 43 | set server_hs [SSL::tls13_secret server hs] 44 | set secret_message [binary format H* "f4[format %X [expr { ([string length $server_hs] / 2) + 33 } ]]0001[set client_rand][set server_hs]"] 45 | call sendSecret $secret_message 46 | 47 | set client_app [SSL::tls13_secret client app] 48 | set secret_message [binary format H* "f4[format %X [expr { ([string length $client_app] / 2) + 33 } ]]0002[set client_rand][set client_app]"] 49 | call sendSecret $secret_message 50 | 51 | set server_app [SSL::tls13_secret server app] 52 | set secret_message [binary format H* "f4[format %X [expr { ([string length $server_app] / 2) + 33 } ]]0003[set client_rand][set server_app]"] 53 | call sendSecret $secret_message 54 | 55 | } 56 | 57 | proc handleSecret {} { 58 | set client_rand [SSL::clientrandom] 59 | set secret [SSL::sessionsecret] 60 | 61 | set secret_message [binary format H* "f35000$client_rand$secret"] 62 | call sendSecret $secret_message 63 | } 64 | 65 | proc sendSecret { secret_message } { 66 | 67 | set length [string length $secret_message] 68 | # Verify that the message is the appropriate length ( 69 | if { $length != 83 && $length != 68 && $length != 84}{ 70 | return 71 | } 72 | 73 | set cmp_unit [TMM::cmp_unit] 74 | 75 | set key "${cmp_unit}_conn_${static::virtual_server}" ;# key for session table 76 | set conn [session lookup dest_addr $key] ;# conn = session table data for $key for this dest_addr 77 | 78 | if { $conn eq "" }{ 79 | set conn [connect -timeout 1000 -idle 300 -status conn_status $static::virtual_server] 80 | if { $conn_status ne "connected" }{ 81 | return 82 | } 83 | session add dest_addr $key "$conn" 300 84 | } else { 85 | # Attempt sideband connection re-use 86 | set conn_info [connect info -status $conn] 87 | set conn_state [lindex [lindex $conn_info 0] 0] 88 | if { $conn_state ne "connected" }{ 89 | set conn [connect -timeout 1000 -idle 300 -status conn_status $static::virtual_server] 90 | if { $conn_status ne "connected" }{ 91 | return 92 | } 93 | session add dest_addr $key "$conn" 300 94 | } 95 | } 96 | 97 | # Send secret message 98 | set send_bytes [send -timeout 1000 -status send_status $conn $secret_message] 99 | recv -timeout 1 $conn 100 | } 101 | -------------------------------------------------------------------------------- /f5_irules/extrahop_shared_secret_proto: -------------------------------------------------------------------------------- 1 | # Updated 1/27/21 2 | # 3 | when CLIENT_ACCEPTED { 4 | TCP::collect 3 5 | } 6 | 7 | when SERVERSSL_HANDSHAKE { 8 | # Send an initial hello 9 | SSL::respond [binary format H* "f00800dec0de0100000000"] 10 | } 11 | 12 | when CLIENT_DATA { 13 | 14 | # Uncomment the following 5 lines for logging/debugging purposes: 15 | #set msg_header [TCP::payload 3] 16 | #binary scan $msg_header H* header_hex 17 | #set msg_type [string range $header_hex 0 1] 18 | #scan [string range $header_hex 2 3] %x msg_length 19 | #log local0. [concat "Processed secret of length" $msg_length "and type" $msg_type ] 20 | 21 | TCP::release 22 | TCP::notify request 23 | TCP::collect 3 24 | } 25 | 26 | when USER_REQUEST { 27 | # We don't expect a response, so let's just signal one and detach to make oneconnect happy 28 | TCP::notify response 29 | LB::detach 30 | } 31 | -------------------------------------------------------------------------------- /migrate_detection_hiding/migrate_detection_hiding_enterprise.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import sys 11 | import logging 12 | import base64 13 | from urllib.parse import urlunparse 14 | 15 | # The hostname of the ExtraHop system you are migrating detection 16 | # hiding rules from 17 | SOURCE_HOST = "" 18 | # The API KEY on the ExtraHop system you are migrating detection 19 | # hiding rules from 20 | SOURCE_API_KEY = "" 21 | # The hostname of the ExtraHop system you are migrating detection 22 | # hiding rules to 23 | TARGET_HOST = "" 24 | # The API KEY on the ExtraHop system you are migrating detection 25 | # hiding rules to 26 | TARGET_API_KEY = "" 27 | 28 | source_headers = { 29 | "Authorization": f"ExtraHop apikey={SOURCE_API_KEY}", 30 | "Content-Type": "application/json", 31 | } 32 | 33 | target_headers = { 34 | "Authorization": f"ExtraHop apikey={TARGET_API_KEY}", 35 | "Content-Type": "application/json", 36 | } 37 | 38 | 39 | def getRules(): 40 | """ 41 | Method that retrieves detection hiding rules from the source 42 | ExtraHop system. 43 | Returns: 44 | rules (list): List of rule objects 45 | """ 46 | url = urlunparse( 47 | ("https", SOURCE_HOST, "/api/v1/detections/rules/hiding", "", "", "") 48 | ) 49 | 50 | r = requests.get(url, headers=source_headers) 51 | if r.status_code == 200: 52 | rules = [] 53 | for rule in r.json(): 54 | if rule["enabled"]: 55 | rules.append(rule) 56 | return rules 57 | else: 58 | logging.error(r.status_code) 59 | logging.error(r.text) 60 | raise RuntimeError("Unable to retrieve rules") 61 | 62 | 63 | def replaceId(participant): 64 | """ 65 | Method that replaces a device or device group ID in a participant object with the 66 | equivalent ID on the target appliance. 67 | 68 | Parameters: 69 | participant (dict): The participant object 70 | Returns: 71 | participant (dict): The updated participant object 72 | """ 73 | if participant["object_type"] == "device": 74 | macaddr = getMac(participant["object_id"]) 75 | if macaddr == None: 76 | return {} 77 | else: 78 | new_id = getDevId(macaddr) 79 | if new_id == -1: 80 | return {} 81 | else: 82 | participant["object_id"] = new_id 83 | elif participant["object_type"] == "device_group": 84 | group_name = getName(participant["object_id"]) 85 | if group_name == "": 86 | return {} 87 | else: 88 | new_id = getGroupId(group_name) 89 | if new_id == -1: 90 | return {} 91 | else: 92 | participant["object_id"] = new_id 93 | elif participant["object_type"] == "network_locality": 94 | networks = getNetworks(participant["object_id"]) 95 | if networks == []: 96 | return {} 97 | else: 98 | new_id = getLocalityId(networks) 99 | if new_id == -1: 100 | return {} 101 | else: 102 | participant["object_id"] = new_id 103 | return participant 104 | 105 | def getLocalityId(networks): 106 | """ 107 | Method that searches the target appliance for a network locality 108 | that contains all of the specified networks and returns 109 | the ID of that locality. 110 | 111 | Parameters: 112 | networks (list): A list of CIDR blocks and IP addresses 113 | 114 | Returns: 115 | int: The numerical ID of the network locality 116 | """ 117 | url = urlunparse( 118 | ("https", TARGET_HOST, "/api/v1/networklocalities", "", "", "") 119 | ) 120 | r = requests.get(url, headers=target_headers) 121 | if r.status_code == 200: 122 | for locality in r.json(): 123 | if set(networks) == set(locality["networks"]): 124 | return locality["id"] 125 | logging.warning( 126 | f"No equivalent network locality exists on the target appliance with the following IP addresses and CIDR blocks {networks}" 127 | ) 128 | return -1 129 | else: 130 | logging.warning(r.status_code) 131 | logging.warning(r.text) 132 | logging.warning( 133 | f"Unable to retrieve network localities from the target appliance" 134 | ) 135 | return -1 136 | 137 | def getNetworks(locality_id): 138 | """ 139 | Method that returns the networks in the specified network locality. 140 | 141 | Parameters: 142 | locality_id (int): The numeric identifier of the locality 143 | 144 | Returns: 145 | list: The list of IP addresses and CIDR blocks 146 | """ 147 | url = urlunparse( 148 | ( 149 | "https", 150 | SOURCE_HOST, 151 | f"/api/v1/networklocalities/{locality_id}", 152 | "", 153 | "", 154 | "", 155 | ) 156 | ) 157 | headers = { 158 | "Authorization": f"ExtraHop apikey={SOURCE_API_KEY}", 159 | "Content-Type": "application/json", 160 | } 161 | r = requests.get(url, headers=headers) 162 | if r.status_code == 200: 163 | return r.json()["networks"] 164 | else: 165 | logging.warning(r.status_code) 166 | logging.warning(r.text) 167 | logging.warning(f"Unable to retrieve networks for {locality_id}") 168 | return [] 169 | 170 | def getMac(dev_id): 171 | """ 172 | Method that retrieves the MAC address for a device 173 | Parameters: 174 | dev_id (int): The numerical identifier of the device 175 | Returns: 176 | str: The MAC address of the device 177 | """ 178 | url = urlunparse( 179 | ("https", SOURCE_HOST, f"/api/v1/devices/{dev_id}", "", "", "") 180 | ) 181 | 182 | r = requests.get(url, headers=source_headers) 183 | if r.status_code == 200: 184 | return r.json()["macaddr"] 185 | else: 186 | logging.warning(r.status_code) 187 | logging.warning(r.text) 188 | logging.warning(f"Unable to retrieve MAC address for {dev_id}") 189 | return None 190 | 191 | 192 | def getDevId(macaddr): 193 | """ 194 | Method that returns the ID of a device with a given MAC address on Target Appliance 195 | Parameters: 196 | macaddr (str): The MAC address of the device 197 | Returns: 198 | int: The numerical ID of the device 199 | """ 200 | url = urlunparse( 201 | ("https", TARGET_HOST, "/api/v1/devices/search", "", "", "") 202 | ) 203 | 204 | data = {"filter": {"field": "macaddr", "operand": macaddr, "operator": "="}} 205 | r = requests.post(url, headers=target_headers, json=data) 206 | if r.status_code == 200: 207 | devices = r.json() 208 | if len(devices) == 0: 209 | logging.warning( 210 | f"No equivalent device exists on Target Appliance for {macaddr}" 211 | ) 212 | return -1 213 | if len(devices) > 1: 214 | # If there are more than one device with the given MAC address, return 215 | # the L2 device. 216 | for device in devices: 217 | if device["is_l3"] == False: 218 | return device["id"] 219 | logging.warning(f"No L2 device found for {macaddr}") 220 | return -1 221 | else: 222 | return devices[0]["id"] 223 | else: 224 | logging.warning(r.status_code) 225 | logging.warning(r.text) 226 | logging.warning(f"Unable to retrieve device ID for {macaddr}") 227 | return -1 228 | 229 | 230 | def getName(group_id): 231 | """ 232 | Method that retrieves the name of a device group. 233 | Parameters: 234 | group_id (int): The numerical identifier of the device group 235 | Returns: 236 | str: The name of the device group 237 | """ 238 | url = urlunparse( 239 | ("https", SOURCE_HOST, f"/api/v1/devicegroups/{group_id}", "", "", "") 240 | ) 241 | 242 | r = requests.get(url, headers=source_headers) 243 | if r.status_code == 200: 244 | return r.json()["name"] 245 | else: 246 | logging.warning(r.status_code) 247 | logging.warning(r.text) 248 | logging.warning(f"Unable to retrieve name for {group_id}") 249 | return "" 250 | 251 | 252 | def getGroupId(group_name): 253 | """ 254 | Method that returns the ID of a device group with a given name 255 | Parameters: 256 | group_name (str): The name of the device group 257 | Returns: 258 | int: The ID of the device group 259 | """ 260 | url = urlunparse(("https", TARGET_HOST, "/api/v1/devicegroups", "", "", "")) 261 | 262 | r = requests.get(url, headers=target_headers) 263 | if r.status_code == 200: 264 | groups = r.json() 265 | for group in groups: 266 | if group["name"] == group_name: 267 | return group["id"] 268 | logging.warning( 269 | f"Unable to find an equivalent device group for {group_name}" 270 | ) 271 | return -1 272 | else: 273 | logging.warning(r.status_code) 274 | logging.warning(r.text) 275 | logging.warning(f"Unable to retrieve device groups") 276 | return -1 277 | 278 | 279 | def makeRule(rule): 280 | """ 281 | Method that creates a detection hiding rule on the Target Appliance. 282 | Parameters: 283 | rule (dict): The rule properties. 284 | Returns: 285 | bool: Indicates whether the rule was successfully created 286 | """ 287 | rule_id = rule["id"] 288 | # Rule descriptions and properties cannot be null in POST requests, so we 289 | # need to remove null descriptions and properties 290 | if rule["description"] == None: 291 | rule.pop("description") 292 | if rule["properties"] == None: 293 | rule.pop("properties") 294 | url = urlunparse( 295 | ("https", TARGET_HOST, "/api/v1/detections/rules/hiding", "", "", "") 296 | ) 297 | 298 | r = requests.post(url, headers=target_headers, json=rule) 299 | if r.status_code == 201: 300 | logging.info(f"Successfully migrated rule {rule_id}") 301 | return True 302 | else: 303 | logging.warning(r.status_code) 304 | logging.warning(r.text) 305 | logging.warning(f"Failed to create rule {rule_id}") 306 | return False 307 | 308 | 309 | def updateParticipants(rule): 310 | """ 311 | Method that updates participant objects in a rule by updating device 312 | and device group IDs. 313 | 314 | Parameters: 315 | rule (dict): The rule to be updated 316 | 317 | Returns: 318 | rule (dict): The updated rule 319 | """ 320 | roles = ["victim", "offender"] 321 | for role in roles: 322 | if rule[role] != "Any": 323 | logging.info( 324 | f"Retrieving equivalent participant IDs for rule: {rule['id']}" 325 | ) 326 | participant = rule[role] 327 | p_type = participant["object_type"] 328 | if ( 329 | p_type == "device" 330 | or p_type == "device_group" 331 | or p_type == "network_locality" 332 | ): 333 | participant = replaceId(participant) 334 | if participant == {}: 335 | return None 336 | else: 337 | rule[role] = participant 338 | elif p_type == "locality_type": 339 | participant["object_locality"] = participant["object_value"] 340 | return rule 341 | 342 | def main(): 343 | rules = getRules() 344 | num_rules = str(len(rules)) 345 | logging.info(f"Migrating {num_rules} detection hiding rules") 346 | 347 | # Find equivalent IDs for participants 348 | updated_rules = [] 349 | not_found = [] 350 | for rule in rules: 351 | new_rule = updateParticipants(rule) 352 | if new_rule: 353 | updated_rules.append(new_rule) 354 | else: 355 | not_found.append(rule["id"]) 356 | c = "y" 357 | # If unable to retrieve equivalent IDs for participants, warn user before 358 | # continuing 359 | if not_found: 360 | total_up = str(len(updated_rules)) 361 | logging.warning( 362 | f"\niFailed to find equivalent participant devices for rules with the following IDs: {not_found}" 363 | ) 364 | logging.warning(f"Do you want to migrate the other {total_up} rules?") 365 | c = input("(y/n)") 366 | if c == "y": 367 | # Create new rules on Target Appliance 368 | for rule in updated_rules: 369 | success = makeRule(rule) 370 | 371 | 372 | if __name__ == "__main__": 373 | logging.basicConfig( 374 | format="%(message)s", 375 | handlers=[logging.StreamHandler(sys.stdout),], 376 | level=logging.INFO, 377 | ) 378 | main() 379 | -------------------------------------------------------------------------------- /migrate_saml/create_local_user_groups.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import os 11 | 12 | # Retrieves the IP address or hostname of the ExtraHop system from an environment variable. 13 | HOST = os.environ['EXTRAHOP_HOST'] 14 | 15 | # Retrieves the API key from an environment variable. 16 | API_KEY = os.environ['EXTRAHOP_API_KEY'] 17 | 18 | USER_MAP_FILE = "user_map.json" 19 | GROUPS_FILE = "user_groups.json" 20 | 21 | 22 | def updateMembers(group, members): 23 | """ 24 | Method that updates the membership of a user group. 25 | 26 | Parameters: 27 | group (dict): The group metadata 28 | members (dict): The members of the group 29 | 30 | Returns: 31 | str: Indicates whether the request was successful 32 | """ 33 | url = HOST + "/api/v1/usergroups/" + group["id"] + "/members" 34 | headers = { 35 | "Content-Type": "application/json", 36 | "Accept": "application/json", 37 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 38 | } 39 | r = requests.put(url, headers=headers, data=json.dumps(members)) 40 | if r.status_code == 204: 41 | return "success" 42 | else: 43 | return r.json 44 | 45 | 46 | def getSamlNames(members, user_map, remote_users): 47 | """ 48 | Method that replaces old remote usernames with new SAML usernames. 49 | 50 | Parameters: 51 | members (dict): The members of a user group 52 | user_map (list): User metadata 53 | remote_users (list): A list of remote user names 54 | 55 | Returns: 56 | members (dict): The members of a user group 57 | """ 58 | users = members["users"] 59 | for user in users: 60 | if user in remote_users: 61 | userIndex = remote_users.index(user) 62 | saml_name = user_map[userIndex]["saml_username"] 63 | members["users"][saml_name] = members["users"].pop(user) 64 | return members 65 | 66 | 67 | # Create a list of deleted remote users 68 | remote_users = [] 69 | with open(USER_MAP_FILE) as json_file: 70 | user_map = json.load(json_file) 71 | for user in user_map: 72 | remote_users.append(user["remote_username"]) 73 | 74 | with open(GROUPS_FILE) as json_file: 75 | groups = json.load(json_file) 76 | 77 | # Create list of local groups with remote users 78 | to_do = [] 79 | for group in groups: 80 | try: 81 | members = group["members"]["users"] 82 | for member in members: 83 | if member in remote_users: 84 | to_do.append(group) 85 | break 86 | except: 87 | continue 88 | groups = to_do 89 | 90 | # Update group membership 91 | success = [] 92 | fail = [] 93 | for group in groups: 94 | members = group["members"] 95 | members = getSamlNames(members, user_map, remote_users) 96 | updated = updateMembers(group, members["users"]) 97 | if updated == "success": 98 | success.append({"group": group, "users": members["users"]}) 99 | else: 100 | fail.append([group, updated]) 101 | 102 | # Print out results of script 103 | if success: 104 | print("Successfully updated membership of the following groups:") 105 | for update in success: 106 | print(update["group"]["name"]) 107 | print(" Members:") 108 | for user in update["users"]: 109 | print(" " + user) 110 | print("") 111 | 112 | if fail: 113 | print("Failed to update ownership of the following groups:") 114 | for failure in fail: 115 | print(" " + failure[0]["name"]) 116 | print(" " + str(failure[1])) 117 | print("") 118 | 119 | if success: 120 | with open(GROUPS_FILE, "w") as outfile: 121 | json.dump(groups, outfile) 122 | -------------------------------------------------------------------------------- /migrate_saml/create_saml_accounts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import csv 11 | import sys 12 | import os 13 | 14 | # Retrieves the IP address or hostname of the ExtraHop system from an environment variable. 15 | HOST = os.environ['EXTRAHOP_HOST'] 16 | 17 | # Retrieves the API key from an environment variable. 18 | API_KEY = os.environ['EXTRAHOP_API_KEY'] 19 | 20 | # Determines whether SAML account names are retrieved from a CSV file. 21 | READ_CSV_FILE = False 22 | # The name of the CSV file that SAML account names are retrieved from if READ_CSV_FILE is set to True. 23 | CSV_FILE = "remote_to_saml.csv" 24 | 25 | USER_FILE = "user_map.json" 26 | 27 | csv_mapping = {} 28 | if READ_CSV_FILE: 29 | with open(CSV_FILE, "rt", encoding="ascii") as f: 30 | reader = csv.reader(f) 31 | for row in reader: 32 | csv_mapping[row.pop()] = row.pop() 33 | 34 | 35 | def generateName(name): 36 | """ 37 | Method that returns the name of the SAML user account for a remote user account. 38 | 39 | Parameters: 40 | name (str): The name of the remote user account 41 | 42 | Returns: 43 | str: The name of the SAML user account 44 | """ 45 | if csv_mapping: 46 | if name in csv_mapping: 47 | return csv_mapping[name] 48 | else: 49 | print("Error: Specified user " + name + " not found in " + CSV_FILE) 50 | sys.exit() 51 | else: 52 | return name + "@extrahop.com" 53 | 54 | 55 | # Function that creates a SAML account for the specified user 56 | def createUser(new_name, user): 57 | """ 58 | Method that creates a SAML user account on the ExtraHop system. 59 | 60 | Parameters: 61 | new_name (str): The name of the SAML user account 62 | user (dict): Metadata about the remote user account 63 | 64 | Returns: 65 | r (Response): The response from the ExtraHop system 66 | """ 67 | url = HOST + "/api/v1/users" 68 | headers = { 69 | "Content-Type": "application/json", 70 | "Accept": "application/json", 71 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 72 | } 73 | user_params = { 74 | "username": new_name, 75 | "enabled": user["enabled"], 76 | "name": user["name"], 77 | "type": "remote", 78 | } 79 | r = requests.post(url, headers=headers, data=json.dumps(user_params)) 80 | return r 81 | 82 | 83 | with open(USER_FILE) as json_file: 84 | user_map = json.load(json_file) 85 | 86 | failed = [] 87 | for user in user_map: 88 | username = user["remote_username"] 89 | new_name = generateName(username) 90 | r = createUser(new_name, user) 91 | if r.status_code == 201: 92 | user["saml_username"] = new_name 93 | print( 94 | "Successfully created new user account for " 95 | + username 96 | + ": " 97 | + new_name 98 | ) 99 | else: 100 | failed.append([username, r.status_code, r.text]) 101 | 102 | if failed: 103 | print("") 104 | print( 105 | "Failed to create duplicate local user accounts for the following users:" 106 | ) 107 | for error in failed: 108 | print("") 109 | for message in error: 110 | print(message) 111 | 112 | with open(USER_FILE, "w") as outfile: 113 | json.dump(user_map, outfile) 114 | -------------------------------------------------------------------------------- /migrate_saml/delete_remote_users.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import os 11 | 12 | # Retrieves the IP address or hostname of the ExtraHop system from an environment variable. 13 | HOST = os.environ['EXTRAHOP_HOST'] 14 | 15 | # Retrieves the API key from an environment variable. 16 | API_KEY = os.environ['EXTRAHOP_API_KEY'] 17 | 18 | 19 | USER_MAP_FILE = "user_map.json" 20 | # Function that deletes a user 21 | def deleteUser(remote_user, saml_user): 22 | """ 23 | Method that deletes a user account. 24 | 25 | Parameters: 26 | remote_user (str): The name of the remote user to delete 27 | saml_user (str): The name of the SAML user to transfer customizations to 28 | 29 | Returns: 30 | str: Indicates whether the request was successful 31 | """ 32 | url = HOST + "/api/v1/users/" + remote_user + "?dest_user=" + saml_user 33 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 34 | r = requests.delete(url, headers=headers) 35 | if r.status_code == 204: 36 | return "success" 37 | else: 38 | return r.json() 39 | 40 | 41 | # Create a list of remote users 42 | remote_users = {} 43 | with open(USER_MAP_FILE) as json_file: 44 | user_map = json.load(json_file) 45 | for user in user_map: 46 | remote_users[user["remote_username"]] = user["saml_username"] 47 | 48 | # Delete remote user accounts 49 | success = [] 50 | fail = [] 51 | for user in remote_users: 52 | updated = deleteUser(user, remote_users[user]) 53 | if updated == "success": 54 | success.append(user) 55 | else: 56 | fail.append([user, updated]) 57 | 58 | # Print out results of script 59 | if success: 60 | print("Successfully deleted the following remote user accounts:") 61 | for update in success: 62 | print(" " + update) 63 | 64 | if fail: 65 | print("Failed to delete the following remote user accounts:") 66 | for failure in fail: 67 | print(" " + failure[0]) 68 | print(" " + str(failure[1])) 69 | print("") 70 | -------------------------------------------------------------------------------- /migrate_saml/remote_to_saml.csv: -------------------------------------------------------------------------------- 1 | example1@extrahop.com,example-one@extrahop.com 2 | example2@extrahop.com,example-two@extrahop.com 3 | -------------------------------------------------------------------------------- /migrate_saml/retrieve_local_user_groups.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import os 11 | 12 | # Retrieves the IP address or hostname of the ExtraHop system from an environment variable. 13 | HOST = os.environ['EXTRAHOP_HOST'] 14 | 15 | # Retrieves the API key from an environment variable. 16 | API_KEY = os.environ['EXTRAHOP_API_KEY'] 17 | 18 | 19 | OUTPUT_FILE = "user_groups.json" 20 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 21 | 22 | 23 | def getGroups(): 24 | """ 25 | Method that retrieves metadata for every user group. 26 | 27 | Returns: 28 | list: List of dictionaries containing user group metadata 29 | """ 30 | url = HOST + "/api/v1/usergroups" 31 | r = requests.get(url, headers=headers) 32 | return r.json() 33 | 34 | 35 | # Function that retrieves members of the specified group 36 | def getMembers(group_id): 37 | """ 38 | Method that retrieves the members of a user group. 39 | 40 | Parameters: 41 | group_id (str): The ID of the user group 42 | 43 | Returns: 44 | list: The list of group members 45 | """ 46 | url = HOST + "/api/v1/usergroups/" + str(group_id) + "/members" 47 | r = requests.get(url, headers=headers) 48 | if r.status_code == 200: 49 | return r.json() 50 | else: 51 | print("Unable to retrieve members of group " + str(group_id)) 52 | print(r.status_code) 53 | print(r.text) 54 | return None 55 | 56 | 57 | groups = getGroups() 58 | final_groups = [] 59 | for group in groups: 60 | if not group["is_remote"]: 61 | group_id = group["id"] 62 | group["members"] = getMembers(group_id) 63 | final_groups.append(group) 64 | 65 | with open(OUTPUT_FILE, "w") as outfile: 66 | json.dump(final_groups, outfile) 67 | -------------------------------------------------------------------------------- /migrate_saml/retrieve_remote_users.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import sys 11 | import os 12 | 13 | # Retrieves the IP address or hostname of the ExtraHop system from an environment variable. 14 | HOST = os.environ['EXTRAHOP_HOST'] 15 | 16 | # Retrieves the API key from an environment variable. 17 | API_KEY = os.environ['EXTRAHOP_API_KEY'] 18 | 19 | OUTPUT_FILE = "user_map.json" 20 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 21 | 22 | 23 | def getUsers(): 24 | """ 25 | Method that retrieves metadata for every user. 26 | 27 | Returns: 28 | list: List of dictionaries containing user metadata 29 | """ 30 | url = HOST + "/api/v1/users" 31 | r = requests.get(url, headers=headers) 32 | return r.json() 33 | 34 | 35 | # Function that checks for duplicate usernames 36 | def checkDuplicates(u_list): 37 | """ 38 | Method that finds duplicate usernames. 39 | 40 | Parameters: 41 | u_list (list): List of usernames 42 | 43 | Returns: 44 | s (list): List of duplicate usernames 45 | """ 46 | checked = [] 47 | duplicates = set() 48 | for user in u_list: 49 | for c in checked: 50 | if user.lower() == c.lower(): 51 | duplicates.add(user) 52 | duplicates.add(c) 53 | checked.append(user) 54 | s = sorted(duplicates, key=str.lower) 55 | return s 56 | 57 | 58 | users = getUsers() 59 | user_map = [] 60 | u_list = [] 61 | for user in users: 62 | if user["type"] != "local": 63 | user["remote_username"] = user["username"] 64 | user_map.append(user) 65 | u_list.append(user["username"]) 66 | 67 | duplicates = checkDuplicates(u_list) 68 | if duplicates: 69 | print("Error: The following duplicate remote usernames were found:") 70 | for user in duplicates: 71 | print(" " + user) 72 | print( 73 | "Local and SAML user accounts cannot share usernames, regardless of case. Rename or delete duplicates before continuing." 74 | ) 75 | sys.exit() 76 | 77 | with open(OUTPUT_FILE, "w") as outfile: 78 | json.dump(user_map, outfile) 79 | -------------------------------------------------------------------------------- /migrate_saml/retrieve_sharing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import os 11 | 12 | # Retrieves the IP address or hostname of the ExtraHop system from an environment variable. 13 | HOST = os.environ['EXTRAHOP_HOST'] 14 | 15 | # Retrieves the API key from an environment variable. 16 | API_KEY = os.environ['EXTRAHOP_API_KEY'] 17 | 18 | # The type of customization metadata to retrieve. 19 | # The following values are valid: 'dashboards', 'activitymaps' 20 | OBJECT_TYPE = "dashboards" 21 | 22 | # The name of the JSON file to save customization metadata in. 23 | # The following values are valid: 'dashboards', 'activitymaps' 24 | OUTPUT_FILE = "dashboards.json" 25 | 26 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 27 | 28 | 29 | def getObjects(): 30 | """ 31 | Method that retrieves metadata for every object. 32 | 33 | Returns: 34 | list: A list of each object 35 | """ 36 | url = HOST + "/api/v1/" + OBJECT_TYPE 37 | r = requests.get(url, headers=headers) 38 | return r.json() 39 | 40 | 41 | # Function that retrieves sharing settings for a specified object ID 42 | def getSharing(object_id): 43 | """ 44 | Method that retrieves sharing settings for an object. 45 | 46 | Parameters: 47 | object_id (str): The ID of the object 48 | 49 | Returns: 50 | dict: The sharing settings of the object 51 | """ 52 | url = HOST + "/api/v1/" + OBJECT_TYPE + "/" + str(object_id) + "/sharing" 53 | r = requests.get(url, headers=headers) 54 | if r.status_code == 200: 55 | return r.json() 56 | else: 57 | print( 58 | "Unable to retrieve sharing information for object ID " 59 | + str(object_id) 60 | ) 61 | print(r.status_code) 62 | print(r.text) 63 | return None 64 | 65 | 66 | eh_objects = getObjects() 67 | for eh_object in eh_objects: 68 | object_id = eh_object["id"] 69 | eh_object["sharing"] = getSharing(object_id) 70 | 71 | with open(OUTPUT_FILE, "w") as outfile: 72 | json.dump(eh_objects, outfile) 73 | -------------------------------------------------------------------------------- /migrate_saml/transfer_sharing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import sys 11 | import os 12 | 13 | # Retrieves the IP address or hostname of the ExtraHop system from an environment variable. 14 | HOST = os.environ['EXTRAHOP_HOST'] 15 | 16 | # Retrieves the API key from an environment variable. 17 | API_KEY = os.environ['EXTRAHOP_API_KEY'] 18 | 19 | # The type of customization to transfer. 20 | # The following values are valid: 'dashboards', 'activitymaps', 'reports' 21 | OBJECT_TYPE = "dashboards" 22 | # The name of the JSON file that includes the customization metadata. 23 | # The following values are valid: 'dashboards.json', 'activity_maps.json', 'reports.json' 24 | OBJECT_FILE = "dashboards.json" 25 | 26 | 27 | USER_MAP_FILE = "user_map.json" 28 | 29 | 30 | def sharedWithRemote(eh_object, remote_users, user_map): 31 | """ 32 | Method that checks to see if the specified object was shared with 33 | deleted remote users. If so, returns a sharing dictionary 34 | with the SAML user account names. 35 | 36 | Parameters: 37 | eh_object (dict): Object metadata 38 | remote_users (list): The names of deleted remote user accounts 39 | user_map (list): User metadata 40 | 41 | Returns: 42 | updated (dict): A sharing dictionary with SAML user account names 43 | """ 44 | sharing = eh_object["sharing"] 45 | updated = {"users": {}} 46 | if sharing != None: 47 | users_shared = sharing["users"] 48 | for user in users_shared: 49 | if user in remote_users: 50 | user_index = remote_users.index(user) 51 | saml_name = user_map[user_index]["saml_username"] 52 | updated["users"][saml_name] = sharing["users"][user] 53 | if updated["users"]: 54 | return updated 55 | else: 56 | return None 57 | 58 | 59 | def updateSharing(eh_object, remoteShares): 60 | """ 61 | Method that updates sharing options for a specified object. 62 | 63 | Parameters: 64 | eh_object (dict): Object metadata 65 | remoteShares (dict): A sharing dictionary with SAML user account names 66 | 67 | Returns: 68 | str: Indicates whether the request was successful 69 | """ 70 | url = ( 71 | HOST 72 | + "/api/v1/" 73 | + OBJECT_TYPE 74 | + "/" 75 | + str(eh_object["id"]) 76 | + "/sharing" 77 | ) 78 | headers = { 79 | "Content-Type": "application/json", 80 | "Accept": "application/json", 81 | "Authorization": "ExtraHop apikey=%s" % API_KEY, 82 | } 83 | r = requests.patch(url, headers=headers, data=json.dumps(remoteShares)) 84 | if r.status_code == 204: 85 | return "success" 86 | else: 87 | return r.json() 88 | 89 | 90 | # Create a list of deleted remote users 91 | remote_users = [] 92 | with open(USER_MAP_FILE) as json_file: 93 | user_map = json.load(json_file) 94 | for user in user_map: 95 | remote_users.append(user["remote_username"]) 96 | 97 | # Extract object metadata from JSON file 98 | with open(OBJECT_FILE) as json_file: 99 | eh_objects = json.load(json_file) 100 | 101 | success = [] 102 | fail = [] 103 | # Restore sharing options for deleted remote users 104 | for eh_object in eh_objects: 105 | remoteShares = sharedWithRemote(eh_object, remote_users, user_map) 106 | if remoteShares: 107 | updated = updateSharing(eh_object, remoteShares) 108 | if updated == "success": 109 | success.append( 110 | {"eh_object": eh_object, "remoteShares": remoteShares} 111 | ) 112 | else: 113 | fail.append([eh_object, updated]) 114 | 115 | # Print out results of script 116 | if success: 117 | print( 118 | "Successfully updated sharing options the following " 119 | + OBJECT_TYPE 120 | + ":" 121 | ) 122 | for update in success: 123 | print(update["eh_object"]["name"]) 124 | print(update["remoteShares"]) 125 | print("") 126 | 127 | if fail: 128 | print("Failed to update ownership of the following " + OBJECT_TYPE + ":") 129 | for failure in fail: 130 | print(" " + failure[0]["name"]) 131 | print(" " + str(failure[1])) 132 | print("") 133 | -------------------------------------------------------------------------------- /ml_api_logger/ml_api_logger.go: -------------------------------------------------------------------------------- 1 | package main 2 | import ( 3 | "fmt" 4 | "io/ioutil" 5 | "net/http" 6 | "os" 7 | ) 8 | 9 | func addyLog(w http.ResponseWriter, request *http.Request) { 10 | defer request.Body.Close() 11 | messageBytes, err := ioutil.ReadAll(request.Body) 12 | message := string(messageBytes[:]) 13 | if err != nil { 14 | fmt.Printf("Error decoding response body: %s", err) 15 | } 16 | fmt.Println(message) 17 | } 18 | 19 | func main() { 20 | loggerIP := os.Getenv("LOGGER_IP") 21 | if loggerIP == "" { 22 | loggerIP = "0.0.0.0" 23 | } 24 | http.HandleFunc("/log", addyLog) 25 | err := http.ListenAndServeTLS( 26 | loggerIP + ":" + os.Getenv("LOGGER_PORT"), 27 | os.Getenv("LOGGER_CERT"), 28 | os.Getenv("LOGGER_KEY"), 29 | nil, 30 | ) 31 | if err != nil { 32 | fmt.Println(err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /py_rx360_auth/py_rx360_auth.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import base64 10 | import json 11 | from urllib.parse import urlunparse 12 | 13 | # The hostname of the Reveal(x) 360 API. This hostname is displayed in Reveal(x) 14 | # 360 on the API Access page under API Endpoint. The hostname does not 15 | # include the /oauth/token. 16 | HOST = "example.api.com" 17 | # The ID of the REST API credentials. 18 | ID = "abcdefg123456789" 19 | # The secret of the REST API credentials. 20 | SECRET = "123456789abcdefg987654321abcdefg" 21 | 22 | 23 | def getToken(): 24 | """ 25 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 26 | 27 | Returns: 28 | str: A temporary API access token 29 | """ 30 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 31 | headers = { 32 | "Authorization": "Basic " + auth, 33 | "Content-Type": "application/x-www-form-urlencoded", 34 | } 35 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 36 | r = requests.post( 37 | url, headers=headers, data="grant_type=client_credentials", 38 | ) 39 | return r.json()["access_token"] 40 | 41 | 42 | def getDevices(token): 43 | """ 44 | Method that sends a request to Reveal(x) 360 and authenticates the request with 45 | a REST API token. The request retrieves 100 active devices from the ExtraHop system. 46 | 47 | Returns 48 | list: The list of active devices 49 | """ 50 | headers = {"Authorization": "Bearer " + token} 51 | url = urlunparse(("https", HOST, "/api/v1/devices/search", "", "", "")) 52 | data = { 53 | "active_from": 1, 54 | "active_until": 0, 55 | "limit": 100 56 | } 57 | r = requests.post(url, headers=headers, json=data) 58 | return r.json() 59 | 60 | 61 | def getDeviceGroups(token): 62 | """ 63 | Method that sends a request to Reveal(x) 360 and authenticates the request with 64 | a REST API token. The request retrieves all device groups from the ExtraHop system. 65 | 66 | Returns 67 | list: The list of device groups 68 | """ 69 | headers = {"Authorization": "Bearer " + token} 70 | url = urlunparse(("https", HOST, "/api/v1/devicegroups", "", "", "")) 71 | r = requests.get(url, headers=headers) 72 | return r.json() 73 | 74 | 75 | token = getToken() 76 | devices = getDevices(token) 77 | print(devices) 78 | device_groups = getDeviceGroups(token) 79 | print(device_groups) 80 | -------------------------------------------------------------------------------- /query_records_explore/query_records_explore.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | import unicodecsv as csv 11 | from urllib.parse import urlunparse 12 | 13 | # The IP address or hostname of the ExtraHop system. Note that this hostname is not the hostname of the connected Explore appliance that the records are stored on. 14 | HOST = "extrahop.example.com" 15 | # The API key. 16 | API_KEY = "123456789abcdefghijklmnop" 17 | # The file that output is written to. 18 | FILENAME = "records.csv" 19 | # If the record query matches more than 100 records, the amount of time after the initial query that the remaining records can be retrieved from the system. 20 | TIME_LIMIT = "1m" 21 | # The record query parameters. 22 | QUERY = { 23 | "context_ttl": TIME_LIMIT, 24 | "from": "-30m", 25 | "filter": { 26 | "field": "ex.isSuspicious", 27 | "operator": "=", 28 | "operand": {"type": "boolean", "value": "true"}, 29 | }, 30 | } 31 | # The record fields that are written to the CSV output file. 32 | COLUMNS = [ 33 | "timestamp", 34 | "sender", 35 | "senderAddr", 36 | "senderPort", 37 | "receiver", 38 | "receiverAddr", 39 | "receiverPort", 40 | "age", 41 | "proto", 42 | "l7proto", 43 | "bytes", 44 | "pkts", 45 | "rto", 46 | "ex", 47 | ] 48 | 49 | 50 | def recordQuery(query): 51 | """ 52 | Method that performs an initial record query on an ExtraHop system. 53 | 54 | Parameters: 55 | query (dict): The record query parameters 56 | 57 | Returns: 58 | dict: The records that matched the query parameters 59 | """ 60 | url = urlunparse(("https", HOST, "/api/v1/records/search", "", "", "")) 61 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 62 | r = requests.post(url, headers=headers, data=json.dumps(query)) 63 | try: 64 | return json.loads(r.text) 65 | except: 66 | print("Record query failed") 67 | print(r.text) 68 | print(r.status_code) 69 | 70 | 71 | # Method that retrieves remaining records from a record query 72 | def continueQuery(cursor): 73 | """ 74 | Method that retrieves remaining records from a record query. 75 | 76 | Parameters: 77 | cursor (str): The unique identifier of the cursor that specifies the next page of results in the query 78 | 79 | Returns: 80 | dict: The records on this page of the query results 81 | """ 82 | url = urlunparse(("https", HOST, "/api/v1/records/cursor", "", "", "")) 83 | headers = {"Authorization": "ExtraHop apikey=%s" % API_KEY} 84 | query = {"cursor": cursor} 85 | r = requests.post(url, headers=headers, data=json.dumps(query)) 86 | try: 87 | return json.loads(r.text) 88 | except: 89 | print("Record query failed") 90 | print(r.text) 91 | print(r.status_code) 92 | 93 | 94 | # Query records 95 | response = recordQuery(QUERY) 96 | records = response["records"] 97 | if "cursor" in response: 98 | response_cursor = response["cursor"] 99 | retrieved = len(records) 100 | while retrieved > 0: 101 | print( 102 | "Retrieved " 103 | + str(len(records)) 104 | + " of " 105 | + str(response["total"]) 106 | + " total records" 107 | ) 108 | response = continueQuery(response_cursor) 109 | newRecords = response["records"] 110 | retrieved = len(newRecords) 111 | records = records + newRecords 112 | 113 | print("Total records retrieved = " + str(len(records))) 114 | 115 | # Simplify and format records for CSV 116 | table = [] 117 | for record in records: 118 | row = {} 119 | fields = record["_source"] 120 | for column in COLUMNS: 121 | try: 122 | value = fields[column] 123 | # Retrieve isSuspicious field from ex object 124 | if column == "ex": 125 | try: 126 | row["isSuspicious"] = value["isSuspicious"] 127 | except: 128 | row[column] = value 129 | # Concatenate values returned as lists 130 | elif type(value) is list: 131 | row[column] = " ".join(value) 132 | # Retrieve values from dict objects 133 | elif type(value) is dict: 134 | try: 135 | # If value is a list, concatenate list 136 | if type(value["value"]) is list: 137 | row[column] = " ".join(value["value"]) 138 | else: 139 | row[column] = value["value"] 140 | except: 141 | row[column] = value 142 | else: 143 | row[column] = value 144 | except: 145 | row[column] = "" 146 | table.append(row) 147 | 148 | 149 | # Write records to csv 150 | with open(FILENAME, "wb") as csvfile: 151 | csvwriter = csv.writer(csvfile, encoding="utf-8") 152 | csvwriter.writerow(list(table[0].keys())) 153 | for row in table: 154 | csvwriter.writerow(list(row.values())) 155 | -------------------------------------------------------------------------------- /query_records_third_party/query_records_third_party.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import base64 9 | import json 10 | import requests 11 | import unicodecsv as csv 12 | from urllib.parse import urlunparse 13 | 14 | # The IP address or hostname of the ExtraHop appliance or RevealX 360 API 15 | HOST = "extrahop.example.com" 16 | 17 | # For RevealX 360 authentication 18 | # The ID of the REST API credentials. 19 | ID = "abcdefg123456789" 20 | # The secret of the REST API credentials. 21 | SECRET = "123456789abcdefg987654321abcdefg" 22 | # A global variable for the temporary API access token (leave blank) 23 | TOKEN = "" 24 | 25 | # For self-managed appliance authentication 26 | # The API key. 27 | API_KEY = "123456789abcdefghijklmnop" 28 | 29 | # The file that output is written to. 30 | FILENAME = "records.csv" 31 | # The maximum number of records to retrieve at a time. 32 | LIMIT = 1000 33 | # The record query parameters. 34 | QUERY = { 35 | "from": 1586273860000, 36 | "until": 1586273860500, 37 | "limit": LIMIT, 38 | "filter": { 39 | "field": "ex.isSuspicious", 40 | "operator": "=", 41 | "operand": {"type": "boolean", "value": "true"}, 42 | }, 43 | "sort": [{"direction": "asc", "field": "ipaddr"}], 44 | } 45 | 46 | # The record fields that are written to the CSV output file. 47 | COLUMNS = [ 48 | "timestamp", 49 | "sender", 50 | "senderAddr", 51 | "senderPort", 52 | "receiver", 53 | "receiverAddr", 54 | "receiverPort", 55 | "age", 56 | "proto", 57 | "l7proto", 58 | "bytes", 59 | "pkts", 60 | "rto", 61 | "ex", 62 | ] 63 | 64 | def getToken(): 65 | """ 66 | Method that generates and retrieves a temporary API access token for RevealX 360 authentication. 67 | 68 | Returns: 69 | str: A temporary API access token 70 | """ 71 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 72 | headers = { 73 | "Authorization": "Basic " + auth, 74 | "Content-Type": "application/x-www-form-urlencoded", 75 | } 76 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 77 | r = requests.post( 78 | url, 79 | headers=headers, 80 | data="grant_type=client_credentials", 81 | ) 82 | try: 83 | return r.json()["access_token"] 84 | except: 85 | print(r.text) 86 | print(r.status_code) 87 | print("Error retrieving token from RevealX 360") 88 | sys.exit() 89 | 90 | def getAuthHeader(force_token_gen=False): 91 | """ 92 | Method that adds an authorization header for a request. For RevealX 360, adds a temporary access 93 | token. For self-managed appliances, adds an API key. 94 | 95 | Parameters: 96 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 97 | 98 | Returns: 99 | str: The value for the header key in the headers dictionary 100 | """ 101 | global TOKEN 102 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 103 | return f"ExtraHop apikey={API_KEY}" 104 | else: 105 | if TOKEN == "" or force_token_gen == True: 106 | TOKEN = getToken() 107 | return f"Bearer {TOKEN}" 108 | 109 | def recordQuery(query): 110 | """ 111 | Method that queries records from the ExtraHop system. 112 | 113 | Parameters: 114 | query (dict): The record query parameters 115 | 116 | Returns: 117 | dict: The records that matched the query parameters 118 | """ 119 | url = urlunparse(("https", HOST, "/api/v1/records/search", "", "", "")) 120 | headers = { 121 | "Authorization": getAuthHeader(), 122 | "Accept": "application/json", 123 | } 124 | r = requests.post(url, headers=headers, data=json.dumps(query)) 125 | try: 126 | return json.loads(r.text) 127 | except: 128 | print("Record query failed") 129 | print(r.text) 130 | print(r.status_code) 131 | 132 | 133 | # Query records 134 | response = recordQuery(QUERY) 135 | total = response["total"] 136 | records = response["records"] 137 | offset = LIMIT 138 | print("Retrieved " + str(len(records)) + " out of " + str(total) + " records") 139 | while total > offset: 140 | QUERY["offset"] = offset 141 | response = recordQuery(QUERY) 142 | new_records = response["records"] 143 | records = records + new_records 144 | offset = offset + LIMIT 145 | print( 146 | "Retrieved " + str(len(records)) + " out of " + str(total) + " records" 147 | ) 148 | 149 | # Simplify and format records for CSV 150 | table = [] 151 | for record in records: 152 | row = {} 153 | fields = record["_source"] 154 | for column in COLUMNS: 155 | try: 156 | value = fields[column] 157 | # Retrieve isSuspicious field from ex object 158 | if column == "ex": 159 | try: 160 | row["isSuspicious"] = value["isSuspicious"] 161 | except: 162 | row[column] = value 163 | # Concatenate values returned as lists 164 | elif type(value) is list: 165 | row[column] = " ".join(value) 166 | # Retrieve values from dict objects 167 | elif type(value) is dict: 168 | try: 169 | # If value is a list, concatenate list 170 | if type(value["value"]) is list: 171 | row[column] = " ".join(value["value"]) 172 | else: 173 | row[column] = value["value"] 174 | except: 175 | row[column] = value 176 | else: 177 | row[column] = value 178 | except: 179 | row[column] = "" 180 | table.append(row) 181 | 182 | # Write records to CSV file 183 | if len(table) > 0: 184 | with open(FILENAME, "wb") as csvfile: 185 | csvwriter = csv.writer(csvfile, encoding="utf-8") 186 | csvwriter.writerow(list(table[0].keys())) 187 | for row in table: 188 | csvwriter.writerow(list(row.values())) 189 | -------------------------------------------------------------------------------- /rollback_firmware/rollback_firmware.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import csv 10 | import logging 11 | import sys 12 | from urllib.parse import urlunparse 13 | 14 | SYSTEM_LIST = "systems.csv" 15 | 16 | 17 | def getRollbackVersion(system): 18 | """ 19 | Method that retrieves and displays which version of firmware is available for rollback. 20 | 21 | Parameters: 22 | system (dict): The object that contains the system host and api key 23 | 24 | Returns: 25 | bool: Indicates whether the firmware can be rolled back on the system 26 | """ 27 | headers = {"Authorization": f"ExtraHop apikey={system['api_key']}"} 28 | url = urlunparse( 29 | ( 30 | "https", 31 | system["host"], 32 | "/api/v1/extrahop/firmware/previous", 33 | "", 34 | "", 35 | "", 36 | ) 37 | ) 38 | 39 | r = requests.get(url, headers=headers) 40 | if r.status_code == 200: 41 | print( 42 | f"{system['host']} will be rolled back to version {r.json()['version']}" 43 | ) 44 | return True 45 | else: 46 | print( 47 | f"{system['host']} cannot be rolled back because {r.json()['error_message']}" 48 | ) 49 | return False 50 | 51 | 52 | def rollbackFirmware(system): 53 | """ 54 | Method that rolls back firmware on an ExtraHop system. 55 | 56 | Parameters: 57 | system (dict): The object that contains the system host and api key 58 | """ 59 | headers = {"Authorization": f"ExtraHop apikey={system['api_key']}"} 60 | url = urlunparse( 61 | ( 62 | "https", 63 | system["host"], 64 | "/api/v1/extrahop/firmware/previous/rollback", 65 | "", 66 | "", 67 | "", 68 | ) 69 | ) 70 | r = requests.post(url, headers=headers) 71 | if r.status_code == 202: 72 | print(f"Started rollback process on {system['host']}") 73 | else: 74 | print( 75 | f"Failed to rollback firmware on {system['host']} because {r.json()['error_message']}" 76 | ) 77 | 78 | 79 | def main(): 80 | # Retrieve URLs and API keys from CSV file 81 | systems = [] 82 | with open(SYSTEM_LIST, "rt", encoding="ascii") as f: 83 | reader = csv.reader(f) 84 | for row in reader: 85 | system = {"host": row[0], "api_key": row[1]} 86 | systems.append(system) 87 | 88 | # Show which firmware versions will be rolled back to 89 | for system in systems: 90 | system["ready"] = getRollbackVersion(system) 91 | c = input("Do you want to continue? (y/n)") 92 | 93 | # Roll back firmware 94 | if c == "y" or c == "yes": 95 | for system in systems: 96 | if system["ready"] == False: 97 | continue 98 | else: 99 | rollbackFirmware(system) 100 | 101 | 102 | if __name__ == "__main__": 103 | logging.basicConfig( 104 | format="%(message)s", 105 | handlers=[logging.StreamHandler(sys.stdout),], 106 | level=logging.INFO, 107 | ) 108 | main() 109 | -------------------------------------------------------------------------------- /rollback_firmware/systems.csv: -------------------------------------------------------------------------------- 1 | extrahop.example.com,123456789abcdefghijklmnop 2 | extrahop.example2.com,123456789abcdefghijklmnop 3 | -------------------------------------------------------------------------------- /search_device/search_device.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import requests 10 | from urllib.parse import urlunparse 11 | import base64 12 | import sys 13 | 14 | # The IP address or hostname of the ExtraHop appliance or Reveal(x) 360 API 15 | HOST = "extrahop.example.com" 16 | 17 | # For Reveal(x) 360 authentication 18 | # The ID of the REST API credentials. 19 | ID = "abcdefg123456789" 20 | # The secret of the REST API credentials. 21 | SECRET = "123456789abcdefg987654321abcdefg" 22 | # A global variable for the temporary API access token (leave blank) 23 | TOKEN = "" 24 | 25 | # For self-managed appliance authentication 26 | # The API key. 27 | API_KEY = "123456789abcdefghijklmnop" 28 | 29 | # An array of IP addresses. 30 | IP_ADDR_LIST = ["10.10.10.200", "10.10.10.201", "10.10.10.202", "10.10.10.203"] 31 | 32 | 33 | def getToken(): 34 | """ 35 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 36 | 37 | Returns: 38 | str: A temporary API access token 39 | """ 40 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 41 | headers = { 42 | "Authorization": "Basic " + auth, 43 | "Content-Type": "application/x-www-form-urlencoded", 44 | } 45 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 46 | r = requests.post( 47 | url, 48 | headers=headers, 49 | data="grant_type=client_credentials", 50 | ) 51 | try: 52 | return r.json()["access_token"] 53 | except: 54 | print(r.text) 55 | print(r.status_code) 56 | print("Error retrieveing token from Reveal(x) 360") 57 | sys.exit() 58 | 59 | 60 | def getAuthHeader(force_token_gen=False): 61 | """ 62 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 63 | token. For self-managed appliances, adds an API key. 64 | 65 | Parameters: 66 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 67 | 68 | Returns: 69 | header (str): The value for the header key in the headers dictionary 70 | """ 71 | global TOKEN 72 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 73 | return f"ExtraHop apikey={API_KEY}" 74 | else: 75 | if TOKEN == "" or force_token_gen == True: 76 | TOKEN = getToken() 77 | return f"Bearer {TOKEN}" 78 | 79 | 80 | def searchDevice(search): 81 | """ 82 | Method that searches the ExtraHop system for a device that 83 | matches the specified search criteria 84 | 85 | Parameters: 86 | search (dict): The device search criteria 87 | 88 | Returns: 89 | dict: The metadata of the device that matches the criteria 90 | """ 91 | url = urlunparse(("https", HOST, "/api/v1/devices/search", "", "", "")) 92 | headers = {"Authorization": getAuthHeader()} 93 | r = requests.post(url, headers=headers, data=json.dumps(search)) 94 | try: 95 | devices = r.json() 96 | if len(devices) == 0: 97 | print("No devices match criteria:") 98 | print(json.dumps(search, indent=4)) 99 | return None 100 | elif len(devices) == 1: 101 | return devices[0] 102 | else: 103 | print(f"Warning: More than one device matches criteria:") 104 | print(json.dumps(search, indent=4)) 105 | return devices[0] 106 | except: 107 | print("Search failed for device with criteria:") 108 | print(json.dumps(search, indent=4)) 109 | print(r.status_code) 110 | print(r.text) 111 | 112 | 113 | for ip in IP_ADDR_LIST: 114 | search_filter = { 115 | "filter": {"field": "ipaddr", "operand": ip, "operator": "="} 116 | } 117 | device = searchDevice(search_filter) 118 | if device: 119 | print(ip) 120 | print(" " + device["discovery_id"]) 121 | -------------------------------------------------------------------------------- /self-managed-sensor-rx360-connect/self-managed-sensor-rx360-connect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2020 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import csv 10 | import json 11 | from urllib.parse import urlunparse 12 | 13 | SENSORS_LIST = "sensors.csv" 14 | 15 | # Read sensor hostnames and connection tokens from CSV 16 | sensors = [] 17 | with open(SENSORS_LIST, "r") as f: 18 | reader = csv.reader(f) 19 | for row in reader: 20 | sensor = {"host": row[0], "api_key": row[1], "token": row[2]} 21 | sensors.append(sensor) 22 | 23 | # Function that connects a sensor to CCP 24 | def connectSensor(sensor): 25 | url = urlunparse( 26 | ("https", sensor["host"], "/api/v1/cloud/connect", "", "", "") 27 | ) 28 | headers = {"Authorization": "ExtraHop apikey=%s" % sensor["api_key"]} 29 | data = {"cloud_token": sensor["token"]} 30 | r = requests.post(url, headers=headers, data=json.dumps(data)) 31 | if r.status_code == 201: 32 | print("Successfully paired " + sensor["host"]) 33 | else: 34 | print("Error! Failed to pair " + sensor["host"]) 35 | print(r.text) 36 | 37 | 38 | for sensor in sensors: 39 | connectSensor(sensor) 40 | -------------------------------------------------------------------------------- /self-managed-sensor-rx360-connect/sensors.csv: -------------------------------------------------------------------------------- 1 | extrahop.example.com,123456789abcdefghijklmnop,561b85-e9092a3a-343fcb03-78c72777-8db70bbd 2 | -------------------------------------------------------------------------------- /specify_custom_make_model/custom_config.csv: -------------------------------------------------------------------------------- 1 | ipaddr,custom_make,custom_model 2 | 10.10.10.200,Example Make,Example Model 3 | 10.10.10.201,Example Make,Example Model 4 | -------------------------------------------------------------------------------- /specify_custom_make_model/specify_custom_make_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import csv 10 | import sys 11 | from urllib.parse import urlunparse 12 | import base64 13 | import sys 14 | 15 | # The IP address or hostname of the ExtraHop appliance or Reveal(x) 360 API 16 | HOST = "extrahop.example.com" 17 | 18 | # For Reveal(x) 360 authentication 19 | # The ID of the REST API credentials. 20 | ID = "abcdefg123456789" 21 | # The secret of the REST API credentials. 22 | SECRET = "123456789abcdefg987654321abcdefg" 23 | # A global variable for the temporary API access token (leave blank) 24 | TOKEN = "" 25 | 26 | # For self-managed appliance authentication 27 | # The API key. 28 | API_KEY = "123456789abcdefghijklmnop" 29 | 30 | # The file that contains the list of IP addresses. 31 | CUSTOM_CONFIG = "custom_config.csv" 32 | 33 | 34 | def getToken(): 35 | """ 36 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 37 | 38 | Returns: 39 | str: A temporary API access token 40 | """ 41 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 42 | headers = { 43 | "Authorization": "Basic " + auth, 44 | "Content-Type": "application/x-www-form-urlencoded", 45 | } 46 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 47 | r = requests.post( 48 | url, 49 | headers=headers, 50 | data="grant_type=client_credentials", 51 | ) 52 | try: 53 | return r.json()["access_token"] 54 | except: 55 | print(r.text) 56 | print(r.status_code) 57 | print("Error retrieveing token from Reveal(x) 360") 58 | sys.exit() 59 | 60 | 61 | def getAuthHeader(force_token_gen=False): 62 | """ 63 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 64 | token. For self-managed appliances, adds an API key. 65 | 66 | Parameters: 67 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 68 | 69 | Returns: 70 | header (str): The value for the header key in the headers dictionary 71 | """ 72 | global TOKEN 73 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 74 | return f"ExtraHop apikey={API_KEY}" 75 | else: 76 | if TOKEN == "" or force_token_gen == True: 77 | TOKEN = getToken() 78 | return f"Bearer {TOKEN}" 79 | 80 | 81 | def readCSV(filepath): 82 | """ 83 | Method that reads a table of IP addresses, models, and makes from a CSV file. 84 | The first row is discarded. 85 | 86 | Parameters: 87 | filepath (str): The path of the CSV file 88 | 89 | Returns: 90 | devices (list): A list of dictionaries containing the custom make and model for each IP address 91 | """ 92 | devices = [] 93 | with open(filepath, "rt", encoding="ascii") as f: 94 | reader = csv.reader(f) 95 | next(reader) 96 | for row in reader: 97 | device = { 98 | "ipaddr": row[0], 99 | "custom_make": row[1], 100 | "custom_model": row[2], 101 | } 102 | devices.append(device) 103 | return devices 104 | 105 | 106 | def getDevicesByIp(ip): 107 | """ 108 | Method that retrieves the devices associated with a specified IP address 109 | 110 | Parameters: 111 | ip (str): The IP address 112 | 113 | Returns: 114 | devices (list): The list of device dictionaries 115 | """ 116 | url = urlunparse( 117 | ( 118 | "https", 119 | HOST, 120 | f"/api/v1/devices/search", 121 | "", 122 | "", 123 | "", 124 | ) 125 | ) 126 | headers = {"Authorization": getAuthHeader()} 127 | data = { 128 | "filter": { 129 | "field": "ipaddr", 130 | "operand": ip, 131 | "operator": "=" 132 | } 133 | } 134 | r = requests.post(url, headers=headers, json=data) 135 | if r.status_code == 200: 136 | devices = [] 137 | for device in r.json(): 138 | devices.append(device) 139 | return devices 140 | else: 141 | print("Unable to retrieve devices") 142 | print(r.status_code) 143 | print(r.text) 144 | sys.exit() 145 | 146 | 147 | def specifyMakeModel(device): 148 | """ 149 | Method that specifies a custom make and model for a device. 150 | 151 | Parameters: 152 | device (dict): The device dictionary 153 | """ 154 | headers = {"Authorization": getAuthHeader()} 155 | data = { 156 | "custom_make": device["custom_make"], 157 | "custom_model": device["custom_model"], 158 | } 159 | for dev_id in device["ids"]: 160 | url = urlunparse( 161 | ( 162 | "https", 163 | HOST, 164 | f"/api/v1/devices/{dev_id}", 165 | "", 166 | "", 167 | "", 168 | ) 169 | ) 170 | r = requests.patch(url, headers=headers, json=data) 171 | if r.status_code == 204: 172 | print(f"Successfully updated device {device['display_name']}") 173 | print(f" Custom Make: {device['custom_make']}") 174 | print(f" Custom Model: {device['custom_model']}") 175 | else: 176 | print( 177 | f"Failed to update custom make and model for {device['display_name']}" 178 | ) 179 | print(r.status_code) 180 | print(r.text) 181 | 182 | 183 | def main(): 184 | devices = readCSV(CUSTOM_CONFIG) 185 | # Retrieve IDs of devices with the specified IPs 186 | for device in devices: 187 | device_specs = getDevicesByIp(device["ipaddr"]) 188 | ids = [] 189 | for spec in device_specs: 190 | # Only add IDs for devices that do not already have the specified custom make and model 191 | if ( 192 | device["custom_make"] != spec["custom_make"] 193 | and device["custom_model"] != spec["custom_model"] 194 | ): 195 | ids.append(spec["id"]) 196 | device["ids"] = ids 197 | device["display_name"] = spec["display_name"] 198 | for device in devices: 199 | if device["ids"]: 200 | specifyMakeModel(device) 201 | else: 202 | print( 203 | f"Skipping {device['display_name']} because the device has already been assigned the specified custom make and model: {device['custom_make']} {device['custom_model']}" 204 | ) 205 | 206 | 207 | if __name__ == "__main__": 208 | main() 209 | -------------------------------------------------------------------------------- /specify_high_value/ip_list.csv: -------------------------------------------------------------------------------- 1 | 10.10.10.200,10.10.10.201,10.10.10.202,10.10.10.203 2 | -------------------------------------------------------------------------------- /specify_high_value/specify_high_value.py: -------------------------------------------------------------------------------- 1 | # This file is subject to the terms and conditions defined in 2 | # file 'LICENSE', which is part of this source code package. 3 | 4 | import requests 5 | import csv 6 | import sys 7 | from urllib.parse import urlunparse 8 | import base64 9 | import sys 10 | 11 | # The IP address or hostname of the ExtraHop appliance or Reveal(x) 360 API 12 | HOST = "extrahop.example.com" 13 | 14 | # For Reveal(x) 360 authentication 15 | # The ID of the REST API credentials. 16 | ID = "abcdefg123456789" 17 | # The secret of the REST API credentials. 18 | SECRET = "123456789abcdefg987654321abcdefg" 19 | # A global variable for the temporary API access token (leave blank) 20 | TOKEN = "" 21 | 22 | # For self-managed appliance authentication 23 | # The API key. 24 | API_KEY = "123456789abcdefghijklmnop" 25 | 26 | # The file that contains the list of IP addresses. 27 | IP_LIST = "ip_list.csv" 28 | 29 | 30 | def getToken(): 31 | """ 32 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 33 | 34 | Returns: 35 | str: A temporary API access token 36 | """ 37 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 38 | headers = { 39 | "Authorization": "Basic " + auth, 40 | "Content-Type": "application/x-www-form-urlencoded", 41 | } 42 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 43 | r = requests.post( 44 | url, 45 | headers=headers, 46 | data="grant_type=client_credentials", 47 | ) 48 | try: 49 | return r.json()["access_token"] 50 | except: 51 | print(r.text) 52 | print(r.status_code) 53 | print("Error retrieveing token from Reveal(x) 360") 54 | sys.exit() 55 | 56 | 57 | def getAuthHeader(force_token_gen=False): 58 | """ 59 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 60 | token. For self-managed appliances, adds an API key. 61 | 62 | Parameters: 63 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 64 | 65 | Returns: 66 | header (str): The value for the header key in the headers dictionary 67 | """ 68 | global TOKEN 69 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 70 | return f"ExtraHop apikey={API_KEY}" 71 | else: 72 | if TOKEN == "" or force_token_gen == True: 73 | TOKEN = getToken() 74 | return f"Bearer {TOKEN}" 75 | 76 | 77 | def readCSV(filepath): 78 | """ 79 | Method that reads a list of values from a CSV file 80 | 81 | Parameters: 82 | filepath (str): The path of the CSV file 83 | 84 | Returns: 85 | values (list): The list of values 86 | """ 87 | values = [] 88 | with open(IP_LIST, "rt", encoding="ascii") as f: 89 | reader = csv.reader(f) 90 | for row in reader: 91 | for item in row: 92 | values.append(item) 93 | return values 94 | 95 | 96 | def getDevicesByIp(ip): 97 | """ 98 | Method that retrieves the devices with a specified IP address 99 | 100 | Parameters: 101 | ip (str): The IP address 102 | 103 | Returns: 104 | devices (list): The device objects 105 | """ 106 | url = urlunparse( 107 | ( 108 | "https", 109 | HOST, 110 | f"/api/v1/devices/search", 111 | "", 112 | "", 113 | "", 114 | ) 115 | ) 116 | headers = {"Authorization": getAuthHeader()} 117 | data = { 118 | "filter": { 119 | "field": "ipaddr", 120 | "operand": ip, 121 | "operator": "=" 122 | } 123 | } 124 | r = requests.post(url, headers=headers, json=data) 125 | if r.status_code == 200: 126 | devices = [] 127 | for device in r.json(): 128 | devices.append(device) 129 | return devices 130 | else: 131 | print("Unable to retrieve tags") 132 | print(r.status_code) 133 | print(r.text) 134 | sys.exit() 135 | 136 | 137 | def idHighValue(device): 138 | """ 139 | Method that specifies a device as high value. 140 | 141 | Parameters: 142 | device (dict): The device dictionary 143 | """ 144 | url = urlunparse( 145 | ( 146 | "https", 147 | HOST, 148 | f"/api/v1/devices/{device['id']}", 149 | "", 150 | "", 151 | "", 152 | ) 153 | ) 154 | headers = {"Authorization": getAuthHeader()} 155 | data = {"custom_criticality": "critical"} 156 | r = requests.patch(url, headers=headers, json=data) 157 | if r.status_code == 204: 158 | print(f"Successfully specified {device['display_name']} as high value") 159 | else: 160 | print(f"Failed to specify {device['display_name']} as high value") 161 | print(r.status_code) 162 | print(r.text) 163 | 164 | 165 | def main(): 166 | device_ips = readCSV(IP_LIST) 167 | devices = [] 168 | # Retrieve IDs of devices with the specified IPs 169 | for ip in device_ips: 170 | devices += getDevicesByIp(ip) 171 | for device in devices: 172 | if device["custom_criticality"] != "critical": 173 | idHighValue(device) 174 | else: 175 | print( 176 | f"Skipping {device['display_name']} because the device has already been specified as high value" 177 | ) 178 | 179 | 180 | if __name__ == "__main__": 181 | main() 182 | -------------------------------------------------------------------------------- /sunburst/.gitignore: -------------------------------------------------------------------------------- 1 | output.csv 2 | -------------------------------------------------------------------------------- /sunburst/README.md: -------------------------------------------------------------------------------- 1 | # SUNBURST 2 | 3 | ## sunburst_detect.py 4 | 5 | This Python script searches the ExtraHop system for the following metrics. 6 | * All DNS queries that reference the Command and Control domains associated with the SUNBURST backdoor attack, such as avsvmcloud[.]com. 7 | * Every time that a device on your network contacted an IP address associated with the SUNBURST backdoor attack. 8 | 9 | You must run the script with Python version 3.6 or later. 10 | 11 | After you have downloaded the script and the threats.json file, run one of the following commands from the directory that you saved the script to. 12 | 13 | Windows 14 | 15 | ``` 16 | py .\sunburst_detect.py -t HOST -a API_KEY 17 | ``` 18 | 19 | Linux and Mac OS X 20 | 21 | ``` 22 | python3 ./sunburst_detect.py -t HOST -a API_KEY 23 | ``` 24 | 25 | *Note*: In the above command, replace the following configuration variables with information from your environment: 26 | 27 | * *HOST*: The hostname of your ExtraHop system 28 | * *API_KEY*: Your API key. If you do not have an API key, see [Generate an API key](https://docs.extrahop.com/current/rest-api-guide/#generate-an-api-key). 29 | 30 | By default the script searches from 2020-07-31 to the current date. You can specify a different time period with the --from-time and --until-time parameters. For example, values of --from-time 2020-11-01 --until-time 2020-12-01 searches from November 1st to December 1st. 31 | 32 | The command displays output similar to the following text: 33 | 34 | ``` 35 | Getting all active devices between 2020-07-31 00:00:00 - 2020-12-18 16:46:32 36 | Requesting 0 37 | Requesting 1000 38 | ... 39 | Querying against 6402 devices 40 | Fetching application host metrics. 41 | Processing 3055 stats 42 | Getting device host metrics: 2020-12-17 16:46:32 - 2020-12-18 16:46:32 43 | getting metrics for 1-200 of 6402 devices 44 | getting metrics for 201-400 of 6402 devices 45 | ... 46 | ``` 47 | 48 | First, the script retrieves the list of all devices in blocks of 1000. Then the script searches the devices in blocks of 200, one day at a time. 49 | 50 | After the command completes, if there are any DNS or IP address matches, the command displays output similar to the following text: 51 | 52 | ``` 53 | Found Sunburst host indicators in application metrics (see output.csv). 54 | ``` 55 | 56 | The output file (output.csv) contains details about DNS or IP address matches in the following CSV format: 57 | 58 | ``` 59 | time,object_type,object_id,name,ipaddr,macaddr,indicator,count,uri 60 | 2020-12-17 17:45:00,device,1510,exampledomain.com,10.4.1.6,00:12:34:56:78:90,www.avsvmcloud.com,2, 61 | ``` 62 | 63 | For more information, see [How to Hunt for, Detect, and Respond to SUNBURST](https://www.extrahop.com/company/blog/2020/detect-and-respond-to-sunburst/) and 64 | [Analyzing the SolarWinds Orion SUNBURST Attack Campaign For Threat Intelligence](https://www.extrahop.com/company/blog/2020/analyzing-sunburst/). 65 | 66 | ## threats.json 67 | 68 | This file contains a list of suspicious IP addresses associated with the SUNBURST backdoor attack. 69 | -------------------------------------------------------------------------------- /tag_device/ip_list.csv: -------------------------------------------------------------------------------- 1 | 10.10.10.200,10.10.10.201,10.10.10.202,10.10.10.203 2 | -------------------------------------------------------------------------------- /tag_device/tag_device.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import json 10 | import csv 11 | import sys 12 | from urllib.parse import urlunparse 13 | import base64 14 | 15 | # The IP address or hostname of the ExtraHop appliance or Reveal(x) 360 API 16 | HOST = "extrahop.example.com" 17 | 18 | # For Reveal(x) 360 authentication 19 | # The ID of the REST API credentials. 20 | ID = "abcdefg123456789" 21 | # The secret of the REST API credentials. 22 | SECRET = "123456789abcdefg987654321abcdefg" 23 | # A global variable for the temporary API access token (leave blank) 24 | TOKEN = "" 25 | 26 | # For self-managed appliance authentication 27 | # The API key. 28 | API_KEY = "123456789abcdefghijklmnop" 29 | 30 | # The name of the tag. 31 | TAG = "new-tag" 32 | # The file that contains the list of IP addresses. 33 | IP_LIST = "ip_list.csv" 34 | 35 | 36 | def getToken(): 37 | """ 38 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 39 | 40 | Returns: 41 | str: A temporary API access token 42 | """ 43 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 44 | headers = { 45 | "Authorization": "Basic " + auth, 46 | "Content-Type": "application/x-www-form-urlencoded", 47 | } 48 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 49 | r = requests.post( 50 | url, 51 | headers=headers, 52 | data="grant_type=client_credentials", 53 | ) 54 | try: 55 | return r.json()["access_token"] 56 | except: 57 | print(r.text) 58 | print(r.status_code) 59 | print("Error retrieveing token from Reveal(x) 360") 60 | sys.exit() 61 | 62 | 63 | def getAuthHeader(force_token_gen=False): 64 | """ 65 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 66 | token. For self-managed appliances, adds an API key. 67 | 68 | Parameters: 69 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 70 | 71 | Returns: 72 | header (str): The value for the header key in the headers dictionary 73 | """ 74 | global TOKEN 75 | if API_KEY != "123456789abcdefghijklmnop" and API_KEY != "": 76 | return f"ExtraHop apikey={API_KEY}" 77 | else: 78 | if TOKEN == "" or force_token_gen == True: 79 | TOKEN = getToken() 80 | return f"Bearer {TOKEN}" 81 | 82 | 83 | def readCSV(filepath): 84 | """ 85 | Method that reads a list of values from a CSV file 86 | 87 | Parameters: 88 | filepath (str): The path of the CSV file 89 | 90 | Returns: 91 | values (list): The list of values 92 | """ 93 | values = [] 94 | with open(IP_LIST, "rt", encoding="ascii") as f: 95 | reader = csv.reader(f) 96 | for row in reader: 97 | for item in row: 98 | values.append(item) 99 | return values 100 | 101 | 102 | def getTagId(tag_name): 103 | """ 104 | Method that retrieves the ID of a device tag. 105 | 106 | Parameters: 107 | tag (str): The name of the device tag 108 | 109 | Returns: 110 | tag_id (str): The ID of the device tag 111 | """ 112 | url = urlunparse(("https", HOST, "/api/v1/tags", "", "", "")) 113 | headers = {"Authorization": getAuthHeader()} 114 | r = requests.get(url, headers=headers) 115 | if r.status_code == 200: 116 | for tag in r.json(): 117 | if tag["name"] == tag_name: 118 | return tag["id"] 119 | else: 120 | print("Unable to retrieve tags") 121 | print(r.status_code) 122 | print(r.text) 123 | sys.exit() 124 | 125 | 126 | def getDevicesByIp(ip): 127 | """ 128 | Method that retrieves the devices with a specified IP address 129 | 130 | Parameters: 131 | ip (str): The IP address 132 | 133 | Returns: 134 | devices (list): The device objects 135 | """ 136 | url = urlunparse( 137 | ( 138 | "https", 139 | HOST, 140 | f"/api/v1/devices/search", 141 | "", 142 | "", 143 | "", 144 | ) 145 | ) 146 | headers = {"Authorization": getAuthHeader()} 147 | data = { 148 | "filter": { 149 | "field": "ipaddr", 150 | "operand": ip, 151 | "operator": "=" 152 | } 153 | } 154 | r = requests.post(url, headers=headers, json=data) 155 | if r.status_code == 200: 156 | devices = [] 157 | for device in r.json(): 158 | devices.append(device) 159 | return devices 160 | else: 161 | print("Unable to retrieve tags") 162 | print(r.status_code) 163 | print(r.text) 164 | sys.exit() 165 | 166 | 167 | # Add the tag to each device 168 | def assignTag(tag_id, tag_name, devices): 169 | """ 170 | Method that assigns a tag to a list of devices 171 | 172 | Parameters: 173 | tag_id (int): The ID of the tag 174 | tag_name (str): The name of the tag 175 | devices (list): The list of device dictionaries 176 | """ 177 | ids = [device["id"] for device in devices] 178 | data = {"assign": ids} 179 | url = urlunparse( 180 | ("https", HOST, f"/api/v1/tags/{tag_id}/devices", "", "", "") 181 | ) 182 | headers = {"Authorization": getAuthHeader()} 183 | r = requests.post(url, headers=headers, json=data) 184 | if r.status_code == 204: 185 | print(f"Assigned {tag_name} tag to the following devices:") 186 | for device in devices: 187 | print(f' {device["display_name"]}') 188 | elif r.status_code == 207: 189 | print("Assigned {tag_name} tag to a limited number of devices.") 190 | print(json.dumps(r.json(), indent=4)) 191 | else: 192 | print(f"Failed to assign tag: {tag_name}") 193 | print(r.status_code) 194 | print(r.text) 195 | sys.exit() 196 | 197 | 198 | def createTag(tag): 199 | """ 200 | Method that creates a tag on the ExtraHop system. 201 | 202 | Parameters: 203 | tag (str): The name of the tag 204 | """ 205 | url = urlunparse(("https", HOST, "/api/v1/tags", "", "", "")) 206 | headers = {"Authorization": getAuthHeader()} 207 | data = {"name": TAG} 208 | r = requests.post(url, headers=headers, json=data) 209 | if r.status_code == 201: 210 | print(f"Created tag {tag}") 211 | else: 212 | print("Failed to create tag") 213 | print(r.status_code) 214 | print(r.text) 215 | sys.exit() 216 | 217 | 218 | def main(): 219 | # If the tag does not already exist, create it 220 | tag_id = getTagId(TAG) 221 | if tag_id: 222 | print(f"{TAG} already exists") 223 | else: 224 | createTag(TAG) 225 | tag_id = getTagId(TAG) 226 | device_ips = readCSV(IP_LIST) 227 | devices = [] 228 | # Retrieve IDs of devices with the specified IPs 229 | for ip in device_ips: 230 | devices += getDevicesByIp(ip) 231 | assignTag(tag_id, TAG, devices) 232 | 233 | 234 | if __name__ == "__main__": 235 | main() 236 | -------------------------------------------------------------------------------- /update_network_localities/create_network_localities.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.8 2 | 3 | # COPYRIGHT 2022 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | from typing import Dict, List, Union 8 | 9 | import argparse 10 | import csv 11 | import requests 12 | import sys 13 | import util 14 | 15 | from ipaddress import ip_network 16 | from urllib.parse import urlunparse 17 | 18 | 19 | def validate_and_transform_entry(entry: dict) -> dict: 20 | """ 21 | Function that transforms data from the CSV file into a format that can be 22 | sent to the REST API. 23 | 24 | Parameters: 25 | entry (dict): The original data from the CSV row 26 | 27 | Returns: 28 | cleaned_entry (dict): The transformed data 29 | """ 30 | cleaned_entry = {} 31 | cleaned_entry["name"] = entry.get("name", "") 32 | 33 | # Networks 34 | networks = entry.get("networks") 35 | try: 36 | nets = [ip_network(n, strict=False) for n in networks.split("|")] 37 | except Exception: 38 | raise Exception( 39 | f"Failed to convert network list: {networks}. Check that they are " 40 | "formatted as one or more valid IP addresses delimited by pipes" 41 | ) 42 | cleaned_entry["networks"] = [str(n) for n in nets] 43 | 44 | external = entry.get("external", "") 45 | # The string "true" (case insensitive) is interpreted as true. All other 46 | # strings are interpreted as false. 47 | cleaned_entry["external"] = True if external.lower() == "true" else False 48 | 49 | cleaned_entry["description"] = entry.get("description", "") 50 | 51 | return cleaned_entry 52 | 53 | 54 | parser = argparse.ArgumentParser(description="Create Network Locality Entries") 55 | parser.add_argument( 56 | "host", 57 | type=str, 58 | help="The hostname of the console, sensor, or Reveal(x) 360 API.", 59 | ) 60 | parser.add_argument( 61 | "--input", 62 | "-i", 63 | type=str, 64 | default="localities.csv", 65 | help="The path of the CSV file.", 66 | ) 67 | parser.add_argument( 68 | "--group", 69 | "-g", 70 | type=str, 71 | choices=["description", "external"], 72 | default=None, 73 | help="Consolidates network localities based on the specified field. If two " 74 | "or more localities have the same value for the field in the CSV file, " 75 | "those localities are consolidated into a single locality on the target " 76 | "console, sensor, or Reveal(x) 360.", 77 | ) 78 | util.add_api_args(parser) 79 | args = parser.parse_args() 80 | 81 | url = urlunparse(("https", args.host, "/api/v1/networklocalities", "", "", "")) 82 | headers = { 83 | "Content-Type": "application/json", 84 | "Authorization": util.get_auth_header(args), 85 | } 86 | 87 | try: 88 | # Delete all network locality entries on the sensor or console 89 | rsp = requests.get(url, headers=headers) 90 | entries = rsp.json() 91 | for entry in entries: 92 | id = entry["id"] 93 | rsp = requests.delete(f"{url}/{id}", headers=headers) 94 | if entries: 95 | print( 96 | "Preparing for locality entry upload by deleting existing entries" 97 | ) 98 | 99 | # Transform data from the CSV file into a format that can be sent to the 100 | # REST API 101 | valid_entries: List[Dict] = [] 102 | with open(args.input, "r") as file: 103 | csv_file = csv.DictReader(file) 104 | 105 | for row in csv_file: 106 | try: 107 | cleaned_entry = validate_and_transform_entry(row) 108 | except Exception as e: 109 | print(e) 110 | sys.exit() 111 | valid_entries.append(cleaned_entry) 112 | 113 | # Combine entries by the specified group parameter 114 | if args.group: 115 | print(f"Grouping entries by '{args.group}' field") 116 | 117 | grouped: Dict[Union[str, bool], Dict] = dict() 118 | for entry in valid_entries: 119 | key = entry[args.group] 120 | if key in grouped: 121 | grouped[key]["networks"].extend(entry["networks"]) 122 | else: 123 | grouped[key] = entry 124 | if (name := entry.get("name")) and not grouped[key]["name"]: 125 | grouped[key]["name"] = name 126 | valid_entries = list(grouped.values()) 127 | 128 | # Create new network localities on the sensor or console 129 | for entry in valid_entries: 130 | rsp = requests.post(url, headers=headers, json=entry) 131 | if rsp.ok: 132 | print(f"Successfully uploaded entry {entry['name']}") 133 | else: 134 | print(f"Failed to upload entry {entry['name']}") 135 | print(f"{rsp.status_code}: {rsp.text}") 136 | sys.exit() 137 | 138 | except KeyboardInterrupt: 139 | sys.exit() 140 | -------------------------------------------------------------------------------- /update_network_localities/example_input.csv: -------------------------------------------------------------------------------- 1 | networks,external,description,name 2 | 2001:0db8:85a3:0000:0000:8a2e:0370:7334,True,,External Ipv6 3 | 192.168.1.1|192.68.1.2|192.168.1.3,False,group1,Group 2 4 | 123.45.6.5|123.45.6.6,False,group2,[auto]: Internal - 123.45.6.5 5 | 1.2.3.4,False,,[auto]: Internal - 1.2.3.4 6 | 192.168.1.12/24,False,group1,Servers 7 | -------------------------------------------------------------------------------- /update_network_localities/example_output.csv: -------------------------------------------------------------------------------- 1 | networks,external,description,name 2 | 2001:0db8:85a3:0000:0000:8a2e:0370:7334,True,,[auto]: External - 2001:0db8:85a3:0000:0000:8a2e:0370:7334 3 | 192.168.1.2,False,group1,[auto]: Internal - 192.168.1.2 4 | 192.168.1.1,False,group1,[auto]: Internal - 192.168.1.1 5 | 123.45.6.5,False,group2,[auto]: Internal - 123.45.6.5 6 | 1.2.3.4,False,,[auto]: Internal - 1.2.3.4 7 | 123.45.6.6,True,group2,[auto]: External - 123.45.6.6 8 | -------------------------------------------------------------------------------- /update_network_localities/retrieve_network_localities.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.8 2 | 3 | # COPYRIGHT 2022 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | import argparse 8 | import csv 9 | import requests 10 | import util 11 | 12 | from urllib.parse import urlunparse 13 | 14 | parser = argparse.ArgumentParser(description="Retrieve Network Locality Entries") 15 | parser.add_argument("host", type=str, help="The hostname of the console, sensor, or Reveal(x) 360 API.") 16 | parser.add_argument( 17 | "--output", 18 | "-o", 19 | type=str, 20 | default="localities.csv", 21 | help="The filename of the CSV file that output is saved to.", 22 | ) 23 | util.add_api_args(parser) 24 | args = parser.parse_args() 25 | 26 | # Retrieve network localities 27 | url = urlunparse(("https", args.host, "/api/v1/networklocalities", "", "", "")) 28 | headers = { 29 | "Accept": "application/json", 30 | "Authorization": util.get_auth_header(args) 31 | } 32 | 33 | rsp = requests.get(url, headers=headers) 34 | localities = rsp.json() 35 | 36 | 37 | def get_name(locality): 38 | """ 39 | After 9.0, a unique name is required for network locality names. 40 | 41 | On 9.0+ appliances, use the network locality name configured on the 42 | appliance. 43 | 44 | Otherwise, automatically generate a name for the network locality entry. 45 | """ 46 | if (name := locality.get("name")) is not None: 47 | return name 48 | 49 | external_str = "External" if locality["external"] else "Internal" 50 | auto_name = f"[auto]: {external_str} - {locality['network']}" 51 | # If the generated name is longer than 40 characters, truncate the name. 52 | # Locality names cannot be longer than 40 characters. 53 | if len(auto_name) > 40: 54 | trunc_name = auto_name[:40] 55 | print( 56 | f"Warning: {auto_name} is greater than 40 characters. The " 57 | f"name is truncated to {trunc_name} in the CSV output file." 58 | ) 59 | auto_name = trunc_name 60 | return auto_name 61 | 62 | 63 | def get_networks(locality): 64 | """ 65 | After 9.0, network locality entries can contain more than one network. 66 | """ 67 | 68 | if (networks := locality.get("networks")) is not None: 69 | return ("|").join(networks) 70 | 71 | return locality["network"] 72 | 73 | 74 | # Write network localities to CSV file 75 | csv_headers = ("networks", "external", "description", "name") 76 | with open(args.output, "w") as output: 77 | csv_writer = csv.writer(output) 78 | csv_writer.writerow(csv_headers) 79 | 80 | for locality in localities: 81 | networks = get_networks(locality) 82 | row = [networks] 83 | 84 | row.append(locality.get("external")) 85 | row.append(locality.get("description")) 86 | 87 | name = get_name(locality) 88 | row.append(name) 89 | 90 | csv_writer.writerow(row) 91 | 92 | print("Successfully downloaded network localities.") 93 | -------------------------------------------------------------------------------- /update_network_localities/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2022 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | import requests 8 | from urllib.parse import urlunparse 9 | import base64 10 | import sys 11 | 12 | 13 | def add_api_args(parser): 14 | parser.add_argument( 15 | "--apikey", 16 | type=str, 17 | default="", 18 | help="The REST API key for the console or sensor.", 19 | ) 20 | parser.add_argument( 21 | "--id", 22 | type=str, 23 | default="", 24 | help="The ID of the Reveal(x) 360 REST API credentials.", 25 | ) 26 | parser.add_argument( 27 | "--secret", 28 | type=str, 29 | default="", 30 | help="The secret of the Reveal(x) 360 REST API credentials.", 31 | ) 32 | 33 | 34 | def get_token(args): 35 | """ 36 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 37 | 38 | Returns: 39 | str: A temporary API access token 40 | """ 41 | auth = base64.b64encode(bytes(args.id + ":" + args.secret, "utf-8")).decode( 42 | "utf-8" 43 | ) 44 | headers = { 45 | "Authorization": "Basic " + auth, 46 | "Content-Type": "application/x-www-form-urlencoded", 47 | } 48 | url = urlunparse(("https", args.host, "/oauth2/token", "", "", "")) 49 | r = requests.post( 50 | url, 51 | headers=headers, 52 | data="grant_type=client_credentials", 53 | ) 54 | try: 55 | return r.json()["access_token"] 56 | except: 57 | print(r.text) 58 | print(r.status_code) 59 | print("Error retrieving token from Reveal(x) 360") 60 | sys.exit() 61 | 62 | 63 | def get_auth_header(args): 64 | """ 65 | Method that adds an authorization header for a request. For Reveal(x) 360, adds a temporary access 66 | token. For self-managed appliances, adds an API key. 67 | 68 | Parameters: 69 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 70 | 71 | Returns: 72 | header (str): The value for the header key in the headers dictionary 73 | """ 74 | if args.apikey: 75 | return f"ExtraHop apikey={args.apikey}" 76 | else: 77 | token = get_token(args) 78 | return f"Bearer {token}" 79 | -------------------------------------------------------------------------------- /upgrade_system/systems.csv: -------------------------------------------------------------------------------- 1 | extrahop.example.com,123456789abcdefghijklmnop,eda-complete.tar 2 | extrahop.example2.com,123456789abcdefghijklmnop,eca-complete.tar 3 | -------------------------------------------------------------------------------- /upgrade_system/upgrade_system.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import os 9 | import requests 10 | import csv 11 | from urllib.parse import urlunparse 12 | import threading 13 | import time 14 | from packaging import version 15 | 16 | SYSTEM_LIST = "systems.csv" 17 | # The maximum number of times to retry uploading the firmware 18 | MAX_RETRIES = 5 19 | # The maximum number of concurrent threads 20 | MAX_THREADS = 2 21 | # The number of minutes to wait between checking upgrade progress 22 | # This variable is valid only for upgrades from firmware 8.5 and later 23 | WAIT = 0.5 24 | 25 | # Retrieve URLs, API keys, and firmware file paths 26 | systems = [] 27 | with open(SYSTEM_LIST, "rt", encoding="ascii") as f: 28 | reader = csv.reader(f) 29 | for row in reader: 30 | system = {"host": row[0], "api_key": row[1], "firmware": row[2]} 31 | systems.append(system) 32 | 33 | 34 | class Upgrader(threading.Thread): 35 | """ 36 | A class for a thread that upgrades an appliance. 37 | 38 | Attributes 39 | ---------- 40 | host : str 41 | The IP address or hostname of the ExtraHop system 42 | api_key : str 43 | An API key on the ExtraHop system 44 | firmware : str 45 | The path of the firmware .tar file 46 | """ 47 | 48 | def __init__(self, host, api_key, firmware): 49 | threading.Thread.__init__(self) 50 | self.host = host 51 | self.api_key = api_key 52 | self.firmware = firmware 53 | 54 | def run(self): 55 | print(f"Starting upgrade thread for {self.host}") 56 | upload_success = uploadFirmware( 57 | self.host, self.api_key, self.firmware, 0 58 | ) 59 | if upload_success: 60 | job_location = upgradeFirmware(self.host, self.api_key) 61 | if job_location: 62 | monitorStatus(self.host, self.api_key, job_location) 63 | 64 | 65 | def uploadFirmware(host, api_key, firmware, retry): 66 | """ 67 | Method that uploads firmware to an ExtraHop system. 68 | 69 | Parameters: 70 | host (str): The IP address or hostname of the ExtraHop system 71 | api_key (str): An API key on the ExtraHop system 72 | firmware (str): The path of the firmware .tar file 73 | 74 | Returns: 75 | bool: Indicates whether the upload was successful 76 | """ 77 | headers = { 78 | "Authorization": "ExtraHop apikey=%s" % api_key, 79 | "Content-Type": "application/vnd.extrahop.firmware", 80 | } 81 | url = urlunparse(("https", host, "/api/v1/extrahop/firmware", "", "", "")) 82 | file_path = os.path.join(firmware) 83 | data = open(file_path, "rb") 84 | print(f"Uploading firmware to {host}") 85 | r = requests.post(url, data=data, headers=headers) 86 | if r.status_code == 201: 87 | print(f"Uploaded firmware to {host}") 88 | return True 89 | elif retry < MAX_RETRIES: 90 | print(f"Failed to upload firmware to {host}") 91 | print("Retrying firmware upload") 92 | uploadFirmware(host, api_key, firmware, retry + 1) 93 | else: 94 | print(f"Firmware upload to {host} failed") 95 | print(r.text) 96 | return False 97 | 98 | 99 | # Function that upgrades firmware on system 100 | def upgradeFirmware(host, api_key): 101 | """ 102 | Method that upgrades firmware on an ExtraHop system 103 | 104 | Parameters: 105 | host (str): The IP address or hostname of the ExtraHop system 106 | api_key (str): An API key on the ExtraHop system 107 | 108 | Returns: 109 | str: The relative URL of the job status 110 | """ 111 | headers = {"Authorization": "ExtraHop apikey=%s" % api_key} 112 | url = urlunparse( 113 | ("https", host, "/api/v1/extrahop/firmware/latest/upgrade", "", "", "") 114 | ) 115 | r = requests.post(url, headers=headers) 116 | if r.status_code == 202: 117 | print(f"Started firmware upgrade process on {host}") 118 | if "Location" in r.headers: 119 | return r.headers["Location"] 120 | else: 121 | return None 122 | else: 123 | print(f"Failed to upgrade firmware on {host}") 124 | print(r.status_code) 125 | return None 126 | 127 | 128 | def monitorStatus(host, api_key, job_location): 129 | """ 130 | Method that periodically retrieves the status of a job until the job 131 | is completed. 132 | 133 | Parameters: 134 | host (str): The IP address or hostname of the ExtraHop system 135 | api_key (str): An API key on the ExtraHop system 136 | job_location (str): The relative URL of the job status 137 | """ 138 | headers = {"Authorization": "ExtraHop apikey=%s" % api_key} 139 | url = urlunparse(("https", host, job_location, "", "", "")) 140 | while True: 141 | # Catch connection errors 142 | try: 143 | r = requests.get(url, headers=headers) 144 | except requests.exceptions.ConnectionError: 145 | print(f"Connection refused by {host}") 146 | time.sleep(WAIT * 60) 147 | continue 148 | if r.status_code == 200: 149 | parsed_resp = r.json() 150 | status = parsed_resp["status"] 151 | if status == "DONE": 152 | print(f"Upgrade for {host} is {status}.") 153 | return 154 | elif status == "FAILED": 155 | print(f"Upgrade for {host} {status}.") 156 | return 157 | elif status == "TIMEOUT": 158 | print( 159 | f"Upgrade for {host} timed out because the job took longer than expected.\nTo verify the progress of the upgrade, go to the ExtraHop system in a web browser." 160 | ) 161 | return 162 | else: 163 | print(f"Upgrade for {host} is {status}.") 164 | else: 165 | print(f"Failed to retrieve status for upgrade on {host}.") 166 | print(r.status_code) 167 | try: 168 | print(r.json()) 169 | except: 170 | pass 171 | time.sleep(WAIT * 60) 172 | 173 | 174 | def getFirmware(host, api_key): 175 | """ 176 | Method that retrieves the version of firmware running on 177 | the ExtraHop system. 178 | 179 | Parameters: 180 | host (str): The IP address or hostname of the ExtraHop system 181 | api_key (str): An API key on the ExtraHop system 182 | 183 | Returns: 184 | str: The firmware version 185 | """ 186 | headers = {"Authorization": "ExtraHop apikey=%s" % api_key} 187 | url = urlunparse(("https", host, "/api/v1/extrahop/", "", "", "")) 188 | r = requests.get(url, headers=headers) 189 | if r.status_code == 200: 190 | parsed_resp = r.json() 191 | return parsed_resp["version"] 192 | else: 193 | print(f"Failed to retrieve firmware version from {host}") 194 | print(r.status_code) 195 | return "8.4.0" 196 | 197 | 198 | if __name__ == "__main__": 199 | for system in systems: 200 | host = system["host"] 201 | api_key = system["api_key"] 202 | firmware = system["firmware"] 203 | 204 | while threading.activeCount() > MAX_THREADS: 205 | time.sleep(0.1) 206 | 207 | u = Upgrader(host, api_key, firmware) 208 | u.start() 209 | -------------------------------------------------------------------------------- /upgrade_system_cloud/systems.csv: -------------------------------------------------------------------------------- 1 | extrahop.example.com,123456789abcdefghijklmnop 2 | extrahop.example2.com,123456789abcdefghijklmnop 3 | -------------------------------------------------------------------------------- /upgrade_system_cloud/upgrade_system_cloud.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2022 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import csv 10 | from urllib.parse import urlunparse 11 | import threading 12 | import time 13 | import sys 14 | import argparse 15 | import json 16 | 17 | 18 | # The system config list 19 | SYSTEM_LIST = "systems.csv" 20 | 21 | 22 | def parse_arguments(): 23 | """ 24 | Function that parses command line arguments. 25 | 26 | Returns: 27 | argparse.Namespace: An object containing the argument values 28 | """ 29 | argparser = argparse.ArgumentParser() 30 | group = argparser.add_mutually_exclusive_group(required=True) 31 | group.add_argument( 32 | "--latest-hotfix", 33 | dest="latest_hotfix", 34 | action="store_true", 35 | help="Upgrade to the latest version in the current release", 36 | ) 37 | group.add_argument( 38 | "--latest-release", 39 | dest="latest_release", 40 | action="store_true", 41 | help="Upgrade to the latest version in the latest release", 42 | ) 43 | group.add_argument( 44 | "--version", 45 | dest="version", 46 | help="Upgrade to the specified version", 47 | ) 48 | argparser.add_argument( 49 | "--force", 50 | dest="force", 51 | action="store_true", 52 | help="Upgrade systems without prompting for confirmation", 53 | ) 54 | argparser.add_argument( 55 | "--max-threads", 56 | dest="max_threads", 57 | default=2, 58 | type=int, 59 | help="The maximum number of concurrent threads", 60 | ) 61 | argparser.add_argument( 62 | "--wait", 63 | dest="wait", 64 | default=0.5, 65 | type=float, 66 | help="The number of minutes to wait between checking upgrade progress", 67 | ) 68 | return argparser.parse_args() 69 | 70 | 71 | def getNextFirmware(host, api_key, firmware_target): 72 | """ 73 | Function that retrieves the firmware version to upgrade the ExtraHop system to. 74 | 75 | Parameters: 76 | host (str): The IP address or hostname of the ExtraHop system 77 | api_key (str): An API key on the ExtraHop system 78 | firmware_target (str): The command-line argument that specifies the target firmware version 79 | 80 | Returns: 81 | n_firmware (str): The firmware version 82 | """ 83 | url = urlunparse( 84 | ("https", host, "/api/v1/extrahop/firmware/next", "", "", "") 85 | ) 86 | headers = { 87 | "Authorization": f"ExtraHop apikey={api_key}", 88 | } 89 | r = requests.get(url, headers=headers) 90 | if r.status_code == 200: 91 | n_firmware = parseReleases(r.json(), firmware_target, host) 92 | return n_firmware 93 | else: 94 | try: 95 | print(json.dumps(r.json())) 96 | except: 97 | pass 98 | raise RuntimeError(f"Failed to retrieve list of available versions from {host}. \nStatus code:{r.status_code}") 99 | 100 | 101 | def parseReleases(releases, firmware_target, host): 102 | """ 103 | Function that parses the list of firmware versions that you can upgrade the ExtraHop system to 104 | and selects the correct version based on the specified firmware_target. 105 | 106 | Parameters: 107 | releases (list): The list of releases that that you can upgrade the ExtraHop system to 108 | firmware_target (str): The command-line argument that specifies the target firmware version 109 | host (str): The IP address or hostname of the ExtraHop system 110 | 111 | Returns: 112 | version (str): The version to upgrade to 113 | """ 114 | if not releases: 115 | return None 116 | if firmware_target == "latest-release": 117 | return releases[0]["versions"][0] 118 | elif firmware_target == "latest-hotfix": 119 | for release in releases: 120 | if release["current_release"] == True: 121 | return release["versions"][0] 122 | return None 123 | else: 124 | for release in releases: 125 | for version in release["versions"]: 126 | if firmware_target == version: 127 | return version 128 | return None 129 | 130 | 131 | class Upgrader(threading.Thread): 132 | """ 133 | A class for a thread that upgrades an appliance. 134 | 135 | Attributes 136 | ---------- 137 | host : str 138 | The IP address or hostname of the ExtraHop system 139 | api_key : str 140 | An API key on the ExtraHop system 141 | firmware : str 142 | The firmware version to upgrade the ExtraHop system to 143 | wait : float 144 | The number of minutes to wait between checking upgrade progress 145 | """ 146 | 147 | def __init__(self, host, api_key, firmware, wait): 148 | threading.Thread.__init__(self) 149 | self.host = host 150 | self.api_key = api_key 151 | self.firmware = firmware 152 | self.wait = wait 153 | 154 | def run(self): 155 | print(f"Starting upgrade thread for {self.host}") 156 | download_job = self._downloadFirmware() 157 | if download_job: 158 | download_success = self._monitorStatus( 159 | download_job, "Firmware download" 160 | ) 161 | if download_success: 162 | upgrade_job = self._upgradeFirmware() 163 | if upgrade_job: 164 | upgrade_success = self._monitorStatus( 165 | upgrade_job, "Upgrade" 166 | ) 167 | 168 | def _downloadFirmware(self): 169 | """ 170 | Method that downloads firmware from ExtraHop Cloud Services to an ExtraHop system. 171 | 172 | Returns: 173 | str: The relative URL of the job status 174 | """ 175 | headers = {"Authorization": "ExtraHop apikey=%s" % self.api_key} 176 | url = urlunparse( 177 | ( 178 | "https", 179 | self.host, 180 | "/api/v1/extrahop/firmware/download/version", 181 | "", 182 | "", 183 | "", 184 | ) 185 | ) 186 | data = {"version": self.firmware} 187 | r = requests.post(url, json=data, headers=headers) 188 | if r.status_code == 202: 189 | print(f"Started firmware download process on {self.host}") 190 | if "Location" in r.headers: 191 | return r.headers["Location"] 192 | else: 193 | return None 194 | else: 195 | print(f"Firmware download to {self.host} failed") 196 | print(r.text) 197 | return None 198 | 199 | def _upgradeFirmware(self): 200 | """ 201 | Method that upgrades firmware on an ExtraHop system 202 | 203 | Returns: 204 | str: The relative URL of the job status 205 | """ 206 | headers = {"Authorization": "ExtraHop apikey=%s" % self.api_key} 207 | url = urlunparse( 208 | ( 209 | "https", 210 | self.host, 211 | "/api/v1/extrahop/firmware/latest/upgrade", 212 | "", 213 | "", 214 | "", 215 | ) 216 | ) 217 | r = requests.post(url, headers=headers) 218 | if r.status_code == 202: 219 | print(f"Started firmware upgrade process on {self.host}") 220 | if "Location" in r.headers: 221 | return r.headers["Location"] 222 | else: 223 | return None 224 | else: 225 | print(f"Failed to upgrade firmware on {self.host}") 226 | print(r.status_code) 227 | return None 228 | 229 | def _monitorStatus(self, job_location, job_name="Job"): 230 | """ 231 | Method that periodically retrieves the status of a job until the job 232 | is completed. 233 | 234 | Parameters: 235 | job_location (str): The relative URL of the job status 236 | 237 | Returns: 238 | bool: Indicates whether the job completed successfully 239 | """ 240 | headers = {"Authorization": "ExtraHop apikey=%s" % self.api_key} 241 | url = urlunparse(("https", self.host, job_location, "", "", "")) 242 | while True: 243 | # Catch connection errors 244 | try: 245 | r = requests.get(url, headers=headers) 246 | except requests.exceptions.ConnectionError: 247 | print(f"Connection refused by {self.host}") 248 | time.sleep(self.wait * 60) 249 | continue 250 | if r.status_code == 200: 251 | parsed_resp = r.json() 252 | status = parsed_resp["status"] 253 | if status == "DONE": 254 | print(f"{job_name} for {self.host} is {status}.") 255 | return True 256 | elif status == "FAILED": 257 | print(f"{job_name} for {self.host} {status}.") 258 | print(json.dumps(parsed_resp, indent=4)) 259 | return False 260 | elif status == "TIMEOUT": 261 | print( 262 | f"{job_name} for {self.host} timed out because the job took longer than expected." 263 | ) 264 | return False 265 | else: 266 | print(f"{job_name} for {self.host} is {status}.") 267 | else: 268 | print( 269 | f"Failed to retrieve status for {job_name} on {self.host}." 270 | ) 271 | print(r.status_code) 272 | try: 273 | print(r.json()) 274 | except: 275 | pass 276 | time.sleep(self.wait * 60) 277 | 278 | 279 | if __name__ == "__main__": 280 | target_firmware = "" 281 | force = False 282 | args = parse_arguments() 283 | if args.latest_hotfix: 284 | firmware_target = "latest-hotfix" 285 | elif args.latest_release: 286 | firmware_target = "latest-release" 287 | else: 288 | firmware_target = args.version 289 | if args.force: 290 | force = True 291 | max_threads = args.max_threads 292 | wait = args.wait 293 | systems = [] 294 | no_upgrade = [] 295 | with open(SYSTEM_LIST, "rt", encoding="ascii") as f: 296 | reader = csv.reader(f) 297 | for row in reader: 298 | system = {"host": row[0], "api_key": row[1]} 299 | system["next_firmware"] = getNextFirmware( 300 | system["host"], system["api_key"], firmware_target 301 | ) 302 | if system["next_firmware"]: 303 | systems.append(system) 304 | else: 305 | no_upgrade.append(system) 306 | 307 | # Print systems that are already upgraded 308 | if no_upgrade: 309 | if firmware_target == "latest-hotfix": 310 | print( 311 | f"The following systems have already been upgraded to the latest hotfix:" 312 | ) 313 | elif firmware_target == "latest-release": 314 | print( 315 | f"The following systems have already been upgraded to the latest release:" 316 | ) 317 | else: 318 | print( 319 | f"The following systems cannot be upgraded to version {firmware_target}:" 320 | ) 321 | for system in no_upgrade: 322 | print(f" {system['host']}") 323 | 324 | # If there are no systems to upgrade, exit 325 | if not systems: 326 | sys.exit() 327 | 328 | # If force has not been specified, ask for confirmation before continuing 329 | if not force: 330 | print("The following systems will be upgraded:") 331 | for system in systems: 332 | print(f" {system['host']}: {system['next_firmware']}") 333 | print("Do you want to continue?") 334 | c = input("y/n") 335 | if c.lower() != "y" and c.lower() != "yes": 336 | sys.exit() 337 | 338 | # Create an upgrade thread for each system 339 | for system in systems: 340 | while threading.activeCount() > max_threads: 341 | time.sleep(0.1) 342 | 343 | u = Upgrader( 344 | system["host"], system["api_key"], system["next_firmware"], wait 345 | ) 346 | u.start() 347 | -------------------------------------------------------------------------------- /upgrade_system_url/systems.csv: -------------------------------------------------------------------------------- 1 | extrahop.example.com,123456789abcdefghijklmnop,https://example.extrahop.com/eda/8.7.1.tar 2 | extrahop.example2.com,123456789abcdefghijklmnop,https://example.extrahop.com/eda/8.7.1.tar 3 | -------------------------------------------------------------------------------- /upgrade_system_url/upgrade_system_url.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import requests 9 | import csv 10 | from urllib.parse import urlunparse 11 | import threading 12 | import time 13 | import json 14 | import argparse 15 | 16 | SYSTEM_LIST = "systems.csv" 17 | 18 | 19 | def parse_arguments(): 20 | """ 21 | Function that parses command line arguments. 22 | 23 | Returns: 24 | argparse.Namespace: An object containing the argument values 25 | """ 26 | argparser = argparse.ArgumentParser() 27 | argparser.add_argument( 28 | "--max-threads", 29 | dest="max_threads", 30 | default=2, 31 | type=int, 32 | help="The maximum number of concurrent threads", 33 | ) 34 | argparser.add_argument( 35 | "--wait", 36 | dest="wait", 37 | default=0.5, 38 | type=float, 39 | help="The number of minutes to wait between checking upgrade progress", 40 | ) 41 | return argparser.parse_args() 42 | 43 | 44 | class Upgrader(threading.Thread): 45 | """ 46 | A class for a thread that upgrades an appliance. 47 | 48 | Attributes 49 | ---------- 50 | host : str 51 | The IP address or hostname of the ExtraHop system 52 | api_key : str 53 | An API key on the ExtraHop system 54 | firmware : str 55 | The URL to retrieve the firmware from 56 | wait : float 57 | The number of minutes to wait between checking upgrade progress 58 | """ 59 | 60 | def __init__(self, host, api_key, firmware, wait): 61 | threading.Thread.__init__(self) 62 | self.host = host 63 | self.api_key = api_key 64 | self.firmware = firmware 65 | self.wait = wait 66 | 67 | def run(self): 68 | print(f"Starting upgrade thread for {self.host}") 69 | job_location = self._upgradeFirmware() 70 | if job_location: 71 | self._monitorStatus(job_location, "Firmware upgrade") 72 | 73 | def _upgradeFirmware(self): 74 | """ 75 | Method that downloads and upgrades firmware on an ExtraHop system. 76 | 77 | Returns: 78 | str: The relative URL of the job status 79 | """ 80 | headers = {"Authorization": "ExtraHop apikey=%s" % self.api_key} 81 | url = urlunparse( 82 | ( 83 | "https", 84 | self.host, 85 | "/api/v1/extrahop/firmware/download/url", 86 | "", 87 | "", 88 | "", 89 | ) 90 | ) 91 | data = {"firmware_url": self.firmware, "upgrade": True} 92 | r = requests.post(url, json=data, headers=headers) 93 | if r.status_code == 202: 94 | print(f"Started firmware upgrade process on {self.host}") 95 | if "Location" in r.headers: 96 | return r.headers["Location"] 97 | else: 98 | return None 99 | else: 100 | print(f"Failed to upgrade firmware on {self.host}") 101 | print(r.status_code) 102 | try: 103 | print(r.json()) 104 | except: 105 | pass 106 | return None 107 | 108 | def _monitorStatus(self, job_location, job_name="Job"): 109 | """ 110 | Method that periodically retrieves the status of a job until the job 111 | is completed. 112 | 113 | Parameters: 114 | job_location (str): The relative URL of the job status 115 | 116 | Returns: 117 | bool: Indicates whether the job completed successfully 118 | """ 119 | headers = {"Authorization": "ExtraHop apikey=%s" % self.api_key} 120 | url = urlunparse(("https", self.host, job_location, "", "", "")) 121 | step_number = None 122 | total_steps = None 123 | step_description = None 124 | while True: 125 | # Catch connection errors 126 | try: 127 | r = requests.get(url, headers=headers) 128 | except requests.exceptions.ConnectionError: 129 | print(f"Connection refused by {self.host}") 130 | time.sleep(self.wait * 60) 131 | continue 132 | if r.status_code == 200: 133 | parsed_resp = r.json() 134 | status = parsed_resp["status"] 135 | if status == "DONE": 136 | print(f"{job_name} for {self.host} is {status}.") 137 | return True 138 | elif status == "FAILED": 139 | print(f"{job_name} for {self.host} {status}.") 140 | print(json.dumps(parsed_resp, indent=4)) 141 | return False 142 | elif status == "TIMEOUT": 143 | print( 144 | f"{job_name} for {self.host} timed out because the job took longer than expected." 145 | ) 146 | return False 147 | else: 148 | try: 149 | step_number = parsed_resp["step_number"] 150 | total_steps = parsed_resp["total_steps"] 151 | step_description = parsed_resp["step_description"] 152 | except: 153 | pass 154 | print(f"{job_name} for {self.host} is {status}.") 155 | if step_number: 156 | print(f" Step {step_number} of {total_steps}: {step_description}") 157 | else: 158 | print( 159 | f"Failed to retrieve status for {job_name} on {self.host}." 160 | ) 161 | print(r.status_code) 162 | try: 163 | print(r.json()) 164 | except: 165 | pass 166 | time.sleep(self.wait * 60) 167 | 168 | 169 | if __name__ == "__main__": 170 | args = parse_arguments() 171 | max_threads = args.max_threads 172 | wait = args.wait 173 | # Retrieve appliance URLs, API keys, and firmware URLs 174 | systems = [] 175 | with open(SYSTEM_LIST, "rt", encoding="ascii") as f: 176 | reader = csv.reader(f) 177 | for row in reader: 178 | system = {"host": row[0], "api_key": row[1], "firmware": row[2]} 179 | systems.append(system) 180 | 181 | for system in systems: 182 | host = system["host"] 183 | api_key = system["api_key"] 184 | firmware = system["firmware"] 185 | 186 | while threading.activeCount() > max_threads: 187 | time.sleep(0.1) 188 | 189 | u = Upgrader(host, api_key, firmware, wait) 190 | u.start() 191 | -------------------------------------------------------------------------------- /upload_ids_rules/ids.csv: -------------------------------------------------------------------------------- 1 | host,api_key,ids_file 2 | extrahop.example.com,123456789abcdefghijklmnop,ids-bundle.tar.gz.aes 3 | -------------------------------------------------------------------------------- /upload_ids_rules/upload_ids_rules.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from urllib.parse import urlunparse 3 | import csv 4 | 5 | # The path of the CSV file 6 | CSV_PATH = "ids.csv" 7 | 8 | 9 | def readCSV(filepath): 10 | """ 11 | Function that reads a list of ExtraHop system hostnames, 12 | API keys, and IDS filenames from a CSV file 13 | 14 | Parameters: 15 | filepath (str): The path of the CSV file 16 | 17 | Returns: 18 | list: A list of objects containing the hostnames, API keys, and filenames 19 | """ 20 | with open(filepath, "r") as f: 21 | csv_reader = csv.DictReader(f) 22 | return [row for row in csv_reader] 23 | 24 | 25 | def uploadIds(host, api_key, ids_file_path): 26 | """ 27 | Function that uploads a specified IDS file to an ExtraHop appliance. 28 | The function can upload IDS ruleset files to sensors and IDS resource 29 | files to consoles. 30 | 31 | Parameters: 32 | host (str): The hostname or IP address of the appliance 33 | api_key (str): The API key for the appliance 34 | ids_file (str): The path of the IDS ruleset or resource file 35 | """ 36 | url = urlunparse( 37 | ("https", host, "/api/v1/extrahop/cloudresources", "", "", "") 38 | ) 39 | headers = { 40 | "Authorization": f"ExtraHop apikey={api_key}", 41 | "accept": "application/json", 42 | } 43 | with open(ids_file_path, "rb") as ids_file: 44 | r = requests.post( 45 | url, 46 | data=ids_file, 47 | headers=headers, 48 | ) 49 | if r.status_code == 202: 50 | print(f"{ids_file_path} was successfully uploaded to {host}") 51 | print() 52 | else: 53 | print("IDS upload failed") 54 | print(r.status_code) 55 | print(r.text) 56 | print() 57 | 58 | 59 | def main(): 60 | appliances = readCSV(CSV_PATH) 61 | for appliance in appliances: 62 | uploadIds( 63 | appliance["host"], appliance["api_key"], appliance["ids_file"] 64 | ) 65 | 66 | 67 | if __name__ == "__main__": 68 | main() 69 | -------------------------------------------------------------------------------- /upload_stix/systems.csv: -------------------------------------------------------------------------------- 1 | extrahop.example.com/,123456789abcdefghijklmnop 2 | extrahop.example2.com/,123456789abcdefghijklmnop 3 | 4 | -------------------------------------------------------------------------------- /upload_stix/upload_stix.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import os 10 | import requests 11 | import csv 12 | from urllib.parse import urlunparse 13 | 14 | # The path of the CSV file with the HTTPS URLs and API keys of the systems. 15 | SYSTEM_LIST = "systems.csv" 16 | # The path of the directory that contains the STIX files. 17 | STIX_DIR = "stix_dir" 18 | 19 | # Read system URLs and API keys from CSV file 20 | systems = [] 21 | with open(SYSTEM_LIST, "rt", encoding="ascii") as f: 22 | reader = csv.reader(f) 23 | for row in reader: 24 | system = {"host": row[0], "api_key": row[1]} 25 | systems.append(system) 26 | 27 | 28 | def getCollections(host, api_key): 29 | """ 30 | Method that retrieves every threat collection on the ExtraHop system. 31 | Parameters: 32 | host (str): The IP address or hostname of the ExtraHop system 33 | 34 | Returns: 35 | list: A list of threat collection dictionaries 36 | """ 37 | headers = {"Authorization": "ExtraHop apikey=%s" % api_key} 38 | url = urlunparse(("https", host, "/api/v1/threatcollections", "", "", "")) 39 | r = requests.get(url, headers=headers) 40 | if r.status_code == 200: 41 | return r.json() 42 | else: 43 | print("Error: Unable to retrieve existing threat collections.") 44 | print(r.status_code) 45 | print(r.text) 46 | sys.exit() 47 | 48 | 49 | # Function that checks which STIX files have already been uploaded 50 | def check_files(collections): 51 | """ 52 | Method that finds out which STIX files in the STIX_DIR directory 53 | have already been uploaded to the ExtraHop system. 54 | 55 | Parameters: 56 | collections (list): A list of threat collection dictionaries 57 | 58 | Returns: 59 | upload_list (list): A list of STIX filenames that have not been uploaded 60 | skip_list (list): A list of threat collection dictionaries for the STIX files that have already been uploaded 61 | """ 62 | upload_list = [] 63 | skip_list = [] 64 | # Get a list of all stix files in the STIX_DIR directory 65 | upload_list = [] 66 | collection_names = [c["name"] for c in collections] 67 | for dir, subdirs, files in os.walk(STIX_DIR): 68 | for file in files: 69 | if file.endswith((".tar")) or file.endswith(".tgz"): 70 | name = file.split(".")[0] 71 | if name in collection_names: 72 | obj = {} 73 | for c in collections: 74 | if c["name"] == name: 75 | obj["user_key"] = c["user_key"] 76 | obj["filename"] = file 77 | skip_list.append(obj) 78 | else: 79 | upload_list.append(file) 80 | return upload_list, skip_list 81 | 82 | 83 | def process_files(upload_files, skip_list, host, api_key): 84 | """ 85 | Method that processes each file in the STIX_DIR directory. If a file has not been 86 | uploaded before, the method creates a new threat collection for the file. If a 87 | file has been uploaded before, the method updates the threat collection for that 88 | file with with the latest file contents. 89 | 90 | Parameters: 91 | upload_files (list): A list of STIX filenames to upload 92 | skip_list (list): A list of threat collection dictionaries for the STIX files that have already been uploaded 93 | host (str): The IP address or hostname of the ExtraHop system 94 | """ 95 | for f in upload_files: 96 | upload_new(f"{STIX_DIR}/{f}", host, api_key) 97 | for s in skip_list: 98 | update_old(f"{STIX_DIR}/{s['filename']}", s["user_key"], host, api_key) 99 | 100 | 101 | def upload_new(file_path, host, api_key): 102 | """ 103 | Method that uploads a new threat collection. 104 | 105 | Parameters: 106 | file_path (str): The filepath of the STIX file 107 | host (str): The IP address or hostname of the ExtraHop system 108 | """ 109 | print("Uploading " + file_path + " on " + host) 110 | url = urlunparse(("https", host, "/api/v1/threatcollections", "", "", "")) 111 | name = file_path.split("/")[-1] 112 | name = name.split(".")[0] 113 | files = {"file": open(file_path, "rb")} 114 | values = {"name": name} 115 | headers = {"Authorization": "ExtraHop apikey=%s" % api_key} 116 | r = requests.post( 117 | url, data=values, files=files, headers=headers 118 | ) 119 | if r.status_code == 204: 120 | print("Upload complete") 121 | else: 122 | print("Upload failed") 123 | print(r.status_code) 124 | print(r.text) 125 | 126 | 127 | # Function that updates an existing threat collection 128 | def update_old(file_path, user_key, host, api_key): 129 | """ 130 | Method that updates an existing threat collection. 131 | 132 | Parameters: 133 | file (str): The filenpath of the STIX file 134 | user_key (str): The user key of the threat collection 135 | host (str): The IP address or hostname of the appliance 136 | """ 137 | print("Updating " + file_path + " on " + host) 138 | url = urlunparse( 139 | ( 140 | "https", 141 | host, 142 | f"api/v1/threatcollections/~{str(user_key)}", 143 | "", 144 | "", 145 | "", 146 | ) 147 | ) 148 | files = {"file": open(file_path, "rb")} 149 | headers = {"Authorization": "ExtraHop apikey=%s" % api_key} 150 | r = requests.put(url, files=files, headers=headers) 151 | if r.status_code == 204: 152 | print("Update complete") 153 | else: 154 | print("Update failed") 155 | print(r.status_code) 156 | print(r.text) 157 | 158 | 159 | # Process STIX files for each system 160 | for system in systems: 161 | host = system["host"] 162 | api_key = system["api_key"] 163 | collections = getCollections(host, api_key) 164 | update_files, skip_list = check_files(collections) 165 | process_files(update_files, skip_list, host, api_key) 166 | -------------------------------------------------------------------------------- /upload_stix/upload_stix_rx360.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # COPYRIGHT 2021 BY EXTRAHOP NETWORKS, INC. 4 | # 5 | # This file is subject to the terms and conditions defined in 6 | # file 'LICENSE', which is part of this source code package. 7 | 8 | import json 9 | import os 10 | import requests 11 | import csv 12 | from urllib.parse import urlunparse 13 | import base64 14 | import sys 15 | 16 | # The IP address or hostname of the Reveal(x) 360 API 17 | HOST = "extrahop.example.com" 18 | 19 | # The ID of the REST API credentials. 20 | ID = "abcdefg123456789" 21 | # The secret of the REST API credentials. 22 | SECRET = "123456789abcdefg987654321abcdefg" 23 | # A global variable for the temporary API access token (leave blank) 24 | TOKEN = "" 25 | 26 | # The path of the directory that contains the STIX files. 27 | STIX_DIR = "stix_dir" 28 | 29 | 30 | def getToken(): 31 | """ 32 | Method that generates and retrieves a temporary API access token for Reveal(x) 360 authentication. 33 | 34 | Returns: 35 | str: A temporary API access token 36 | """ 37 | auth = base64.b64encode(bytes(ID + ":" + SECRET, "utf-8")).decode("utf-8") 38 | headers = { 39 | "Authorization": "Basic " + auth, 40 | "Content-Type": "application/x-www-form-urlencoded", 41 | } 42 | url = urlunparse(("https", HOST, "/oauth2/token", "", "", "")) 43 | r = requests.post( 44 | url, 45 | headers=headers, 46 | data="grant_type=client_credentials", 47 | ) 48 | try: 49 | return r.json()["access_token"] 50 | except: 51 | print(r.text) 52 | print(r.status_code) 53 | print("Error retrieveing token from Reveal(x) 360") 54 | sys.exit() 55 | 56 | 57 | def getAuthHeader(force_token_gen=False): 58 | """ 59 | Method that adds an authorization header for a request. 60 | 61 | Parameters: 62 | force_token_gen (bool): If true, always generates a new temporary API access token for the request 63 | 64 | Returns: 65 | header (str): The value for the header key in the headers dictionary 66 | """ 67 | global TOKEN 68 | if TOKEN == "" or force_token_gen == True: 69 | TOKEN = getToken() 70 | return f"Bearer {TOKEN}" 71 | 72 | 73 | def getCollections(host): 74 | """ 75 | Method that retrieves every threat collection on the ExtraHop system. 76 | Parameters: 77 | host (str): The IP address or hostname of the ExtraHop system 78 | 79 | Returns: 80 | list: A list of threat collection dictionaries 81 | """ 82 | headers = {"Authorization": getAuthHeader()} 83 | url = urlunparse(("https", host, "/api/v1/threatcollections", "", "", "")) 84 | r = requests.get(url, headers=headers) 85 | if r.status_code == 200: 86 | return r.json() 87 | else: 88 | print("Error: Unable to retrieve existing threat collections.") 89 | print(r.status_code) 90 | print(r.text) 91 | sys.exit() 92 | 93 | 94 | # Function that checks which STIX files have already been uploaded 95 | def check_files(collections): 96 | """ 97 | Method that finds out which STIX files in the STIX_DIR directory 98 | have already been uploaded to the ExtraHop system. 99 | 100 | Parameters: 101 | collections (list): A list of threat collection dictionaries 102 | 103 | Returns: 104 | upload_list (list): A list of STIX filenames that have not been uploaded 105 | skip_list (list): A list of threat collection dictionaries for the STIX files that have already been uploaded 106 | """ 107 | upload_list = [] 108 | skip_list = [] 109 | # Get a list of all stix files in the STIX_DIR directory 110 | upload_list = [] 111 | collection_names = [c["name"] for c in collections] 112 | for dir, subdirs, files in os.walk(STIX_DIR): 113 | for file in files: 114 | if file.endswith((".tar")) or file.endswith(".tgz"): 115 | name = file.split(".")[0] 116 | if name in collection_names: 117 | obj = {} 118 | for c in collections: 119 | if c["name"] == name: 120 | obj["user_key"] = c["user_key"] 121 | obj["filename"] = file 122 | skip_list.append(obj) 123 | else: 124 | upload_list.append(file) 125 | return upload_list, skip_list 126 | 127 | 128 | def process_files(upload_files, skip_list, host): 129 | """ 130 | Method that processes each file in the STIX_DIR directory. If a file has not been 131 | uploaded before, the method creates a new threat collection for the file. If a 132 | file has been uploaded before, the method updates the threat collection for that 133 | file with with the latest file contents. 134 | 135 | Parameters: 136 | upload_files (list): A list of STIX filenames to upload 137 | skip_list (list): A list of threat collection dictionaries for the STIX files that have already been uploaded 138 | host (str): The IP address or hostname of the ExtraHop system 139 | """ 140 | for f in upload_files: 141 | upload_new(f"{STIX_DIR}/{f}", host) 142 | for s in skip_list: 143 | update_old(f"{STIX_DIR}/{s['filename']}", s["user_key"], host) 144 | 145 | 146 | def upload_new(file_path, host): 147 | """ 148 | Method that uploads a new threat collection. 149 | 150 | Parameters: 151 | file_path (str): The filepath of the STIX file 152 | host (str): The IP address or hostname of the ExtraHop system 153 | """ 154 | print("Uploading " + file_path + " on " + host) 155 | url = urlunparse(("https", host, "/api/v1/threatcollections", "", "", "")) 156 | name = file_path.split("/")[-1] 157 | name = name.split(".")[0] 158 | files = {"file": open(file_path, "rb")} 159 | values = {"name": name} 160 | headers = {"Authorization": getAuthHeader()} 161 | r = requests.post(url, data=values, files=files, headers=headers) 162 | if r.status_code == 204: 163 | print("Upload complete") 164 | else: 165 | print("Upload failed") 166 | print(r.status_code) 167 | print(r.text) 168 | 169 | 170 | # Function that updates an existing threat collection 171 | def update_old(file_path, user_key, host): 172 | """ 173 | Method that updates an existing threat collection. 174 | 175 | Parameters: 176 | file (str): The filenpath of the STIX file 177 | user_key (str): The user key of the threat collection 178 | host (str): The IP address or hostname of the Reveal(x) 360 API 179 | """ 180 | print("Updating " + file_path + " on " + host) 181 | url = urlunparse( 182 | ( 183 | "https", 184 | HOST, 185 | f"api/v1/threatcollections/~{str(user_key)}", 186 | "", 187 | "", 188 | "", 189 | ) 190 | ) 191 | files = {"file": open(file_path, "rb")} 192 | headers = {"Authorization": getAuthHeader()} 193 | r = requests.put(url, files=files, headers=headers) 194 | if r.status_code == 204: 195 | print("Update complete") 196 | else: 197 | print("Update failed") 198 | print(r.status_code) 199 | print(r.text) 200 | 201 | 202 | # Process STIX files 203 | collections = getCollections(HOST) 204 | upload_files, skip_list = check_files(collections) 205 | print(f"{str(len(upload_files))} to upload") 206 | print(f"{str(len(skip_list))} to update") 207 | process_files(upload_files, skip_list, HOST) 208 | --------------------------------------------------------------------------------