├── redcanary ├── __init__.py └── responseutils │ ├── __init__.py │ └── common.py ├── .github └── CODEOWNERS ├── LICENSE.txt ├── setup.py ├── .gitignore ├── README.md ├── cblr-basic.py ├── process-util.py ├── usb-util.py ├── sensor-util.py ├── timeline.py └── network-util.py /redcanary/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /redcanary/responseutils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @redcanaryco/team-community 2 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | The MIT License 3 | 4 | Copyright (c) 2016 Red Canary, Inc. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | import os 5 | 6 | 7 | def read(fname): 8 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 9 | 10 | 11 | def find_scripts(): 12 | scripts = [] 13 | exclude = ['setup.py'] 14 | for file in os.scandir('.'): 15 | if file.name.endswith('.py') and file.is_file() and (file.name not in exclude): 16 | scripts.append(file.name) 17 | return scripts 18 | 19 | 20 | setup( 21 | name='redcanary-response-utils', 22 | author='Keith McCammon', 23 | author_email='keith@redcanary.com', 24 | url='https://github.com/redcanaryco/redcanary-response-utils', 25 | license='MIT', 26 | packages=find_packages(), 27 | scripts=find_scripts(), 28 | description='Tools to automate and/or expedite response.', 29 | version='0.1', 30 | classifiers=[ 31 | 'Development Status :: 5 - Production/Stable', 32 | 'Intended Audience :: Developers', 33 | 'License :: Freely Distributable', 34 | 'License :: OSI Approved :: MIT License', 35 | 'Operating System :: OS Independent', 36 | 'Programming Language :: Python', 37 | ], 38 | install_requires=[ 39 | 'cbapi' 40 | ] 41 | ) 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # Project things 92 | *.csv 93 | .carbonblack 94 | sensors.csv 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Tools to automate and/or expedite response. 2 | 3 | ### Setup 4 | 5 | ``` 6 | git clone git@github.com:redcanaryco/redcanary-response-utils.git 7 | 8 | mkvirtualenv redcanary-response-utils 9 | 10 | python setup.py develop 11 | 12 | 13 | ./sensor-util.py 14 | 15 | ``` 16 | 17 | ### cblr-basic.py 18 | Platforms: Carbon Black (Response) 19 | 20 | Execute a basic response plan targeting a single endpoint. 21 | Performs the following actions: 22 | 23 | 1. Isolate the endpoint. 24 | 2. Kill associated processes. 25 | 3. Ban offending binary file(s). 26 | 27 | ### network-util.py 28 | Platforms: Carbon Black (Response) 29 | 30 | Enumerate network connections based on a wide variety of criteria. Includes 31 | support for: 32 | 33 | - process- and connection-based whitelists 34 | - filtering by host type (Workstation or Server) 35 | - more 36 | 37 | ### process-util.py 38 | Platforms: Carbon Black (Response) 39 | 40 | Enumerate processes. This is a performant alternative to timeline.py if you 41 | wish to quickly examine process start events only. 42 | 43 | ### sensor-util.py 44 | Platforms: Carbon Black (Response) 45 | 46 | Enumerate sensors and output metadata, to include endpoint health. 47 | 48 | ### timeline.py 49 | Platforms: Carbon Black (Response) 50 | 51 | Generate a timeline of activity associated with a user, endpoint, or other 52 | limiting criteria. 53 | 54 | ### usb-util.py 55 | Platforms: Carbon Black (Response) 56 | 57 | Enumerate USB mass storage devices. 58 | 59 | NOTE: Only supports enumeration of devices on Windows endpoints. 60 | -------------------------------------------------------------------------------- /redcanary/responseutils/common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import sys 5 | 6 | 7 | def build_cli_parser(description="Red Canary example script"): 8 | parser = argparse.ArgumentParser(description=description) 9 | 10 | parser.add_argument("--profile", type=str, action="store", 11 | help="The credentials.response profile to use.") 12 | 13 | # File output 14 | parser.add_argument("--prefix", type=str, action="store", 15 | help="Output filename prefix.") 16 | parser.add_argument("--append", type=str, action="store", 17 | help="Append to output file.") 18 | 19 | # Time 20 | parser.add_argument("--days", type=int, action="store", 21 | help="Number of days to search.") 22 | parser.add_argument("--minutes", type=int, action="store", 23 | help="Number of days to search.") 24 | 25 | # Cb Response inputs 26 | cbr = parser.add_mutually_exclusive_group(required=False) 27 | cbr.add_argument("--queryfile", type=str, action="store", 28 | help="File containing queries, one per line.") 29 | cbr.add_argument('--query', type=str, action="store", 30 | help="A single Cb query to execute.") 31 | 32 | return parser 33 | 34 | 35 | def convert_timestamp(datetime_obj): 36 | try: 37 | ret = datetime_obj.strftime('%Y%m%d-%H%M%S') 38 | except ValueError: 39 | ret = '00000000-000000' 40 | 41 | return ret 42 | 43 | 44 | def log_err(msg): 45 | """Format msg as an ERROR and print to stderr. 46 | """ 47 | msg = 'ERROR: {0}\n'.format(msg) 48 | sys.stderr.write(msg) 49 | 50 | 51 | def log_info(msg): 52 | """Format msg and print to stdout. 53 | """ 54 | msg = '{0}\n'.format(msg) 55 | sys.stdout.write(msg) 56 | -------------------------------------------------------------------------------- /cblr-basic.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | 4 | from cbapi.response import BannedHash 5 | from cbapi.response.models import Process, Sensor 6 | from cbapi.response.rest_api import CbEnterpriseResponseAPI 7 | 8 | logging.basicConfig(level=logging.INFO) 9 | 10 | 11 | def main(): 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--hostname", type=str, action="store", 15 | help="Target a specific hostname.") 16 | parser.add_argument("--process-name", type=str, action="store", 17 | help="Name of the process to kill.") 18 | args = parser.parse_args() 19 | 20 | # Connect to Cb Response 21 | cb = CbEnterpriseResponseAPI() 22 | 23 | # Target hostname and process 24 | sensor = cb.select(Sensor).where("hostname:{0}".format(args.hostname))[0] 25 | target_process = args.process_name 26 | 27 | # Isolate sensor 28 | sensor.network_isolation_enabled = True 29 | sensor.save() 30 | 31 | # Initiate Live Response session 32 | cblr = cb.live_response.request_session(sensor.id) 33 | 34 | # Find processes by name, then kill them. 35 | process_list = cblr.list_processes() 36 | target_pids = [proc['pid'] for proc in process_list if target_process in proc['path']] 37 | for pid in target_pids: 38 | cblr.kill_process(pid) 39 | 40 | # Ban the hash 41 | process_list = cb.select(Process).where("process_name:{0}".format(args.process_name)) 42 | target_md5s = set() 43 | for process in process_list: 44 | target_md5s.add(process.process_md5) 45 | 46 | for md5 in target_md5s: 47 | banned_hash = cb.create(BannedHash) 48 | banned_hash.md5hash = md5 49 | banned_hash.text = "Banned by Joe Dirt" 50 | banned_hash.enabled = True 51 | banned_hash.save() 52 | 53 | 54 | if __name__ == '__main__': 55 | 56 | main() 57 | 58 | 59 | -------------------------------------------------------------------------------- /process-util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import csv 5 | import os 6 | import sys 7 | from datetime import datetime 8 | 9 | # Local helpers 10 | from redcanary.responseutils.common import * 11 | 12 | # Carbon Black 13 | from cbapi.response import CbEnterpriseResponseAPI 14 | from cbapi.response.models import Process, Sensor 15 | from cbapi.errors import * 16 | 17 | 18 | if sys.version_info.major >= 3: 19 | _python3 = True 20 | else: 21 | _python3 = False 22 | 23 | 24 | def get_process_details(process): 25 | timestamp = convert_timestamp(process.start) 26 | hostname = process.hostname.lower() 27 | username = process.username.lower() 28 | path = process.path 29 | cmdline = process.cmdline 30 | 31 | try: 32 | process_md5 = process.process_md5 33 | except: 34 | process_md5 = '' 35 | 36 | return [timestamp, 37 | hostname, 38 | username, 39 | path, 40 | cmdline, 41 | process_md5, 42 | process.childproc_count, 43 | process.filemod_count, 44 | process.modload_count, 45 | process.netconn_count, 46 | process.webui_link, 47 | process.parent_name 48 | ] 49 | 50 | 51 | def process_search(cb_conn, query, query_base=None, groupby=None): 52 | if query_base != None: 53 | query += query_base 54 | 55 | query_result = cb_conn.select(Process).where(query) 56 | 57 | if groupby != None: 58 | query_result = query_result.group_by(groupby) 59 | 60 | query_result_len = len(query_result) 61 | log_info('Total results: {0}'.format(query_result_len)) 62 | 63 | results = [] 64 | 65 | try: 66 | process_counter = 0 67 | for process in query_result: 68 | process_counter += 1 69 | if process_counter % 100 == 0: 70 | log_info('Processing {0} of {1}'.format(process_counter, query_result_len)) 71 | 72 | results.append(get_process_details(process)) 73 | except KeyboardInterrupt: 74 | log_info("Caught CTRL-C. Returning what we have . . .") 75 | 76 | return tuple(results) 77 | 78 | 79 | def main(): 80 | parser = build_cli_parser("Process utility") 81 | parser.add_argument("--groupby", type=str, action="store", 82 | help="Group process results") 83 | args = parser.parse_args() 84 | 85 | # BEGIN Common 86 | if args.prefix: 87 | output_filename = '{0}-processes.csv'.format(args.prefix) 88 | else: 89 | output_filename = 'processes.csv' 90 | 91 | if args.append == True or args.queryfile is not None: 92 | file_mode = 'a' 93 | else: 94 | file_mode = 'w' 95 | 96 | if args.days: 97 | query_base = ' start:-{0}m'.format(args.days*1440) 98 | elif args.minutes: 99 | query_base = ' start:-{0}m'.format(args.minutes) 100 | else: 101 | query_base = '' 102 | 103 | if args.profile: 104 | cb = CbEnterpriseResponseAPI(profile=args.profile) 105 | else: 106 | cb = CbEnterpriseResponseAPI() 107 | 108 | if args.groupby: 109 | groupby = args.groupby 110 | else: 111 | groupby = None 112 | 113 | queries = [] 114 | if args.query: 115 | queries.append(args.query) 116 | elif args.queryfile: 117 | with open(args.queryfile, 'r') as f: 118 | for query in f.readlines(): 119 | queries.append(query.strip()) 120 | f.close() 121 | else: 122 | queries.append('') 123 | # END Common 124 | 125 | output_file = open(output_filename, file_mode) 126 | writer = csv.writer(output_file) 127 | writer.writerow(["proc_timestamp", 128 | "proc_hostname", 129 | "proc_username", 130 | "proc_path", 131 | "proc_cmdline", 132 | "proc_md5", 133 | "proc_child_count", 134 | "proc_filemod_count", 135 | "proc_modload_count", 136 | "proc_netconn_count", 137 | "proc_url", 138 | "parent_name" 139 | ]) 140 | 141 | for query in queries: 142 | result_set = process_search(cb, query, query_base, groupby) 143 | 144 | for row in result_set: 145 | if _python3 == False: 146 | row = [col.encode('utf8') if isinstance(col, unicode) else col for col in row] 147 | writer.writerow(row) 148 | 149 | output_file.close() 150 | 151 | 152 | if __name__ == '__main__': 153 | 154 | sys.exit(main()) 155 | -------------------------------------------------------------------------------- /usb-util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | OVERVIEW 5 | 6 | Extract USB mass storage device events from Cb Response. 7 | """ 8 | 9 | import argparse 10 | import csv 11 | import json 12 | import os 13 | import sys 14 | 15 | from redcanary.responseutils.common import * 16 | 17 | from cbapi.response import CbEnterpriseResponseAPI 18 | from cbapi.response.models import Process 19 | 20 | 21 | if sys.version_info.major >= 3: 22 | _python3 = True 23 | else: 24 | _python3 = False 25 | 26 | 27 | # Use these to find registry events of interest. Disk device class is probably 28 | # what you want, but you may also choose to fool with the volume device class 29 | # as well: '{53f5630d-b6bf-11d0-94f2-00a0c91efb8b}' 30 | match_guid = ['{53f56307-b6bf-11d0-94f2-00a0c91efb8b}'] 31 | 32 | search_terms = ["registry\\machine\\system\\currentcontrolset\\control\\deviceclasses\\{53f5630d-b6bf-11d0-94f2-00a0c91efb8b}\\*", 33 | "registry\\machine\\currentcontrolset\\control\\deviceclasses\\{53f56307-b6bf-11d0-94f2-00a0c91efb8b}\\*"] 34 | 35 | 36 | class USBEvent: 37 | def __init__(self, path): 38 | self.path = path 39 | 40 | self.vendor = '' 41 | self.product = '' 42 | self.version = '' 43 | self.serial = '' 44 | #self.drive_letter = '' 45 | #self.volume_name = '' 46 | 47 | self.parse() 48 | 49 | def __repr__(self): 50 | for k,v in self.__dict__.iteritems(): 51 | print('{0},{1}'.format(k, v)) 52 | 53 | def parse(self): 54 | path = self.path.split('usbstor#disk&')[1] 55 | fields = path.split('&') 56 | self.vendor = fields[0].split('ven_')[1] 57 | self.product = fields[1].split('prod_')[1] 58 | 59 | if self.vendor == 'drobo': 60 | # Drobo doesn't provide a version 61 | drobo_fields = self.product.split('#') 62 | self.product = drobo_fields[0] 63 | self.serial = drobo_fields[1] 64 | else: 65 | self.version = fields[2].split('#')[0].split('rev_')[1] 66 | self.serial = fields[2].split('#')[1] 67 | 68 | 69 | def usbstor_search(cb_conn, query, query_base=None, timestamps=False): 70 | if query_base is not None: 71 | query += query_base 72 | 73 | query_result = cb_conn.select(Process).where(query) 74 | query_result_len = len(query_result) 75 | 76 | results = set() 77 | 78 | for proc in query_result: 79 | for regmod in proc.regmods: 80 | #TODO: Convert time boundary (minutes) and check against the 81 | # regmod event time to speed things up 82 | for guid in match_guid: 83 | if guid in regmod.path and 'usbstor#disk&' in regmod.path: 84 | usb_result = USBEvent(regmod.path) 85 | 86 | output_fields = [proc.hostname, 87 | usb_result.vendor, 88 | usb_result.product, 89 | usb_result.version, 90 | usb_result.serial] 91 | if timestamps == True: 92 | output_fields.insert(0, convert_timestamp(regmod.timestamp)) 93 | 94 | results.add(tuple(output_fields)) 95 | 96 | return results 97 | 98 | 99 | def main(): 100 | parser = build_cli_parser("USB utility") 101 | 102 | # Output options 103 | parser.add_argument("--timestamps", action="store_true", 104 | help="Include timestamps in results.") 105 | 106 | args = parser.parse_args() 107 | 108 | if args.queryfile: 109 | sys.exit("queryfile not supported in this utility") 110 | 111 | if args.prefix: 112 | output_filename = '%s-usbstor.csv' % args.prefix 113 | else: 114 | output_filename = 'usbstor.csv' 115 | 116 | if args.profile: 117 | cb = CbEnterpriseResponseAPI(profile=args.profile) 118 | else: 119 | cb = CbEnterpriseResponseAPI() 120 | 121 | output_file = open(output_filename, 'w') 122 | writer = csv.writer(output_file, quoting=csv.QUOTE_ALL) 123 | 124 | header_row = ['endpoint', 'vendor', 'product', 'version', 'serial'] 125 | if args.timestamps == True: 126 | header_row.insert(0, 'timestamp') 127 | writer.writerow(header_row) 128 | 129 | for term in search_terms: 130 | query = 'process_name:ntoskrnl.exe regmod:%s' % term 131 | 132 | if args.days: 133 | query += ' last_update:-%dm' % (args.days*1440) 134 | elif args.minutes: 135 | query += ' last_update:-%dm' % args.minutes 136 | 137 | results = usbstor_search(cb, query, query_base=args.query, timestamps=args.timestamps) 138 | 139 | for row in results: 140 | if _python3 == False: 141 | row = [col.encode('utf8') if isinstance(col, unicode) else col for col in list(row)] 142 | writer.writerow(row) 143 | 144 | output_file.close() 145 | 146 | 147 | if __name__ == '__main__': 148 | 149 | sys.exit(main()) 150 | -------------------------------------------------------------------------------- /sensor-util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | OVERVIEW 5 | 6 | Extract selected sensor information from Cb Response. 7 | """ 8 | 9 | import argparse 10 | import csv 11 | import os 12 | import sys 13 | 14 | from cbapi.response import CbEnterpriseResponseAPI 15 | from cbapi.response.models import Process, Sensor 16 | 17 | 18 | if sys.version_info.major >= 3: 19 | _python3 = True 20 | else: 21 | _python3 = False 22 | 23 | 24 | def log_err(msg): 25 | """Format msg as an ERROR and print to stderr. 26 | """ 27 | msg = 'ERROR: {0}\n'.format(msg) 28 | sys.stderr.write(msg) 29 | 30 | 31 | def log_info(msg): 32 | """Format msg and print to stdout. 33 | """ 34 | msg = '{0}\n'.format(msg) 35 | sys.stdout.write(msg) 36 | 37 | 38 | def main(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument("--profile", type=str, action="store", 41 | help="The credentials.response profile to use.") 42 | 43 | # File output 44 | parser.add_argument("--prefix", type=str, action="store", 45 | help="Output filename prefix.") 46 | 47 | # Cb Response Sensor query paramaters 48 | s = parser.add_mutually_exclusive_group(required=False) 49 | s.add_argument("--group-id", type=int, action="store", 50 | help="Target sensor group based on numeric ID.") 51 | s.add_argument("--hostname", type=str, action="store", 52 | help="Target sensor matching hostname.") 53 | s.add_argument("--ip", type=str, action="store", 54 | help="Target sensor matching IP address (dotted quad).") 55 | 56 | # Health checking 57 | parser.add_argument("--process-count", action="store_true", 58 | help="Count processes associated with this sensor.") 59 | parser.add_argument("--tamper-count", action="store_true", 60 | help="Count tamper events associated with this sensor.") 61 | 62 | parser.add_argument("--checkin-ip", action="store_true", 63 | help="Return the latest public IP associated with the sensor.") 64 | 65 | args = parser.parse_args() 66 | 67 | if args.prefix: 68 | output_filename = '%s-sensors.csv' % args.prefix 69 | else: 70 | output_filename = 'sensors.csv' 71 | 72 | if args.profile: 73 | cb = CbEnterpriseResponseAPI(profile=args.profile) 74 | else: 75 | cb = CbEnterpriseResponseAPI() 76 | 77 | output_file = open(output_filename, 'w') 78 | writer = csv.writer(output_file, quoting=csv.QUOTE_ALL) 79 | 80 | header_row = ['computer_name', 81 | 'computer_dns_name', 82 | 'sensor_group_id', 83 | 'os', 84 | 'os_type', 85 | 'computer_sid', 86 | 'last_checkin_time', 87 | 'registration_time', 88 | 'network_adapters', 89 | 'id', 90 | 'group_id', 91 | 'group_name', 92 | 'num_eventlog_mb', 93 | 'num_storefiles_mb', 94 | 'systemvolume_free_size', 95 | 'systemvolume_total_size', 96 | 'health', 97 | 'commit_charge_mb', 98 | 'build_version_string', 99 | 'process_count', 100 | 'tamper_count', 101 | 'clock_delta', 102 | 'checkin_ip'] 103 | writer.writerow(header_row) 104 | 105 | query_base = None 106 | if args.group_id: 107 | query_base = 'groupid:{0}'.format(args.group_id) 108 | elif args.hostname: 109 | query_base = 'hostname:{0}'.format(args.hostname) 110 | elif args.ip: 111 | query_base = 'ip:{0}'.format(args.ip) 112 | 113 | if query_base is None: 114 | sensors = cb.select(Sensor) 115 | else: 116 | sensors = cb.select(Sensor).where(query_base) 117 | 118 | num_sensors = len(sensors) 119 | log_info("Found {0} sensors".format(num_sensors)) 120 | 121 | counter = 1 122 | for sensor in sensors: 123 | if counter % 10 == 0: 124 | print("{0} of {1}".format(counter, num_sensors)) 125 | 126 | if len(sensor.resource_status) > 0: 127 | commit_charge = "{0:.2f}".format(float(sensor.resource_status[0]['commit_charge'])/1024/1024) 128 | else: 129 | commit_charge = '' 130 | num_eventlog_mb = "{0:.2f}".format(float(sensor.num_eventlog_bytes)/1024/1024) 131 | num_storefiles_mb = "{0:.2f}".format(float(sensor.num_storefiles_bytes)/1024/1024) 132 | systemvolume_free_size = "{0:.2f}".format(float(sensor.systemvolume_free_size)/1024/1024) 133 | systemvolume_total_size = "{0:.2f}".format(float(sensor.systemvolume_total_size)/1024/1024) 134 | 135 | if args.process_count == True: 136 | process_count = len(cb.select(Process).where('sensor_id:{0}'.format(sensor.id))) 137 | else: 138 | process_count = '' 139 | 140 | if args.checkin_ip == True: 141 | try: 142 | checkin_ip = cb.select(Process).where('sensor_id:{0}'.format(sensor.id)).first().comms_ip 143 | except AttributeError: 144 | checkin_ip = '' 145 | else: 146 | checkin_ip = '' 147 | 148 | if args.tamper_count == True: 149 | tamper_count = len(cb.select(Process).where('tampered:true AND sensor_id:{0}'.format(sensor.id))) 150 | else: 151 | tamper_count = '' 152 | 153 | output_fields = [sensor.computer_name.lower(), 154 | sensor.computer_dns_name.lower(), 155 | sensor.group_id, 156 | sensor.os, 157 | sensor.os_type, 158 | sensor.computer_sid, 159 | sensor.last_checkin_time, 160 | sensor.registration_time, 161 | sensor.network_adapters, 162 | sensor.id, 163 | sensor.group_id, 164 | sensor.group.name, 165 | num_eventlog_mb, 166 | num_storefiles_mb, 167 | systemvolume_free_size, 168 | systemvolume_total_size, 169 | sensor.sensor_health_message, 170 | commit_charge, 171 | sensor.build_version_string, 172 | process_count, 173 | tamper_count, 174 | sensor.clock_delta, 175 | checkin_ip] 176 | 177 | if _python3 == False: 178 | row = [col.encode('utf8') if isinstance(col, unicode) else col for col in output_fields] 179 | else: 180 | row = output_fields 181 | writer.writerow(row) 182 | 183 | counter += 1 184 | 185 | output_file.close() 186 | 187 | 188 | if __name__ == '__main__': 189 | 190 | sys.exit(main()) 191 | -------------------------------------------------------------------------------- /timeline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import csv 5 | import os 6 | import sys 7 | from datetime import datetime 8 | 9 | # Red Canary 10 | from redcanary.responseutils.common import * 11 | 12 | # Carbon Black 13 | from cbapi.response import CbEnterpriseResponseAPI 14 | from cbapi.response.models import Process, Sensor 15 | from cbapi.errors import * 16 | 17 | 18 | if sys.version_info.major >= 3: 19 | _python3 = True 20 | else: 21 | _python3 = False 22 | 23 | 24 | def process_search(cb_conn, query, query_base=None, 25 | filemods=None, netconns=None, 26 | processes=None, regmods=None): 27 | 28 | if query_base != None: 29 | query += query_base 30 | 31 | log_info("QUERY: {0}".format(query)) 32 | 33 | query_result = cb_conn.select(Process).where(query).group_by("id") 34 | query_result_len = len(query_result) 35 | log_info('Total results: {0}'.format(query_result_len)) 36 | 37 | results = [] 38 | 39 | try: 40 | process_counter = 0 41 | for proc in query_result: 42 | process_counter += 1 43 | if process_counter % 10 == 0: 44 | log_info('Processing {0} of {1}'.format(process_counter, query_result_len)) 45 | 46 | hostname = proc.hostname.lower() 47 | username = proc.username.lower() 48 | path = proc.path 49 | cmdline = proc.cmdline 50 | 51 | try: 52 | process_md5 = path.process_md5 53 | except: 54 | process_md5 = '' 55 | 56 | parent_name = proc.parent_name 57 | 58 | if processes == True: 59 | results.append(('process', 60 | convert_timestamp(proc.start), 61 | hostname, 62 | username, 63 | path, 64 | cmdline, 65 | process_md5, 66 | parent_name, 67 | proc.childproc_count, 68 | proc.webui_link 69 | )) 70 | 71 | if netconns == True: 72 | for netconn in proc.all_netconns(): 73 | results.append(('netconn', 74 | convert_timestamp(netconn.timestamp), 75 | hostname, 76 | username, 77 | path, 78 | cmdline, 79 | process_md5, 80 | parent_name, 81 | proc.childproc_count, 82 | proc.webui_link, 83 | netconn.domain, 84 | netconn.remote_ip, 85 | netconn.remote_port, 86 | netconn.local_ip, 87 | netconn.local_port, 88 | netconn.proto, 89 | netconn.direction 90 | )) 91 | 92 | if filemods == True: 93 | for filemod in proc.all_filemods(): 94 | results.append(('filemod', 95 | convert_timestamp(filemod.timestamp), 96 | hostname, 97 | username, 98 | path, 99 | cmdline, 100 | process_md5, 101 | parent_name, 102 | proc.childproc_count, 103 | proc.webui_link, 104 | '','','','','','','', # netconn 105 | filemod.path, 106 | filemod.type, 107 | filemod.md5 108 | )) 109 | 110 | if regmods == True: 111 | for regmod in proc.all_regmods(): 112 | results.append(('regmod', 113 | convert_timestamp(regmod.timestamp), 114 | hostname, 115 | username, 116 | path, 117 | cmdline, 118 | process_md5, 119 | parent_name, 120 | proc.childproc_count, 121 | proc.webui_link, 122 | '','','','','','','', # netconn 123 | '','','', # filemod 124 | regmod.path, 125 | regmod.type 126 | )) 127 | 128 | 129 | except KeyboardInterrupt: 130 | log_info("Caught CTRL-C. Returning what we have . . .") 131 | 132 | return results 133 | 134 | 135 | def main(): 136 | parser = build_cli_parser("Timeline utility") 137 | 138 | # Output options 139 | output_events = parser.add_argument_group('output_events', 140 | "If any output type is set, all other types will be suppressed unless they are explicitly set as well.") 141 | output_events.add_argument("--filemods", action="store_true", 142 | help="Output file modification records.") 143 | output_events.add_argument("--netconns", action="store_true", 144 | help="Output network connection records.") 145 | output_events.add_argument("--processes", action="store_true", 146 | help="Output process start records.") 147 | output_events.add_argument("--regmods", action="store_true", 148 | help="Output registry modification records.") 149 | 150 | args = parser.parse_args() 151 | 152 | if args.prefix: 153 | filename = '{0}-timeline.csv'.format(args.prefix) 154 | else: 155 | filename = 'timeline.csv' 156 | 157 | if args.append == True or args.queryfile is not None: 158 | file_mode = 'a' 159 | else: 160 | file_mode = 'w' 161 | 162 | if args.days: 163 | query_base = ' start:-{0}m'.format(args.days*1440) 164 | elif args.minutes: 165 | query_base = ' start:-{0}m'.format(args.minutes) 166 | else: 167 | query_base = '' 168 | 169 | # This is horrible. All are False by default. If all are False, then set 170 | # all to True. If any are set to True, then evaluate each independently. 171 | # If you're reading this and know of a cleaner way to do this, ideally via 172 | # argparse foolery, by all means . . . 173 | if args.filemods == False and \ 174 | args.netconns == False and \ 175 | args.processes == False and \ 176 | args.regmods == False: 177 | (filemods, netconns, processes, regmods) = (True, True, True, True) 178 | else: 179 | filemods = args.filemods 180 | netconns = args.netconns 181 | processes = args.processes 182 | regmods = args.regmods 183 | 184 | if args.profile: 185 | cb = CbEnterpriseResponseAPI(profile=args.profile) 186 | else: 187 | cb = CbEnterpriseResponseAPI() 188 | 189 | queries = [] 190 | if args.query: 191 | queries.append(args.query) 192 | elif args.queryfile: 193 | with open(args.queryfile, 'r') as f: 194 | for query in f.readlines(): 195 | queries.append(query.strip()) 196 | f.close() 197 | else: 198 | queries.append('') 199 | 200 | file = open(filename, file_mode) 201 | writer = csv.writer(file) 202 | writer.writerow(["event_type", 203 | "timestamp", 204 | "hostname", 205 | "username", 206 | "path", 207 | "cmdline", 208 | "process_md5", 209 | "parent", 210 | "childproc_count", 211 | "url", 212 | "netconn_domain", 213 | "netconn_remote_ip", 214 | "netconn_remote_port", 215 | "netconn_local_ip", 216 | "netconn_local_port", 217 | "netconn_proto", 218 | "netconn_direction", 219 | "filemod_path", 220 | "filemod_type", 221 | "filemod_md5", 222 | "regmod_path", 223 | "regmod_type" 224 | ]) 225 | 226 | for query in queries: 227 | result_set = process_search(cb, query, query_base, filemods, netconns, 228 | processes, regmods) 229 | 230 | for row in result_set: 231 | if _python3 == False: 232 | row = [col.encode('utf8') if isinstance(col, unicode) else col for col in row] 233 | writer.writerow(row) 234 | 235 | file.close() 236 | 237 | 238 | if __name__ == '__main__': 239 | 240 | sys.exit(main()) 241 | -------------------------------------------------------------------------------- /network-util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | OVERVIEW 5 | 6 | Extract network connection data from Cb Response based on one or more 7 | criteria: 8 | 9 | - Time (past N minutes, days) 10 | - CbR query match 11 | - CbR query whitelist 12 | - IP whitelist w/ one or more of: specific addresses, loopback, multicast. 13 | - Host type: workstation or server 14 | 15 | Results are written to a file named netconns.csv. 16 | 17 | USE CASES AND EXAMPLES 18 | 19 | --> Show network connections within the past hour 20 | 21 | ./netconn-util.py --minutes 60 22 | 23 | --> Show the above, but do not inspect any web browser processes 24 | 25 | ./netconn-util.py --minutes 60 --whitelist whitelist.browsers 26 | 27 | --> Show network connections associated with a given user 28 | 29 | ./netconn-util.py --minutes 60 --query 'username:joeuser' 30 | 31 | --> Show network connections from some system processes: 32 | 33 | ./netconn-util.py --minutes 60 --query 'process_name:explorer.exe or 34 | process_name:svchost.exe' 35 | 36 | --> Show inbound network connections 37 | 38 | ./netconn-util.py --minutes 60 --inbound 39 | 40 | --> Show inbound network connections to workstations 41 | 42 | ./netconn-util.py --minutes 60 --inbound --workstations 43 | 44 | WHITELISTS 45 | 46 | Whitelists are text files with one query term per line, all of which will be 47 | added to the query that is eventually executed. Thus, you must be mindful of 48 | whitelist size so as not to exceed the maximum length of a CbER query. 49 | 50 | Whitelists are used as much for performance as they are for accuracy. That 51 | meaning, you can and should use them to avoid iterating over thousands of 52 | network connections for process that do not matter to your inquiry. 53 | 54 | As an example, the contents of whitelist.browsers above may look like: 55 | 56 | -process_name:firefox.exe 57 | -process_name:firefox 58 | -process_name:chrome.exe 59 | -process_name:chrome 60 | -process_name:iexplore.exe 61 | 62 | CREDITS 63 | 64 | - Many early improvements from TS. 65 | """ 66 | 67 | import argparse 68 | import csv 69 | import ipaddress 70 | import os 71 | import re 72 | import sys 73 | from datetime import datetime 74 | 75 | # Red Canary 76 | from redcanary.responseutils.common import * 77 | 78 | from cbapi.response import CbEnterpriseResponseAPI 79 | from cbapi.response.models import Process, Sensor 80 | 81 | 82 | if sys.version_info.major >= 3: 83 | _python3 = True 84 | else: 85 | _python3 = False 86 | 87 | 88 | def build_whitelist(filename): 89 | f = open(filename, 'rb') 90 | terms = f.readlines() 91 | f.close() 92 | 93 | whitelist = '' 94 | for term in terms: 95 | whitelist += ' {0}'.format(term.strip()) 96 | 97 | return whitelist 98 | 99 | 100 | def get_hosts(filename): 101 | f = open(filename, 'rb') 102 | hosts = f.readlines() 103 | f.close() 104 | 105 | filtered_hosts = set() 106 | for host in hosts: 107 | host = host.strip() 108 | 109 | # Filter out invalid IPs 110 | if not re.match(r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$', host): 111 | continue 112 | 113 | filtered_hosts.add(host) 114 | 115 | return filtered_hosts 116 | 117 | 118 | def process_search(cb_conn, query, query_base=None, limit=None, 119 | direction=None, loopback=None, ignore_hosts=None, 120 | ignore_private_dest=False, 121 | multicast=None, tcp=True, udp=True, 122 | domain=None): 123 | 124 | re_multicast = re.compile(r'2(?:2[4-9]|3\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]\d?|0)){3}') 125 | 126 | if query_base != None: 127 | query += query_base 128 | 129 | log_info("QUERY: {0}".format(query)) 130 | query_result = cb_conn.select(Process).where(query).group_by("id") 131 | query_result_len = len(query_result) 132 | log_info("RESULT COUNT: {0}".format(query_result_len)) 133 | 134 | results = set() 135 | 136 | try: 137 | process_counter = 0 138 | for proc in query_result: 139 | process_counter += 1 140 | if process_counter % 100 == 0: 141 | log_info('Processing {0} of {1} results'.format(process_counter, query_result_len)) 142 | 143 | if proc.netconn_count == 0: 144 | continue 145 | 146 | path = proc.path 147 | hostname = proc.hostname.lower() 148 | username = proc.username.lower() 149 | 150 | inspect_counter = 0 151 | for netconn in proc.all_netconns(): 152 | inspect_counter += 1 153 | if inspect_counter > limit: 154 | log_info("Reached inspect-limit ({0})".format(limit)) 155 | break 156 | 157 | if netconn.local_ip != "": 158 | src_ip = ipaddress.ip_address(netconn.local_ip) 159 | if netconn.remote_ip != "": 160 | dst_ip = ipaddress.ip_address(netconn.remote_ip) 161 | 162 | if direction is not None and direction != netconn.direction: 163 | continue 164 | elif ignore_private_dest == True and dst_ip.is_private == True: 165 | continue 166 | elif tcp == False and netconn.proto == 'IPPROTO_TCP': 167 | continue 168 | elif udp == False and netconn.proto == 'IPPROTO_UDP': 169 | continue 170 | elif loopback == False and (src_ip.is_loopback == True or \ 171 | dst_ip.is_loopback == True): 172 | continue 173 | elif ignore_hosts and netconn.remote_ip in ignore_hosts: 174 | continue 175 | elif multicast == False and src_ip.is_multicast == True: 176 | continue 177 | elif domain is not None and domain not in netconn.domain: 178 | continue 179 | 180 | results.add((convert_timestamp(netconn.timestamp), 181 | path, 182 | hostname, 183 | username, 184 | netconn.domain, 185 | netconn.proto, 186 | netconn.direction, 187 | netconn.local_ip, 188 | netconn.local_port, 189 | netconn.remote_ip, 190 | netconn.remote_port, 191 | proc.webui_link 192 | )) 193 | except KeyboardInterrupt: 194 | print("Caught CTRL-C. Returning what we have . . .") 195 | 196 | return results 197 | 198 | 199 | def main(): 200 | parser = build_cli_parser("Network utility") 201 | 202 | # Non-exlusive query terms. Note that we're passing them this way because 203 | # we can't risk the user passing terms that Cb can't search (i.e., a 204 | # process-level term plus an event-level term). These are joined by AND, 205 | # not OR. 206 | parser.add_argument("--hostname", type=str, action="store", 207 | help="Search for hostname") 208 | parser.add_argument("--username", type=str, action="store", 209 | help="Search for username") 210 | 211 | # Whitelist conditions 212 | parser.add_argument("--whitelist", type=str, action="store", 213 | help="Path to whitelist file.") 214 | parser.add_argument("--ignore-hosts", type=str, action="store", 215 | help="Path to file listing IPs to ignore traffic to/from.") 216 | parser.add_argument("--noloopback", action="store_false", 217 | help="Ignore connections to and from 127.0.0.1.") 218 | parser.add_argument("--nomulticast", action="store_false", 219 | help="Ignore multicast connections") 220 | parser.add_argument("--ignore-private-dest", action="store_true", 221 | help="Ignore connections to RFC1918 networks.") 222 | 223 | # Traffic attributes 224 | d = parser.add_mutually_exclusive_group(required=False) 225 | d.add_argument("--inbound", action="store_true", 226 | help="Report only inbound netconns.") 227 | d.add_argument('--outbound', action="store_true", 228 | help="Report only outbound netconns.") 229 | 230 | p = parser.add_mutually_exclusive_group(required=False) 231 | p.add_argument("--tcp", action="store_true", 232 | help="Report only UDP netconns.") 233 | p.add_argument('--udp', action="store_true", 234 | help="Report only TCP netconns.") 235 | 236 | # Endpoint attributes 237 | t = parser.add_mutually_exclusive_group(required=False) 238 | t.add_argument("--workstations", action="store_true", 239 | help="Only process workstations.") 240 | t.add_argument("--servers", action="store_true", 241 | help="Only process servers.") 242 | 243 | # Query and inspection limiting 244 | parser.add_argument("--inspect-limit", dest="inspect_limit", type=int, 245 | action="store", default="5000", 246 | help="Limit netconns per process that we inspect (default: 5000.") 247 | 248 | # Shortcuts for speed 249 | parser.add_argument("--domain", dest="domain", action="store", 250 | help="Quick search for only those events with a domain match.") 251 | parser.add_argument("--port", dest="port", action="store", 252 | help="Quick search for only those events involving a specific port.") 253 | parser.add_argument("--ipaddr", dest="ipaddr", action="store", 254 | help="Quick search for only those events with an IP match.") 255 | 256 | args = parser.parse_args() 257 | 258 | if args.prefix: 259 | output_filename = '%s-netconns.csv' % args.prefix 260 | else: 261 | output_filename = 'netconns.csv' 262 | 263 | if args.append == True or args.queryfile is not None: 264 | file_mode = 'a' 265 | else: 266 | file_mode = 'w' 267 | 268 | # Query buildup 269 | if args.days: 270 | query_base = ' start:-%dm' % (args.days*1440) 271 | elif args.minutes: 272 | query_base = ' start:-%dm' % args.minutes 273 | else: 274 | query_base = '' 275 | 276 | if args.servers: 277 | query_base += ' (host_type:"domain_controller" OR host_type:"server")' 278 | elif args.workstations: 279 | query_base += ' host_type:"workstation"' 280 | 281 | if args.hostname: 282 | query_base += ' hostname:{0}'.format(args.hostname) 283 | if args.username: 284 | query_base += ' username:{0}'.format(args.username) 285 | 286 | if args.whitelist: 287 | query_base += build_whitelist(args.whitelist) 288 | 289 | if args.ignore_hosts: 290 | ignore_hosts = get_hosts(args.ignore_hosts) 291 | 292 | if args.domain: 293 | query_base += ' domain:%s' % args.domain 294 | elif args.ipaddr: 295 | query_base += ' ipaddr:%s' % args.ipaddr 296 | elif args.port: 297 | query_base += ' ipport:%s' % args.port 298 | else: 299 | query_base += ' netconn_count:[1 TO *]' 300 | 301 | if args.inbound: 302 | direction = 'Inbound' 303 | elif args.outbound: 304 | direction = 'Outbound' 305 | else: 306 | direction = None 307 | 308 | udp = True 309 | tcp = True 310 | if args.tcp and not args.udp: 311 | udp = False 312 | elif args.udp and not args.tcp: 313 | tcp = False 314 | 315 | # Connect and stage queries 316 | if args.profile: 317 | cb = CbEnterpriseResponseAPI(profile=args.profile) 318 | else: 319 | cb = CbEnterpriseResponseAPI() 320 | 321 | # TODO - Update this routine to guard against impossible queries. 322 | queries = [] 323 | if args.query: 324 | queries.append(args.query) 325 | elif args.queryfile: 326 | with open(args.queryfile, 'r') as f: 327 | for query in f.readlines(): 328 | if ':' in query: 329 | queries.append(query.strip()) 330 | f.close() 331 | else: 332 | queries.append('') 333 | 334 | # Main routine and output 335 | output_file = open(output_filename, file_mode) 336 | writer = csv.writer(output_file) 337 | if args.append is False: 338 | writer.writerow(["timestamp", 339 | "path", 340 | "hostname", 341 | "username", 342 | "domain", 343 | "proto", 344 | "direction", 345 | "local_ip", 346 | "local_port", 347 | "remote_ip", 348 | "remote_port"]) 349 | 350 | for query in queries: 351 | result_set = process_search(cb, query, query_base, 352 | limit=args.inspect_limit, 353 | direction=direction, 354 | loopback=args.noloopback, 355 | ignore_hosts=args.ignore_hosts, 356 | ignore_private_dest=args.ignore_private_dest, 357 | multicast=args.nomulticast, 358 | tcp=tcp, udp=udp, 359 | domain=args.domain) 360 | 361 | for r in result_set: 362 | row = list(r) 363 | if _python3 == False: 364 | row = [col.encode('utf8') if isinstance(col, unicode) else col for col in row] 365 | writer.writerow(row) 366 | 367 | output_file.close() 368 | 369 | 370 | if __name__ == '__main__': 371 | 372 | sys.exit(main()) 373 | --------------------------------------------------------------------------------