├── .dockerignore ├── Procfile ├── .gitignore ├── images ├── newapp.png ├── newbot.png ├── aws_iam.png ├── bot_check.png ├── copytoken.png ├── bot_health.png ├── meraki_key.png ├── postman_org.png ├── a4e_new_token.png ├── a4e_show_token.png ├── aws_iam_users.png ├── enterdetails.png ├── meraki_clients.png ├── meraki_combine.png ├── meraki_profile.png ├── a4e_create_token.png ├── a4e_token_config.png ├── aws_iam_adduser.png ├── aws_s3_lifecycle.png ├── meraki_networks.png ├── spark_call_users.png ├── spark_get_token.png ├── umbrella_roaming.png ├── aws_s3_lifecycle_1.png ├── aws_s3_lifecycle_2.png ├── meraki_all_networks.png ├── meraki_client_show.png ├── meraki_sm_clients.png ├── aws_s3_delete_readme.png ├── meraki_client_rename.png ├── meraki_client_select.png ├── meraki_sm_client_save.png ├── meraki_sm_client_show.png ├── meraki_sm_client_tag.png ├── spark_call_user_show.png ├── aws_iam_adduser_complete.png ├── aws_iam_adduser_review.png ├── meraki_combine_confirm.png ├── meraki_enable_api_access.png ├── meraki_sm_client_rename.png ├── meraki_sm_client_select.png ├── spark_call_select_user.png ├── spark_call_user_select.png ├── umbrella_roaming_rename.png ├── umbrella_roaming_select.png ├── aws_iam_adduser_permissions.png └── spark_call_verify_privileges.png ├── Dockerfile ├── docker-compose.yml ├── start.sh ├── requirements.txt ├── .env.sample ├── LICENSE ├── app.json ├── cico_common.py ├── cico_a4e.py ├── umbrella_log_collector.py ├── app.py ├── cico_umbrella.py ├── meraki_dashboard_link_parser.py ├── cico_spark_call.py ├── cico_combined.py ├── README.md └── cico_meraki.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .env -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: python3 app.py 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | venv 3 | .idea/ 4 | __pycache__/ 5 | -------------------------------------------------------------------------------- /images/newapp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/newapp.png -------------------------------------------------------------------------------- /images/newbot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/newbot.png -------------------------------------------------------------------------------- /images/aws_iam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_iam.png -------------------------------------------------------------------------------- /images/bot_check.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/bot_check.png -------------------------------------------------------------------------------- /images/copytoken.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/copytoken.png -------------------------------------------------------------------------------- /images/bot_health.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/bot_health.png -------------------------------------------------------------------------------- /images/meraki_key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_key.png -------------------------------------------------------------------------------- /images/postman_org.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/postman_org.png -------------------------------------------------------------------------------- /images/a4e_new_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/a4e_new_token.png -------------------------------------------------------------------------------- /images/a4e_show_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/a4e_show_token.png -------------------------------------------------------------------------------- /images/aws_iam_users.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_iam_users.png -------------------------------------------------------------------------------- /images/enterdetails.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/enterdetails.png -------------------------------------------------------------------------------- /images/meraki_clients.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_clients.png -------------------------------------------------------------------------------- /images/meraki_combine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_combine.png -------------------------------------------------------------------------------- /images/meraki_profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_profile.png -------------------------------------------------------------------------------- /images/a4e_create_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/a4e_create_token.png -------------------------------------------------------------------------------- /images/a4e_token_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/a4e_token_config.png -------------------------------------------------------------------------------- /images/aws_iam_adduser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_iam_adduser.png -------------------------------------------------------------------------------- /images/aws_s3_lifecycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_s3_lifecycle.png -------------------------------------------------------------------------------- /images/meraki_networks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_networks.png -------------------------------------------------------------------------------- /images/spark_call_users.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/spark_call_users.png -------------------------------------------------------------------------------- /images/spark_get_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/spark_get_token.png -------------------------------------------------------------------------------- /images/umbrella_roaming.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/umbrella_roaming.png -------------------------------------------------------------------------------- /images/aws_s3_lifecycle_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_s3_lifecycle_1.png -------------------------------------------------------------------------------- /images/aws_s3_lifecycle_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_s3_lifecycle_2.png -------------------------------------------------------------------------------- /images/meraki_all_networks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_all_networks.png -------------------------------------------------------------------------------- /images/meraki_client_show.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_client_show.png -------------------------------------------------------------------------------- /images/meraki_sm_clients.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_sm_clients.png -------------------------------------------------------------------------------- /images/aws_s3_delete_readme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_s3_delete_readme.png -------------------------------------------------------------------------------- /images/meraki_client_rename.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_client_rename.png -------------------------------------------------------------------------------- /images/meraki_client_select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_client_select.png -------------------------------------------------------------------------------- /images/meraki_sm_client_save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_sm_client_save.png -------------------------------------------------------------------------------- /images/meraki_sm_client_show.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_sm_client_show.png -------------------------------------------------------------------------------- /images/meraki_sm_client_tag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_sm_client_tag.png -------------------------------------------------------------------------------- /images/spark_call_user_show.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/spark_call_user_show.png -------------------------------------------------------------------------------- /images/aws_iam_adduser_complete.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_iam_adduser_complete.png -------------------------------------------------------------------------------- /images/aws_iam_adduser_review.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_iam_adduser_review.png -------------------------------------------------------------------------------- /images/meraki_combine_confirm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_combine_confirm.png -------------------------------------------------------------------------------- /images/meraki_enable_api_access.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_enable_api_access.png -------------------------------------------------------------------------------- /images/meraki_sm_client_rename.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_sm_client_rename.png -------------------------------------------------------------------------------- /images/meraki_sm_client_select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/meraki_sm_client_select.png -------------------------------------------------------------------------------- /images/spark_call_select_user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/spark_call_select_user.png -------------------------------------------------------------------------------- /images/spark_call_user_select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/spark_call_user_select.png -------------------------------------------------------------------------------- /images/umbrella_roaming_rename.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/umbrella_roaming_rename.png -------------------------------------------------------------------------------- /images/umbrella_roaming_select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/umbrella_roaming_select.png -------------------------------------------------------------------------------- /images/aws_iam_adduser_permissions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/aws_iam_adduser_permissions.png -------------------------------------------------------------------------------- /images/spark_call_verify_privileges.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meraki/spark-operations-bot/HEAD/images/spark_call_verify_privileges.png -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt ./ 6 | RUN pip install --no-cache-dir -r requirements.txt 7 | 8 | COPY . /app 9 | 10 | CMD [ "python3", "app.py" ] -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | sparkopsbot: 4 | ports: 5 | - "5000:5000" 6 | image: joshand/spark-operations-bot 7 | container_name: spark-operations-bot 8 | env_file: 9 | - .env -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # pull down docker image 4 | docker pull joshand/spark-operations-bot 5 | 6 | # make sure docker image is not already running 7 | docker-compose down 8 | 9 | # start docker image 10 | docker-compose up -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | APScheduler==3.4.0 2 | Flask==0.12.2 3 | Jinja2==2.9.6 4 | MarkupSafe==1.0 5 | Werkzeug==0.12.2 6 | boto3==1.4.7 7 | botocore==1.7.26 8 | certifi==2017.7.27.1 9 | chardet==3.0.4 10 | ciscosparkapi==0.10.1 11 | ciscosparkbot==0.6.3 12 | click==6.7 13 | docutils==0.14 14 | future==0.16.0 15 | gevent==1.2.2 16 | greenlet==0.4.12 17 | grequests==0.3.0 18 | idna==2.5 19 | itsdangerous==0.24 20 | jmespath==0.9.3 21 | python-dateutil==2.6.1 22 | requests==2.18.4 23 | requests-toolbelt==0.8.0 24 | s3transfer==0.1.11 25 | six==1.11.0 26 | urllib3==1.22 27 | wheel==0.30.0 -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | //SPARK_BOT_URL=https://mypublicsite.io *(your public facing webserver, or the forwarding address from ngrok)* 2 | //SPARK_BOT_TOKEN= 3 | //SPARK_BOT_EMAIL= 4 | //SPARK_BOT_APP_NAME= 5 | //SPARK_BOT_HELP_MSG= 6 | //MERAKI_API_TOKEN= 7 | //MERAKI_ORG= 8 | //MERAKI_HTTP_USERNAME= 9 | //MERAKI_HTTP_PASSWORD= 10 | //SPARK_API_TOKEN= 11 | //S3_BUCKET= 12 | //S3_ACCESS_KEY_ID= 13 | //S3_SECRET_ACCESS_KEY= 14 | //A4E_CLIENT_ID= 15 | //A4E_CLIENT_SECRET= -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 imapex 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Cisco Spark Operations Bot", 3 | "description": "Cisco Spark Operations Bot", 4 | "repository": "https://github.com/meraki/spark-operations-bot", 5 | "addons": [], 6 | "env": { 7 | "SPARK_BOT_URL": { 8 | "description": "The address at which your bot can be reached. The public address of your Heroku app will be 'https://.herokuapp.com", 9 | "required": true 10 | }, 11 | "SPARK_BOT_TOKEN": { 12 | "description": "The bot's access token from Cisco Spark. Create a new bot account at https://developer.ciscospark.com/add-bot.html if you do not have one.", 13 | "required": true 14 | }, 15 | "SPARK_BOT_EMAIL": { 16 | "description": "The bot's email address from Cisco Spark.", 17 | "required": true 18 | }, 19 | "SPARK_BOT_APP_NAME": { 20 | "description": "The bot's name from Cisco Spark.", 21 | "required": true 22 | }, 23 | "SPARK_BOT_HELP_MSG": { 24 | "description": "(Optional) Used to customize Bot Help Banner", 25 | "required": false 26 | }, 27 | "MERAKI_API_TOKEN": { 28 | "description": "(Required for Meraki Support) The API token from the Meraki Dashboard.", 29 | "required": false 30 | }, 31 | "MERAKI_ORG": { 32 | "description": "(Required for Meraki Support) The Organization ID from the Meraki API.", 33 | "required": false 34 | }, 35 | "MERAKI_HTTP_USERNAME": { 36 | "description": "(Optional for Meraki Support) Meraki Dashboard username. Used to generate cross-launch links from Spark client to Dashboard.", 37 | "required": false 38 | }, 39 | "MERAKI_HTTP_PASSWORD": { 40 | "description": "(Optional for Meraki Support) Meraki Dashboard password. Used to generate cross-launch links from Spark client to Dashboard.", 41 | "required": false 42 | }, 43 | "SPARK_API_TOKEN": { 44 | "description": "(Required for Spark Call Support) API Token for Cisco Spark Call Admin user.", 45 | "required": false 46 | }, 47 | "S3_BUCKET": { 48 | "description": "(Required for Umbrella Support) Amazon S3 Bucket name for Umbrella log import.", 49 | "required": false 50 | }, 51 | "S3_ACCESS_KEY_ID": { 52 | "description": "(Required for Umbrella Support) Amazon S3 Access Key ID for Umbrella log import.", 53 | "required": false 54 | }, 55 | "S3_SECRET_ACCESS_KEY": { 56 | "description": "(Required for Umbrella Support) Amazon S3 Secret Access Key for Umbrella log import.", 57 | "required": false 58 | }, 59 | "A4E_CLIENT_ID": { 60 | "description": "(Required for AMP for Endpoints Support) AMP for Endpoints 3rd Party API Client ID.", 61 | "required": false 62 | }, 63 | "A4E_CLIENT_SECRET": { 64 | "description": "(Required for AMP for Endpoints Support) AMP for Endpoints API Key.", 65 | "required": false 66 | } 67 | }, 68 | "keywords": ["meraki", "spark", "cisco", "ciscospark", "umbrella", "opendns", "bot", "botkit", "python", "a4e", "amp4endpoints", "amp"] 69 | } -------------------------------------------------------------------------------- /cico_common.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This module is specifically for common functions that are expected to be used in multiple places across the 3 | various modules 4 | ''' 5 | import os 6 | 7 | 8 | # ======================================================== 9 | # Load required parameters from environment variables 10 | # ======================================================== 11 | 12 | meraki_api_token = os.getenv("MERAKI_API_TOKEN") 13 | meraki_org = os.getenv("MERAKI_ORG") 14 | meraki_http_username = os.getenv("MERAKI_HTTP_USERNAME") 15 | meraki_http_password = os.getenv("MERAKI_HTTP_PASSWORD") 16 | spark_api_token = os.getenv("SPARK_API_TOKEN") 17 | s3_bucket = os.getenv("S3_BUCKET") 18 | s3_key = os.getenv("S3_ACCESS_KEY_ID") 19 | s3_secret = os.getenv("S3_SECRET_ACCESS_KEY") 20 | a4e_client_id = os.getenv("A4E_CLIENT_ID") 21 | a4e_client_secret = os.getenv("A4E_CLIENT_SECRET") 22 | 23 | 24 | # ======================================================== 25 | # Initialize Program - Function Definitions 26 | # ======================================================== 27 | 28 | 29 | def meraki_support(): 30 | ''' 31 | This function is used to check whether the Meraki environment variables have been set. It will return true 32 | if they have and false if they have not 33 | 34 | :return: true/false based on whether or not Meraki support is available 35 | ''' 36 | if meraki_api_token: # and meraki_org: --removed: org auto-selection has been added, so org not required-- 37 | return True 38 | else: 39 | return False 40 | 41 | 42 | def meraki_dashboard_support(): 43 | ''' 44 | This function is used to check whether the Meraki dashboard environment variables have been set. It will return true 45 | if they have and false if they have not. These variables are optional, and are used to build better cross-launch 46 | links to the dashboard 47 | 48 | :return: true/false based on whether or not Meraki dashboard support is available 49 | ''' 50 | if meraki_http_password and meraki_http_username: 51 | return True 52 | else: 53 | return False 54 | 55 | 56 | def spark_call_support(): 57 | ''' 58 | This function is used to check whether the Spark Call environment variables have been set. It will return true 59 | if they have and false if they have not 60 | 61 | :return: true/false based on whether or not Spark Call support is available 62 | ''' 63 | if spark_api_token: 64 | return True 65 | else: 66 | return False 67 | 68 | 69 | def umbrella_support(): 70 | ''' 71 | This function is used to check whether the Umbrella (S3) environment variables have been set. It will return true 72 | if they have and false if they have not 73 | 74 | :return: true/false based on whether or not Umbrella support is available 75 | ''' 76 | if s3_bucket and s3_key and s3_secret: 77 | return True 78 | else: 79 | return False 80 | 81 | def a4e_support(): 82 | ''' 83 | This function is used to check whether the Amp for Endpoints environment variables have been set. It will return 84 | true if they have and false if they have not 85 | 86 | :return: true/false based on whether or not A4E support is available 87 | ''' 88 | if a4e_client_id and a4e_client_secret: 89 | return True 90 | else: 91 | return False -------------------------------------------------------------------------------- /cico_a4e.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import os 3 | import json 4 | 5 | 6 | a4e_client_id = os.getenv("A4E_CLIENT_ID") 7 | a4e_client_secret = os.getenv("A4E_CLIENT_SECRET") 8 | header = {"Accept-Encoding": "gzip"} 9 | 10 | 11 | def get_a4e_events(): 12 | # Get a list of all A4E events 13 | url = "https://api.amp.cisco.com/v1/events?event_type[]=1090519054&event_type[]=553648143&event_type[]=2164260880&event_type[]=553648145" 14 | evlist = requests.get(url, headers=header, auth=(a4e_client_id, a4e_client_secret)) 15 | evjson = json.loads(evlist.content.decode("utf-8")) 16 | return evjson 17 | 18 | 19 | def get_a4e_health(incoming_msg, rettype): 20 | # Get a list of all events 21 | evjson = get_a4e_events() 22 | totalevents = evjson["metadata"]["results"]["current_item_count"] 23 | 24 | processed_events = 0 25 | threat_detected_count = 0 26 | threat_quarantined_count = 0 27 | threat_quarantine_failed_count = 0 28 | threat_detected_excluded_count = 0 29 | erricon = "" 30 | 31 | retmsg = "

AMP for Endpoints Details:

" 32 | retmsg += "AMP for Endpoints Dashboard
    " 33 | for ev in evjson["data"]: 34 | if ev["event_type_id"] == 1090519054: 35 | threat_detected_count += 1 36 | elif ev["event_type_id"] == 553648143: 37 | threat_quarantined_count += 1 38 | elif ev["event_type_id"] == 2164260880: 39 | threat_quarantine_failed_count += 1 40 | erricon = chr(0x2757) + chr(0xFE0F) 41 | elif ev["event_type_id"] == 553648145: 42 | threat_detected_excluded_count += 1 43 | 44 | processed_events += 1 45 | retmsg += "
  • " + str(threat_detected_count) + " threat(s) detected. (" + str(threat_detected_excluded_count) + " in excluded locations.)
    • " 46 | # retmsg += "
  • " + str(threat_detected_excluded_count) + " threat(s) detected in excluded locations.
    • " 47 | retmsg += "
  • " + str(threat_quarantined_count) + " threat(s) quarantined.
    • " 48 | retmsg += "
  • " + str(threat_quarantine_failed_count) + " threat(s) quarantine failed." + erricon + "
    • " 49 | retmsg += "
Processed " + str(processed_events) + " of " + str(totalevents) + " threat event(s)." 50 | 51 | return retmsg 52 | 53 | 54 | def get_a4e_clients(incoming_msg, rettype): 55 | cmdlist = incoming_msg.text.split(" ") 56 | client_id = cmdlist[len(cmdlist)-1] 57 | 58 | evjson = get_a4e_events() 59 | totalevents = evjson["metadata"]["results"]["current_item_count"] 60 | 61 | processed_events = 0 62 | hostarr = {} 63 | erricon = "" 64 | 65 | for ev in evjson["data"]: 66 | compname = ev["computer"]["hostname"].upper() 67 | if compname not in hostarr: 68 | hostarr[compname] = {"threat_detected_count": 0, "threat_quarantined_count": 0, "threat_quarantine_failed_count": 0, "threat_detected_excluded_count": 0} 69 | 70 | if ev["event_type_id"] == 1090519054: 71 | hostarr[compname]["threat_detected_count"] += 1 72 | elif ev["event_type_id"] == 553648143: 73 | hostarr[compname]["threat_quarantined_count"] += 1 74 | elif ev["event_type_id"] == 2164260880: 75 | hostarr[compname]["threat_quarantine_failed_count"] += 1 76 | erricon = chr(0x2757) + chr(0xFE0F) 77 | elif ev["event_type_id"] == 553648145: 78 | hostarr[compname]["threat_detected_excluded_count"] += 1 79 | 80 | processed_events += 1 81 | 82 | if rettype == "json": 83 | return {"aggregate": {"total_events": totalevents, "processed_events": processed_events}, "clients": hostarr} 84 | else: 85 | retmsg = "

AMP for Endpoints Stats:

    " 86 | for cli in hostarr: 87 | if cli == client_id: 88 | retmsg += "
  • " + str(hostarr[cli]["threat_detected_count"]) + " threat(s) detected. (" + str(hostarr[cli]["threat_detected_excluded_count"]) + " in excluded locations.)
    • " 89 | retmsg += "
  • " + str(hostarr[cli]["threat_quarantined_count"]) + " threat(s) quarantined.
    • " 90 | retmsg += "
  • " + str(hostarr[cli]["threat_quarantine_failed_count"]) + " threat(s) quarantine failed." + erricon + "
    • " 91 | retmsg += "
Processed " + str(processed_events) + " of " + str(totalevents) + " threat event(s)." 92 | 93 | return retmsg 94 | 95 | 96 | def get_a4e_health_html(incoming_msg): 97 | return get_a4e_health(incoming_msg, "html") 98 | 99 | 100 | def get_a4e_clients_html(incoming_msg): 101 | return get_a4e_clients(incoming_msg, "html") 102 | -------------------------------------------------------------------------------- /umbrella_log_collector.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This module is specifically for Umbrella log collection operations. This is for the Amazon S3 API. 3 | ''' 4 | import os 5 | import boto3 6 | from pathlib import Path 7 | 8 | # ======================================================== 9 | # Load required parameters from environment variables 10 | # ======================================================== 11 | 12 | s3_bucket = os.getenv("S3_BUCKET") 13 | s3_key = os.getenv("S3_ACCESS_KEY_ID") 14 | s3_secret = os.getenv("S3_SECRET_ACCESS_KEY") 15 | 16 | if not s3_bucket or not s3_key or not s3_secret: 17 | print("umbrella_log_collector.py - Missing Environment Variable.") 18 | if not s3_bucket: 19 | print("S3_BUCKET") 20 | if not s3_key: 21 | print("S3_ACCESS_KEY_ID") 22 | if not s3_secret: 23 | print("S3_SECRET_ACCESS_KEY") 24 | 25 | # ======================================================== 26 | # Initialize Program - Function Definitions 27 | # ======================================================== 28 | 29 | 30 | def download_dir(client, resource, dist, local='/tmp', bucket=''): 31 | ''' 32 | This code was taken from here: 33 | https://stackoverflow.com/questions/31918960/boto3-to-download-all-files-from-a-s3-bucket 34 | 35 | :param client: 36 | :param resource: 37 | :param dist: 38 | :param local: 39 | :param bucket: 40 | :return: Nothing. This function will download files to the local filesystem. 41 | ''' 42 | 43 | paginator = client.get_paginator('list_objects') 44 | for result in paginator.paginate(Bucket=bucket, Delimiter='/', Prefix=dist): 45 | if result.get('CommonPrefixes') is not None: 46 | for subdir in result.get('CommonPrefixes'): 47 | download_dir(client, resource, subdir.get('Prefix'), local, bucket) 48 | if result.get('Contents') is not None: 49 | for file in result.get('Contents'): 50 | print(file.get('Key')) 51 | if not os.path.exists(os.path.dirname(local + os.sep + file.get('Key'))): 52 | os.makedirs(os.path.dirname(local + os.sep + file.get('Key'))) 53 | 54 | my_file = Path(local + os.sep + file.get('Key')) 55 | if my_file.is_file(): 56 | # already exists, don't download again 57 | pass 58 | else: 59 | resource.meta.client.download_file(bucket, file.get('Key'), local + os.sep + file.get('Key')) 60 | 61 | 62 | def cleanup_files(cl, dist, local='/tmp'): 63 | ''' 64 | Check to see if any files / directories need to be removed from the file system. First, take a list of all objects 65 | in the bucket, then compare to the file system. If anything exists on the filesystem that is not in the object 66 | list, delete. 67 | 68 | :param cl: boto3 client 69 | :param dist: String. Subfolder to clean up. 70 | :param local: String. Base path to clean up. (so it's local/dist or local\dist) 71 | :return: Nothing 72 | ''' 73 | 74 | # Get list of all AWS S3 objects 75 | s3flist = [] 76 | try: 77 | objd = cl.list_objects_v2(Bucket=s3_bucket) 78 | except: 79 | objd = None 80 | print("Error Attempting to load S3 Objects") 81 | 82 | if objd: 83 | for x in objd["Contents"]: 84 | s3flist.append(x["Key"]) 85 | 86 | # Get list of all filesystem objects 87 | try: 88 | flist = os.listdir(local + os.sep + dist) 89 | # Iterate list of fs objects 90 | for fdir in flist: 91 | # Everything in the base path should be a directory. Only continue if this is a directory object 92 | if os.path.isdir(local + os.sep + dist + fdir): 93 | # Check to see how many files are in this directory 94 | flist2 = os.listdir(local + os.sep + dist + fdir) 95 | # If 0 files, delete directory 96 | if len(flist2) == 0: 97 | print("removing empty directory " + local + os.sep + dist + fdir) 98 | os.rmdir(local + os.sep + dist + fdir) 99 | 100 | # Iterate list of files 101 | for fn in flist2: 102 | fpath = dist + fdir + os.sep + fn 103 | # If this file is not in the Amazon S3 object list, then we want to delete 104 | if fpath not in s3flist: 105 | print("delete " + local + os.sep + fpath) 106 | os.remove(local + os.sep + fpath) 107 | except: 108 | print("Error: unable to clean up...") 109 | 110 | 111 | def get_logs(): 112 | ''' 113 | Main entry point for log collection. 114 | 115 | :return: Nothing 116 | ''' 117 | 118 | print("Parsing Umbrella logs...") 119 | cl = boto3.client( 120 | 's3', 121 | aws_access_key_id=s3_key, 122 | aws_secret_access_key=s3_secret, 123 | region_name="us-east-1" 124 | ) 125 | rs = boto3.resource( 126 | 's3', 127 | aws_access_key_id=s3_key, 128 | aws_secret_access_key=s3_secret, 129 | region_name="us-east-1" 130 | ) 131 | 132 | try: 133 | download_dir(cl, rs, 'dnslogs/', '/tmp', s3_bucket) 134 | except: 135 | print("Error Loading Logs...") 136 | 137 | cleanup_files(cl, 'dnslogs/', '/tmp') 138 | # print("Sleeping for 5 minutes...") 139 | # time.sleep(60*5) # Delay for 5 minute (60 seconds * 5 minutes). 140 | 141 | if __name__ == '__main__': 142 | get_logs() -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | This is the main entry point for the bot. All modules will be loaded from here. There are a number of environment 4 | variables that are required for this bot to function. 5 | See the README at https://github.com/meraki/spark-operations-bot 6 | ''' 7 | import os 8 | from ciscosparkbot import SparkBot 9 | import cico_meraki 10 | import cico_spark_call 11 | import cico_combined 12 | import cico_common 13 | import cico_umbrella 14 | import cico_a4e 15 | import sys 16 | import atexit 17 | from apscheduler.schedulers.background import BackgroundScheduler 18 | import umbrella_log_collector 19 | import meraki_dashboard_link_parser 20 | 21 | 22 | # ======================================================== 23 | # Load required parameters from environment variables 24 | # ======================================================== 25 | 26 | # If there is a PORT environment variable, use that to map the Flask port. Used for Heroku. 27 | # If no port set, use default of 5000. 28 | app_port = os.getenv("PORT") 29 | if not app_port: 30 | app_port = 5000 31 | else: 32 | app_port = int(app_port) 33 | 34 | # Load additional environment variables 35 | bot_email = os.getenv("SPARK_BOT_EMAIL") 36 | spark_token = os.getenv("SPARK_BOT_TOKEN") 37 | bot_url = os.getenv("SPARK_BOT_URL") 38 | bot_app_name = os.getenv("SPARK_BOT_APP_NAME") 39 | bot_help = os.getenv("SPARK_BOT_HELP_MSG") 40 | 41 | # If any of the bot environment variables are missing, terminate the application 42 | if not bot_email or not spark_token or not bot_url or not bot_app_name: 43 | print("app.py - Missing Environment Variable.") 44 | if not bot_email: 45 | print("SPARK_BOT_EMAIL") 46 | if not spark_token: 47 | print("SPARK_BOT_TOKEN") 48 | if not bot_url: 49 | print("SPARK_BOT_URL") 50 | if not bot_app_name: 51 | print("SPARK_BOT_APP_NAME") 52 | sys.exit() 53 | 54 | 55 | # ======================================================== 56 | # Monkey Patch Spark Bot send_help method to customize header 57 | # ======================================================== 58 | def new_send_help(self, post_data): 59 | """ 60 | Construct a help message for users. 61 | :param post_data: 62 | :return: 63 | """ 64 | if bot_help: 65 | message = bot_help + "\n" 66 | else: 67 | message = "Hello! " 68 | message += "I understand the following commands: \n" 69 | for c in self.commands.items(): 70 | if c[1]["help"][0] != "*": 71 | message += "* **%s**: %s \n" % (c[0], c[1]["help"]) 72 | return message 73 | 74 | 75 | SparkBot.send_help = new_send_help 76 | 77 | 78 | # ======================================================== 79 | # Initialize Program - Run any pre-flight actions required 80 | # ======================================================== 81 | 82 | # This function is called by the scheduler to download logs from Amazon S3 (for Umbrella) 83 | def job_function(): 84 | umbrella_log_collector.get_logs() 85 | 86 | 87 | # Check to see if a dashboard username and password has been provided. If so, scrape the dashboard to build 88 | # cross-launch resources to use for the bot, otherwise initialize to None 89 | if cico_common.meraki_dashboard_support(): 90 | print("Attempting to resolve Dashboard references...") 91 | dbmap = meraki_dashboard_link_parser.get_meraki_http_info() 92 | cico_meraki.meraki_dashboard_map = dbmap 93 | print("Dbmap=", dbmap) 94 | else: 95 | cico_meraki.meraki_dashboard_map = None 96 | 97 | 98 | # If the Umbrella environment variables (aka Amazon S3) have been configured, enable the job scheduler to run every 99 | # 5 minutes to download logs. 100 | if cico_common.umbrella_support(): 101 | cron = BackgroundScheduler() 102 | 103 | # Explicitly kick off the background thread 104 | cron.start() 105 | job = cron.add_job(job_function, 'interval', minutes=5) 106 | print("Beginning Umbrella Log Collection...") 107 | job_function() 108 | 109 | # Shutdown your cron thread if the web process is stopped 110 | atexit.register(lambda: cron.shutdown(wait=False)) 111 | 112 | # ======================================================== 113 | # Initialize Bot - Register commands and start web server 114 | # ======================================================== 115 | 116 | # Create a new bot 117 | bot = SparkBot(bot_app_name, spark_bot_token=spark_token, 118 | spark_bot_url=bot_url, spark_bot_email=bot_email, default_action="help", debug=True) 119 | 120 | bot.add_command('help', 'Get help.', bot.send_help) 121 | bot.remove_command('/echo') 122 | bot.remove_command('/help') 123 | 124 | # Add bot commands. 125 | # If Meraki environment variables have been enabled, add Meraki-specifc commands. 126 | if cico_common.meraki_support(): 127 | bot.add_command('meraki-health', 'Get health of Meraki environment.', cico_meraki.get_meraki_health_html) 128 | bot.add_command('meraki-check', 'Check Meraki user status.', cico_meraki.get_meraki_clients_html) 129 | # If Spark Call environment variables have been enabled, add Spark Call-specifc commands. 130 | if cico_common.spark_call_support(): 131 | bot.add_command('spark-health', 'Get health of Spark environment.', cico_spark_call.get_spark_call_health_html) 132 | bot.add_command('spark-check', 'Check Spark user status.', cico_spark_call.get_spark_call_clients_html) 133 | # If Umbrella (S3) environment variables have been enabled, add Umbrella-specifc commands. 134 | if cico_common.umbrella_support(): 135 | bot.add_command('umbrella-health', 'Get health of Umbrella envrionment.', cico_umbrella.get_umbrella_health_html) 136 | bot.add_command('umbrella-check', 'Check Umbrella user status.', cico_umbrella.get_umbrella_clients_html) 137 | # If Amp for Endpoints environment variables have been enabled, add A4E-specifc commands. 138 | if cico_common.a4e_support(): 139 | bot.add_command('a4e-health', 'Get health of AMP for Endpoints envrionment.', cico_a4e.get_a4e_health_html) 140 | bot.add_command('a4e-check', 'Check AMP for Endpoints user status.', cico_a4e.get_a4e_clients_html) 141 | # Add generic commands. 142 | bot.add_command('health', 'Get health of entire environment.', cico_combined.get_health) 143 | bot.add_command('check', 'Get user status.', cico_combined.get_clients) 144 | 145 | 146 | # Run Bot 147 | bot.run(host='0.0.0.0', port=app_port) -------------------------------------------------------------------------------- /cico_umbrella.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This module is specifically for Umbrella-related operations. This is for parsing S3 logs as there isn't a 3 | relevant Umbrella API. 4 | ''' 5 | import os 6 | import gzip 7 | import io 8 | from stat import S_ISREG, S_ISDIR, ST_CTIME, ST_MODE 9 | 10 | # ======================================================== 11 | # Load required parameters from environment variables 12 | # ======================================================== 13 | 14 | umbrella_over_dash = os.getenv("UMBRELLA_OVERRIDE_DASHBOARD") 15 | 16 | # ======================================================== 17 | # Initialize Program - Function Definitions 18 | # ======================================================== 19 | 20 | 21 | def parse_umbrella_logs(): 22 | ''' 23 | This function will parse the local Umbrella logs to generate aggregate and user-specific stats 24 | 25 | :return: Dictionary. Dict of all stats found in logs. 26 | ''' 27 | 28 | frstats = {"Aggregate": {"Total": 0}, "Users": {}} 29 | local = "/tmp" 30 | dist = "dnslogs/" 31 | 32 | # Load data from log files. We are sorting from newest to oldest, looking for files in specific directories 33 | try: 34 | entries = (os.path.join(local + os.sep + dist, fn) for fn in os.listdir(local + os.sep + dist)) 35 | except: 36 | return {"Error": "Unable to load Umbrella data from Log Files."} 37 | entries = ((os.stat(path), path) for path in entries) 38 | entries = ((stat[ST_CTIME], path) 39 | for stat, path in entries if S_ISDIR(stat[ST_MODE])) 40 | for cdate, fdir in sorted(entries, reverse=True): 41 | #print(fdir, cdate) 42 | entries2 = (os.path.join(fdir, fn) for fn in os.listdir(fdir)) 43 | entries2 = ((os.stat(path), path) for path in entries2) 44 | entries2 = ((stat[ST_CTIME], path) 45 | for stat, path in entries2 if S_ISREG(stat[ST_MODE])) 46 | for cdate, fn in sorted(entries2, reverse=True): 47 | #print(fn, cdate) 48 | # We have now iterated to a specific file. Open it and read the data in. 49 | with open(fn, 'rb') as fin: 50 | data = io.BytesIO(fin.read()) 51 | 52 | # These files are gzipped, so we will need to decompress 53 | data.seek(0) 54 | decompressedFile = gzip.GzipFile(fileobj=data, mode='rb') 55 | # Read the file data and parse 56 | filedata = decompressedFile.read().decode("utf-8") 57 | # "2017-10-07 00:41:56","hankaaron@ciscodcloudpov.com","hankaaron@ciscodcloudpov.com","108.221.201.58","108.221.201.58","Blocked","1 (A)","NOERROR","internetbadguys.com.","Phishing" 58 | filelist = filedata.split("\n") 59 | # Iterate lines in the file, from bottom to top. We are doing this since we want to collect the 5 most 60 | # recent threats for a given client, and most recent threats will be at the bottom 61 | for f in reversed(filelist): 62 | # We should have a file. Make sure. 63 | if f: 64 | # Parse data and split, then create dictionary for record 65 | fr = f[1:-1].split('","') 66 | urec = {"Timestamp": fr[0], "InternalIp": fr[3], "Domain": fr[8], "Categories": fr[9]} 67 | 68 | # New user. Initialize basic constructs for them. fr[1] represents a device name. 69 | if fr[1] not in frstats["Users"]: 70 | frstats["Users"][fr[1]] = {} 71 | frstats["Users"][fr[1]]["Blocked"] = [] 72 | frstats["Users"][fr[1]]["Aggregate"] = {} 73 | frstats["Users"][fr[1]]["Aggregate"]["Total"] = 0 74 | 75 | # If this entry is for a Blocked record, capture additional stats for that 76 | if fr[5] == "Blocked": 77 | # We will capture the aggregate number of blocks for this type of event. fr[9] represents 78 | # the category of the block. If this is a new category, set to 1, otherwise increment 79 | if fr[9] in frstats["Aggregate"]: 80 | frstats["Aggregate"][fr[9]] += 1 81 | else: 82 | frstats["Aggregate"][fr[9]] = 1 83 | 84 | # If we have fewer than 5 Blocked events already tagged, we want to append any Blocked 85 | # events to the blocked list for the user. 86 | if len(frstats["Users"][fr[1]]["Blocked"]) < 5: 87 | frstats["Users"][fr[1]]["Blocked"].append(urec) 88 | 89 | # We will capture the user-specific number of blocks for this type of event. fr[9] 90 | # represents the category of the block. If this is a new category, set to 1, otherwise 91 | # increment 92 | if fr[9] in frstats["Users"][fr[1]]["Aggregate"]: 93 | frstats["Users"][fr[1]]["Aggregate"][fr[9]] += 1 94 | else: 95 | frstats["Users"][fr[1]]["Aggregate"][fr[9]] = 1 96 | 97 | # Add aggregate total and user-specific total 98 | frstats["Aggregate"]["Total"] += 1 99 | frstats["Users"][fr[1]]["Aggregate"]["Total"] += 1 100 | 101 | #print(frstats) 102 | return frstats 103 | 104 | 105 | def get_umbrella_health(incoming_msg, rettype): 106 | ''' 107 | This function will return health data for the Umbrella network (based on local logs) 108 | 109 | :param incoming_msg: String. this is the message that is posted in Spark. The client's username will be parsed 110 | out from this. 111 | :param rettype: String. html or json 112 | :return: String (if rettype = html). This is a fully formatted string that will be sent back to Spark 113 | Dictionary (if rettype = json). Raw data that is expected to be consumed in cico_combined 114 | ''' 115 | 116 | # Parse logs to get relevant data 117 | logdata = parse_umbrella_logs() 118 | 119 | retmsg = "

Umbrella Details (Last 24 hours):

" 120 | if umbrella_over_dash: 121 | retmsg += "Umbrella Dashboard