33 | 34 | #set OS environment variables for script. Change these for your deployment 35 | 36 | MY_PROJECT=MY_PROJECT 37 | GCS_FUNCTION=ExampleGCSFunction 38 | 39 | GCS_BUCKET=example-bucket-xxxx/ 40 | 41 | HEC_URL=URL-OR-IP-AND-PORT-FOR-HEC 42 | GCS_TOKEN=TOKEN-0000-0000-0000-0000 43 | 44 | RETRY_FUNCTON=ExamplePubSubRetry 45 | RETRY_TOPIC=ExamplePubSubRetryTopic 46 | RETRY_SUBSCRIPTION=ExamplePubSubRetryTopic-sub 47 | RETRY_TRIGGER_PUBSUB=ExampleRetryTrigger 48 | RETRY_SCHEDULE=ExampleRetrySchedule 49 | 50 | #this section is specific for this example only; give the bucket a global unique id 51 | 52 | gsutil mb gs://$GCS_BUCKET 53 | 54 | 55 | #the clone command only needs to be done once for all of the examples 56 | git clone https://github.com/splunk/splunk-gcp-functions.git 57 | 58 | cd splunk-gcp-functions/GCS 59 | 60 | #create function 61 | 62 | gcloud functions deploy $GCS_FUNCTION --runtime python37 \ 63 | --trigger-bucket=$GCS_BUCKET --entry-point=hello_gcs \ 64 | --allow-unauthenticated --timeout=300 --memory=2048MB\ 65 | --set-env-vars=HEC_URL=$HEC_URL,HEC_TOKEN=$GCS_TOKEN,PROJECTID=$MY_PROJECT,RETRY_TOPIC=$RETRY_TOPIC 66 | 67 | 68 | #This is a common section for all examples 69 | #Doesn't need to be repeated for all unless you wish to have separate PubSub Topics for retrying different events. 70 | 71 | gcloud pubsub topics create $RETRY_TOPIC 72 | 73 | gcloud pubsub subscriptions create --topic $RETRY_TOPIC $RETRY_SUBSCRIPTION --ack-deadline=240 74 | cd ../Retry 75 | 76 | #create Retry function 77 | 78 | gcloud functions deploy $RETRY_FUNCTON --runtime python37 \ 79 | --trigger-topic=$RETRY_TRIGGER_PUBSUB --entry-point=hello_pubsub --allow-unauthenticated --timeout=240\ 80 | --set-env-vars=HEC_URL=PROJECTID=$MY_PROJECT,SUBSCRIPTION=$RETRY_SUBSCRIPTION,RETRY_TRIGGER_TOPIC=$RETRY_TRIGGER_PUBSUB 81 | 82 | gcloud pubsub topics create $RETRY_TRIGGER_PUBSUB 83 | 84 | gcloud scheduler jobs create pubsub $RETRY_SCHEDULE --schedule "*/10 * * * *" --topic $RETRY_TRIGGER_PUBSUB --message-body "Retry" --project $MY_PROJECT 85 | 86 |87 | -------------------------------------------------------------------------------- /Alert/Installguide.md: -------------------------------------------------------------------------------- 1 | # GCP Cloud Functions – Installation / Setup Guide 2 | 3 | ## Alert Function 4 | (version 0.1) 5 | 6 | ## **Function Flow process** 7 | 8 | **Normal Flow:** 9 | Stackdriver Alert -> WebHook -> GCP Function -> HEC 10 | 11 | **Error Flow:** 12 | Stackdriver Alert -> WebHook -> GCP Function -> PubSub Topic (error:RetryTopic) 13 | Cloud Schedule -> PubSub Topic (Trigger) -> GCP Function(->Pull from PubSub Retry Topic)-> HEC 14 | 15 | ## **Pre-requisites** 16 | 17 | HEC set-up on a Splunk instance (load balancer needed for a cluster) 18 | HEC token/input MUST allow access to all indexes noted in the environment variables if the default token index is being over-ridden 19 | This function requires a sourcetype of google:gcp:alert to be set-up on the Splunk instance (see below) 20 | This function requires Cloud Functions API to be enabled. 21 | Set up Stackdriver Alert; create a Notification as a Web Hook to the URL of the Function 22 | Set up a PubSub Topic for error messages (Note the name of the topic - this will be used in the Environment variables later) 23 | 24 | ## **Function Dependencies:** 25 | Alert Function requires the Retry Function. Install and set up the Retry Function first 26 | 27 | 28 | ## Install with gcloud CLI 29 | 30 | This is a beta release. Cloud CLI scripts to follow shortly 31 | 32 | 33 | ## **Manual Setup** 34 | 1. Create a new Cloud Function 35 | 2. Name your function – note the url for the function - you will need it later for the Stackdriver Alert 36 | 3. Set the Trigger to be HTTP 37 | 4. Add the code: 38 | 5. Select Inline editor as source 39 | 6. Select the Runtime as Python 3.7 40 | 7. Copy the function code into the main.py 41 | 8. Copy the content of requirements.txt into the requirements.txt tab 42 | 9. Click on “Show variables like environment, networking, timeouts and more” to open up more options 43 | 10. Select the region where you want the function to run 44 | 11. Click on the + Add variable to open up the Environment variables entry 45 | 12. Add the Environment variables and values described in the table below 46 | 13. Click Deploy 47 | 14. You will need to install the Retry function if you wish to have a recovery for any events that failed to write to Splunk. See install guide for that function. 48 | 15. Create your Alert in Stackdriver. Set the Notification to send webhook to the url of the function 49 | 50 | ## **Function Environment Variables** 51 | 52 |
| Variable | Value |
| HEC_URL | Hostname/IP address and port number for URL for Splunk HEC (Load balancer required for cluster) 54 | e.g. mysplunkinstance.splunk.com:8088 or 113.114.115.192:8088 |
| HEC_TOKEN | HEC Token for the input. Generate on Splunk instance. |
| PROJECTID | Project ID for where the Retry Topic exists |
| HOST | Host value that Splunk will assign for the Alert event. Defaults to GCP_Alert_Function |
| SPLUNK_SOURCETYPE | Sourcetype that will be given to the event (defaults to google:gcp:alert) |
| SPLUNK_SOURCE | If set, this will be assigned to the “Source” of the event. If not set, defaults to "Stackdriver Alert:policyname" |
| INDEX | If this is set, its value can be set to over-ride the HEC token index. (defaults to no value – i.e. HEC token set index name) |
| RETRY_TOPIC | Name of Topic to send event to on any failure scenario for the function |
70 | [google:gcp:alert] 71 | category = Custom 72 | pulldown_type = 1 73 | DATETIME_CONFIG = 74 | INDEXED_EXTRACTIONS = json 75 | LINE_BREAKER = ([\r\n]+) 76 | AUTO_KV_JSON = false 77 | KV_MODE=none 78 | NO_BINARY_CHECK = true 79 | disabled = false 80 | TRUNCATE=0 81 |82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /Assets/Installguide.md: -------------------------------------------------------------------------------- 1 | # GCP Cloud Functions – Installation / Setup Guide 2 | 3 | # Assets Function 4 | (0.2) 5 | 6 | ## **Pre-requisites** 7 | HEC set-up on a Splunk instance (load balancer needed for a cluster) 8 | (note that if the over-ride token/URL has been set in the function environment variables, the destination for this must match where the source of the failed function originated) 9 | This function will require a sourcetype to be created on your Splunk instance. An example sourcetype is available in props.conf in this folder. 10 | This function requires Cloud Functions API to be enabled. 11 | This function requires Cloud Assets API to be enabled on the Project you will be requesting the Asset inventory from. (https://cloud.google.com/asset-inventory/docs/quickstart and click on
| Variable | Value |
| PARENT | Project ID, Organisation ID or Folder to collect the assets information - This can only be an organization number (such as "organizations/123"), a project ID (such as "projects/my-project-id"), a project number (such as "projects/12345"), or a folder number (such as "folders/123") |
| GCS_FILE_PATH | GCS path to bucket where the Assets inventory will be written.
65 | Enter the full path and initial prefix to this bucket and object - eg. gs://my_asset_bucket_for_project/asset_file 66 | Note that unixtime will be added to the filename on writing from the function |
curl -k "https://mysplunkserver.example.com:8088/services/collector" \
12 | -H "Authorization: Splunk CF179AE4-3C99-45F5-A7CC-3284AA91CF67" \
13 | -d '{"event": "Hello, world!", "sourcetype": "manual"}'
14 | 2) Check that your HEC_URL environment variable is correct. You don't need /services/collector for example. Just use IPaddress:Port or Hostname:Port. Note that for Splunk Cloud customers, there is a specific URL for HEC - this is usually in the format of http-inputs-mysplunkcloud.splunkcloud.com. (There is no need for a port number).
15 | 3) Do not use the ACK on the HEC settings
16 |
17 | **My PubSub Function is hitting maximum executions limit - what's gone wrong?**
18 |
19 | This is likely to be caused by the PubSub Function ingesting its own logs. This will cause an infinate loop / race. To stop this, edit the Log Export / Sink that the PubSub topic is subscribing to and make sure the filter excludes the PubSub Function from its logs. An easy way to resolve this is by using the filter resource.labels.function_name!=ExamplePubSubFunction (changing the name to your function name).
20 | Another possibilty is that your log export filter is too broad, and the number of of events is very large. Consider your filter design, and create more than one pubsub function if necessary to read from different topics/log exports to reduce the load on one function.
21 |
22 |
23 | **My metrics has a gap between groups of metrics in Splunk**
24 |
25 | This is normally caused by the Metrics Schedule and the Interval setting (TIME_INTERVAL) for the Metrics functions not being the same. For example, the schedule is 10mins whereas the metrics interval is 5. The TIME_INTERVAL setting should match that of the Schedule period.
26 | If the settings are the same, then examine the function log and search for errors - if you see function timeouts or memory limit exceeded, this indicates that you need to increase the memory allocated to the function and function timeout (usually due to a large number of metrics being requested). Alternatively, reduce the time interval, and the number of metrics for the function (for example, split the list over more than one function).
27 |
28 | **I have no metrics arriving in Splunk**
29 |
30 | If you want to sent your metrics to a metrics index, make sure that your HEC input specifies a metrics index.
31 | Also note the previous issue, where increasing the memory allocation and timeout for your function may resolve the issue (and/or reduce TIME_INTERVAL).
32 |
33 | **Some of my events in Splunk are not complete / truncated**
34 |
35 | This usually occurs due to the size of the event coming from Stackdriver - if they are very large, they will be truncated if you have only the default settings for TRUNCATE on the sourcetype (set to 10000). Some of the container logs for example can be 16K. You should update your sourcetype to add TRUNCATE=0.
36 |
37 | **My events from GCS are not being split properly in Splunk**
38 |
39 | This is usually down to your sourcetype for the HEC input not being set properly, or you have multiple sourcetypes going into the same GCS bucket. The Function currently only supports one sourcetype per GCS Bucket. Make sure you have the correct sourcetype on the HEC input setting.
40 | The other potential issue is that you have not set the LINE_BREAKER regex environment variable in the function settings. By default, it will break events up from the file by newline only. If you have multi-line events, make sure you set the LINE_BREAKER to have the same regex values as the Splunk sourcetype's settings in props.conf (you may need to consult with your Splunk admin). It is important also to make sure to set BEFORE=TRUE if the break is done before the LINE_BREAKER regex.
41 |
42 | **My events are not evenly distributed across my indexer cluster**
43 |
44 | This is typically down to 2 reasons:
45 | 1) Your Load Balancer has been set to have sticky sessions. Disable this if possible
46 | 2) You are only sending events to one indexer (one HEC URL which is one of the Indexers). If you don't have a Load balancer, consider using a Heavy Forwarder in front of your Indexer cluster, as the functions currently only support sending to 1 HEC URL per function.
47 |
48 |
--------------------------------------------------------------------------------
/Examples/Examples-cleanup.md:
--------------------------------------------------------------------------------
1 | # Example Cleanup
2 |
3 | The script below cleans up (destructively) the examples created.
4 |
5 | ## Warning
6 | **THIS CANNOT BE UNDONE!!!**
7 | Make sure you don't want to keep any remaining data!!!
8 | The following commands also deletes all objects stored within the buckets. These objects cannot be recovered. If you want to avoid accidentally deleting objects, use the replace ``gsutil rm`` with the ``gsutil rb`` command below, which only deletes a bucket if the bucket is empty.
9 |
10 |
11 | Update the Highlighted items, and save as a shell script (e.g. cleanup.sh).
12 | Before running the script make sure you set permissions eg ``chmod +x cleanup.sh``
13 | To run the script, use the example number as the script argument :
14 | For example 1 use ``./cleanup.sh 1`` 24 | #run this script from where you originally ran the original examples from 25 | #use the same environment variable values as for the example builds 26 | 27 | #Example-1 Start 28 | PUBSUB_FUNCTION=ExamplePubSubFunction 29 | 30 | PUBSUB_TOPIC=ExamplePubSubLogsTopic 31 | PUBSUB_SINK1=ExampleSinkForFunctions 32 | PUBSUB_SINK2=ExampleSinkNoFunctions 33 | #Example-1 End (note the retry needs to be cleaned up below also) 34 | 35 | #Example-2a/b Start 36 | METRICS_FUNCTIONa=ExampleMetricsEventsFunction 37 | METRICS_TRIGGER=ExampleMetricsTriggerTopic 38 | METRICS_SCHEDULE=ExampleMetricsSchedule 39 | METRICS_FUNCTIONb=ExampleMetricsFunction 40 | #Example 2a/b End 41 | 42 | #Example 3 Start 43 | GCS_FUNCTION=ExampleGCSFunction 44 | GCS_BUCKET=example-bucket-xxxx/ 45 | #Example 3 End 46 | 47 | #Example 4 Start 48 | ASSETS_FUNCTION=ExampleAssetsFunction 49 | 50 | GCS_ASSETS_BUCKET=example-assets-bucket-xxxx/ 51 | GCS_FUNCTION=ExampleGCSAssetsFunction 52 | 53 | ASSETS_SCHEDULE=ExampleAssetsSchedule 54 | ASSETS_TRIGGER_PUBSUB=ExampleAssetsTrigger 55 | #Example 4 End 56 | 57 | #Common for all examples# 58 | RETRY_FUNCTON=ExamplePubSubRetry 59 | RETRY_TOPIC=ExamplePubSubRetryTopic 60 | RETRY_SUBSCRIPTION=ExamplePubSubRetryTopic-sub 61 | RETRY_TRIGGER_PUBSUB=ExampleRetryTrigger 62 | RETRY_SCHEDULE=ExampleRetrySchedule 63 | #End of common 64 | 65 | #remove git project clone (all examples) 66 | rm -r splunk-gcp-functions 67 | 68 | 69 | case $1 in 70 | 1) 71 | CLEAN=1 72 | ;; 73 | 2a) 74 | CLEAN=2 75 | ;; 76 | 2b) 77 | CLEAN=3 78 | ;; 79 | 2) 80 | CLEAN=4 81 | ;; 82 | 3) 83 | CLEAN=5 84 | ;; 85 | 4) 86 | CLEAN=6 87 | ;; 88 | ALL) 89 | CLEAN=0 90 | ;; 91 | *) 92 | CLEAN=0 93 | ;; 94 | esac 95 | 96 | #Example 1 97 | if [ $CLEAN -eq 1 ] || [ $CLEAN -eq 0 ] 98 | then 99 | gcloud functions delete $PUBSUB_FUNCTION --quiet 100 | gcloud logging sinks delete $PUBSUB_SINK1 --quiet 101 | gcloud logging sinks delete $PUBSUB_SINK2 --quiet 102 | gcloud pubsub topics delete $PUBSUB_TOPIC --quiet 103 | fi 104 | 105 | #Example 2a 106 | if [ $CLEAN -eq 2 ] || [ $CLEAN -eq 4 ] || [ $CLEAN -eq 0 ] 107 | then 108 | gcloud functions delete $METRICS_FUNCTIONa --quiet 109 | fi 110 | 111 | #Example 2b 112 | if [ $CLEAN -eq 3 ] || [ $CLEAN -eq 4 ] || [ $CLEAN -eq 0 ] 113 | then 114 | gcloud functions delete $METRICS_FUNCTIONb --quiet 115 | fi 116 | 117 | #Examples 2a/2b 118 | if [ $CLEAN -eq 2 ] || [ $CLEAN -eq 3 ] || [ $CLEAN -eq 4 ] || [ $CLEAN -eq 0 ] 119 | then 120 | gcloud pubsub topics delete $METRICS_TRIGGER --quiet 121 | gcloud scheduler jobs delete $METRICS_SCHEDULE --quiet 122 | fi 123 | 124 | #Example 3 125 | if [ $CLEAN -eq 5 ] || [ $CLEAN -eq 0 ] 126 | then 127 | gcloud functions delete $GCS_FUNCTION --quiet 128 | gsutil rm -r gs://$GCS_BUCKET 129 | fi 130 | 131 | #Example 4 132 | if [ $CLEAN -eq 6 ] || [ $CLEAN -eq 0 ] 133 | then 134 | gcloud functions delete $ASSETS_FUNCTION --quiet 135 | gcloud pubsub topics delete $ASSETS_TRIGGER_PUBSUB --quiet 136 | gcloud scheduler jobs delete $ASSETS_SCHEDULE --quiet 137 | gsutil rm -r gs://$GCS_ASSETS_BUCKET 138 | fi 139 | 140 | #Common for All 141 | if [ $# -eq 2 ] 142 | then 143 | if [ $2 == 'R' ] 144 | then 145 | CLEAN=0 146 | fi 147 | fi 148 | #Common for All 149 | if [ $CLEAN -eq 0 ] 150 | then 151 | gcloud functions delete $RETRY_FUNCTON --quiet 152 | gcloud scheduler jobs delete $RETRY_SCHEDULE --quiet 153 | gcloud pubsub subscriptions delete $RETRY_SUBSCRIPTION --quiet 154 | gcloud pubsub topics delete $RETRY_TOPIC --quiet 155 | gcloud pubsub topics delete $RETRY_TRIGGER_PUBSUB --quiet 156 | fi 157 | 158 | 159 |160 | 161 | -------------------------------------------------------------------------------- /Examples/Example-2b-Metrics.md: -------------------------------------------------------------------------------- 1 | # Example 2b Metrics Collection (Metrics Index) 2 | 3 | This example will create a Cloud Schedule which triggers the Metrics Function (via a PubSub Topic). The function will send the metrics into Splunk HEC as a metric format (into an metric index). The script will also create a retry PubSub Topic, and set up a Function to retry any failed messages to HEC. 4 | (If you have already created any other examples, the Cloud Schedule and PubSub Trigger topic doesn't need to be re-created) 5 | 6 | #### PubSub Topics: 7 | 8 | **ExampleRetryTrigger** : This topic is common between all functions and triggers retries based on Cloud Schedule 9 | 10 | **ExamplePubSubRetryTopic** : This topic can be common between all functions. This topic will collect failed writes from the Functions to HEC 11 | 12 | **ExampleRetryTrigger** : This topic can be common between all functions and triggers retries based on Cloud Schedule 13 | 14 | 15 | #### GCP Functions: 16 | 17 | **ExampleMetricsFunction** : Function to pull sample of metrics from compute. Formatted as a metric into metrics index via HEC 18 | 19 | **ExampleMetricRetryTopic** : Retry Function to pull any failed messages from ExampleMetricsFunction 20 | 21 | 22 | #### Cloud Scheduler 23 | 24 | **ExampleMetricsSchedule** : Schedule for Running Events (5mins) - Common for all metrics examples 25 | **ExampleRetry** : Retry Schedule (10mins) - common for all examples 26 | 27 | 28 | ## CLI Example 29 | 30 | (run in bash or the Cloud Shell) 31 | 32 | **Note that you will need to change values in bold in the scripts below to identify your project id, Log-Sink Service Account, HEC URL and HEC Token** 33 | You can also change the OS environment variables in the first section to fit your needs 34 | Note to use your Project ID, and not Project Name / Number 35 | 36 | When running the scripts the first time in a new project, if asked, accept the queries to create/initialise services 37 | 38 |
39 | 40 | 41 | #set OS environment variables for script. Change these for your deployment 42 | 43 | MY_PROJECT=MY_PROJECT 44 | METRICS_FUNCTION=ExampleMetricsFunction 45 | METRICS_TRIGGER=ExampleMetricsTriggerTopic 46 | METRICS_SCHEDULE=ExampleMetricsSchedule 47 | 48 | HEC_URL=URL-OR-IP-AND-PORT-FOR-HEC 49 | METRICS_TOKEN=TOKEN-0000-0000-0000-0000 50 | 51 | RETRY_FUNCTON=ExamplePubSubRetry 52 | RETRY_TOPIC=ExamplePubSubRetryTopic 53 | RETRY_SUBSCRIPTION=ExamplePubSubRetryTopic-sub 54 | RETRY_TRIGGER_PUBSUB=ExampleRetryTrigger 55 | RETRY_SCHEDULE=ExampleRetrySchedule 56 | 57 | 58 | #This Schedule and topic only needs to be created once for all metrics functions unless you want different schedules. 59 | #Note:Match the schedule to the value in the TIME_INTERVAL environment variable below 60 | #This example assumes a 5 minute schedule 61 | 62 | gcloud pubsub topics create $METRICS_TRIGGER 63 | 64 | gcloud scheduler jobs create pubsub $METRICS_SCHEDULE --schedule "*/5 * * * *" --topic $METRICS_TRIGGER --message-body "RunMetric" --project $MY_PROJECT 65 | 66 | # ..End of common Metric trigger section 67 | 68 | 69 | #this command only needs to be done once for all of the examples 70 | git clone https://github.com/splunk/splunk-gcp-functions.git 71 | 72 | cd splunk-gcp-functions/Metrics 73 | 74 | #create function 75 | 76 | #this could be replaced by a static yaml file with the env variables set: 77 | 78 | echo -e "HEC_URL: $HEC_URL\\nHEC_TOKEN: $METRICS_TOKEN\\nPROJECTID: $MY_PROJECT\\nTIME_INTERVAL: '5'\\nRETRY_TOPIC: $RETRY_TOPIC\\nMETRIC_INDEX_TYPE: METRICS\\nMETRICS_LIST: '[\"compute.googleapis.com/instance/cpu/utilization\",\"compute.googleapis.com/instance/disk/read_ops_count\",\"compute.googleapis.com/instance/disk/write_bytes_count\",\"compute.googleapis.com/instance/disk/write_ops_count\",\"compute.googleapis.com/instance/network/received_bytes_count\",\"compute.googleapis.com/instance/network/received_packets_count\",\"compute.googleapis.com/instance/network/sent_bytes_count\",\"compute.googleapis.com/instance/network/sent_packets_count\",\"compute.googleapis.com/instance/uptime\"]'" > EnvMVars.yaml 79 | 80 | gcloud functions deploy $METRICS_FUNCTION --runtime python37 \ 81 | --trigger-topic=$METRICS_TRIGGER --entry-point=hello_pubsub --allow-unauthenticated \ 82 | --env-vars-file EnvMVars.yaml 83 | 84 | 85 | #This is a common section for all examples 86 | #Doesn't need to be repeated for all unless you wish to have separate PubSub Topics for retrying different events. 87 | 88 | gcloud pubsub topics create $RETRY_TOPIC 89 | 90 | gcloud pubsub subscriptions create --topic $RETRY_TOPIC $RETRY_SUBSCRIPTION --ack-deadline=240 91 | cd ../Retry 92 | 93 | #create Retry function 94 | 95 | gcloud functions deploy $RETRY_FUNCTON --runtime python37 \ 96 | --trigger-topic=$RETRY_TRIGGER_PUBSUB --entry-point=hello_pubsub --allow-unauthenticated --timeout=120\ 97 | --set-env-vars=PROJECTID=$MY_PROJECT,SUBSCRIPTION=$RETRY_SUBSCRIPTION,RETRY_TRIGGER_TOPIC=$RETRY_TRIGGER_PUBSUB 98 | 99 | gcloud pubsub topics create $RETRY_TRIGGER_PUBSUB 100 | 101 | gcloud scheduler jobs create pubsub $RETRY_SCHEDULE --schedule "*/10 * * * *" --topic $RETRY_TRIGGER_PUBSUB --message-body "Retry" --project $MY_PROJECT 102 | 103 | 104 |-------------------------------------------------------------------------------- /Examples/Example-2a-Metrics.md: -------------------------------------------------------------------------------- 1 | # Example 2a Metrics Collection (Event Index) 2 | 3 | This example will create a Cloud Schedule which triggers the Metrics Function (via a PubSub Topic). The function will send the metrics into Splunk HEC as an Event format (into an Event index). The script will also create a retry PubSub Topic, and set up a Function to retry any failed messages to HEC. 4 | (If you have already created any other examples, the Cloud Schedule and PubSub Trigger topic doesn't need to be re-created) 5 | 6 | #### PubSub Topics: 7 | 8 | **ExampleRetryTrigger** : This topic is common between all functions and triggers retries based on Cloud Schedule 9 | 10 | **ExamplePubSubRetryTopic** : This topic can be common between all functions. This topic will collect failed writes from the Functions to HEC 11 | 12 | **ExampleRetryTrigger** : This topic can be common between all functions and triggers retries based on Cloud Schedule 13 | 14 | 15 | 16 | #### GCP Functions: 17 | 18 | **ExampleMetricsEventFunction** : Function to pull sample of metrics from compute. Formatted as an Event into HEC 19 | 20 | **ExampleEventsRetryTopic** : Retry Function to pull any failed messages from ExampleMetricsFunction 21 | 22 | 23 | #### Cloud Scheduler 24 | 25 | **ExampleMetricsSchedule** : Schedule for Running Events (5mins) 26 | **ExampleRetry** : Retry Schedule (10mins) 27 | 28 | 29 | ## CLI Example 30 | 31 | (run in bash or the Cloud Shell) 32 | 33 | **Note that you will need to change values in bold in the scripts below to identify your project id, Log-Sink Service Account, HEC URL and HEC Token** 34 | You can also change the OS environment variables in the first section to fit your needs 35 | Note to use your Project ID, and not Project Name / Number 36 | 37 | When running the scripts the first time in a new project, if asked, accept the queries to create/initialise services 38 | 39 |
40 | 41 | #set OS environment variables for script. Change these for your deployment 42 | 43 | MY_PROJECT=MY_PROJECT 44 | METRICS_FUNCTION=ExampleMetricsEventsFunction 45 | METRICS_TRIGGER=ExampleMetricsTriggerTopic 46 | METRICS_SCHEDULE=ExampleMetricsSchedule 47 | 48 | HEC_URL=URL-OR-IP-AND-PORT-FOR-HEC 49 | METRICS_TOKEN=TOKEN-0000-0000-0000-0000 50 | 51 | RETRY_FUNCTON=ExamplePubSubRetry 52 | RETRY_TOPIC=ExamplePubSubRetryTopic 53 | RETRY_SUBSCRIPTION=ExamplePubSubRetryTopic-sub 54 | RETRY_TRIGGER_PUBSUB=ExampleRetryTrigger 55 | RETRY_SCHEDULE=ExampleRetrySchedule 56 | 57 | 58 | #This Schedule and topic only needs to be created once for all metrics functions unless you want different schedules. 59 | #Note:Match the schedule to the value in the TIME_INTERVAL environment variable below 60 | #This example assumes a 5 minute schedule 61 | 62 | #This Schedule and topic only needs to be created once for all metrics functions unless you want different schedules. 63 | #Note:Match the schedule to the value in the TIME_INTERVAL environment variable below 64 | #This example assumes a 5 minute schedule 65 | 66 | gcloud pubsub topics create $METRICS_TRIGGER 67 | 68 | gcloud scheduler jobs create pubsub $METRICS_SCHEDULE --schedule "*/5 * * * *" --topic $METRICS_TRIGGER --message-body "RunMetric" --project $MY_PROJECT 69 | 70 | # ..End of common Metric trigger section 71 | 72 | #the clone command only needs to be done once for all of the examples 73 | git clone https://github.com/splunk/splunk-gcp-functions.git 74 | 75 | 76 | cd splunk-gcp-functions/Metrics 77 | 78 | #create function 79 | 80 | #this could be replaced by a static yaml file with the env variables set: 81 | 82 | echo -e "HEC_URL: $HEC_URL\\nHEC_TOKEN: $METRICS_TOKEN\\nPROJECTID: $MY_PROJECT\\nTIME_INTERVAL: '5'\\nRETRY_TOPIC: $RETRY_TOPIC\\nMETRICS_LIST: '[\"compute.googleapis.com/instance/cpu/utilization\",\"compute.googleapis.com/instance/disk/read_ops_count\",\"compute.googleapis.com/instance/disk/write_bytes_count\",\"compute.googleapis.com/instance/disk/write_ops_count\",\"compute.googleapis.com/instance/network/received_bytes_count\",\"compute.googleapis.com/instance/network/received_packets_count\",\"compute.googleapis.com/instance/network/sent_bytes_count\",\"compute.googleapis.com/instance/network/sent_packets_count\",\"compute.googleapis.com/instance/uptime\"]'" > EnvEVars.yaml 83 | 84 | gcloud functions deploy $METRICS_FUNCTION --runtime python37 \ 85 | --trigger-topic=$METRICS_TRIGGER --entry-point=hello_pubsub --allow-unauthenticated \ 86 | --env-vars-file EnvEVars.yaml 87 | 88 | 89 | #This is a common section for all examples 90 | #Doesn't need to be repeated for all unless you wish to have separate PubSub Topics for retrying different events. 91 | 92 | gcloud pubsub topics create $RETRY_TOPIC 93 | 94 | gcloud pubsub subscriptions create --topic $RETRY_TOPIC $RETRY_SUBSCRIPTION --ack-deadline=240 95 | cd ../Retry 96 | 97 | #create Retry function 98 | 99 | gcloud functions deploy $RETRY_FUNCTON --runtime python37 \ 100 | --trigger-topic=$RETRY_TRIGGER_PUBSUB --entry-point=hello_pubsub --allow-unauthenticated --timeout=240\ 101 | --set-env-vars=PROJECTID=$MY_PROJECT,SUBSCRIPTION=$RETRY_SUBSCRIPTION,RETRY_TRIGGER_TOPIC=$RETRY_TRIGGER_PUBSUB 102 | 103 | gcloud pubsub topics create $RETRY_TRIGGER_PUBSUB 104 | 105 | gcloud scheduler jobs create pubsub $RETRY_SCHEDULE --schedule "*/10 * * * *" --topic $RETRY_TRIGGER_PUBSUB --message-body "Retry" --project $MY_PROJECT 106 | 107 | 108 |-------------------------------------------------------------------------------- /Alert/main.py: -------------------------------------------------------------------------------- 1 | #GCP - AlertFunction v0.1.0 2 | 3 | '''MIT License 4 | Copyright (c) 2019 Splunk 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 8 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 11 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 12 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 13 | SOFTWARE. ''' 14 | 15 | import base64 16 | import argparse 17 | import os 18 | import pprint 19 | import time 20 | import json 21 | import re 22 | 23 | from datetime import datetime 24 | from datetime import date 25 | 26 | import time 27 | import requests 28 | from requests.adapters import HTTPAdapter 29 | import urllib3 30 | ##turns off the warning that is generated below because using self signed ssl cert 31 | urllib3.disable_warnings() 32 | 33 | 34 | def hello_world(request): 35 | """Responds to HTTP request from Alert Webhook. 36 | Args: 37 | request (flask.Request): HTTP request object. 38 | Returns: 39 | The response text or any set of values that can be turned into a 40 | Response object using 41 | `make_response
| Sink | Description | Filter |
| ExampleSinkFunctions | Selects all GCP Function logs. Important note that it filters out the PubSub Function!! | resource.labels.function_name!="ExamplePubSub" |
| ExampleSinkNoFunctions | Selects all Kubernetes/containers logs | protoPayload.serviceName="container.googleapis.com" |
39 | 40 | #set OS environment variables for script. Change these for your deployment 41 | 42 | MY_PROJECT=MY_PROJECT 43 | PUBSUB_FUNCTION=ExamplePubSubFunction 44 | 45 | PUBSUB_TOPIC=ExamplePubSubLogsTopic 46 | PUBSUB_SINK1=ExampleSinkForFunctions 47 | PUBSUB_SINK2=ExampleSinkNoFunctions 48 | 49 | HEC_URL=URL-OR-IP-AND-PORT-FOR-HEC 50 | PUBSUB_TOKEN=TOKEN-0000-0000-0000-0000 51 | 52 | RETRY_FUNCTON=ExamplePubSubRetry 53 | RETRY_TOPIC=ExamplePubSubRetryTopic 54 | RETRY_SUBSCRIPTION=ExamplePubSubRetryTopic-sub 55 | RETRY_TRIGGER_PUBSUB=ExampleRetryTrigger 56 | RETRY_SCHEDULE=ExampleRetrySchedule 57 | 58 | 59 | 60 | #this section is specific for this example only 61 | 62 | gcloud pubsub topics create $PUBSUB_TOPIC 63 | 64 | #create log-sinks... 65 | 66 | #MAKE NOTE OF THIS SINK - IT ENSURES THAT THERE IS NO RECORDING OF THE FUNCTIONS OWN LOGS!!! 67 | 68 | gcloud logging sinks create $PUBSUB_SINK1 \ 69 | pubsub.googleapis.com/projects/$MY_PROJECT/topics/$PUBSUB_TOPIC \ 70 | --log-filter="resource.labels.function_name!=$PUBSUB_FUNCTION" 71 | 72 | LOG_SINK_SERVICE_ACCOUNT=`gcloud logging sinks describe $PUBSUB_SINK1 --format="value(writerIdentity)"` 73 | 74 | #the last command will return the LOG_SINK_SERVICE_ACCOUNT 75 | gcloud pubsub topics add-iam-policy-binding $PUBSUB_TOPIC \ 76 | --member $LOG_SINK_SERVICE_ACCOUNT --role roles/pubsub.publisher 77 | 78 | # THIS SINK WILL GET ALL LOGS OTHER THAN CLOUD FUNCTIONS - BEWARE IT MAY HAVE HIGH VOLUME!!! 79 | 80 | gcloud logging sinks create $PUBSUB_SINK2 \ 81 | pubsub.googleapis.com/projects/$MY_PROJECT/topics/$PUBSUB_TOPIC \ 82 | --log-filter="resource.type!=cloud_function" 83 | 84 | LOG_SINK_SERVICE_ACCOUNT=`gcloud logging sinks describe $PUBSUB_SINK2 --format="value(writerIdentity)"` 85 | 86 | #the last command will return the LOG_SINK_SERVICE_ACCOUNT 87 | gcloud pubsub topics add-iam-policy-binding $PUBSUB_TOPIC \ 88 | --member $LOG_SINK_SERVICE_ACCOUNT --role roles/pubsub.publisher 89 | 90 | #the clone command only needs to be done once for all of the examples 91 | git clone https://github.com/splunk/splunk-gcp-functions.git 92 | 93 | cd splunk-gcp-functions/PubSubFunction 94 | 95 | #create function 96 | 97 | gcloud functions deploy $PUBSUB_FUNCTION --runtime python37 \ 98 | --trigger-topic=$PUBSUB_TOPIC --entry-point=hello_pubsub \ 99 | --allow-unauthenticated \ 100 | --set-env-vars=HEC_URL=$HEC_URL,HEC_TOKEN=$PUBSUB_TOKEN,PROJECTID=$MY_PROJECT,RETRY_TOPIC=$RETRY_TOPIC 101 | 102 | 103 | #This is a common section for all examples 104 | #Doesn't need to be repeated for all unless you wish to have separate PubSub Topics for retrying different events. 105 | 106 | gcloud pubsub topics create $RETRY_TOPIC 107 | 108 | gcloud pubsub subscriptions create --topic $RETRY_TOPIC $RETRY_SUBSCRIPTION --ack-deadline=240 109 | cd ../Retry 110 | 111 | #create Retry function 112 | 113 | gcloud functions deploy $RETRY_FUNCTON --runtime python37 \ 114 | --trigger-topic=$RETRY_TRIGGER_PUBSUB --entry-point=hello_pubsub --allow-unauthenticated --timeout=240\ 115 | --set-env-vars=PROJECTID=$MY_PROJECT,SUBSCRIPTION=$RETRY_SUBSCRIPTION,RETRY_TRIGGER_TOPIC=$RETRY_TRIGGER_PUBSUB 116 | 117 | gcloud pubsub topics create $RETRY_TRIGGER_PUBSUB 118 | 119 | gcloud scheduler jobs create pubsub $RETRY_SCHEDULE --schedule "*/5 * * * *" --topic $RETRY_TRIGGER_PUBSUB --message-body "Retry" --project $MY_PROJECT 120 | 121 |122 | -------------------------------------------------------------------------------- /Examples/Example-4-Assets.md: -------------------------------------------------------------------------------- 1 | # Example 4: Assets Function 2 | 3 | This example will create 3 PubSub Topics, create the Assets and a GCS Function with a Retry Function, a GCS example bucket. 2 Cloud Schedules are also created to trigger the Assets and Retry Functions (via PubSub Topic). Note that the Retry Schedule and Retry Trigger and Retry Topic is common between all of examples and doesn't need to be repeated if you build more than one example. 4 | 5 | Note: This function requires Cloud Assets API to be enabled on the Project you will be requesting the Asset inventory from. Do this before deploying the functions. 6 | Also, ensure that the Cloud Schedule is enabled before running the script. (create a dummy schedule to confirm this beforehand) 7 | 8 | #### PubSub Topics Created: 9 | 10 | **ExampleAssetsTopic** : This topic is created as a trigger for the function 11 | 12 | **ExamplePubSubRetryTopic** : This topic can be common between all functions. This topic will collect failed writes from ExamplePubSub to HEC 13 | 14 | **ExampleRetryTrigger** : This topic can be common between all functions and triggers retries based on Cloud Schedule 15 | 16 | #### GCP Functions Created: 17 | 18 | **ExampleAssets** : Function to call the Assets API 19 | 20 | **ExampleGCSAssets** : GCS Function pulling from an ExampleAssetsBucket 21 | 22 | **ExampleRetry** : Retry Function to pull any failed messages from ExamplePubSub (can be re-used across all examples) 23 | 24 | ## GCS Bucket 25 | 26 | **example-assets-bucket-xxxx** : Example GCS Bucket to store the Assets files - note you will need to change the name to make sure that the bucket name is globally unique. 27 | 28 | 29 | ## CLI Example Scripts 30 | (run in bash or the Cloud Shell) 31 | 32 | **Note that you will need to change values in bold in the scripts below to identify your project id, HEC URL, token and GCS Bucket** 33 | You can also change the OS environment variables in the first section to fit your needs 34 |
43 |
44 | #set OS environment variables for script. Change these for your deployment
45 |
46 | MY_PROJECT=MY_PROJECT
47 | PARENT=projects/$MY_PROJECT
48 | ASSETS_FUNCTION=ExampleAssetsFunction
49 | # remember to give the bucket a global unique id. The file bath contains the object prefix for the object created by the asset function
50 | GCS_ASSETS_BUCKET=example-assets-bucket-xxxx
51 | GCS_FILE_PATH=gs://$GCS_ASSETS_BUCKET/example-assets
52 | GCS_FUNCTION=ExampleGCSAssetsFunction
53 |
54 | HEC_URL=URL-OR-IP-AND-PORT-FOR-HEC
55 | ASSETS_TOKEN=TOKEN-0000-0000-0000-0000
56 |
57 | ASSETS_SCHEDULE=ExampleAssetsSchedule
58 | ASSETS_TRIGGER_PUBSUB=ExampleAssetsTrigger
59 |
60 | RETRY_FUNCTON=ExamplePubSubRetry
61 | RETRY_TOPIC=ExamplePubSubRetryTopic
62 | RETRY_SUBSCRIPTION=ExamplePubSubRetryTopic-sub
63 | RETRY_TRIGGER_PUBSUB=ExampleRetryTrigger
64 | RETRY_SCHEDULE=ExampleRetrySchedule
65 |
66 | #this section is specific for this example only;
67 |
68 | #make sure that the function has access to view the assets.
69 | gcloud projects add-iam-policy-binding $MY_PROJECT \
70 | --member serviceAccount:$MY_PROJECT@appspot.gserviceaccount.com \
71 | --role roles/cloudasset.viewer
72 |
73 | gsutil mb gs://$GCS_ASSETS_BUCKET
74 |
75 | #the clone command only needs to be done once for all of the examples
76 | git clone https://github.com/splunk/splunk-gcp-functions.git
77 |
78 | cd splunk-gcp-functions/Assets
79 |
80 | #create triggers
81 | gcloud pubsub topics create $ASSETS_TRIGGER_PUBSUB
82 |
83 | gcloud scheduler jobs create pubsub $ASSETS_SCHEDULE --schedule "0 */6 * * *" --topic $ASSETS_TRIGGER_PUBSUB --message-body "Assets" --project $MY_PROJECT
84 |
85 |
86 | #create function
87 | gcloud functions deploy $ASSETS_FUNCTION --runtime python37 \
88 | --trigger-topic=$ASSETS_TRIGGER_PUBSUB --entry-point=hello_pubsub \
89 | --allow-unauthenticated \
90 | --set-env-vars=PARENT=$PARENT,GCS_FILE_PATH=$GCS_FILE_PATH
91 |
92 |
93 | cd ../GCS
94 |
95 | #create function
96 |
97 | gcloud functions deploy $GCS_FUNCTION --runtime python37 \
98 | --trigger-bucket=$GCS_ASSETS_BUCKET --entry-point=hello_gcs --timeout=120\
99 | --allow-unauthenticated --timeout=120\
100 | --set-env-vars=HEC_URL=$HEC_URL,HEC_TOKEN=$ASSETS_TOKEN,PROJECTID=$MY_PROJECT,RETRY_TOPIC=$RETRY_TOPIC,BEFORE=TRUE,LINE_BREAKER='{\"name\":\"\/\/'
101 |
102 |
103 | #This is a common section for all examples
104 | #Doesn't need to be repeated for all unless you wish to have separate PubSub Topics for retrying different events.
105 |
106 | gcloud pubsub topics create $RETRY_TOPIC
107 |
108 | gcloud pubsub subscriptions create --topic $RETRY_TOPIC $RETRY_SUBSCRIPTION --ack-deadline=240
109 | cd ../Retry
110 |
111 | #create Retry function
112 |
113 | gcloud functions deploy $RETRY_FUNCTON --runtime python37 \
114 | --trigger-topic=$RETRY_TRIGGER_PUBSUB --entry-point=hello_pubsub --allow-unauthenticated --timeout=240\
115 | --set-env-vars=PROJECTID=$MY_PROJECT,SUBSCRIPTION=$RETRY_SUBSCRIPTION,RETRY_TRIGGER_TOPIC=$RETRY_TRIGGER_PUBSUB
116 |
117 | gcloud pubsub topics create $RETRY_TRIGGER_PUBSUB
118 |
119 | gcloud scheduler jobs create pubsub $RETRY_SCHEDULE --schedule "*/10 * * * *" --topic $RETRY_TRIGGER_PUBSUB --message-body "Retry" --project $MY_PROJECT
120 |
121 |
122 |
--------------------------------------------------------------------------------
/Examples/README.md:
--------------------------------------------------------------------------------
1 | # Example Configuration builds
2 |
3 | The files here contain scripts can be executed to build a full sample configurations using all of the functions in this library. The following configurations are created:
4 | (Note that there are some common sections for these examples, which do not need to be re-run if one of the other examples has been created. This is noted in the scripts)
5 |
6 | To run the examples, you can either run directly from the Cloud Shell in the GCP console (click **>_** to activate Cloud Shell), or by downloading the SDK or Quickstart onto your host/local machine (see here - https://cloud.google.com/sdk/install)
7 |
8 | Make sure you have installed git on the host running the example scripts (GCP's Cloud Shell already has this installed).
9 |
10 | Please refer to the individual function documentation for any pre-requisites before running the examples.
11 |
12 | ## Example 1: PubSub
13 |
14 | This example will create 2 example Log Export Sinks, 2 PubSub Topics and use the PubSub Function with a Retry Function. A Cloud Schedule is also created to trigger the Retry Function (via PubSub Topic). Note that this Schedule and Topic is common between all of examples and doesn't need to be repeated if you build more than one example.
15 |
16 | ## Example 2a: Metrics Collection (Event Index)
17 |
18 | This example will create a Cloud Schedule which triggers the Metrics Function (via a PubSub Topic). The function will send the metrics into Splunk HEC as an Event format (into an Event index). The script will also create a retry PubSub Topic, and set up a Function to retry any failed messages to HEC.
19 | If you have already created any other examples, the Cloud Schedule and PubSub Trigger topic doesn't need to be re-created.
20 |
21 | ## Example 2b: Metrics Collection (Metrics Index)
22 |
23 | This example is a clone of example 2a, but this function will send the metrics into Splunk's Metrics Index. It creates a Cloud Schedule which triggers the Metrics Function (via a PubSub Topic). The script will also create a retry PubSub Topic, and set up a Function to retry any failed messages to HEC.
24 | Note that in practice, only one Cloud Schedule would be needed for metrics unless there is a need to have different schedules/intervals. If you want to run both examples, the section to create the Cloud Schedule for Metrics and its trigger PubSub Topic can be ignored. In the same way, if you have already created any other examples, the Cloud Schedule and PubSub Trigger topic doesn't need to be re-created.
25 |
26 |
27 | ## Example 3: GCS
28 |
29 | This example creates a Function that is trigged by an object being created in GCS. The script also creates a Retry Topic for any failed messages to Splunk HEC. A Retry Function is created to send any failed messages. It will also create a Cloud Schedule and PubSub Trigger - if you have already created any other examples, these don't need to be re-created.
30 |
31 |
32 | ## Example 4: Assets
33 |
34 | The example creates a function to collect asset information periodically, writing this into a GCS Bucket. The function is triggered by a PubSub Topic (called via Cloud Schedule). The example also builds a GCS Function as per Exanmple 3 to collect this asset data and post to Splunk.
35 |
36 | ## Example Cleanup
37 |
38 | The Examples can be cleaned up by copying and saving the script in the cleanup page. Update the variables (bucket names) highlighted in the script. Note that if you have changed the variables in any way, remember to change these for the cleanup, otherwise you may have services or components remaining after runing the script. **Note that this is a destructive process that cannot be undone - take care not to delete buckets or topics that contain data you wish to keep.**
39 |
40 | ## What the Scripts create...
41 |
42 | #### Log export Sinks:
43 |
44 | | Sink | Description | Filter |
| ExampleSinkFunctions | Selects all GCP Function logs. Important note that it filters out the PubSub Function!! | resource.labels.function_name!="ExamplePubSub" |
| ExampleSinkNoFunctions | Selects all Kubernetes/containers logs | protoPayload.serviceName="container.googleapis.com" |
| Variable | Value |
| HEC_URL | Hostname/IP address and port number for URL for Splunk HEC (Load balancer required for cluster) 73 | e.g. mysplunkinstance.splunk.com:8088 or 113.114.115.192:8088 |
| HEC_TOKEN | HEC Token for the input. Generate on Splunk instance. 75 | (make note of HEC token requirements above) |
| HOST | Set the host metadata for the events. Defaults to Host set by the Splunk HEC Token settings if not defined. Default is not set |
| LINE_BREAKER | Enter the regex for the line breaking for the events in the bucket objects. 78 | Defaults to \n (newline) |
| BEFORE | Set this to TRUE if you want to break BEFORE the line breaker, or FALSE if you want to break After the line breaker. 80 | Defaults to FALSE |
| PROJECTID | Project ID for where the Retry Topic exists |
| RETRY_TOPIC | Name of Topic to send event to on any failure scenario for the function |
| BATCH | Size of Batch to send to HEC. Reduce this if you want less events per batch to be sent to Splunk. Increasing can improve throughput, but keep within HEC limits. Default 70k (if not set). Note that this needs to be AT LEAST the size of the largest event in the data being sent, otherwise the function will crash |
| THREADS | Number of worker threads to send payload to HEC. Use only if issues with overload on HEC. Default 127 (i.e. 128 threads) |
| EXCLUDE | If set, this should contain the Regex for any object key (filename) to exclude/ignore in the bucket. By default this is not set. e.g \d{9}.json would exclude all files ending with 9 numbers and .json |
| Variable | Value |
| HEC_URL | Hostname/IP address and port number for URL for Splunk HEC (Load balancer required for cluster) 69 | e.g. mysplunkinstance.splunk.com:8088 or 113.114.115.192:8088 |
| HEC_TOKEN | HEC Token for the input. Generate on Splunk instance. |
| PROJECTID | Project ID for where the Retry Topic exists |
| HOST | Host value that Splunk will assign for the PubSub event. Defaults to GCPFunction |
| SPLUNK_SOURCETYPE | Sourcetype that will be given to the event (defaults to google:gcp:pubsub:message) |
| SPLUNK_SOURCE | If set, this will be assigned to the “Source” of the event. If not set, defaults to PubSub topic |
| INDEX | If this is set, its value can be set to over-ride the HEC token index. If this is set to LOGNAME then another environment variable with the name of the log needs to be set with an index name e.g. if you want all logs from “cloudaudit.googleapis.com%2Factivity” to be sent to index ActivityIX, you need to create an environment variable with the name “activity” with the value of ActivityIX. 76 | Note to use the value after “%2F”, or if the log doesn’t have that, use the value after /logs/ (eg. A logname of projects/projname/logs/OSConfigAgent would have variable set to OSConfigAgent) 77 | Notes:HEC Token must have set access to the indexes noted here 78 | Wildcard values are not accepted 79 | (defaults to no value – i.e. HEC token set index name) |
| logname | A variable with a log name (ending only) will override the HEC token index for the event. Note that INDEX needs to be set to LOGNAME for this to be used. Use logname after /logs/ or if name has “%2F” in the name, use the logname after “%2F” 81 | Examples: 82 | cloudaudit.googleapis.com%2Factivity -> use activity 83 | /logs/OSConfigAgent -> use OSConfigAgent 84 | (defaults to no value) |
| COMPATIBLE | Set this to TRUE to maintain compatibility with Add-On. If not TRUE, event payload will be exact copy of PubSub event. Default is TRUE |
| RETRY_TOPIC | Name of Topic to send event to on any failure scenario for the function |
| Variable | Value |
| PROJECTID | Project ID for where the Retry Topic exists |
| SUBSCRIPTION | Name of the subscription that pulls from the Retry/Error PubSub Topic. |
| RETRY_TRIGGER_TOPIC | Name of the Retry Trigger Topic that triggers the function. Used to spawn more re-tries. |
| HEC_URL | OVER-RIDE Hostname/IP address and port number for URL for Splunk HEC (Load balancer required for cluster) 40 | e.g. mysplunkinstance.splunk.com:8088 or 113.114.115.192:8088 41 | This will point the destination of the message to a different URL to the originating message 42 | Default is original message URL. Do not set if you wish to keep original destination |
| HEC_TOKEN | HEC Token to OVER-RIDE the original destination for the event. Generate on Splunk instance. 44 | Note that this should be set on the Splunk side to be the same Index-type the token used for the function that is using this as a retry i.e. if a metrics index was the destination of the original, this over-ride token should indicate a metrics index also 45 | Default is original message Token. Do not set if you wish to keep original destination |
| EVENT_TYPE | Only Set if HEC_TOKEN and HEC_URL are set to over-ride the original event. Valid values: METRICS, EVENT, RAW
47 | METRIC : use this for Metrics going into Metrics index 48 | EVENT : use this for Metrics going into Event index 49 | RAW : use this for GCS Function re-try. 50 | Do not set if you wish to keep to the original destination |
| BATCH | Number of events to pull from Retry PubSub in one call. Note that higher numbers here will potentially result in a higher execution time. Take care not to set this too high - you may need to make the pubsub subcription Acknowledgement deadline for the retry topic higher if this is set to a large number (>200) and also adjust the function timeout to accomodate. General guidance is the function can recall approx 40k pubsub events in 5 minutes function call with a 256M Function allocation (multiple spawned functions) Default = 100 (Recommended) |
| TIMEOUT | Time in seconds for the function to stop pulling from Retry PubSub in one call (unless there are no events to retry). Note that if this is set higher than the function timeout, the function potentially will exit with a timeout - this could also result in some messages being sent more than once to Splunk. Guideance is to be same value as function timeout. Note: To avoid function timeouts where possible, the actual max execution will generally aim to be a few seconds less than the value set here. Default = 240 |
| Variable | Value |
| HEC_URL | Hostname/IP address and port number for URL for Splunk HEC (Load balancer required for cluster) 70 | e.g. mysplunkinstance.splunk.com:8088 or 113.114.115.192:8088 |
| HEC_TOKEN | HEC Token for the input. Generate on Splunk instance. 72 | Ideally this should be the same as the token used for the function that is using this as a retry 73 | |
| PROJECTID | Project ID for where the Retry Topic exists |
| METRICS_LIST | A list of metrics for the function to pull. Enclose the comma separated list with square brackets. Use full names for the metrics. For example: 76 | ["compute.googleapis.com/instance/cpu/utilization","compute.googleapis.com/instance/disk/read_ops_count"] 77 | |
| TIME_INTERVAL | Time interval for the function to retrieve metrics for (in minutes). This is retrospective – i.e a setting of 5 will retrieve metrics from the last 5 minutes. Running 5, 10 or 15 minute intervals is a recommended setting; larger values may cause function timeouts, in which case you will need to adjust the function timeout setting |
| HOST | Hostname you wish to give the event 80 | Defaults to GCPMetricsFunction 81 | |
| SPLUNK_SOURCETYPE | Sourcetype to assign to the events. Note that this is only used if the metric is going into an event index. 83 | Defaults to google:gcp:monitoring 84 | |
| METRIC_INDEX_TYPE | Sets the type of metrics index that is being sent to. This should be METRICS for metrics index, or EVENT for event index.The event format is compatible with the GCP Add-On metrics. 86 | Defaults to EVENT 87 | |
| RETRY_TOPIC | Name of Topic to send event/metric to on any failure scenario for the function |
106 | ["compute.googleapis.com/instance/cpu/utilization","compute.googleapis.com/instance/disk/read_ops_count","compute.googleapis.com/instance/disk/read_bytes_count","compute.googleapis.com/instance/disk/write_bytes_count","compute.googleapis.com/instance/disk/write_ops_count","compute.googleapis.com/instance/network/received_bytes_count","compute.googleapis.com/instance/network/received_packets_count","compute.googleapis.com/instance/network/sent_bytes_count","compute.googleapis.com/instance/network/sent_packets_count","compute.googleapis.com/instance/uptime","compute.googleapis.com/firewall/dropped_bytes_count","compute.googleapis.com/firewall/dropped_packets_count"] 107 |108 | 109 | ### Cloud Functions: 110 | 111 |
112 | ["cloudfunctions.googleapis.com/function/active_instances","cloudfunctions.googleapis.com/function/execution_count","cloudfunctions.googleapis.com/function/execution_times","cloudfunctions.googleapis.com/function/network_egress","cloudfunctions.googleapis.com/function/user_memory_bytes"] 113 |114 | 115 | ### Containers / Kubernetes 116 | 117 |
118 | ["container.googleapis.com/container/cpu/utilization","container.googleapis.com/container/disk/bytes_used","container.googleapis.com/container/accelerator/duty_cycle","container.googleapis.com/container/accelerator/memory_total","container.googleapis.com/container/accelerator/memory_used","container.googleapis.com/container/accelerator/request","container.googleapis.com/container/cpu/reserved_cores","container.googleapis.com/container/cpu/usage_time","container.googleapis.com/container/disk/bytes_total","container.googleapis.com/container/disk/bytes_used","container.googleapis.com/container/disk/inodes_free","container.googleapis.com/container/disk/inodes_total","container.googleapis.com/container/memory/bytes_total","container.googleapis.com/container/memory/bytes_used","container.googleapis.com/container/uptime"] 119 | 120 | ["kubernetes.io/container/accelerator/duty_cycle", "kubernetes.io/container/accelerator/memory_total", "kubernetes.io/container/accelerator/memory_used", "kubernetes.io/container/accelerator/request", "kubernetes.io/container/cpu/core_usage_time", "kubernetes.io/container/cpu/limit_cores", "kubernetes.io/container/cpu/limit_utilization", "kubernetes.io/container/cpu/request_cores", "kubernetes.io/container/cpu/request_utilization", "kubernetes.io/container/ephemeral_storage/limit_bytes", "kubernetes.io/container/ephemeral_storage/request_bytes", "kubernetes.io/container/ephemeral_storage/used_bytes", "kubernetes.io/container/memory/limit_bytes", "kubernetes.io/container/memory/limit_utilization", "kubernetes.io/container/memory/page_fault_count", "kubernetes.io/container/memory/request_bytes", "kubernetes.io/container/memory/request_utilization", "kubernetes.io/container/memory/used_bytes", "kubernetes.io/container/restart_count", "kubernetes.io/container/uptime"] 121 | 122 | ["kubernetes.io/node/cpu/allocatable_cores", "kubernetes.io/node/cpu/allocatable_utilization", "kubernetes.io/node/cpu/core_usage_time", "kubernetes.io/node/cpu/total_cores", "kubernetes.io/node/ephemeral_storage/allocatable_bytes", "kubernetes.io/node/ephemeral_storage/inodes_free", "kubernetes.io/node/ephemeral_storage/inodes_total", "kubernetes.io/node/ephemeral_storage/total_bytes", "kubernetes.io/node/ephemeral_storage/used_bytes", "kubernetes.io/node/memory/allocatable_bytes", "kubernetes.io/node/memory/allocatable_utilization", "kubernetes.io/node/memory/total_bytes", "kubernetes.io/node/memory/used_bytes", "kubernetes.io/node/network/received_bytes_count", "kubernetes.io/node/network/sent_bytes_count", "kubernetes.io/node/pid_limit", "kubernetes.io/node/pid_used", "kubernetes.io/node_daemon/cpu/core_usage_time", "kubernetes.io/node_daemon/memory/used_bytes"] 123 | 124 | ["kubernetes.io/pod/network/received_bytes_count", "kubernetes.io/pod/network/sent_bytes_count", "kubernetes.io/pod/volume/total_bytes", "kubernetes.io/pod/volume/used_bytes", "kubernetes.io/pod/volume/utilization"] 125 | 126 |127 | 128 | ### Storage 129 |
130 | [storage.googleapis.com/api/request_count",storage.googleapis.com/network/received_bytes_count",storage.googleapis.com/network/sent_bytes_count",storage.googleapis.com/storage/object_count"] 131 |132 | 133 | ### Logging 134 | 135 |
136 | ["logging.googleapis.com/billing/bytes_ingested","logging.googleapis.com/billing/monthly_bytes_ingested","logging.googleapis.com/byte_count","logging.googleapis.com/exports/byte_count","logging.googleapis.com/exports/error_count","logging.googleapis.com/exports/log_entry_count","logging.googleapis.com/log_entry_count","logging.googleapis.com/logs_based_metrics_error_count","logging.googleapis.com/metric_throttled","logging.googleapis.com/time_series_count"] 137 |138 | 139 | ### Monitoring 140 | 141 |
142 | ["monitoring.googleapis.com/billing/bytes_ingested","monitoring.googleapis.com/stats/num_time_series","monitoring.googleapis.com/uptime_check/content_mismatch","monitoring.googleapis.com/uptime_check/error_code","monitoring.googleapis.com/uptime_check/http_status","monitoring.googleapis.com/uptime_check/request_latency"] 143 |144 | 145 | ### PubSub 146 | 147 |
148 | ["pubsub.googleapis.com/snapshot/backlog_bytes","pubsub.googleapis.com/snapshot/backlog_bytes_by_region","pubsub.googleapis.com/snapshot/config_updates_count","pubsub.googleapis.com/snapshot/num_messages","pubsub.googleapis.com/snapshot/num_messages_by_region","pubsub.googleapis.com/snapshot/oldest_message_age","pubsub.googleapis.com/snapshot/oldest_message_age_by_region","pubsub.googleapis.com/subscription/ack_message_count","pubsub.googleapis.com/subscription/backlog_bytes","pubsub.googleapis.com/subscription/byte_cost","pubsub.googleapis.com/subscription/config_updates_count","pubsub.googleapis.com/subscription/mod_ack_deadline_message_count","pubsub.googleapis.com/subscription/mod_ack_deadline_message_operation_count","pubsub.googleapis.com/subscription/mod_ack_deadline_request_count","pubsub.googleapis.com/subscription/num_outstanding_messages","pubsub.googleapis.com/subscription/num_undelivered_messages","pubsub.googleapis.com/subscription/oldest_unacked_message_age_by_region","pubsub.googleapis.com/subscription/pull_ack_message_operation_count","pubsub.googleapis.com/subscription/pull_ack_request_count","pubsub.googleapis.com/subscription/pull_message_operation_count","pubsub.googleapis.com/subscription/pull_message_operation_count","pubsub.googleapis.com/subscription/push_request_count","pubsub.googleapis.com/subscription/push_request_latencies","pubsub.googleapis.com/subscription/sent_message_count","pubsub.googleapis.com/topic/message_sizes","pubsub.googleapis.com/topic/num_unacked_messages_by_region","pubsub.googleapis.com/topic/oldest_unacked_message_age_by_region"] 149 |150 | 151 | 152 | 153 | -------------------------------------------------------------------------------- /Metrics/main.py: -------------------------------------------------------------------------------- 1 | #GCPMetricsFunction v0.7.2 2 | #All-in-one metrics function 3 | 4 | '''MIT License 5 | Copyright (c) 2019 Splunk 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 12 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 13 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 14 | SOFTWARE. ''' 15 | 16 | import base64 17 | import argparse 18 | import os 19 | import pprint 20 | import time 21 | import json 22 | import re 23 | import threading 24 | from threading import Thread 25 | from queue import Queue 26 | 27 | from google.cloud import monitoring_v3 28 | from datetime import datetime 29 | 30 | from datetime import date 31 | 32 | import time 33 | import requests 34 | from requests.adapters import HTTPAdapter 35 | import urllib3 36 | ##turns off the warning that is generated below because using self signed ssl cert 37 | urllib3.disable_warnings() 38 | 39 | 40 | #threadsafe HEC Events list 41 | class HECMessages: 42 | def __init__(self): 43 | self.HECevents = [] 44 | self._lock = threading.Lock() 45 | 46 | def locked_update(self, HECevent): 47 | with self._lock: 48 | self.HECevents.append(HECevent) 49 | 50 | """Triggered from a message on a Cloud Pub/Sub topic. 51 | Args: 52 | event (dict): Event payload. 53 | context (google.cloud.functions.Context): Metadata for the event. 54 | These values are ignored as used only as a Trigger for this function 55 | """ 56 | 57 | def hello_pubsub(event, context): 58 | 59 | HEC_Pack_size=20 # number of events per http post to HEC. Max size = 5MB by default on HEC 60 | now = time.time() 61 | #HECevents=[] 62 | HECevents=HECMessages() #create threadsafe message list 63 | 64 | metricslist=json.loads(os.environ['METRICS_LIST']) 65 | try: 66 | payloadType=os.environ['METRIC_INDEX_TYPE'] 67 | except: 68 | payloadType='EVENT' 69 | #print(metricslist) 70 | 71 | workers=len(metricslist) 72 | if workers>8: 73 | workers=8 74 | 75 | metricsq=Queue() 76 | for x in range(workers): 77 | worker = BuilderThreadWorker(metricsq) 78 | # Set as daemon thread 79 | worker.daemon = True 80 | worker.start() 81 | 82 | for metric in metricslist: 83 | metricsq.put((metric, now, HECevents,payloadType)) 84 | 85 | #wait for all of the builds to complete 86 | metricsq.join() 87 | 88 | message_counter=0 89 | package='' 90 | flushed=0 91 | 92 | workers=int(round(len(HECevents.HECevents)/HEC_Pack_size)) 93 | queue = Queue() 94 | threadcount=10 95 | if workers