├── AzureFunctionForSplunk ├── host.json ├── ActivityLogCategories.json ├── MetricsCategories.json ├── AzureFunctionForSplunk.csproj ├── EhActivityLogsExt.cs ├── EhLadTelemetryExt.cs ├── EhMetricsExt.cs ├── EhWadTelemetryExt.cs ├── EhDiagnosticLogsExt.cs ├── FaultProcessor.cs ├── DiagnosticLogCategories.json ├── AzMonMessages.cs ├── .gitignore ├── Runner.cs ├── AzMonMessage.cs ├── Utils.cs └── SplunkEventMessages.cs ├── images ├── AzureFunctionPlusHEC.PNG ├── diagnosticLogSetting.png └── diagnosticLogFunction.png ├── LICENSE ├── AzureFunctionForSplunk.sln ├── .gitattributes ├── .gitignore └── README.md /AzureFunctionForSplunk/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0" 3 | } -------------------------------------------------------------------------------- /images/AzureFunctionPlusHEC.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/AzureFunctionforSplunkVS/HEAD/images/AzureFunctionPlusHEC.PNG -------------------------------------------------------------------------------- /images/diagnosticLogSetting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/AzureFunctionforSplunkVS/HEAD/images/diagnosticLogSetting.png -------------------------------------------------------------------------------- /images/diagnosticLogFunction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/AzureFunctionforSplunkVS/HEAD/images/diagnosticLogFunction.png -------------------------------------------------------------------------------- /AzureFunctionForSplunk/ActivityLogCategories.json: -------------------------------------------------------------------------------- 1 | { 2 | "administrative": "amal:administrative", 3 | "ascrecommendation": "amal:asc:recommendation", 4 | "servicehealth": "amal:serviceHealth", 5 | "resourcehealth": "amal:resourceHealth", 6 | "autoscalesettings": "amal:autoscaleSettings", 7 | "ascalert": "amal:asc:alert", 8 | "insights": "amal:insights", 9 | "security": "amal:security", 10 | "windowseventlogstable": "wineventlog:security" 11 | } -------------------------------------------------------------------------------- /AzureFunctionForSplunk/MetricsCategories.json: -------------------------------------------------------------------------------- 1 | { 2 | "MICROSOFT.COMPUTE/VIRTUALMACHINES": "amm:compute:vm", 3 | "MICROSOFT.WEB/SITES": "amm:web:sites", 4 | "MICROSOFT.SQL/SERVERS/DATABASES": "amm:sqlserver:database", 5 | "MICROSOFT.SQL/SERVERS": "amm:sqlserver:server", 6 | "MICROSOFT.EVENTHUB/NAMESPACES": "amm:eventhub:namespace", 7 | "MICROSOFT.SERVICEBUS/NAMESPACES": "amm:sb:namespace", 8 | "MICROSOFT.WEB/SERVERFARMS": "amm:web:serverfarm", 9 | "MICROSOFT.BATCH/BATCHACCOUNTS": "amm:batch:accounts" 10 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28307.705 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureFunctionForSplunk", "AzureFunctionForSplunk\AzureFunctionForSplunk.csproj", "{70EE11BD-9CAF-432C-97B7-3F4B11E7A2A8}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {70EE11BD-9CAF-432C-97B7-3F4B11E7A2A8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {70EE11BD-9CAF-432C-97B7-3F4B11E7A2A8}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {70EE11BD-9CAF-432C-97B7-3F4B11E7A2A8}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {70EE11BD-9CAF-432C-97B7-3F4B11E7A2A8}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {63CE4300-29B2-405B-9FE3-A9BEB03D9C11} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/AzureFunctionForSplunk.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | netcoreapp2.1 4 | v2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | Always 15 | 16 | 17 | Always 18 | 19 | 20 | PreserveNewest 21 | 22 | 23 | PreserveNewest 24 | Never 25 | 26 | 27 | Always 28 | 29 | 30 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/EhActivityLogsExt.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs; 28 | using Microsoft.Extensions.Logging; 29 | using System.Threading.Tasks; 30 | 31 | namespace AzureFunctionForSplunk 32 | { 33 | public static class EhActivityLogsExt 34 | { 35 | [FunctionName("EhActivityLogsExt")] 36 | public static async Task Run( 37 | [EventHubTrigger("%input-hub-name-activity-log%", Connection = "hubConnection", ConsumerGroup = "%consumer-group-activity-log%")]string[] messages, 38 | [EventHub("%output-hub-name-proxy%", Connection = "outputHubConnection")]IAsyncCollector outputEvents, 39 | IBinder blobFaultBinder, 40 | IBinder incomingBatchBinder, 41 | Binder queueFaultBinder, 42 | ILogger log) 43 | { 44 | var runner = new Runner(); 45 | await runner.Run(messages, blobFaultBinder, queueFaultBinder, incomingBatchBinder, outputEvents, log); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/EhLadTelemetryExt.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs; 28 | using Microsoft.Azure.WebJobs.Host; 29 | using System.Threading.Tasks; 30 | using Microsoft.Extensions.Logging; 31 | 32 | namespace AzureFunctionForSplunk 33 | { 34 | public static class EhLadTelemetryExt 35 | { 36 | [FunctionName("EhLadTelemetryExt")] 37 | public static async Task Run( 38 | [EventHubTrigger("%input-hub-name-lad%", Connection = "hubConnection", ConsumerGroup = "%consumer-group-lad%")]string[] messages, 39 | [EventHub("%output-hub-name-proxy%", Connection = "outputHubConnection")]IAsyncCollector outputEvents, 40 | IBinder blobFaultBinder, 41 | IBinder incomingBatchBinder, 42 | Binder queueFaultBinder, 43 | ILogger log) 44 | { 45 | var runner = new Runner(); 46 | await runner.Run(messages, blobFaultBinder, queueFaultBinder, incomingBatchBinder, outputEvents, log); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/EhMetricsExt.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs; 28 | using Microsoft.Azure.WebJobs.Host; 29 | using System.Threading.Tasks; 30 | using Microsoft.Extensions.Logging; 31 | 32 | namespace AzureFunctionForSplunk 33 | { 34 | public static class EhMetricsExt 35 | { 36 | [FunctionName("EhMetricsExt")] 37 | public static async Task Run( 38 | [EventHubTrigger("%input-hub-name-metrics%", Connection = "hubConnection", ConsumerGroup = "%consumer-group-metrics%")]string[] messages, 39 | [EventHub("%output-hub-name-proxy%", Connection = "outputHubConnection")]IAsyncCollector outputEvents, 40 | IBinder blobFaultBinder, 41 | IBinder incomingBatchBinder, 42 | Binder queueFaultBinder, 43 | ILogger log) 44 | { 45 | var runner = new Runner(); 46 | await runner.Run(messages, blobFaultBinder, queueFaultBinder, incomingBatchBinder, outputEvents, log); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/EhWadTelemetryExt.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs; 28 | using Microsoft.Azure.WebJobs.Host; 29 | using System.Threading.Tasks; 30 | using Microsoft.Extensions.Logging; 31 | 32 | namespace AzureFunctionForSplunk 33 | { 34 | public static class EhWadTelemetryExt 35 | { 36 | [FunctionName("EhWadTelemetryExt")] 37 | public static async Task Run( 38 | [EventHubTrigger("%input-hub-name-wad%", Connection = "hubConnection", ConsumerGroup = "%consumer-group-wad%")]string[] messages, 39 | [EventHub("%output-hub-name-proxy%", Connection = "outputHubConnection")]IAsyncCollector outputEvents, 40 | IBinder blobFaultBinder, 41 | IBinder incomingBatchBinder, 42 | Binder queueFaultBinder, 43 | ILogger log) 44 | { 45 | var runner = new Runner(); 46 | await runner.Run(messages, blobFaultBinder, queueFaultBinder, incomingBatchBinder, outputEvents, log); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/EhDiagnosticLogsExt.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs; 28 | using Microsoft.Azure.WebJobs.Host; 29 | using System.Threading.Tasks; 30 | using Microsoft.Extensions.Logging; 31 | 32 | namespace AzureFunctionForSplunk 33 | { 34 | public static class EhDiagnosticLogsExt 35 | { 36 | [FunctionName("EhDiagnosticLogsExt")] 37 | public static async Task Run( 38 | [EventHubTrigger("%input-hub-name-diagnostic-logs%", Connection = "hubConnection", ConsumerGroup = "%consumer-group-diagnostic-logs%")]string[] messages, 39 | [EventHub("%output-hub-name-proxy%", Connection = "outputHubConnection")]IAsyncCollector outputEvents, 40 | IBinder blobFaultBinder, 41 | IBinder incomingBatchBinder, 42 | Binder queueFaultBinder, 43 | ILogger log) 44 | { 45 | var runner = new Runner(); 46 | await runner.Run(messages, blobFaultBinder, queueFaultBinder, incomingBatchBinder, outputEvents, log); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/FaultProcessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using Microsoft.Azure.WebJobs; 4 | using Microsoft.Azure.WebJobs.Host; 5 | using Microsoft.WindowsAzure.Storage.Blob; 6 | using System.Threading.Tasks; 7 | using Newtonsoft.Json; 8 | using System.Collections.Generic; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace AzureFunctionForSplunk 12 | { 13 | public static class FaultProcessor 14 | { 15 | [FunctionName("FaultProcessor")] 16 | public static async Task Run( 17 | [QueueTrigger("%input-hub-name-faults%", Connection = "AzureWebJobsStorage")]string fault, 18 | [EventHub("%output-hub-name-proxy%", Connection = "outputHubConnection")]IAsyncCollector outputEvents, 19 | IBinder blobFaultBinder, 20 | ILogger log) 21 | { 22 | string outputBinding = Utils.getEnvironmentVariable("outputBinding").ToLower(); 23 | if (outputBinding.Length == 0) 24 | { 25 | log.LogError("Value for outputBinding is required. Permitted values are: 'proxy', 'hec', 'eventhub'."); 26 | return; 27 | } 28 | 29 | var faultData = JsonConvert.DeserializeObject(fault); 30 | 31 | var blobReader = await blobFaultBinder.BindAsync( 32 | new BlobAttribute($"transmission-faults/{faultData.id}", FileAccess.ReadWrite)); 33 | 34 | var json = await blobReader.DownloadTextAsync(); 35 | 36 | try 37 | { 38 | List faultMessages = await Task>.Factory.StartNew(() => JsonConvert.DeserializeObject>(json)); 39 | 40 | switch (outputBinding) 41 | { 42 | case "hec": 43 | await Utils.obHEC(faultMessages, log); 44 | break; 45 | case "proxy": 46 | await Utils.obProxy(faultMessages, log); 47 | break; 48 | case "eventhub": 49 | await Utils.obEventhub(faultMessages, outputEvents, log); 50 | break; 51 | } 52 | 53 | } 54 | catch (Exception ex) 55 | { 56 | log.LogError(ex.Message); 57 | log.LogError($"FaultProcessor failed to transmit: {faultData.id}"); 58 | throw new Exception("FaultProcessor failed to transmit"); 59 | } 60 | 61 | await blobReader.DeleteAsync(); 62 | 63 | log.LogInformation($"C# Queue trigger function FaultProcessor processed: {faultData.id}"); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/DiagnosticLogCategories.json: -------------------------------------------------------------------------------- 1 | { 2 | "MICROSOFT.AUTOMATION/AUTOMATIONACCOUNTS/JOBLOGS": "amdl:auto:acct:jobLogs", 3 | "MICROSOFT.AUTOMATION/AUTOMATIONACCOUNTS/JOBSTREAMS": "amdl:auto:acct:jobStreams", 4 | "MICROSOFT.BATCH/BATCHACCOUNTS/SERVICELOG": "amdl:btch:acct:serviceLog", 5 | "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/KUBE-APISERVER": "amdl:aks:cluster", 6 | "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/KUBE-AUDIT": "amdl:aks:audit", 7 | "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/KUBE-CONTROLLER-MANAGER": "amdl:aks:manager", 8 | "MICROSOFT.DATALAKEANALYTICS/ACCOUNTS/AUDIT": "amdl:dalk:acct:audit", 9 | "MICROSOFT.DATALAKEANALYTICS/ACCOUNTS/REQUESTS": "amdl:dalk:acct:requests", 10 | "MICROSOFT.DATALAKESTORE/ACCOUNTS/AUDIT": "amdl:dlst:acct:audit", 11 | "MICROSOFT.DATALAKESTORE/ACCOUNTS/REQUESTS": "amdl:dlst:acct:requests", 12 | "MICROSOFT.EVENTHUB/NAMESPACES/ARCHIVELOGS": "amdl:eh:ns:archiveLogs", 13 | "MICROSOFT.EVENTHUB/NAMESPACES/OPERATIONALLOGS": "amdl:eh:ns:operationalLogs", 14 | "MICROSOFT.KEYVAULT/VAULTS/AUDITEVENT": "amdl:keyv:vaul:auditEvent", 15 | "MICROSOFT.LOGIC/WORKFLOWS/WORKFLOWRUNTIME": "amdl:logc:wkfl:workflowRuntime", 16 | "MICROSOFT.LOGIC/INTEGRATIONACCOUNTS/INTEGRATIONACCOUNTTRACKINGEVENTS": "amdl:logc:acct:tracking", 17 | "MICROSOFT.NETWORK/AZUREFIREWALLS/AZUREFIREWALLNETWORKRULE": "amdl:net:firewall", 18 | "MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/NETWORKSECURITYGROUPEVENT": "amdl:net:nsg:event", 19 | "MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/NETWORKSECURITYGROUPRULECOUNTER": "amdl:net:nsg:rule", 20 | "MICROSOFT.NETWORK/LOADBALANCERS/LOADBALANCERALERTEVENT": "amdl:net:lb:alert", 21 | "MICROSOFT.NETWORK/LOADBALANCERS/LOADBALANCERPROBEHEALTHSTATUS": "amdl:net:lb:probeHealth", 22 | "MICROSOFT.NETWORK/APPLICATIONGATEWAYS/APPLICATIONGATEWAYACCESSLOG": "amdl:net:ag:access", 23 | "MICROSOFT.NETWORK/APPLICATIONGATEWAYS/APPLICATIONGATEWAYPERFORMANCELOG": "amdl:net:ag:perf", 24 | "MICROSOFT.NETWORK/APPLICATIONGATEWAYS/APPLICATIONGATEWAYFIREWALLLOG": "amdl:net:ag:firewall", 25 | "MICROSOFT.NETWORK/VIRTUALNETWORKGATEWAYS/P2SDIAGNOSTICLOG": "amdl:net:vnetgateway", 26 | "MICROSOFT.NETWORK/VIRTUALNETWORKGATEWAYS/IKEDIAGNOSTICLOG": "amdl:net:vnetgateway", 27 | "MICROSOFT.SEARCH/SEARCHSERVICES/OPERATIONLOGS": "amdl:srch:srch:operationLogs", 28 | "MICROSOFT.SERVERMANAGEMENT/NODES/REQUESTLOGS": "amdl:srvr:node:requestLogs", 29 | "MICROSOFT.SERVICEBUS/NAMESPACES/OPERATIONALLOGS": "amdl:sb:ns:operationalLogs", 30 | "MICROSOFT.SQL/SERVERS/DATABASES/QUERYSTORERUNTIMESTATISTICS": "amdl:sql:db:stats", 31 | "MICROSOFT.STREAMANALYTICS/STREAMINGJOBS/EXECUTION": "amdl:sa:jobs:execution", 32 | "MICROSOFT.STREAMANALYTICS/STREAMINGJOBS/AUTHORING": "amdl:sa:jobs:authoring", 33 | "MICROSOFT.SECURITYGRAPH/ALERT": "amdl:securitygraph:alert", 34 | "MICROSOFT.AADIAM/AUDIT": "amdl:aadal:audit", 35 | "MICROSOFT.AADIAM/SIGNIN": "amdl:aadal:signin" 36 | } -------------------------------------------------------------------------------- /AzureFunctionForSplunk/AzMonMessages.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs.Host; 28 | using Newtonsoft.Json; 29 | using System.Collections.Generic; 30 | using Microsoft.Extensions.Logging; 31 | 32 | namespace AzureFunctionForSplunk 33 | { 34 | public abstract class AzMonMessages 35 | { 36 | public ILogger Log { get; set; } 37 | 38 | public virtual List DecomposeIncomingBatch(string[] messages) 39 | { 40 | List decomposed = new List(); 41 | 42 | foreach (var message in messages) 43 | { 44 | dynamic obj = JsonConvert.DeserializeObject>(message); 45 | 46 | if (((IDictionary)obj).ContainsKey("records")) 47 | { 48 | var records = obj["records"]; 49 | 50 | foreach (var record in records) 51 | { 52 | string stringRecord = record.ToString(); 53 | 54 | decomposed.Add(stringRecord); 55 | } 56 | } else 57 | { 58 | Log.LogError("AzMonMessages: invalid message structure, missing 'records'"); 59 | } 60 | } 61 | 62 | return decomposed; 63 | } 64 | 65 | public AzMonMessages(ILogger log) 66 | { 67 | Log = log; 68 | } 69 | 70 | } 71 | 72 | public class ActivityLogMessages : AzMonMessages 73 | { 74 | public ActivityLogMessages(ILogger log) : base(log) { } 75 | } 76 | 77 | public class DiagnosticLogMessages : AzMonMessages 78 | { 79 | public DiagnosticLogMessages(ILogger log) : base(log) { } 80 | } 81 | 82 | public class MetricMessages : AzMonMessages 83 | { 84 | public MetricMessages(ILogger log) : base(log) { } 85 | } 86 | 87 | public class WadMessages : AzMonMessages 88 | { 89 | public WadMessages(ILogger log): base(log) { } 90 | } 91 | 92 | public class LadMessages : AzMonMessages 93 | { 94 | public LadMessages(ILogger log) : base(log) { } 95 | 96 | public override List DecomposeIncomingBatch(string[] messages) 97 | { 98 | List decomposed = new List(); 99 | 100 | foreach (var record in messages) 101 | { 102 | string stringRecord = record.ToString(); 103 | 104 | decomposed.Add(stringRecord); 105 | } 106 | 107 | return decomposed; 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /AzureFunctionForSplunk/Runner.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs; 28 | using Microsoft.Azure.WebJobs.Host; 29 | using Microsoft.WindowsAzure.Storage.Blob; 30 | using Microsoft.Extensions.Logging; 31 | using Microsoft.WindowsAzure.Storage.Queue; 32 | using System.Threading.Tasks; 33 | using System.Collections.Generic; 34 | using System; 35 | using System.IO; 36 | using Newtonsoft.Json; 37 | 38 | namespace AzureFunctionForSplunk 39 | { 40 | public class Runner 41 | { 42 | public async Task Run( 43 | string[] messages, 44 | IBinder blobFaultBinder, 45 | Binder queueFaultBinder, 46 | IBinder incomingBatchBinder, 47 | IAsyncCollector outputEvents, 48 | ILogger log) 49 | { 50 | var batchId = Guid.NewGuid().ToString(); 51 | 52 | bool logIncoming = Utils.getEnvironmentVariable("logIncomingBatches").ToLower() == "true"; 53 | 54 | var azMonMsgs = (AzMonMessages)Activator.CreateInstance(typeof(T1), log); 55 | List decomposed = null; 56 | 57 | try 58 | { 59 | decomposed = azMonMsgs.DecomposeIncomingBatch(messages); 60 | 61 | if (logIncoming) 62 | { 63 | try 64 | { 65 | var blobWriter = await incomingBatchBinder.BindAsync( 66 | new BlobAttribute($"transmission-incoming/{batchId}", FileAccess.ReadWrite)); 67 | 68 | await blobWriter.UploadTextAsync(String.Join(",", messages)); 69 | } 70 | catch (Exception exIncomingBlob) 71 | { 72 | log.LogError($"Failed to log the incoming transmission blob: {batchId}. {exIncomingBlob.Message}"); 73 | throw exIncomingBlob; 74 | } 75 | } 76 | 77 | } 78 | catch (Exception) 79 | { 80 | throw; 81 | } 82 | 83 | if (decomposed.Count > 0) 84 | { 85 | var splunkMsgs = (SplunkEventMessages)Activator.CreateInstance(typeof(T2), outputEvents, log); 86 | try 87 | { 88 | splunkMsgs.Ingest(decomposed.ToArray()); 89 | await splunkMsgs.Emit(); 90 | } 91 | catch (Exception exEmit) 92 | { 93 | 94 | try 95 | { 96 | var blobWriter = await blobFaultBinder.BindAsync( 97 | new BlobAttribute($"transmission-faults/{batchId}", FileAccess.ReadWrite)); 98 | 99 | string json = await Task.Factory.StartNew(() => JsonConvert.SerializeObject(splunkMsgs.splunkEventMessages)); 100 | await blobWriter.UploadTextAsync(json); 101 | } 102 | catch (Exception exFaultBlob) 103 | { 104 | log.LogError($"Failed to write the fault blob: {batchId}. {exFaultBlob.Message}"); 105 | throw exFaultBlob; 106 | } 107 | 108 | try 109 | { 110 | var qMsg = new TransmissionFaultMessage { id = batchId, type = typeof(T2).ToString() }; 111 | string qMsgJson = JsonConvert.SerializeObject(qMsg); 112 | 113 | var queueWriter = await queueFaultBinder.BindAsync( 114 | new QueueAttribute("transmission-faults")); 115 | await queueWriter.AddMessageAsync(new CloudQueueMessage(qMsgJson)); 116 | } 117 | catch (Exception exFaultQueue) 118 | { 119 | log.LogError($"Failed to write the fault queue: {batchId}. {exFaultQueue.Message}"); 120 | throw exFaultQueue; 121 | } 122 | 123 | log.LogError($"Error emitting messages to output binding: {exEmit.Message}. The messages were held in the fault processor queue for handling once the error is resolved."); 124 | throw exEmit; 125 | } 126 | } 127 | 128 | log.LogInformation($"C# Event Hub trigger function processed a batch of messages: {messages.Length}"); 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # MSTest test Results 33 | [Tt]est[Rr]esult*/ 34 | [Bb]uild[Ll]og.* 35 | 36 | # NUNIT 37 | *.VisualState.xml 38 | TestResult.xml 39 | 40 | # Build Results of an ATL Project 41 | [Dd]ebugPS/ 42 | [Rr]eleasePS/ 43 | dlldata.c 44 | 45 | # .NET Core 46 | project.lock.json 47 | project.fragment.lock.json 48 | artifacts/ 49 | **/Properties/launchSettings.json 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # Visual Studio code coverage results 117 | *.coverage 118 | *.coveragexml 119 | 120 | # NCrunch 121 | _NCrunch_* 122 | .*crunch*.local.xml 123 | nCrunchTemp_* 124 | 125 | # MightyMoose 126 | *.mm.* 127 | AutoTest.Net/ 128 | 129 | # Web workbench (sass) 130 | .sass-cache/ 131 | 132 | # Installshield output folder 133 | [Ee]xpress/ 134 | 135 | # DocProject is a documentation generator add-in 136 | DocProject/buildhelp/ 137 | DocProject/Help/*.HxT 138 | DocProject/Help/*.HxC 139 | DocProject/Help/*.hhc 140 | DocProject/Help/*.hhk 141 | DocProject/Help/*.hhp 142 | DocProject/Help/Html2 143 | DocProject/Help/html 144 | 145 | # Click-Once directory 146 | publish/ 147 | 148 | # Publish Web Output 149 | *.[Pp]ublish.xml 150 | *.azurePubxml 151 | # TODO: Comment the next line if you want to checkin your web deploy settings 152 | # but database connection strings (with potential passwords) will be unencrypted 153 | *.pubxml 154 | *.publishproj 155 | 156 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 157 | # checkin your Azure Web App publish settings, but sensitive information contained 158 | # in these scripts will be unencrypted 159 | PublishScripts/ 160 | 161 | # NuGet Packages 162 | *.nupkg 163 | # The packages folder can be ignored because of Package Restore 164 | **/packages/* 165 | # except build/, which is used as an MSBuild target. 166 | !**/packages/build/ 167 | # Uncomment if necessary however generally it will be regenerated when needed 168 | #!**/packages/repositories.config 169 | # NuGet v3's project.json files produces more ignorable files 170 | *.nuget.props 171 | *.nuget.targets 172 | 173 | # Microsoft Azure Build Output 174 | csx/ 175 | *.build.csdef 176 | 177 | # Microsoft Azure Emulator 178 | ecf/ 179 | rcf/ 180 | 181 | # Windows Store app package directories and files 182 | AppPackages/ 183 | BundleArtifacts/ 184 | Package.StoreAssociation.xml 185 | _pkginfo.txt 186 | 187 | # Visual Studio cache files 188 | # files ending in .cache can be ignored 189 | *.[Cc]ache 190 | # but keep track of directories ending in .cache 191 | !*.[Cc]ache/ 192 | 193 | # Others 194 | ClientBin/ 195 | ~$* 196 | *~ 197 | *.dbmdl 198 | *.dbproj.schemaview 199 | *.jfm 200 | *.pfx 201 | *.publishsettings 202 | orleans.codegen.cs 203 | 204 | # Since there are multiple workflows, uncomment next line to ignore bower_components 205 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 206 | #bower_components/ 207 | 208 | # RIA/Silverlight projects 209 | Generated_Code/ 210 | 211 | # Backup & report files from converting an old project file 212 | # to a newer Visual Studio version. Backup files are not needed, 213 | # because we have git ;-) 214 | _UpgradeReport_Files/ 215 | Backup*/ 216 | UpgradeLog*.XML 217 | UpgradeLog*.htm 218 | 219 | # SQL Server files 220 | *.mdf 221 | *.ldf 222 | *.ndf 223 | 224 | # Business Intelligence projects 225 | *.rdl.data 226 | *.bim.layout 227 | *.bim_*.settings 228 | 229 | # Microsoft Fakes 230 | FakesAssemblies/ 231 | 232 | # GhostDoc plugin setting file 233 | *.GhostDoc.xml 234 | 235 | # Node.js Tools for Visual Studio 236 | .ntvs_analysis.dat 237 | node_modules/ 238 | 239 | # Typescript v1 declaration files 240 | typings/ 241 | 242 | # Visual Studio 6 build log 243 | *.plg 244 | 245 | # Visual Studio 6 workspace options file 246 | *.opt 247 | 248 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 249 | *.vbw 250 | 251 | # Visual Studio LightSwitch build output 252 | **/*.HTMLClient/GeneratedArtifacts 253 | **/*.DesktopClient/GeneratedArtifacts 254 | **/*.DesktopClient/ModelManifest.xml 255 | **/*.Server/GeneratedArtifacts 256 | **/*.Server/ModelManifest.xml 257 | _Pvt_Extensions 258 | 259 | # Paket dependency manager 260 | .paket/paket.exe 261 | paket-files/ 262 | 263 | # FAKE - F# Make 264 | .fake/ 265 | 266 | # JetBrains Rider 267 | .idea/ 268 | *.sln.iml 269 | 270 | # CodeRush 271 | .cr/ 272 | 273 | # Python Tools for Visual Studio (PTVS) 274 | __pycache__/ 275 | *.pyc 276 | 277 | # Cake - Uncomment if you are using it 278 | # tools/** 279 | # !tools/packages.config 280 | 281 | # Telerik's JustMock configuration file 282 | *.jmconfig 283 | 284 | # BizTalk build output 285 | *.btp.cs 286 | *.btm.cs 287 | *.odx.cs 288 | *.xsd.cs 289 | 290 | AzureFunctionForSplunk/host.json 291 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/AzMonMessage.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using System; 28 | using System.Collections.Generic; 29 | using System.Dynamic; 30 | using System.Text.RegularExpressions; 31 | 32 | namespace AzureFunctionForSplunk 33 | { 34 | public class AzMonMessage 35 | { 36 | private ExpandoObject _Message; 37 | protected ExpandoObject Message 38 | { 39 | get 40 | { 41 | return _Message; 42 | } 43 | set 44 | { 45 | this._Message = value; 46 | MessageTime = ((dynamic)value).time; 47 | } 48 | } 49 | 50 | protected string ResourceId { get; set; } 51 | public string SubscriptionId { get; set; } 52 | public string ResourceType { get; set; } 53 | public string ResourceName { get; set; } 54 | public string ResourceGroup { get; set; } 55 | public string SplunkSourceType { get; set; } 56 | public DateTime MessageTime { get; set; } 57 | public string TenantId { get; set; } 58 | public string ProviderName { get; set; } 59 | 60 | public AzMonMessage() 61 | { 62 | SubscriptionId = ""; 63 | ResourceId = ""; 64 | ResourceGroup = ""; 65 | ResourceName = ""; 66 | ResourceType = ""; 67 | SplunkSourceType = ""; 68 | TenantId = ""; 69 | } 70 | 71 | public string GetSplunkEventFromMessage() 72 | { 73 | ExpandoObject o = new ExpandoObject(); 74 | ((IDictionary)o).Add("sourcetype", SplunkSourceType); 75 | ((IDictionary)o).Add("time", unixTime().ToString("0.000")); 76 | ((IDictionary)o).Add("event", Message); 77 | string json = Newtonsoft.Json.JsonConvert.SerializeObject(o); 78 | 79 | return json; 80 | } 81 | 82 | double unixTime() 83 | { 84 | double unixTimestamp = MessageTime.Ticks - new DateTime(1970, 1, 1).Ticks; 85 | unixTimestamp /= TimeSpan.TicksPerSecond; 86 | return unixTimestamp; 87 | } 88 | 89 | protected void GetStandardProperties() 90 | { 91 | string pattern; 92 | 93 | pattern = @"SUBSCRIPTIONS\/(.*?)\/"; 94 | Match m = Regex.Match(ResourceId.ToUpper(), pattern); 95 | SubscriptionId = m.Groups[1].Value; 96 | 97 | pattern = @"SUBSCRIPTIONS\/(?:.*?)\/RESOURCEGROUPS\/(.*?)(\/|\Z)"; 98 | m = Regex.Match(ResourceId.ToUpper(), pattern); 99 | ResourceGroup = m.Groups[1].Value; 100 | 101 | pattern = @"PROVIDERS\/(?:.*?\/.*?\/)(.*?)(?:\/|$)"; 102 | m = Regex.Match(ResourceId.ToUpper(), pattern); 103 | ResourceName = m.Groups[1].Value; 104 | 105 | pattern = @"PROVIDERS\/(.*?\/.*?)(?:\/)(?:.*\/)(.*DATABASES)"; 106 | m = Regex.Match(ResourceId.ToUpper(), pattern); 107 | var group1 = m.Groups[1].Value; 108 | var group2 = m.Groups[2].Value; 109 | if (group2 == "DATABASES") 110 | { 111 | ResourceType = group1 + "/" + group2; 112 | } 113 | else 114 | { 115 | pattern = @"PROVIDERS\/(.*?\/.*?)(?:\/)"; 116 | m = Regex.Match(ResourceId.ToUpper(), pattern); 117 | ResourceType = m.Groups[1].Value; 118 | } 119 | } 120 | 121 | protected void AddStandardProperties(string prefix) 122 | { 123 | if (TenantId != "") 124 | { 125 | ((IDictionary)Message).Add($"{prefix}_TenantId", TenantId); 126 | } 127 | if (SubscriptionId != "") 128 | { 129 | ((IDictionary)Message).Add($"{prefix}_SubscriptionId", SubscriptionId); 130 | } 131 | if (ResourceGroup != "") 132 | { 133 | ((IDictionary)Message).Add($"{prefix}_ResourceGroup", ResourceGroup); 134 | } 135 | if (ResourceType != "") 136 | { 137 | ((IDictionary)Message).Add($"{prefix}_ResourceType", ResourceType); 138 | } 139 | if (ResourceName != "") 140 | { 141 | ((IDictionary)Message).Add($"{prefix}_ResourceName", ResourceName); 142 | } 143 | } 144 | 145 | } 146 | 147 | public class AzMonActivityLog : AzMonMessage 148 | { 149 | public AzMonActivityLog(dynamic message, string sourceType) 150 | { 151 | Message = message; 152 | 153 | if (((IDictionary)message).ContainsKey("resourceId")) 154 | { 155 | ResourceId = message.resourceId; 156 | } 157 | else if (((IDictionary)message).ContainsKey("resourceid")) 158 | { 159 | ResourceId = message.resourceid; 160 | } 161 | else 162 | { 163 | throw new Exception("Unable to extract resourceid or resourceId from the message."); 164 | } 165 | 166 | if (((IDictionary)message).ContainsKey("tenantId")) 167 | { 168 | TenantId = message.tenantId; 169 | 170 | var pattern = @"PROVIDERS/(.*?)(?:$)"; 171 | Match m = Regex.Match(ResourceId.ToUpper(), pattern); 172 | ProviderName = m.Groups[1].Value; 173 | } 174 | 175 | SplunkSourceType = sourceType; 176 | base.GetStandardProperties(); 177 | base.AddStandardProperties("amal"); 178 | } 179 | } 180 | 181 | public class AzMonDiagnosticLog : AzMonMessage 182 | { 183 | public AzMonDiagnosticLog(dynamic message) 184 | { 185 | Message = message; 186 | 187 | if (((IDictionary)message).ContainsKey("resourceId")) 188 | { 189 | ResourceId = message.resourceId; 190 | } 191 | else if (((IDictionary)message).ContainsKey("resourceid")) 192 | { 193 | ResourceId = message.resourceid; 194 | } 195 | else 196 | { 197 | throw new Exception("Unable to extract resourceid or resourceId from the message."); 198 | } 199 | 200 | if (((IDictionary)message).ContainsKey("tenantId")) 201 | { 202 | TenantId = message.tenantId; 203 | 204 | var pattern = @"PROVIDERS/(.*?)(?:$)"; 205 | Match m = Regex.Match(ResourceId.ToUpper(), pattern); 206 | ProviderName = m.Groups[1].Value; 207 | } 208 | 209 | base.GetStandardProperties(); 210 | base.AddStandardProperties("amdl"); 211 | } 212 | } 213 | 214 | public class AzMonMetric : AzMonMessage 215 | { 216 | public AzMonMetric(dynamic message, string sourceType) 217 | { 218 | Message = message; 219 | 220 | if (((IDictionary)message).ContainsKey("resourceId")) 221 | { 222 | ResourceId = message.resourceId; 223 | } 224 | else if (((IDictionary)message).ContainsKey("resourceid")) 225 | { 226 | ResourceId = message.resourceid; 227 | } 228 | else 229 | { 230 | throw new Exception("Unable to extract resourceid or resourceId from the message."); 231 | } 232 | 233 | SplunkSourceType = sourceType; 234 | base.GetStandardProperties(); 235 | base.AddStandardProperties("amm"); 236 | } 237 | } 238 | 239 | public class LadAzMonMetric : AzMonMessage 240 | { 241 | public LadAzMonMetric(dynamic message) 242 | { 243 | Message = message; 244 | 245 | if (((IDictionary)message).ContainsKey("resourceId")) 246 | { 247 | ResourceId = message.resourceId; 248 | } 249 | else if (((IDictionary)message).ContainsKey("resourceid")) 250 | { 251 | ResourceId = message.resourceid; 252 | } 253 | else 254 | { 255 | throw new Exception("Unable to extract resourceid or resourceId from the message."); 256 | } 257 | 258 | SplunkSourceType = "azlm:compute:vm"; 259 | base.GetStandardProperties(); 260 | base.AddStandardProperties("azlm"); 261 | } 262 | } 263 | 264 | public class LadAzMonLog : AzMonMessage 265 | { 266 | public LadAzMonLog(dynamic message) 267 | { 268 | Message = message; 269 | 270 | if (((IDictionary)message).ContainsKey("resourceId")) 271 | { 272 | ResourceId = message.resourceId; 273 | } 274 | else if (((IDictionary)message).ContainsKey("resourceid")) 275 | { 276 | ResourceId = message.resourceid; 277 | } 278 | else 279 | { 280 | throw new Exception("Unable to extract resourceid or resourceId from the message."); 281 | } 282 | 283 | SplunkSourceType = "azll:compute:vm"; 284 | base.GetStandardProperties(); 285 | base.AddStandardProperties("azll"); 286 | } 287 | } 288 | 289 | public class WadAzMonMetric : AzMonMessage 290 | { 291 | public WadAzMonMetric(dynamic message) 292 | { 293 | Message = message; 294 | 295 | ResourceType = "MICROSOFT.COMPUTE/VIRTUALMACHINES"; 296 | 297 | SplunkSourceType = "azwm:compute:vm"; 298 | 299 | if (((IDictionary)message).ContainsKey("dimensions")) 300 | { 301 | var dimensions = message.dimensions; 302 | if (((IDictionary)dimensions).ContainsKey("RoleInstance")) 303 | { 304 | string theName = message.dimensions.RoleInstance; 305 | 306 | // if it's there at all, RoleInstance starts with _ 307 | if (theName.Length > 1) ResourceName = theName.Substring(1); 308 | } 309 | } 310 | 311 | AddStandardProperties("azwm"); 312 | } 313 | 314 | } 315 | 316 | public class WadAzMonLog : AzMonMessage 317 | { 318 | public WadAzMonLog(dynamic message) 319 | { 320 | Message = message; 321 | 322 | ResourceType = "MICROSOFT.COMPUTE/VIRTUALMACHINES"; 323 | 324 | SplunkSourceType = "azwm:compute:vm"; 325 | 326 | if (((IDictionary)message).ContainsKey("properties")) 327 | { 328 | var properties = message.properties; 329 | if (((IDictionary)properties).ContainsKey("RoleInstance")) 330 | { 331 | string theName = message.properties.RoleInstance; 332 | 333 | // if it's there at all, RoleInstance starts with _ 334 | if (theName.Length > 1) ResourceName = theName.Substring(1); 335 | } 336 | } 337 | 338 | AddStandardProperties("azwl"); 339 | } 340 | 341 | } 342 | } 343 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## NOTE 2 | 3 | Due to lack of resources to maintain and changes in underlying dependencies, this repo is archived. It can still be forked. 4 | 5 | # AzureFunctionForSplunkVS 6 | Azure Function sends Azure Monitor telemetry to Splunk - coded in C# / Visual Studio 2017. 7 | 8 | [![Deploy to Azure](http://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fsebastus%2FAzureFunctionDeployment%2FSplunkVS%2FazureDeploy.json) 9 | 10 | # Azure Function For Splunk 11 | Azure Function code that sends telemetry from Azure resources to a Splunk Enterprise or Splunk Cloud instance. 12 | 13 | It consumes Metrics, Diagnostic Logs and the Activity Log according to the techniques defined by Azure Monitor, which provides highly granular and real-time monitoring data for Azure resources, and passes those selected by the user's configuration along to Splunk. 14 | 15 | Here are a few resources if you want to learn more about Azure Monitor:
16 | * [Overview of Azure Monitor](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview) 17 | * [Overview of Azure Diagnostic Logs](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-of-diagnostic-logs) 18 | * [Overview of the Azure Activity Log](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-activity-logs) 19 | * [Overview of Metrics in Microsoft Azure](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-metrics) 20 | 21 | ## Important Security Note 22 | The HEC endpoint for a Splunk instance is SSL encrypted. This function CAN ignore the validity of the certificate. To do so, do not provide App Setting 'splunkCertThumbprint' or leave it blank. To ENABLE cert validation, make the value of that setting the thumbprint of the cert. If you provide the cert thumbprint, the splunkAddress must be https://**whatever**. If you do not provide the cert thumbprint, the splunkAddress must be http://**whatever**. 23 | 24 | ## Solution Overview 25 | 26 | At a high level, the Azure Functions approach to delivering telemetry to Splunk does this: 27 | * Azure resources deliver telemetry to event hubs 28 | * Azure Function is triggered by these messages 29 | * Azure Function delivers the messages to Splunk 30 | 31 | Azure Functions are arranged hierarchically as a Function App containing individual functions within. An individual function is triggered by a single event hub. Regarding logs from Azure Monitor, each log category CAN BE sent to its own hub. Each Azure Resource Provider that emits logs may emit more than one log category. Similarly, metrics are sent to a hub as configured by the user. Hence, there MAY BE many hubs for the Function App to watch over. BUT, you can configure all diagnostic logs to go to the same hub. This practice is recommended for simplicity's sake. 32 | 33 | ## Adding additional hubs 34 | 35 | If you choose to allow Azure services with diagnostic logs to create their default hubs, you will need to create additional functions (in addition to the list just below) - one per additional hub. The reason for this is that each function is triggered by exactly one hub. 36 | 37 | Do this by copying EhDiagnosticLogsExt.cs and name the copy according to the new event hub. For example, if you wanted to use the default hub for Workflow Runtime messages, the default hub name is 'insights-logs-workflowruntime'. You could name your new function 'EhWorkflowRuntimeExt', for example. This is a copy of the code: 38 | 39 | ![DiagnosticLogFunction](images/diagnosticLogFunction.png) 40 | 41 | Change the first box to 'EhWorkflowRuntimeExt', the second to the same, and the third to something like "%input-hub-name-workflow-runtime%". Then, in the settings create a new one like the following: 42 | 43 | ![DiagnosticLogSettings](images/diagnosticLogSetting.png) 44 | 45 | Make the setting key match what you put in the 3rd box above ("%input-hub-name-workflow-runtime%") and the value should be the name of your new hub (e.g. insights-logs-workflowruntime). Rebuild and deploy the function app. This is as simple as making your own fork of the code, add the new function & customize it, push and merge your changes to your fork, then use the "Deploy to Azure" (customized to point to your fork) button here in the README.md (above). 46 | 47 | ### Functions in the Function App 48 | * EhActivityLogsExt - consumes Azure Monitor Activity Logs 49 | * EhDiagnosticLogsExt - consumes Azure Monitor Diagnostic Logs 50 | * EhLadTelemetryExt - consumes telemetry from Azure Linux VMs 51 | * EhMetricsExt - consumes Azure Monitor Metrics 52 | * EhWadTelemetryExt - consumes telemetry from Azure Windows VMs 53 | * FaultProcessor - consumes queue messages from faulted transmissions 54 | 55 | The Activity Log transmits to a hub named 'Insights-Operational-Logs'. This will be configurable at some point, but for now, the function should be configured to listen to that hub. 56 | 57 | The solution leverages the capacity of an Azure Function to be triggered by arrival of messages to an Event Hub. The messages are aggregated by the Azure Functions back end so they arrive at the function already in a batch where the size of the batch depends on current message volume and settings. The batch is examined, the properties of each event are augmented, and then the events are sent via the selected output binding to the Splunk instance. 58 | 59 | ### Cloud-based Splunk using HTTP Event Collector Output Binding 60 | 61 | ![AzureFunctionPlusHEC](images/AzureFunctionPlusHEC.PNG) 62 | The image shows only Splunk VM, but the solution targets Splunk Cloud as well. The Splunk VM may be Splunk Enterprise or a Forwarder. 63 | 64 | ## Installation and Configuration 65 | 66 | Installation and Configuration tasks for the overall solution fall into a few buckets: 67 | 68 | * Diagnostics profiles 69 | * Event hubs 70 | * Splunk instance 71 | * Azure Function 72 | 73 | ### Diagnostics Profiles 74 | Each resource to be monitored must have a diagnostics profile created for it. This can be done in the portal, but more likely you'll want to write a script to configure existing resources and update your solution templates to create these profiles upon creation of the resource. Here's a place to start: 75 | 76 | [Automatically enable Diagnostic Settings at resource creation using a Resource Manager template](https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-enable-diagnostic-logs-using-template) 77 | 78 | ### Diagnostic Profiles for VMs 79 | Each VM to be monitored by the function app requires configuration artifacts: 80 | * Diagnostic Extension designed for the OS 81 | * public configuration file - tells the extension which metrics and logs you want to emit from the VM 82 | * private configuration file - contains credentials for the targets of the VMs telemetry 83 | 84 | For Linux VMs, guidance on installing the extension and guidance on designing the configuration files is in this document 85 | [Use Linux Diagnostic Extension to monitor metrics and logs](https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-linux) 86 | For Windows VMs, that same guidance is here: 87 | [Use PowerShell to enable Azure Diagnostics in a virtual machine running Windows](https://docs.microsoft.com/en-us/azure/virtual-machines/extensions/diagnostics-windows) 88 | 89 | The private configuration file for a Linux VM will require a SAS token. A sample of C# code that generates a suitable SAS token is [here](https://github.com/sebastus/GenerateSasForEh). The sasURL in the protected config file will look something like this: 90 | 91 | ``` 92 | https://namespace.servicebus.windows.net/insights-telemetry-lad?sr=https%3a%2f%2fnamespace.servicebus.windows.net%2finsights-telemetry-lad&sig=wsVxC%2f%2bm7vRhOZjm%2fJMEWTX%2by0sOil6z%2bFqwoWrstkQ%3d&se=1562845709&skn=RootManageSharedAccessKey 93 | ``` 94 | 95 | ### Event hubs 96 | 97 | As mentioned, logs and metrics are sent through event hubs. Event hubs are created automatically by the Azure resource providers that need to write the information, so at the outset all you need to do is create the Event Hub Namespace. Here's how to do this in the portal: 98 | 99 | [Create an Event Hubs namespace and an event hub using the Azure portal](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create) 100 | 101 | You will need to provide credentials to the Azure Function so it can read the hubs. On one end of the security spectrum you could provide the RootManageSharedAccessKey to all functions for all hubs within the namespace. At the other end of the spectrum (following the principal of least required authority) you can create a policy for each hub with Listen access and provide that credential on a function-by-function basis. 102 | 103 | An example of copying the connection string (NOT just the key) associated with the RootManageSharedAccessKey policy is given on [this page](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create) at the bottom of the page. 104 | 105 | To create a least permissions policy: 106 | * Select the hub from the list of hubs on the event hub namespace blade 107 | * Select "Shared access policies" 108 | * Click "+ Add" 109 | * Give it a name, select "Listen", click "Create" button. 110 | * Once it's created, re-enter the properties for that new policy and copy the connection string (NOT just the key). 111 | 112 | ### Splunk Instance 113 | 114 | #### Using HEC output binding 115 | Configuration of the Splunk instance amounts to opening the HEC endpoint and creating/copying the authentication token. The endpoint address and token value must be entered as settings into the Function App. 116 | 117 | Instructions for opening the endpoint and creating/copying the token are on this Splunk webpage: 118 | 119 | [HTTP Event Collector walkthrough](http://dev.splunk.com/view/event-collector/SP-CAAAE7F#usinghttpeventcollector) 120 | 121 | ### Azure Function 122 | 123 | There are several ways to create an Azure Function and load your code into it. Here's one such example: 124 | 125 | [Create your first function using the Azure CLI](https://docs.microsoft.com/en-us/azure/azure-functions/functions-create-first-azure-function-azure-cli) 126 | 127 | This technique requires that your code be referencable in a github repo, and this is exactly what we need. 128 | 129 | Because the repo needs to contain settings specific to your installation, I recommend you fork this repo and make your changes there. Then provide the address of your fork in the example above to populate your function app. 130 | 131 | Note that the actual settings are not in the code. These are provided by you in the portal. 132 | 133 | If you want to automate the creation of your Azure Function (recommended), there is a solution template that accomplishes this located here: 134 | 135 | [Azure Function Deployment ARM template.](https://github.com/sebastus/AzureFunctionDeployment/tree/SplunkVS) 136 | 137 | Use the SplunkVS branch in the link. It's configured specifically for this function. 138 | 139 | Or, just click the "Deploy to Azure" button at the top of this page. 140 | 141 | Once the Function App exists, check and correct application settings. The settings are created automatically if you use the ARM template (the 'Deploy to Azure' button.) 142 | 143 | You will then need to create an Azure Storage Queue in the storage account. Its name is case-sensitive: "transmission-faults". On the "Function app settings" page, switch "Function app edit mode" to Read/Write and then disable the FaultProcessor function on the Functions page. 144 | 145 | #### Using HEC output binding 146 | 147 | * hubConnection - connection string for the hub namespace 148 | * input-hub-name-activity-logs - should be set to 'insights-operational-logs' 149 | * input-hub-name-diagnostics-logs - set to name of hub for diagnostic logs, e.g. insights-diagnostic-logs (or your other custom choice) 150 | * input-hub-name-metrics - 'insights-metrics-pt1m' 151 | * outputBinding - HEC 152 | * splunkAddress - e.g. https://YOURVM.SOMEREGION.cloudapp.azure.com:8088/services/collector/event 153 | * splunkToken - e.g. 5F1B2C8F-YOUR-GUID-HERE-CE29A659E7D1 154 | * splunkCertThumbprint - leave blank to ignore cert validity. This is a good choice if you haven't yet installed your own cert on Splunk Enterprise. Set it to the thumbprint of your cert once it's installed in Splunk Enterprise. 155 | 156 | 157 | # Contributing 158 | 159 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 160 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 161 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 162 | 163 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 164 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 165 | provided by the bot. You will only need to do this once across all repos using our CLA. 166 | 167 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 168 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 169 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 170 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/Utils.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.Services.AppAuthentication; 28 | using Microsoft.Azure.WebJobs; 29 | using Microsoft.Extensions.Logging; 30 | using Newtonsoft.Json; 31 | using System; 32 | using System.Collections.Generic; 33 | using System.IO; 34 | using System.Net; 35 | using System.Net.Http; 36 | using System.Net.Security; 37 | using System.Security.Cryptography.X509Certificates; 38 | using System.Text; 39 | using System.Threading.Tasks; 40 | 41 | namespace AzureFunctionForSplunk 42 | { 43 | public class TransmissionFaultMessage 44 | { 45 | public string id { get; set; } 46 | public string type { get; set; } 47 | 48 | } 49 | 50 | public class Utils 51 | { 52 | static string splunkCertThumbprint { get; set; } 53 | 54 | static Utils() 55 | { 56 | splunkCertThumbprint = getEnvironmentVariable("splunkCertThumbprint"); 57 | } 58 | 59 | public static string getEnvironmentVariable(string name) 60 | { 61 | var result = System.Environment.GetEnvironmentVariable(name, EnvironmentVariableTarget.Process); 62 | if (result == null) 63 | return ""; 64 | 65 | return result; 66 | } 67 | 68 | public static string getFilename(string basename) 69 | { 70 | 71 | var filename = ""; 72 | var home = getEnvironmentVariable("HOME"); 73 | if (home.Length == 0) 74 | { 75 | filename = "../../../" + basename; 76 | } 77 | else 78 | { 79 | filename = home + "\\site\\wwwroot\\" + basename; 80 | } 81 | return filename; 82 | } 83 | 84 | public static Dictionary GetDictionary(string filename) 85 | { 86 | Dictionary dictionary; 87 | try 88 | { 89 | string json = File.ReadAllText(filename); 90 | 91 | dictionary = JsonConvert.DeserializeObject>(json); 92 | } 93 | catch (Exception) 94 | { 95 | dictionary = new Dictionary(); 96 | throw; 97 | } 98 | 99 | return dictionary; 100 | } 101 | 102 | public static string GetDictionaryValue(string key, Dictionary dictionary) 103 | { 104 | string value = ""; 105 | if (dictionary.TryGetValue(key, out value)) 106 | { 107 | return value; 108 | } else 109 | { 110 | return null; 111 | } 112 | } 113 | 114 | public class SingleHttpClientInstance 115 | { 116 | private static readonly HttpClient HttpClient; 117 | 118 | static SingleHttpClientInstance() 119 | { 120 | var handler = new SocketsHttpHandler 121 | { 122 | SslOptions = new SslClientAuthenticationOptions 123 | { 124 | RemoteCertificateValidationCallback = ValidateMyCert, 125 | EnabledSslProtocols = System.Security.Authentication.SslProtocols.Tls12 126 | } 127 | }; 128 | 129 | HttpClient = new HttpClient(handler); 130 | } 131 | 132 | public static async Task SendToService(HttpRequestMessage req) 133 | { 134 | HttpResponseMessage response = await HttpClient.SendAsync(req); 135 | return response; 136 | } 137 | } 138 | 139 | //public static bool ValidateMyCert(object sender, X509Certificate cert, X509Chain chain, SslPolicyErrors sslErr) 140 | //{ 141 | // // if user has not configured a cert, anything goes 142 | // if (string.IsNullOrWhiteSpace(splunkCertThumbprint)) 143 | // return true; 144 | 145 | // // if user has configured a cert, must match 146 | // var thumbprint = cert.GetCertHashString(); 147 | // if (thumbprint == splunkCertThumbprint) 148 | // return true; 149 | 150 | // return false; 151 | //} 152 | 153 | public static bool ValidateMyCert(object sender, X509Certificate cert, X509Chain chain, SslPolicyErrors sslErr) 154 | { 155 | // if user has not configured a cert, anything goes 156 | if (string.IsNullOrWhiteSpace(splunkCertThumbprint)) 157 | return true; 158 | 159 | // if user has configured a cert, must match 160 | var numcerts = chain.ChainElements.Count; 161 | var cacert = chain.ChainElements[numcerts - 1].Certificate; 162 | 163 | var thumbprint = cacert.GetCertHashString().ToLower(); 164 | if (thumbprint == splunkCertThumbprint) 165 | return true; 166 | 167 | return false; 168 | } 169 | 170 | public static async Task obEventhub(List standardizedEvents, IAsyncCollector outputEvents, ILogger log) 171 | { 172 | foreach (string item in standardizedEvents) 173 | { 174 | try 175 | { 176 | await outputEvents.AddAsync(item); 177 | } 178 | catch (Exception ex) 179 | { 180 | throw new System.Exception("Sending to event hub output. Unplanned exception: ", ex); 181 | } 182 | } 183 | } 184 | 185 | public static async Task obProxy(List standardizedEvents, ILogger log) 186 | { 187 | string proxyAddress = Utils.getEnvironmentVariable("proxyAddress"); 188 | if (proxyAddress.Length == 0) 189 | { 190 | log.LogError("Address of proxy function is required."); 191 | throw new ArgumentException(); 192 | } 193 | 194 | string serviceResourceIDURI = Utils.getEnvironmentVariable("serviceResourceIDURI"); 195 | if (serviceResourceIDURI.Length == 0) 196 | { 197 | log.LogError("The AAD service resource ID URI (serviceResourceIDURI) of the proxy app is required."); 198 | throw new ArgumentException(); 199 | } 200 | 201 | string astpConnection = ""; 202 | bool devEnvironment = Utils.getEnvironmentVariable("FUNCTIONS_CORETOOLS_ENVIRONMENT").ToLower() == "true"; 203 | if (devEnvironment) 204 | { 205 | astpConnection = Utils.getEnvironmentVariable("astpConnectionString"); 206 | } 207 | // log.LogInformation($"devEnvironment: {devEnvironment}, astpConnection: {astpConnection}"); 208 | 209 | string accessToken = ""; 210 | try 211 | { 212 | var azureServiceTokenProvider = new AzureServiceTokenProvider( 213 | connectionString: astpConnection 214 | ); 215 | 216 | accessToken = await azureServiceTokenProvider.GetAccessTokenAsync(serviceResourceIDURI); 217 | } catch (Exception ex) 218 | { 219 | log.LogError($"Error acquiring token from AzureServiceTokenProvider: {ex.Message}"); 220 | throw; 221 | } 222 | 223 | //ServicePointManager.Expect100Continue = true; 224 | //ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; 225 | //ServicePointManager.ServerCertificateValidationCallback += new RemoteCertificateValidationCallback(ValidateMyCert); 226 | 227 | var client = new SingleHttpClientInstance(); 228 | 229 | StringBuilder bulkTransmission = new StringBuilder(); 230 | foreach (string item in standardizedEvents) 231 | { 232 | bulkTransmission.Append(item); 233 | } 234 | try 235 | { 236 | var httpRequestMessage = new HttpRequestMessage 237 | { 238 | Method = HttpMethod.Post, 239 | RequestUri = new Uri(proxyAddress), 240 | Headers = { 241 | { HttpRequestHeader.Authorization.ToString(), "Bearer " + accessToken } 242 | }, 243 | Content = new StringContent(bulkTransmission.ToString(), Encoding.UTF8) 244 | }; 245 | 246 | HttpResponseMessage response = await SingleHttpClientInstance.SendToService(httpRequestMessage); 247 | if (response.StatusCode != HttpStatusCode.OK) 248 | { 249 | throw new System.Net.Http.HttpRequestException($"StatusCode from Proxy Function: {response.StatusCode}, and reason: {response.ReasonPhrase}"); 250 | } 251 | } 252 | catch (System.Net.Http.HttpRequestException e) 253 | { 254 | throw new System.Net.Http.HttpRequestException("Sending to Proxy Function. Is the service running?", e); 255 | } 256 | catch (Exception f) 257 | { 258 | throw new System.Exception("Sending to Proxy Function. Unplanned exception.", f); 259 | } 260 | } 261 | 262 | public static async Task obHEC(List standardizedEvents, ILogger log) 263 | { 264 | string splunkAddress = Utils.getEnvironmentVariable("splunkAddress"); 265 | string splunkToken = Utils.getEnvironmentVariable("splunkToken"); 266 | if (splunkAddress.Length == 0 || splunkToken.Length == 0) 267 | { 268 | log.LogError("Values for splunkAddress and splunkToken are required."); 269 | throw new ArgumentException(); 270 | } 271 | 272 | if (!string.IsNullOrWhiteSpace(splunkCertThumbprint)) 273 | { 274 | if (!splunkAddress.ToLower().StartsWith("https")) 275 | { 276 | throw new ArgumentException("Having provided a Splunk cert thumbprint, the address must be https://whatever"); 277 | } 278 | } 279 | 280 | //ServicePointManager.Expect100Continue = true; 281 | //ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; 282 | //ServicePointManager.ServerCertificateValidationCallback += new RemoteCertificateValidationCallback(ValidateMyCert); 283 | 284 | var client = new SingleHttpClientInstance(); 285 | foreach (string item in standardizedEvents) 286 | { 287 | try 288 | { 289 | var httpRequestMessage = new HttpRequestMessage 290 | { 291 | Method = HttpMethod.Post, 292 | RequestUri = new Uri(splunkAddress), 293 | Headers = { 294 | { HttpRequestHeader.Authorization.ToString(), "Splunk " + splunkToken } 295 | }, 296 | Content = new StringContent(item, Encoding.UTF8, "application/json") 297 | }; 298 | 299 | HttpResponseMessage response = await SingleHttpClientInstance.SendToService(httpRequestMessage); 300 | if (response.StatusCode != HttpStatusCode.OK) 301 | { 302 | throw new System.Net.Http.HttpRequestException($"StatusCode from Splunk: {response.StatusCode}, and reason: {response.ReasonPhrase}"); 303 | } 304 | } 305 | catch (System.Net.Http.HttpRequestException e) 306 | { 307 | throw new System.Net.Http.HttpRequestException("Sending to Splunk. Is Splunk service running?", e); 308 | } 309 | catch (Exception f) 310 | { 311 | throw new System.Exception("Sending to Splunk. Unplanned exception.", f); 312 | } 313 | } 314 | } 315 | 316 | } 317 | } 318 | -------------------------------------------------------------------------------- /AzureFunctionForSplunk/SplunkEventMessages.cs: -------------------------------------------------------------------------------- 1 | // 2 | // AzureFunctionForSplunkVS 3 | // 4 | // Copyright (c) Microsoft Corporation 5 | // 6 | // All rights reserved. 7 | // 8 | // MIT License 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the ""Software""), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is furnished 15 | // to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | 20 | // THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 22 | // FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 23 | // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 24 | // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | // 27 | using Microsoft.Azure.WebJobs.Host; 28 | using Newtonsoft.Json; 29 | using Newtonsoft.Json.Converters; 30 | using System; 31 | using System.Collections.Generic; 32 | using System.Dynamic; 33 | using System.Threading.Tasks; 34 | using Microsoft.Extensions.Logging; 35 | using Microsoft.Azure.WebJobs; 36 | 37 | namespace AzureFunctionForSplunk 38 | { 39 | public abstract class SplunkEventMessages 40 | { 41 | public ILogger Log { get; set; } 42 | 43 | private string categoryFileName; 44 | 45 | public Dictionary Categories = null; 46 | 47 | public string CategoryFileName 48 | { 49 | set 50 | { 51 | categoryFileName = value; 52 | 53 | var filename = Utils.getFilename(categoryFileName); 54 | 55 | Categories = new Dictionary(); 56 | 57 | try 58 | { 59 | Categories = Utils.GetDictionary(filename); 60 | } 61 | catch (Exception ex) 62 | { 63 | Log.LogError($"Error getting categories json file: {filename}. {ex.Message}"); 64 | } 65 | } 66 | } 67 | 68 | public List azureMonitorMessages { get; set; } 69 | public List splunkEventMessages { get; set; } 70 | public IAsyncCollector eventHubOutputEvents { get; set; } 71 | 72 | public abstract void Ingest(string[] records); 73 | 74 | public async Task Emit() 75 | { 76 | string outputBinding = Utils.getEnvironmentVariable("outputBinding"); 77 | if (outputBinding.Length == 0) 78 | { 79 | Log.LogError("Value for outputBinding is required. Permitted values are: 'proxy', 'hec', 'eventhub'."); 80 | return; 81 | } 82 | 83 | splunkEventMessages = new List(); 84 | foreach (var item in azureMonitorMessages) 85 | { 86 | splunkEventMessages.Add(item.GetSplunkEventFromMessage()); 87 | } 88 | 89 | switch (outputBinding) 90 | { 91 | case "hec": 92 | await Utils.obHEC(splunkEventMessages, Log); 93 | break; 94 | case "proxy": 95 | await Utils.obProxy(splunkEventMessages, Log); 96 | break; 97 | case "eventhub": 98 | await Utils.obEventhub(splunkEventMessages, eventHubOutputEvents, Log); 99 | break; 100 | } 101 | 102 | 103 | } 104 | 105 | public SplunkEventMessages(IAsyncCollector outputEvents, ILogger log) 106 | { 107 | Log = log; 108 | eventHubOutputEvents = outputEvents; 109 | azureMonitorMessages = new List(); 110 | } 111 | } 112 | 113 | public class ActivityLogsSplunkEventMessages: SplunkEventMessages 114 | { 115 | public ActivityLogsSplunkEventMessages(IAsyncCollector outputEvents, ILogger log): base(outputEvents, log) 116 | { 117 | CategoryFileName = "ActivityLogCategories.json"; 118 | } 119 | 120 | public override void Ingest(string[] records) 121 | { 122 | // sourceType depends on the message category 123 | 124 | foreach (var record in records) 125 | { 126 | var expandoConverter = new ExpandoObjectConverter(); 127 | var expandoRecord = JsonConvert.DeserializeObject(record, expandoConverter); 128 | 129 | string operationName = ((IDictionary)expandoRecord)["operationName"].ToString(); 130 | 131 | var splits = operationName.Split('/'); 132 | 133 | string sourceType = ""; 134 | if (splits.Length < 3) 135 | { 136 | // ASC Recommendation 137 | sourceType = Utils.GetDictionaryValue("ascrecommendation", Categories) ?? "amal:asc:recommendation"; 138 | } 139 | else if (splits.Length >= 3) 140 | { 141 | var provider = splits[0].ToUpper(); 142 | var type = splits[1].ToUpper(); 143 | var operation = splits[2].ToUpper(); 144 | 145 | switch (provider) 146 | { 147 | case "MICROSOFT.SERVICEHEALTH": 148 | sourceType = Utils.GetDictionaryValue("servicehealth", Categories) ?? "amal:serviceHealth"; 149 | break; 150 | 151 | case "MICROSOFT.RESOURCEHEALTH": 152 | sourceType = Utils.GetDictionaryValue("resourcehealth", Categories) ?? "amal:resourceHealth"; 153 | break; 154 | 155 | case "MICROSOFT.INSIGHTS": 156 | if (type == "AUTOSCALESETTINGS") 157 | { 158 | sourceType = Utils.GetDictionaryValue("autoscalesettings", Categories) ?? "amal:autoscaleSettings"; 159 | } 160 | else if (type == "ALERTRULES") 161 | { 162 | sourceType = Utils.GetDictionaryValue("ascalert", Categories) ?? "amal:ascAlert"; 163 | } 164 | else 165 | { 166 | sourceType = Utils.GetDictionaryValue("insights", Categories) ?? "amal:insights"; 167 | } 168 | break; 169 | case "MICROSOFT.SECURITY": 170 | if (type == "APPLICATIONWHITELISTINGS") 171 | { 172 | if (operation == "ACTION") 173 | { 174 | sourceType = Utils.GetDictionaryValue("ascalert", Categories) ?? "amal:asc:alert"; 175 | } 176 | else 177 | { 178 | sourceType = Utils.GetDictionaryValue("security", Categories) ?? "amal:security"; 179 | } 180 | } 181 | else if (type == "LOCATIONS") 182 | { 183 | sourceType = Utils.GetDictionaryValue("security", Categories) ?? "amal:security"; 184 | } 185 | else if (type == "TASKS") 186 | { 187 | sourceType = Utils.GetDictionaryValue("ascrecommendation", Categories) ?? "amal:asc:recommendation"; 188 | } 189 | break; 190 | default: 191 | { 192 | // administrative category 193 | sourceType = Utils.GetDictionaryValue("administrative", Categories) ?? "amal:administrative"; 194 | break; 195 | } 196 | } 197 | } 198 | 199 | azureMonitorMessages.Add(new AzMonActivityLog(expandoRecord, sourceType)); 200 | } 201 | } 202 | } 203 | 204 | public class DiagnosticLogsSplunkEventMessages : SplunkEventMessages 205 | { 206 | public DiagnosticLogsSplunkEventMessages(IAsyncCollector outputEvents, ILogger log) : base(outputEvents, log) 207 | { 208 | CategoryFileName = "DiagnosticLogCategories.json"; 209 | } 210 | 211 | public override void Ingest(string[] records) 212 | { 213 | // Subscription-based: sourceType depends on the message category and the ResourceType 214 | // Tenant-based: sourceType depends on the message category and the ProviderType 215 | 216 | foreach (var record in records) 217 | { 218 | var expandoConverter = new ExpandoObjectConverter(); 219 | var expandoRecord = JsonConvert.DeserializeObject(record, expandoConverter); 220 | 221 | var azMonMsg = new AzMonDiagnosticLog(expandoRecord); 222 | 223 | var category = "none"; 224 | if (((IDictionary)expandoRecord).ContainsKey("category")) 225 | { 226 | category = ((IDictionary)expandoRecord)["category"].ToString(); 227 | } 228 | 229 | var resourceType = azMonMsg.ResourceType; 230 | var providerName = azMonMsg.ProviderName; 231 | 232 | var logMessage = ""; 233 | var sourceType = ""; 234 | if (azMonMsg.TenantId.Length > 0) 235 | { 236 | logMessage = $"********* ProviderName: {providerName}"; 237 | sourceType = Utils.GetDictionaryValue(providerName.ToUpper() + "/" + category.ToUpper(), Categories) ?? "amdl:diagnostic"; 238 | } 239 | else 240 | { 241 | logMessage = $"********* ResourceType: {resourceType}"; 242 | sourceType = Utils.GetDictionaryValue(resourceType.ToUpper() + "/" + category.ToUpper(), Categories) ?? "amdl:diagnostic"; 243 | } 244 | 245 | // log categories that aren't yet in the DiagnosticLogCategories.json file. 246 | if (category != "none" && sourceType == "amdl:diagnostic") 247 | { 248 | Log.LogInformation($"{logMessage}, category: {category} *********"); 249 | } 250 | 251 | azMonMsg.SplunkSourceType = sourceType; 252 | 253 | azureMonitorMessages.Add(azMonMsg); 254 | } 255 | } 256 | } 257 | 258 | public class MetricsSplunkEventMessages : SplunkEventMessages 259 | { 260 | public MetricsSplunkEventMessages(IAsyncCollector outputEvents, ILogger log) : base(outputEvents, log) 261 | { 262 | CategoryFileName = "MetricsCategories.json"; 263 | } 264 | 265 | public override void Ingest(string[] records) 266 | { 267 | // sourceType depends on the ResourceType 268 | foreach (var record in records) 269 | { 270 | var expandoConverter = new ExpandoObjectConverter(); 271 | var expandoRecord = JsonConvert.DeserializeObject(record, expandoConverter); 272 | 273 | var azMonMsg = new AzMonDiagnosticLog(expandoRecord); 274 | 275 | var resourceType = azMonMsg.ResourceType; 276 | 277 | var sourceType = Utils.GetDictionaryValue(resourceType, Categories) ?? "amm:metrics"; 278 | 279 | azMonMsg.SplunkSourceType = sourceType; 280 | 281 | azureMonitorMessages.Add(azMonMsg); 282 | } 283 | } 284 | 285 | } 286 | 287 | public class WadSplunkEventMessages : SplunkEventMessages 288 | { 289 | public WadSplunkEventMessages(IAsyncCollector outputEvents, ILogger log) : base(outputEvents, log) { } 290 | 291 | public override void Ingest(string[] records) 292 | { 293 | foreach (var record in records) 294 | { 295 | var expandoConverter = new ExpandoObjectConverter(); 296 | var expandoRecord = JsonConvert.DeserializeObject(record, expandoConverter); 297 | 298 | if (((IDictionary)expandoRecord).ContainsKey("category")) 299 | { 300 | azureMonitorMessages.Add(new WadAzMonLog(expandoRecord)); 301 | } 302 | else if (((IDictionary)expandoRecord).ContainsKey("metricName")) 303 | { 304 | azureMonitorMessages.Add(new WadAzMonMetric(expandoRecord)); 305 | } 306 | } 307 | } 308 | 309 | } 310 | 311 | public class LadSplunkEventMessages : SplunkEventMessages 312 | { 313 | public LadSplunkEventMessages(IAsyncCollector outputEvents, ILogger log) : base(outputEvents, log) { } 314 | 315 | public override void Ingest(string[] records) 316 | { 317 | foreach (var record in records) 318 | { 319 | var expandoConverter = new ExpandoObjectConverter(); 320 | var expandoRecord = JsonConvert.DeserializeObject(record, expandoConverter); 321 | 322 | if (((IDictionary)expandoRecord).ContainsKey("category")) 323 | { 324 | azureMonitorMessages.Add(new LadAzMonLog(expandoRecord)); 325 | } 326 | else if (((IDictionary)expandoRecord).ContainsKey("metricName")) 327 | { 328 | azureMonitorMessages.Add(new LadAzMonMetric(expandoRecord)); 329 | } 330 | } 331 | } 332 | 333 | } 334 | 335 | } 336 | --------------------------------------------------------------------------------