├── .gitignore ├── ActivityFunctionApp.ps1 ├── AutomateDefender ├── Readme.md └── function.zip ├── AzureADGraph.ps1 ├── Enablesubscription.ps1 ├── FunctionApp1.png ├── FunctionApp2.png ├── FunctionApp3.png ├── FunctionApp4.png ├── FunctionApp5.png ├── LICENSE ├── PBI_Report ├── AI Template III.pbit ├── Board IX.pbit ├── DLP Stats Template III.pbit ├── Project VI.pbit └── Readme.md ├── README.md ├── Sentinel ├── AnalyticsRule │ ├── SynchDLPAnalyticRulesOffice.ps1 │ ├── readme.md │ └── ruletemplate.yaml ├── EndPoint │ ├── Functions.zip │ ├── Kudu.jpg │ ├── dlpservice.zip │ ├── endpointruletemplate (3).yaml │ ├── readme.md │ └── ruletemplate.txt ├── EndPointDLP_preview │ ├── Analytics │ │ ├── SynchDLPAnalyticRulesEndpoint.ps1 │ │ ├── endpointruletemplate.yaml │ │ └── readme.md │ ├── DocumentCopy │ │ ├── BlobtoSPO.json │ │ ├── endpointdeploy.ps1 │ │ ├── endpointscr.ps1 │ │ ├── img │ │ │ ├── img1.png │ │ │ ├── img2.png │ │ │ ├── img3.png │ │ │ ├── img4.png │ │ │ └── tst.md │ │ └── readme.md │ ├── QueueDLPEvents.ps1 │ ├── Report │ │ ├── endpoint.json │ │ └── readme.md │ ├── SensitiveInfoType.ps1 │ ├── StoreEndpointDLPEvents.ps1 │ ├── deploysentinelfunction.json │ ├── enablesubscription.ps1 │ ├── endpointdlpservice.zip │ └── readme.md ├── QueueEvents.ps1 ├── Readme.md ├── Report │ ├── Exchange & Teams DLP Report.workbook.json │ ├── SharePoint DLP Report.workbook.json │ ├── img │ │ ├── report1.png │ │ ├── report2.png │ │ ├── report3.png │ │ ├── report4.png │ │ └── test.md │ ├── organization.json │ └── readme.md ├── StoreEvents.ps1 ├── deploySentinelfunction.json ├── deploySentinelfunction.parameters.json ├── dlpservice.zip ├── enablesubscription.ps1 ├── logicapp │ ├── dlpaction.json │ ├── img │ │ ├── blank.md │ │ └── incident1.png │ ├── messageid.ps1 │ └── readme.md ├── mipservice │ ├── MIPService.zip │ ├── MIPmap.csv │ ├── SensitivityLabels.json │ ├── importwatch.ps1 │ └── readme.md └── msgtrace │ ├── ingestmsgtrace.ps1 │ ├── ingestmsgtrace_with_detail.ps1 │ └── readme.md ├── Sentinel_CloudApp ├── Azure_Sentinel_analytics_rules.json ├── EndpointSensitiveInfo.json ├── Label Statistics.json ├── MIPmap.csv └── readme.md ├── Sentinel_Deployment ├── deploymentScript.ps1 ├── functionPackage.zip ├── functionPackage │ ├── Modules │ │ └── AzMon.Ingestion │ │ │ ├── AzMon.Ingestion.psd1 │ │ │ ├── AzMon.Ingestion.psm1 │ │ │ ├── Azure.Core.dll │ │ │ ├── Azure.Identity.dll │ │ │ ├── Azure.Monitor.Ingestion.dll │ │ │ ├── Microsoft.Bcl.AsyncInterfaces.dll │ │ │ ├── Microsoft.Identity.Client.Extensions.Msal.dll │ │ │ ├── Microsoft.Identity.Client.dll │ │ │ ├── Microsoft.IdentityModel.Abstractions.dll │ │ │ ├── System.ClientModel.dll │ │ │ ├── System.Memory.Data.dll │ │ │ └── System.Security.Cryptography.ProtectedData.dll │ ├── QueueDLPEvents │ │ ├── function.json │ │ └── run.ps1 │ ├── StoreDLPEvents │ │ ├── function.json │ │ └── run.ps1 │ ├── SyncDLPAnalyticsRules │ │ ├── function.json │ │ ├── readme.md │ │ ├── run.ps1 │ │ └── workloads.json │ ├── SyncSensitivityLabels │ │ ├── function.json │ │ ├── readme.md │ │ └── run.ps1 │ ├── host.json │ ├── profile.ps1 │ ├── requirements.psd1 │ └── version.info ├── images │ ├── incident.png │ ├── incidentManagement.png │ └── lawid.png ├── main.bicep ├── main.json ├── modules │ ├── customDcrTables.bicep │ ├── functionApp.bicep │ ├── functionAppPE.bicep │ ├── keyVault.bicep │ ├── lawCustomTable.bicep │ ├── lawFunction.bicep │ ├── lawRoleAssignment.bicep │ ├── privateNetwork.bicep │ ├── sentinelRules.bicep │ ├── sentinelWatchlists.bicep │ └── sentinelWorkbooks.bicep ├── nuget.csproj ├── readme.md └── releaseNotes.md ├── V2Function_Investigate ├── EnableSubscription.ps1 ├── QueueEvents.ps1 └── StoreEvents.ps1 └── investigation ├── Enablesubscription.ps1 ├── QueueEvents.ps1 └── StoreEvents.ps1 /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # MSTest test Results 33 | [Tt]est[Rr]esult*/ 34 | [Bb]uild[Ll]og.* 35 | 36 | # NUNIT 37 | *.VisualState.xml 38 | TestResult.xml 39 | 40 | # Build Results of an ATL Project 41 | [Dd]ebugPS/ 42 | [Rr]eleasePS/ 43 | dlldata.c 44 | 45 | # .NET Core 46 | project.lock.json 47 | project.fragment.lock.json 48 | artifacts/ 49 | **/Properties/launchSettings.json 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # Visual Studio code coverage results 117 | *.coverage 118 | *.coveragexml 119 | 120 | # NCrunch 121 | _NCrunch_* 122 | .*crunch*.local.xml 123 | nCrunchTemp_* 124 | 125 | # MightyMoose 126 | *.mm.* 127 | AutoTest.Net/ 128 | 129 | # Web workbench (sass) 130 | .sass-cache/ 131 | 132 | # Installshield output folder 133 | [Ee]xpress/ 134 | 135 | # DocProject is a documentation generator add-in 136 | DocProject/buildhelp/ 137 | DocProject/Help/*.HxT 138 | DocProject/Help/*.HxC 139 | DocProject/Help/*.hhc 140 | DocProject/Help/*.hhk 141 | DocProject/Help/*.hhp 142 | DocProject/Help/Html2 143 | DocProject/Help/html 144 | 145 | # Click-Once directory 146 | publish/ 147 | 148 | # Publish Web Output 149 | *.[Pp]ublish.xml 150 | *.azurePubxml 151 | # TODO: Comment the next line if you want to checkin your web deploy settings 152 | # but database connection strings (with potential passwords) will be unencrypted 153 | *.pubxml 154 | *.publishproj 155 | 156 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 157 | # checkin your Azure Web App publish settings, but sensitive information contained 158 | # in these scripts will be unencrypted 159 | PublishScripts/ 160 | 161 | # NuGet Packages 162 | *.nupkg 163 | # The packages folder can be ignored because of Package Restore 164 | **/packages/* 165 | # except build/, which is used as an MSBuild target. 166 | !**/packages/build/ 167 | # Uncomment if necessary however generally it will be regenerated when needed 168 | #!**/packages/repositories.config 169 | # NuGet v3's project.json files produces more ignorable files 170 | *.nuget.props 171 | *.nuget.targets 172 | 173 | # Microsoft Azure Build Output 174 | csx/ 175 | *.build.csdef 176 | 177 | # Microsoft Azure Emulator 178 | ecf/ 179 | rcf/ 180 | 181 | # Windows Store app package directories and files 182 | AppPackages/ 183 | BundleArtifacts/ 184 | Package.StoreAssociation.xml 185 | _pkginfo.txt 186 | 187 | # Visual Studio cache files 188 | # files ending in .cache can be ignored 189 | *.[Cc]ache 190 | # but keep track of directories ending in .cache 191 | !*.[Cc]ache/ 192 | 193 | # Others 194 | ClientBin/ 195 | ~$* 196 | *~ 197 | *.dbmdl 198 | *.dbproj.schemaview 199 | *.jfm 200 | *.pfx 201 | *.publishsettings 202 | orleans.codegen.cs 203 | 204 | # Since there are multiple workflows, uncomment next line to ignore bower_components 205 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 206 | #bower_components/ 207 | 208 | # RIA/Silverlight projects 209 | Generated_Code/ 210 | 211 | # Backup & report files from converting an old project file 212 | # to a newer Visual Studio version. Backup files are not needed, 213 | # because we have git ;-) 214 | _UpgradeReport_Files/ 215 | Backup*/ 216 | UpgradeLog*.XML 217 | UpgradeLog*.htm 218 | 219 | # SQL Server files 220 | *.mdf 221 | *.ldf 222 | *.ndf 223 | 224 | # Business Intelligence projects 225 | *.rdl.data 226 | *.bim.layout 227 | *.bim_*.settings 228 | 229 | # Microsoft Fakes 230 | FakesAssemblies/ 231 | 232 | # GhostDoc plugin setting file 233 | *.GhostDoc.xml 234 | 235 | # Node.js Tools for Visual Studio 236 | .ntvs_analysis.dat 237 | node_modules/ 238 | 239 | # Typescript v1 declaration files 240 | typings/ 241 | 242 | # Visual Studio 6 build log 243 | *.plg 244 | 245 | # Visual Studio 6 workspace options file 246 | *.opt 247 | 248 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 249 | *.vbw 250 | 251 | # Visual Studio LightSwitch build output 252 | **/*.HTMLClient/GeneratedArtifacts 253 | **/*.DesktopClient/GeneratedArtifacts 254 | **/*.DesktopClient/ModelManifest.xml 255 | **/*.Server/GeneratedArtifacts 256 | **/*.Server/ModelManifest.xml 257 | _Pvt_Extensions 258 | 259 | # Paket dependency manager 260 | .paket/paket.exe 261 | paket-files/ 262 | 263 | # FAKE - F# Make 264 | .fake/ 265 | 266 | # JetBrains Rider 267 | .idea/ 268 | *.sln.iml 269 | 270 | # CodeRush 271 | .cr/ 272 | 273 | # Python Tools for Visual Studio (PTVS) 274 | __pycache__/ 275 | *.pyc 276 | 277 | # Cake - Uncomment if you are using it 278 | # tools/** 279 | # !tools/packages.config 280 | 281 | # Telerik's JustMock configuration file 282 | *.jmconfig 283 | 284 | # BizTalk build output 285 | *.btp.cs 286 | *.btm.cs 287 | *.odx.cs 288 | *.xsd.cs 289 | -------------------------------------------------------------------------------- /ActivityFunctionApp.ps1: -------------------------------------------------------------------------------- 1 | #Enumerators and object to wrap the incoming request 2 | $pageArray = @() 3 | $rawreq = @() 4 | $rawreq = New-Object -TypeName psobject 5 | $rawreq | Add-Member -name Content -value Content -membertype noteproperty 6 | 7 | #Retrieve the content URI 8 | $requestbody = Get-Content $req -Raw | ConvertFrom-Json 9 | $rawreq.content = $requestbody | convertto-json 10 | 11 | #Activity Feed webhook Body to process 12 | $contenttype = $requestBody.contenttype 13 | #$tenantguid = $requestBody.tenantid 14 | $clientIdIn = $requestBody.clientid 15 | $contentId = $requestBody.contentid 16 | $contentUri = $requestBody.contentUri 17 | $contentCreated = $requestBody.contentCreated 18 | $contentExpiration = $requestBody.contentExpiration 19 | 20 | #Sign in Parameters 21 | $ClientID = "YOUR CLIENT ID A HEX” 22 | $ClientSecret = "YOUR CLIENT SECRET 23 | $loginURL = "https://login.windows.net" 24 | $tenantdomain = "YOURDOMAIN.onmicrosoft.com" 25 | $TenantGUID = "YOUR TenantGUID HEX" 26 | $resource = "https://manage.office.com" 27 | 28 | 29 | #Verify that it is the correct ID and import to Cosmos DB 30 | if ($clientIdIn -eq $ClientId ) 31 | { 32 | 33 | # Get an Oauth 2 access token based on client id, secret and tenant domain 34 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 35 | 36 | #oauthtoken in the header 37 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 38 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 39 | 40 | #If more than one page is returned capture and return in pageArray 41 | if ($REQ_HEADERS_NextPageUri) { 42 | 43 | $pageTracker = $true 44 | $pagedReq = $REQ_HEADERS_NextPageUri 45 | 46 | while ($pageTracker -ne $false) 47 | 48 | { 49 | $CurrentPage = Invoke-WebRequest -Headers $headerParams -Uri $pagedReq -UseBasicParsing 50 | $pageArray += $CurrentPage 51 | 52 | if ($CurrentPage.Headers.NextPageUri) 53 | { 54 | $pageTracker = $true 55 | } 56 | Else 57 | { 58 | $pageTracker = $false 59 | } 60 | $pagedReq = $CurrentPage.Headers.NextPageUri 61 | } 62 | } 63 | 64 | 65 | $pageArray += $rawreq 66 | 67 | foreach ($page in $pageArray) 68 | 69 | { 70 | 71 | $request = $page.content | ConvertFrom-Json 72 | 73 | 74 | foreach ( $content in $request) 75 | { 76 | $uri = $content.contentUri + "?PublisherIdentifier=" + $TenantGUID 77 | Invoke-WebRequest -UseBasicParsing -Headers $headerParams -Uri $uri -PassThru -OutFile $outputdocument 78 | } 79 | 80 | } 81 | 82 | } 83 | 84 | Out-File -Encoding Ascii -FilePath $res -inputObject "200 OK" 85 | -------------------------------------------------------------------------------- /AutomateDefender/Readme.md: -------------------------------------------------------------------------------- 1 | # Analyst Assistant 2 | 3 | ## Overview 4 | Analyst Assistant is a PowerShell-based Azure Function code designed to streamline the analysis and management of data security incidents detected by Microsoft Purview DLP. It leverages Azure OpenAI (GPT-4o-mini) to intelligently assess data security incidents, classify their risk levels, and automate incident resolution or escalation based on predefined policies. 5 | 6 | ## Features 7 | - **Automated Incident Retrieval**: Fetches recent security incidents from Microsoft Graph Security API. 8 | - **AI-Powered Analysis**: Utilizes Azure OpenAI to analyze incident data contextually and classify risk levels (high, medium, low). 9 | - **Policy-Based Incident Handling**: Supports multiple predefined policies (e.g., Financial Data, PII Policy, Healthcare) with custom instructions for AI analysis. 10 | - **Incident Resolution Automation**: Automatically resolves low-risk incidents or escalates medium/high-risk incidents. 11 | - **Integration with Power BI**: Sends incident data to Power BI for reporting and visualization. 12 | 13 | ## Prerequisites 14 | - Azure subscription with access to: 15 | - Azure OpenAI Service (GPT-4o-mini deployment) 16 | - Microsoft Graph Security API 17 | - Managed identity or identity with the following permissions: 18 | - user.read.all 19 | - securityincident.readwrite.all 20 | - securityincident.read.all 21 | - Mail.Read 22 | - mail.readbasic 23 | - files.read.all 24 | - sites.read.all 25 | 26 | ## Configuration 27 | Update the following variables in [`run.ps1`](\run.ps1): 28 | 29 | - **Azure OpenAI Endpoint and API Key**: 30 | ```powershell 31 | $openAIEndpoint = "" 32 | $apiKey = "" 33 | 34 | - **DLP Report Mailbox**: 35 | ```powershell 36 | $dlpreportmbx = "" 37 | 38 | - **Policies and Instructions**: 39 | - Customize the $policiesAndInstructions array to define your own policies and AI instructions. 40 | 41 | ## Project Structure 42 | │ 43 | ├── # Main script for incident analysis and handling 44 | ├── spoperm.ps1 # Helper script for SharePoint permissions (optional) 45 | └── odbperm.ps1 # Helper script for OneDrive permissions (optional) 46 | 47 | ## Security Considerations 48 | - Store sensitive information (API keys, tokens) securely using Azure Key Vault or Managed Identities. 49 | - Regularly rotate API keys and tokens. 50 | - Limit permissions and access scopes to the minimum required. 51 | 52 | ## Contributing 53 | Contributions are welcome. Please open an issue or submit a pull request for improvements or bug fixes. 54 | -------------------------------------------------------------------------------- /AutomateDefender/function.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/AutomateDefender/function.zip -------------------------------------------------------------------------------- /AzureADGraph.ps1: -------------------------------------------------------------------------------- 1 | # This script will require the Web Application and permissions setup in Azure Active Directory 2 | # This code should run within a scheduled Azure Function. The Tracker file defines from where to start 3 | 4 | #You can use this to initialize the tracker file 5 | #$Tracker = "D:\home\tracker.log" # change to location of choise 6 | #$timerange= "{0:s}" -f (get-date).AddDays(-7) + "Z" 7 | #out-file -FilePath $Tracker -NoNewline -InputObject $object.signinDateTime 8 | 9 | 10 | 11 | 12 | $ClientID = "YOUR CLIENT ID" # Should be a ~35 character string insert your info here 13 | $ClientSecret = "YOUR CLIENT SECRET" # Should be a ~44 character string insert your info here 14 | $loginURL = "https://login.microsoftonline.com/" 15 | $tenantdomain = "contoso.onmicrosoft.com" #Provide your yourdomain.onmicrosoft.com 16 | $output = $outputEventHubMessage #This is the variable to define for the out put, this sample is an Event hub 17 | $Tracker = "D:\home\tracker.log" # change to location of choise this is the root. 18 | 19 | $Timerange = Get-content $Tracker 20 | 21 | # Get an Oauth 2 access token based on client i, secret and tenant domain 22 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 23 | 24 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 25 | 26 | if ($oauth.access_token -ne $null) { 27 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 28 | 29 | $url = "https://graph.windows.net/$tenantdomain/activities/signinEvents?api-version=beta&`$filter=signinDateTime ge $timerange" 30 | 31 | 32 | Do{ 33 | 34 | $myReport = (Invoke-RestMethod -Method Get -UseBasicParsing -Headers $headerParams -Uri $url ) 35 | 36 | $url = ($myReport.value).'@odata.nextLink' 37 | 38 | #Sorting the array so that the most recent object is last 39 | $Report = $myreport.value | Sort-Object signinDateTimeInMillis, Index 40 | foreach ( $object in $Report) { 41 | 42 | $object | ConvertTo-Json | Out-File $output 43 | out-file -FilePath $Tracker -NoNewline -InputObject $object.signinDateTime 44 | 45 | } 46 | 47 | } while($url -ne $null) 48 | 49 | 50 | } else { 51 | 52 | Write-Host "ERROR: No Access Token" 53 | } -------------------------------------------------------------------------------- /Enablesubscription.ps1: -------------------------------------------------------------------------------- 1 | #Sign in Parameters 2 | $ClientID = "YOUR CLIENT ID A HEX” 3 | $ClientSecret = "YOUR CLIENT SECRET 4 | $loginURL = "https://login.windows.net" 5 | $tenantdomain = "YOURDOMAIN.onmicrosoft.com" 6 | $TenantGUID = "YOUR TenantGUID HEX" 7 | $resource = "https://manage.office.com" 8 | 9 | #Provide the Azure Function address and change the authid to your TenantGUID 10 | $webhookadr = "YOUR WEBHOOK ADDRESS HERE” 11 | $authid = "YOUR TENANT GUID UNLESS YOU HAVE A DEV GUID" 12 | $webhookparam = @{address=$webhookadr;authid=$authid;expiration=""} 13 | $webhook = @{ 'Webhook' = $webhookparam} 14 | $webhookbody = $webhook |ConvertTo-Json 15 | 16 | # Get an Oauth 2 access token based on client id, secret and tenant domain 17 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 18 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 19 | 20 | #Let's put the oauth token in the header, where it belongs 21 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 22 | 23 | 24 | #Let's make sure the subscriptions are startedh 25 | Invoke-RestMethod -Method Post -Headers $headerParams -Body $webhookbody -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.AzureActiveDirectory" -ContentType "application/json; charset=utf-8" 26 | Invoke-RestMethod -Method Post -Headers $headerParams -Body $webhookbody -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.Exchange" -ContentType "application/json; charset=utf-8" 27 | Invoke-RestMethod -Method Post -Headers $headerParams -Body $webhookbody -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.SharePoint" -ContentType "application/json; charset=utf-8" 28 | Invoke-RestMethod -Method Post -Headers $headerParams -Body $webhookbody -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.General" -ContentType "application/json; charset=utf-8" 29 | Invoke-RestMethod -Method Post -Headers $headerParams -Body $webhookbody -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=DLP.All" -ContentType "application/json; charset=utf-8" 30 | -------------------------------------------------------------------------------- /FunctionApp1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/FunctionApp1.png -------------------------------------------------------------------------------- /FunctionApp2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/FunctionApp2.png -------------------------------------------------------------------------------- /FunctionApp3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/FunctionApp3.png -------------------------------------------------------------------------------- /FunctionApp4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/FunctionApp4.png -------------------------------------------------------------------------------- /FunctionApp5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/FunctionApp5.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /PBI_Report/AI Template III.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/PBI_Report/AI Template III.pbit -------------------------------------------------------------------------------- /PBI_Report/Board IX.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/PBI_Report/Board IX.pbit -------------------------------------------------------------------------------- /PBI_Report/DLP Stats Template III.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/PBI_Report/DLP Stats Template III.pbit -------------------------------------------------------------------------------- /PBI_Report/Project VI.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/PBI_Report/Project VI.pbit -------------------------------------------------------------------------------- /PBI_Report/Readme.md: -------------------------------------------------------------------------------- 1 | # Summary of the Step-by-Step Guided Walkthrough 2 | 3 | This guide provides an overview of how to get started with new tooling in Power BI reports, focusing on steps to configure and customize reports for organizational needs. 4 | 5 | ### Key Steps: 6 | 7 | 1. **Download and Open the Report**: 8 | - Obtain the latest report version and open it in Power BI Desktop. 9 | - Approve the use of ArcGIS Maps if prompted. 10 | 11 | 2. **Authentication**: 12 | - Authenticate with `https://api.security.microsoft.com` using an Organizational account. 13 | - Repeat authentication for `https://api.security.microsoft.com/api/advancedhunting`. 14 | 15 | 3. **Data Loading**: 16 | - Allow the system to collect data, which may take time in larger environments. 17 | 18 | 4. **Review and Update Reports**: 19 | - Update KPI diagrams and high-level descriptions to align with your organization’s objectives. 20 | - Filter reports to include only the required Sensitive Information Types (SITs). 21 | 22 | 5. **Customize Report Components**: 23 | - Modify diagrams, KPI measurements, and incident views to reflect relevant data. 24 | - Ensure the mapping of labeled content by updating the MIPLabel table with correct label names and GUIDs. 25 | 26 | 6. **Update Critical Systems Access**: 27 | - Use the SensitiveSystems query to update URLs for systems with high business impact. 28 | - Manually add URLs as needed and apply the changes. 29 | 30 | 7. **Operational Scope Review**: 31 | - Verify the operational scope, ensuring that sensitive information processing is accurately represented for legal entities. 32 | 33 | 8. **Additional Reports**: 34 | - Customize additional reports, such as those for Trust & Reputation, Company & Shareholder Value, and incident analysis. 35 | - Set target values and review incident reporting metrics. 36 | 37 | 9. **Power BI Online Integration**: 38 | - Set up Power BI Online to enable secure, role-based access to the dashboard with scheduled data refreshes. 39 | 40 | ### Final Configuration: 41 | 42 | - **Incident Data Customization**: Adjust the time frame for incident data to fit organizational requirements. 43 | - **Sensitive Information Capture**: Set up custom DLP policies to capture sensitive data in Exchange and SharePoint Online. 44 | 45 | For detailed steps and configurations, refer to the complete guide. https://techcommunity.microsoft.com/t5/security-compliance-and-identity/how-to-build-the-microsoft-purview-extended-report-experience/ba-p/4122028 46 | 47 | ## Contributing 48 | 49 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 50 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 51 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 52 | 53 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 54 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 55 | provided by the bot. You will only need to do this once across all repos using our CLA. 56 | 57 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 58 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 59 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | products: 4 | - office-365 5 | languages: 6 | - powershell 7 | extensions: 8 | contentType: samples 9 | createdDate: 2/1/2018 3:00:56 AM 10 | description: "This sample can be used to process notifications from the Office 365 Activity API directly from an Azure Function." 11 | --- 12 | 13 | # Office 365 activity feed sample 14 | 15 | This sample can be used to process notifications from the Office 365 Activity API directly from an Azure Function. The sample will write the information to Azure Cosmos DB but with very small changes it can be used to write to Event Hubs, Blob Storage and other Azure components. The AzureADGraph is created to run as a scheduled function. It stores the state between the runs in a local file in the Azure function. 16 | 17 | ## Addition 18 | 19 | The Investigation folder contains a sample that uses a Timer based function to import Office 365 Activity data to Cosmos DB. See separate blog post for more information. https://techcommunity.microsoft.com/t5/Security-Privacy-and-Compliance/Using-the-Office-365-Management-Activity-API-and-Power-BI-for/ba-p/189086 20 | 21 | ## Create and Register an App in Azure and delegate the appropriate permissions 22 | 23 | Use this as a guide to create and register the application used for making requests. The “Configure an X.509 certificate to enable service-to-service calls” is not necessary for this test. It is enough to have the application and the appropriate permissions set. Do not forget the step to complete the Administrator consent. 24 | https://msdn.microsoft.com/office-365/get-started-with-office-365-management-apis 25 | 26 | ## Steps to Create the Azure Function 27 | 1. Create your function app and name it appropriately. https://docs.microsoft.com/azure/azure-functions/functions-create-function-app-portalnew 28 | 2. Select to create an Azure Function from within Azure Portal. 29 | 3. Create the HTTP trigger, select PowerShell since the sample is based on PowerShell. 30 | 4. Authorization level is Function 31 | 32 | ![Create Function](./FunctionApp1.png) 33 | 34 | 5. Select to create the function 35 | 36 | 6. Use the code in ActivityFunctionApp.ps1 and modify the secrets to match your tenant and the Application created in previous step. From the Azure App you will need the Client ID, Client Secret, from your tenant you need the tenant name and GUID 37 | 7.Configure the integration of the trigger, below is the configuration I have been using to integrate the trigger. If you change the request parameter, you will have to change the corresponding lines in the code. 38 | 39 | ![Configure Integration](./FunctionApp2.png) 40 | 41 | 8. Configure the Output to COSMOS DB by either defining an existing DB or a new one. 42 | 43 | ![Configure OutPut](./FunctionApp3.png) 44 | 45 | 9. Get the App function key it will be used when you enable the Webhooks. 46 | ![Copy the function key](./FunctionApp4.png) 47 | 48 | ## Enable the subscriptions 49 | 50 | Create a separate Azure Function to kick start the webhooks. This is not a common task it is mainly done when you need to make a change to the O365 Webhooks. You can just as well enable the webhooks from onpremises by running the script. See the enablesubscription.ps1 code for how to enable the Webhooks. You will have to make the same changes as for the Activity App. 51 | When you have enabled the webhooks you will find entries in the invocation log from the WebHooks verifying that your app really is listening. 52 | 53 | ![Invocation Log](./FunctionApp5.png) 54 | 55 | ## Query the data using the SQL interface in Cosmos DB 56 | 57 | You should see the COSMOS DB being populated with Records. You can use your own custom solution to query the data as needed. You can also download all the records to your own custom solution. https://docs.microsoft.com/en-us/azure/cosmos-db/ 58 | 59 | Here are a few sample queries to get you started. 60 | 61 | If you want more information about a user 62 | 63 | ```sql 64 | select * from investigate where investigate.UserId = "user@YOURDOMAIN" order by investigate.CreationTime 65 | ``` 66 | If you need more detail about a specific IP address 67 | 68 | ```sql 69 | select * from investigate where investigate.ClientIP = "127.0.0.1" order by investigate.CreationTime 70 | ``` 71 | If you want to understand more about a specific file 72 | 73 | ```sql 74 | select * from SPO where SPO.SourceFileName = "FILENAME.JPG" order by SPO.CreationTime 75 | ``` 76 | 77 | ## Troubleshooting 78 | 79 | To get more information about the environment and to troubleshoot issues use the Kudu interface. https://.scm.azurewebsites.net/ 80 | 81 | https://docs.microsoft.com/azure/azure-functions/functions-how-to-use-azure-function-app-settings 82 | 83 | ## Contributing 84 | 85 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 86 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 87 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 88 | 89 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 90 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 91 | provided by the bot. You will only need to do this once across all repos using our CLA. 92 | 93 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 94 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 95 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 96 | -------------------------------------------------------------------------------- /Sentinel/AnalyticsRule/SynchDLPAnalyticRulesOffice.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | #Path to the template file 5 | $filepath = "d:\home\" 6 | $dlpyamlfile = $filepath + "ruletemplate.yaml" 7 | 8 | #Sentinel variables 9 | $workspace = "$env:SentinelWorkspace" 10 | 11 | $context = Get-AzContext 12 | $profileR = [Microsoft.Azure.Commands.Common.Authentication.Abstractions.AzureRmProfileProvider]::Instance.Profile 13 | $profileClient = New-Object -TypeName Microsoft.Azure.Commands.ResourceManager.Common.RMProfileClient -ArgumentList ($profileR) 14 | $token = $profileClient.AcquireAccessToken($context.Subscription.TenantId) 15 | $authHeader = @{ 16 | 'Content-Type' = 'application/json' 17 | 'Authorization' = 'Bearer ' + $token.AccessToken 18 | } 19 | 20 | $instance = Get-AzResource -Name $workspace -ResourceType Microsoft.OperationalInsights/workspaces 21 | 22 | $processedPolicies = @() 23 | $dlpyaml = Get-Content $dlpyamlfile 24 | 25 | #Last Policy update 26 | $lastpolicylog = 'd:\home\lastofficepolicy.log' 27 | $lastpolicychange = get-content $lastpolicylog 28 | 29 | #Exchange Credentials 30 | $expass = $env:expass 31 | $exuser = $env:exuser 32 | $password = ConvertTo-SecureString $expass -AsPlainText -Force 33 | $credentials=New-Object -TypeName System.Management.Automation.PSCredential ($exuser, $password) 34 | 35 | 36 | #Connecting to SCC PowerShell 37 | if ($credentials) { 38 | $session = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri https://ps.compliance.protection.outlook.com/powershell-liveid?BasicAuthToOAuthConversion=true -Credential $Credentials -Authentication Basic -AllowRedirection 39 | } 40 | 41 | if ($session) {Import-PSSession $session -CommandName Get-DlpComplianceRule -AllowClobber -DisableNameChecking} 42 | 43 | #Retreiving the DLP Policies in place 44 | $policies = Get-DlpCompliancerule 45 | 46 | #Retreiving the Sentinel Analytic rules 47 | $path = $instance.ResourceId 48 | $urllist = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/alertRules?api-version=2020-01-01" 49 | $rules = Invoke-RestMethod -Method "Get" -Uri $urllist -Headers $authHeader 50 | 51 | # Looping through the policies and create Analytic Rules in Sentinel 52 | foreach ($policy in $policies) { 53 | 54 | if (($processedPolicies -notcontains $policy.ReportSeverityLevel,$policy.ParentPolicyName) -and ((get-date $policy.whenChanged).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") -gt (get-date $lastpolicychange).ToString("yyyy-MM-ddTHH:mm:ss.fffZ"))) 55 | { 56 | 57 | #Updating with the severity and name of the Dlp Policy 58 | 59 | #Updating with the severity and name of the Dlp Policy 60 | 61 | $policyName = $policy.ParentPolicyName + "_" + $policy.ReportSeverityLevel 62 | $updateyaml1 = $dlpyaml -replace "dlppolicyname", $policyName 63 | $updateyaml2 = $updateyaml1 -replace "dlppolicy", $policy.ParentPolicyName 64 | $updateyaml3 = $updateyaml2 -replace "ImmutableID", $policy.ImmutableId 65 | $updateyaml4 = $updateyaml3 -replace "UpdateSeverity", $policy.ReportSeverityLevel 66 | 67 | 68 | $matchexisting = $rules.value | where-object {$_.properties.displayname -eq $policyName} | select-object 69 | 70 | if ($matchexisting) { 71 | $finalyaml = $updateyaml4 -replace "ruleGUID", ($matchexisting.etag -replace '"', "") 72 | $update = $matchexisting.name 73 | $urlupdate = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/alertRules/$update" + '?api-version=2020-01-01' 74 | Invoke-RestMethod -Method "Put" -Uri $urlupdate -Headers $authHeader -body $finalyaml 75 | } 76 | 77 | if (-not $matchexisting) { 78 | $etag = New-Guid 79 | $finalyaml = $updateyaml4 -replace "ruleGUID", $etag 80 | $update = $matchexisting.id 81 | $urlupdate = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/alertRules/$etag" + '?api-version=2020-01-01' 82 | Invoke-RestMethod -Method "Put" -Uri $urlupdate -Headers $authHeader -body $finalyaml 83 | } 84 | 85 | #Keep track of already processed rules by placing in array for if sentence 86 | $processedPolicies += $policy.ReportSeverityLevel,$policy.ParentPolicyName 87 | 88 | Clear-Variable matchexisting 89 | Clear-Variable updateyaml1 90 | Clear-Variable finalyaml 91 | 92 | } 93 | 94 | $dlplastchange = $policies.whenChanged | Sort-Object -Descending 95 | $dlplastchange[0] | Out-File $lastpolicylog -NoNewline 96 | 97 | } 98 | Remove-PSSession $session 99 | -------------------------------------------------------------------------------- /Sentinel/AnalyticsRule/readme.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | products: 4 | - office-365 5 | - microsoft-sentinel 6 | languages: 7 | - powershell 8 | extensions: 9 | contentType: samples 10 | createdDate: 4/24/2020 3:00:56 PM 11 | description: "This sample can be used to create a function that ingest DLP.All logs to Sentinel." 12 | --- 13 | 14 | 15 | # Creating Sentinel Analytic Rules based on Office DLP Policies 16 | The script will generate new Analytic rules used for alerting in connection to Office DLP policies. If the script is run more than once it will update the existing rules. 17 | 18 | ### Prerequisites 19 | 20 | - **MUST** have ingested both SharePoint and Exchange events to Azure Sentinel or rule creation will fail with error 500. 21 | 22 | ### Running the Script 23 | 24 | 1. Please use the instructions in Endpointdlp to deploy the function. 25 | 26 | ## Additional Customization 27 | 28 | If you need to customize the KQL query either modify the associated template file ruletemplate.yaml or create your own custom template. 29 | 30 | 31 | 32 | ## Contributing 33 | 34 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 35 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 36 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 37 | 38 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 39 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 40 | provided by the bot. You will only need to do this once across all repos using our CLA. 41 | 42 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 43 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 44 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 45 | -------------------------------------------------------------------------------- /Sentinel/EndPoint/Functions.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPoint/Functions.zip -------------------------------------------------------------------------------- /Sentinel/EndPoint/Kudu.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPoint/Kudu.jpg -------------------------------------------------------------------------------- /Sentinel/EndPoint/dlpservice.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPoint/dlpservice.zip -------------------------------------------------------------------------------- /Sentinel/EndPoint/endpointruletemplate (3).yaml: -------------------------------------------------------------------------------- 1 | { 2 | "etag": "\"ruleGuid\"", 3 | "type": "Microsoft.SecurityInsights/alertRules", 4 | "kind": "Scheduled", 5 | "properties": { 6 | "displayName": "DLPPOLICYName_EndPoint", 7 | "description": "", 8 | "severity": "UpdateSeverity", 9 | "enabled": true, 10 | "query": "let EndPointAction = datatable(ActionName: string, Action: int) [\r\n \"None\", \"0\",\r\n \"Audit\", \"1\",\r\n \"Warn\", \"2\",\r\n \"WarnAndBypass\", \"3\",\r\n \"Block\", \"4\",\r\n \"Allow\", \"5\"\r\n];\r\nO365DLP_CL\r\n| extend RuleName = tostring(parse_json(tostring(parse_json(PolicyDetails_s)[0].Rules))[0].RuleName)\r\n| extend PolicyName = tostring(parse_json(PolicyDetails_s)[0].PolicyName)\r\n| where PolicyName == \"DLPPOLICYName\"\r\n| extend PolicyId = tostring(parse_json(PolicyDetails_s)[0].PolicyId)\r\n| extend RuleId = tostring(parse_json(tostring(parse_json(PolicyDetails_s)[0].Rules))[0].RuleId)\r\n| where RuleId == \"ImmutableID\"\r\n| extend SensitiveInfoTypeName1 = tostring(parse_json(EndpointMetaData_SensitiveInfoTypeData_s)[0].SensitiveInfoTypeName)\r\n| extend Detected1 = tostring(parse_json(tostring(parse_json(tostring(parse_json(EndpointMetaData_SensitiveInfoTypeData_s)[0].SensitiveInformationDetectionsInfo)).DetectedValues)))\r\n| extend Detected = array_slice(todynamic(Detected1), 0, 5)\r\n| extend Deeplink = strcat(\"https://compliance.microsoft.com/datalossprevention/alerts/eventdeeplink?eventid=\", Id_g, \"&creationtime=\", CreationTime_t)\r\n| extend Action = toint(EndpointMetaData_EnforcementMode_d)\r\n| extend accountsplit = split(UserId_s, \"@\")\r\n| join kind= inner\r\n (\r\n EndPointAction\r\n )\r\n on Action\r\n| project\r\n PolicyId,\r\n SensitiveInfoTypeName1,\r\n UserKey_s,\r\n DocumentName_s,\r\n ObjectId_s,\r\n ClientIP_s,\r\n EndpointMetaData_RMSEncrypted_b,\r\n EndpointMetaData_EnforcementMode_d,\r\n EndpointMetaData_DeviceName_s,\r\n EndpointMetaData_OriginatingDomain_s,\r\n EndpointMetaData_SourceLocationType_d,\r\n PolicyName,\r\n RuleName,\r\n usageLocation_s,\r\n EndpointMetaData_EndpointOperation_s,\r\n EndpointMetaData_Sha256_s,\r\n department_s,\r\n manager_s,\r\n ActionName,\r\n Detected,\r\n Workload_s,\r\n Deeplink,\r\n jobTitle_s,\r\n UserId_s,\r\n accountsplit[0],\r\n accountsplit[1]", 11 | "queryFrequency": "PT5M", 12 | "queryPeriod": "PT5M", 13 | "triggerOperator": "GreaterThan", 14 | "triggerThreshold": 0, 15 | "suppressionDuration": "PT2H", 16 | "suppressionEnabled": false, 17 | "startTimeUtc": null, 18 | "tactics": [ 19 | "Exfiltration" 20 | ], 21 | "techniques": [], 22 | "alertRuleTemplateName": null, 23 | "incidentConfiguration": { 24 | "createIncident": true, 25 | "groupingConfiguration": { 26 | "enabled": true, 27 | "reopenClosedIncident": false, 28 | "lookbackDuration": "PT45M", 29 | "matchingMethod": "Selected", 30 | "groupByEntities": [ 31 | "Account" 32 | ], 33 | "groupByAlertDetails": [ 34 | "DisplayName" 35 | ], 36 | "groupByCustomDetails": [] 37 | } 38 | }, 39 | "eventGroupingSettings": { 40 | "aggregationKind": "AlertPerResult" 41 | }, 42 | "alertDetailsOverride": { 43 | "alertDisplayNameFormat": "{{RuleName}}", 44 | "alertDescriptionFormat": "{{UserId_s}}, {{DocumentName_s}}", 45 | "alertDynamicProperties": [ 46 | { 47 | "alertProperty": "AlertLink", 48 | "value": "Deeplink" 49 | }, 50 | { 51 | "alertProperty": "ProductName", 52 | "value": "Workload_s" 53 | }, 54 | { 55 | "alertProperty": "ProviderName", 56 | "value": "PolicyName" 57 | }, 58 | { 59 | "alertProperty": "ProductComponentName", 60 | "value": "RuleName" 61 | } 62 | ] 63 | }, 64 | "customDetails": { 65 | "User": "UserId_s", 66 | "Department": "department_s", 67 | "Location": "usageLocation_s", 68 | "Detected": "SensitiveInfoTypeName1", 69 | "Action": "EndpointMetaData_EndpointOperation_s", 70 | "DocumentName": "DocumentName_s", 71 | "BlockAction": "ActionName", 72 | "Encrypted": "EndpointMetaData_RMSEncrypted_b", 73 | "Manager": "manager_s", 74 | "DataMatch": "Detected", 75 | "Deeplink": "Deeplink", 76 | "jobtitle": "jobTitle_s" 77 | }, 78 | "entityMappings": [ 79 | { 80 | "entityType": "Account", 81 | "fieldMappings": [ 82 | { 83 | "identifier": "Name", 84 | "columnName": "accountsplit_0" 85 | }, 86 | { 87 | "identifier": "UPNSuffix", 88 | "columnName": "accountsplit_1" 89 | } 90 | ] 91 | }, 92 | { 93 | "entityType": "File", 94 | "fieldMappings": [ 95 | { 96 | "identifier": "Name", 97 | "columnName": "DocumentName_s" 98 | }, 99 | { 100 | "identifier": "Directory", 101 | "columnName": "ObjectId_s" 102 | } 103 | ] 104 | }, 105 | { 106 | "entityType": "Host", 107 | "fieldMappings": [ 108 | { 109 | "identifier": "DnsDomain", 110 | "columnName": "EndpointMetaData_OriginatingDomain_s" 111 | }, 112 | { 113 | "identifier": "HostName", 114 | "columnName": "EndpointMetaData_DeviceName_s" 115 | } 116 | ] 117 | }, 118 | { 119 | "entityType": "IP", 120 | "fieldMappings": [ 121 | { 122 | "identifier": "Address", 123 | "columnName": "ClientIP_s" 124 | } 125 | ] 126 | } 127 | ], 128 | "sentinelEntitiesMappings": null, 129 | "templateVersion": null 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /Sentinel/EndPoint/readme.md: -------------------------------------------------------------------------------- 1 | [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel%2FEndPointDLP_preview%2Fdeploysentinelfunction.json) 2 | 3 | --- 4 | page_type: sample 5 | products: 6 | - office-365 7 | - Sentinel 8 | languages: 9 | - powershellcore 10 | extensions: 11 | - contentType: samples 12 | - createdDate: 09/21/2022 3:00:56 PM 13 | --- 14 | 15 | The reason we haven’t pushed this to the core repo, the new code can break existing setups when running the enablement script. 16 | 17 | 1. Setup the service according to this blog post https://techcommunity.microsoft.com/t5/security-compliance-and-identity/advanced-incident-management-for-office-and-endpoint-dlp-using/ba-p/1811497, replace the endpointdlpservice.zip, with the dlpservice.zip in this repo, you can check the zip for any custom code. (This is step 8., 9. in the blog post) The SHA256 hash is C54BE51AD9609685F8FCD0825453B7CADDFE329A2334CE71A66694B50CDA6FBF. (Check with PowerShell Get-FileHash) 18 | 2. At the end of setting everything up, update the ruletemplate.yaml, and do not run the enablement function again since it will pull the templates from the repo. (The enablement function was meant to simplify but has become a bit of liability since it may brake older implementations) 19 | - Replace the content in ruletemplate.yaml with ruletemplate.yaml in this repo. 20 | - Replace the endpointruletemplate with endpointruletemplate.yaml. 21 | When the files have been replaced, reset the date in lastofficepolicy.log and lastendpointpolicy.log to 2005-08-18T15:32:04.000Z and manually run the synchdlp functions. The updated templates make use of the most recent features in Sentinel and allows for dynamic importance and many other settings. 22 | 23 | The best way to update is using Kudu 24 | 25 | ![Kudu](./Kudu.jpg) 26 | 27 | 28 | In the file functions.zip, you can find a few sample functions that can be customized for your workflow requirements. They are merely a starting point. 29 | 30 | • ExportContent, exports evidence around the individual in the case from Office Activities etc… 31 | 32 | • NotifyManager, sends the approval email to line manager as an example and updates the Sentinel incident. 33 | 34 | • StartTeams, creates a Teams conversation in the channel designated in the app. 35 | 36 | • WeeklyDlp, sends a weekly digest to the managers about DLP events in their organization. 37 | 38 | You run the import of the templates by using https://portal.azure.com/#create/Microsoft.Template, build your own template and load file. 39 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/Analytics/SynchDLPAnalyticRulesEndpoint.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | #Path to the template file 5 | $filepath = "d:\home\" 6 | $dlpyamlfile = $filepath + "endpointruletemplate.yaml" 7 | 8 | #Sentinel variables 9 | $workspace = "$env:SentinelWorkspace" 10 | 11 | $context = Get-AzContext 12 | $profileR = [Microsoft.Azure.Commands.Common.Authentication.Abstractions.AzureRmProfileProvider]::Instance.Profile 13 | $profileClient = New-Object -TypeName Microsoft.Azure.Commands.ResourceManager.Common.RMProfileClient -ArgumentList ($profileR) 14 | $token = $profileClient.AcquireAccessToken($context.Subscription.TenantId) 15 | $authHeader = @{ 16 | 'Content-Type' = 'application/json' 17 | 'Authorization' = 'Bearer ' + $token.AccessToken 18 | } 19 | 20 | $instance = Get-AzResource -Name $workspace -ResourceType Microsoft.OperationalInsights/workspaces 21 | 22 | $processedPolicies = @() 23 | $dlpyaml = Get-Content $dlpyamlfile 24 | 25 | #Last Policy update 26 | $lastpolicylog = "d:\home\lastendpointpolicy.log" 27 | $lastpolicychange = Get-Content $lastpolicylog 28 | 29 | #Exchange Credentials 30 | $expass = $env:expass 31 | $exuser = $env:exuser 32 | $password = ConvertTo-SecureString $expass -AsPlainText -Force 33 | $credentials=New-Object -TypeName System.Management.Automation.PSCredential ($exuser, $password) 34 | 35 | 36 | #Connecting to SCC PowerShell 37 | if ($credentials) { 38 | $session = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri https://ps.compliance.protection.outlook.com/powershell-liveid?BasicAuthToOAuthConversion=true -Credential $Credentials -Authentication Basic -AllowRedirection 39 | } 40 | 41 | if ($session) {Import-PSSession $session -CommandName Get-DlpComplianceRule -AllowClobber -DisableNameChecking} 42 | if (-not ($session)) {throw 'Failed to connect to Sentinel Workspace'} 43 | #Retreiving the DLP Policies in place 44 | $policies = Get-DlpCompliancerule | Where-Object workload -match "endpointdevices" 45 | 46 | #Retreiving the Sentinel Analytic rules 47 | $path = $instance.ResourceId 48 | $urllist = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/alertRules?api-version=2020-01-01" 49 | $rules = Invoke-RestMethod -Method "Get" -Uri $urllist -Headers $authHeader 50 | if (-not ($rules)) {throw 'Failed to connect to Sentinel Workspace'} 51 | 52 | # Looping through the policies and create Analytic Rules in Sentinel 53 | foreach ($policy in $policies) { 54 | 55 | if (($processedPolicies -notcontains $policy.ReportSeverityLevel,$policy.ParentPolicyName) -and ((get-date $policy.whenChanged).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") -gt (get-date $lastpolicychange).ToString("yyyy-MM-ddTHH:mm:ss.fffZ"))) 56 | { 57 | 58 | #Updating with the severity and name of the Dlp Policy 59 | 60 | $policyName = $policy.ParentPolicyName + "_" + $policy.ReportSeverityLevel 61 | $updateyaml1 = $dlpyaml -replace "dlppolicyname", $policyName 62 | $updateyaml2 = $updateyaml1 -replace "dlppolicy", $policy.ParentPolicyName 63 | $updateyaml3 = $updateyaml2 -replace "ImmutableID", $policy.ImmutableId 64 | $updateyaml4 = $updateyaml3 -replace "UpdateSeverity", $policy.ReportSeverityLevel 65 | 66 | 67 | $matchexisting = $rules.value | where-object {$_.properties.displayname -eq $policyName + "_EndPoint"} | select-object 68 | 69 | if ($matchexisting) { 70 | $finalyaml = $updateyaml4 -replace "ruleGUID", ($matchexisting.etag -replace '"', "") 71 | $update = $matchexisting.name 72 | $urlupdate = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/alertRules/$update" + '?api-version=2020-01-01' 73 | Invoke-RestMethod -Method "Put" -Uri $urlupdate -Headers $authHeader -body $finalyaml 74 | } 75 | 76 | if (-not $matchexisting) { 77 | $etag = New-Guid 78 | $finalyaml = $updateyaml4 -replace "ruleGUID", $etag 79 | $update = $matchexisting.id 80 | $urlupdate = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/alertRules/$etag" + '?api-version=2020-01-01' 81 | Invoke-RestMethod -Method "Put" -Uri $urlupdate -Headers $authHeader -body $finalyaml 82 | } 83 | 84 | #Keep track of already processed rules by placing in array for if sentence 85 | $processedPolicies += $policy.ReportSeverityLevel,$policy.ParentPolicyName 86 | 87 | Clear-Variable updateyaml1 88 | Clear-Variable finalyaml 89 | 90 | } 91 | $dlplastchange = $policies.whenChanged | Sort-Object -Descending 92 | $dlplastchange[0] | Out-File $lastpolicylog -NoNewline 93 | 94 | } 95 | Remove-PSSession $session 96 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/Analytics/endpointruletemplate.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "Scheduled", 3 | "etag": "\"ruleGuid\"", 4 | "properties": { 5 | "severity": "UpdateSeverity", 6 | "query": "endpointdlp_CL \r\n| where TimeGenerated >= ago(5m) \r\n| mv-apply SensitiveInfoTypeId = todynamic(SensitiveInfoTypeData_s) on (where SensitiveInfoTypeData_s != \"\" 7 | )\r\n| extend SensitiveInfoTypeIdnew = tostring(parse_json(SensitiveInfoTypeId).SensitiveInfoTypeId)\r\n| join kind = inner \r\n ( \r\n DLPSensitivity_CL \r\n | where TimeGenerated >= ago(24h) \r\n | extend SensitiveInfoTypeIdnew = tostring(Id_g)\r\n | summarize arg_max(TimeGenerated, *) by SensitiveInfoTypeIdnew \r\n )\r\n on SensitiveInfoTypeIdnew\r\n| where PolicyMatchInfo_RuleId_g == \"ImmutableID\"\r\n| where Operation_s contains \"FileCopiedToRemovableMedia\" or Operation_s contains \"FileUploadedToCloud\" or Operation_s contains \"FileCopiedToClipboard\" or Operation_s contains \"FilePrinted\" or Operation_s contains \"FileCopiedToNetworkShare\"\r\n| extend AccountCustomEntity = UserKey_s\r\n| extend HostCustomEntity = DeviceName_s\r\n| extend IPCustomEntity = ClientIP_s\r\n| project department_s, AccountCustomEntity, Name_s, DocumentName_s, HostCustomEntity, IPCustomEntity,Operation_s, RMSEncrypted_b, Application_s", 8 | "queryFrequency": "PT5M", 9 | "queryPeriod": "P1D", 10 | "triggerOperator": "GreaterThan", 11 | "triggerThreshold": 0, 12 | "suppressionDuration": "PT2H", 13 | "suppressionEnabled": false, 14 | "incidentConfiguration": { 15 | "createIncident": true, 16 | "groupingConfiguration": { 17 | "enabled": true, 18 | "reopenClosedIncident": false, 19 | "lookbackDuration": "PT45M", 20 | "entitiesMatchingMethod": "Custom", 21 | "groupByEntities": [ 22 | "Account" 23 | ] 24 | } 25 | }, 26 | "eventGroupingSettings": { 27 | "aggregationKind": "AlertPerResult" 28 | }, 29 | "displayName": "DLPPOLICYName_EndPoint", 30 | "enabled": true, 31 | "description": "", 32 | "tactics": [ 33 | "Exfiltration" 34 | ], 35 | "alertRuleTemplateName": null 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/Analytics/readme.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | products: 4 | - office-365 5 | - microsoft-sentinel 6 | languages: 7 | - powershell 8 | extensions: 9 | contentType: samples 10 | createdDate: 8/20/2020 3:00:56 PM 11 | description: "This sample can be used to create a function that ingest DLP.All logs to Sentinel." 12 | --- 13 | 14 | 15 | # Creating Sentinel Analytic Rules based on Office DLP Policies 16 | The script will generate new Analytic rules used for alerting in connection to Office DLP policies. If the script is run more than once it will update the existing rules. 17 | 18 | 19 | - **MUST** have ingested both SharePoint and Exchange events to Azure Sentinel or rule creation will fail with error 500. 20 | 21 | ### Running the Script 22 | 23 | 1. Please use the instructions in the Endpoint DLP preview to deploy the code. 24 | 25 | ## Additional Customization 26 | 27 | If you need to customize the KQL query either modify the associated template file ruletemplate.yaml or create your own custom template. 28 | 29 | 30 | 31 | ## Contributing 32 | 33 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 34 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 35 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 36 | 37 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 38 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 39 | provided by the bot. You will only need to do this once across all repos using our CLA. 40 | 41 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 42 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 43 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 44 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/endpointdeploy.ps1: -------------------------------------------------------------------------------- 1 | # Deployment of scheduler for the endpoint script 2 | $logfile = "$env:temp\EndpointTask.log" 3 | if (!(test-path $logfile)) { 4 | new-item -path $logfile -ItemType File 5 | } 6 | Start-Transcript $logfile 7 | $taskName = "DLPAlert" 8 | $Path = 'PowerShell.exe' 9 | $Arguments = "C:\program files\microsoft\endpointscr.ps1" 10 | 11 | $Service = new-object -ComObject ("Schedule.Service") 12 | $Service.Connect() 13 | $RootFolder = $Service.GetFolder("\") 14 | $TaskDefinition = $Service.NewTask(0) # TaskDefinition object https://msdn.microsoft.com/en-us/library/windows/desktop/aa382542(v=vs.85).aspx 15 | $TaskDefinition.RegistrationInfo.Description = '' 16 | $TaskDefinition.Settings.Enabled = $True 17 | $TaskDefinition.Settings.AllowDemandStart = $True 18 | $TaskDefinition.Settings.DisallowStartIfOnBatteries = $False 19 | $Triggers = $TaskDefinition.Triggers 20 | $Trigger = $Triggers.Create(0) ## 0 is an event trigger https://msdn.microsoft.com/en-us/library/windows/desktop/aa383898(v=vs.85).aspx 21 | $Trigger.Enabled = $true 22 | $Trigger.Id = '1134' # 8003 is for disconnections and 8001 is for connections 23 | $Trigger.Subscription = "" 24 | $Trigger = $Triggers.Create(0) 25 | $Trigger.Enabled = $true 26 | $Trigger.Id = '1133' # 8003 is for disconnections and 8001 is for connections 27 | $Trigger.Subscription = "" 28 | $Action = $TaskDefinition.Actions.Create(0) 29 | $Action.Path = $Path 30 | $action.Arguments = $Arguments 31 | $RootFolder.RegisterTaskDefinition($taskName, $TaskDefinition, 6, "System", $null, 5) 32 | 33 | Stop-Transcript 34 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/endpointscr.ps1: -------------------------------------------------------------------------------- 1 | $logfile = "$env:temp\enpointdlpUL.log" 2 | $storedtimepath = "$env:temp\endpointtime.log" 3 | if (!(test-path $logfile)) { 4 | new-item -path $logfile -ItemType File 5 | } 6 | if (!(test-path $storedtimepath)) { 7 | new-item -path $storedtimepath -itemtype File 8 | } 9 | Start-Transcript -path $logfile -Append 10 | $info = @{} 11 | $storedtime = Get-Content $storedtimepath 12 | $now = (get-date).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") 13 | if ($null -eq $storedtime) { 14 | (get-date $now) | Out-File $storedtimepath 15 | $storedtime = Get-Content $storedtimepath 16 | } 17 | 18 | #Function to store content to Azure Storage Blob 19 | Function copy-toazblob { 20 | param( 21 | $copypath, 22 | $policy, 23 | $computername, 24 | $timestamp, 25 | $storedtimepath 26 | ) 27 | $file = $copypath 28 | 29 | #Get the File-Name without path 30 | $name = $policy + "--" + $computername + "-" + $timestamp + "-" + ((Get-Item $file).Name) 31 | 32 | Copy-Item $file $env:Temp 33 | $upload = $env:Temp + "\" + ((Get-Item $file).Name) 34 | 35 | #The target URL wit SAS Token, consider using Azure Key vault and rotate the token. This is poc code. 36 | $uri = "Https://sampleblob.blob.core.windows.net/endpoint/documents/$($name)? SAS Token" 37 | 38 | #Define required Headers 39 | $headers = @{'x-ms-blob-type' = 'BlockBlob' } 40 | 41 | #Upload File... 42 | $response = Invoke-WebRequest -Uri $uri -Method Put -Headers $headers -InFile $upload -UseBasicParsing 43 | $name 44 | $StatusCode = $Response.StatusCode 45 | $StatusCode = 201 46 | $StatusCode | Out-File "$env:temp\endpointstatus.log" 47 | if ($StatusCode -eq '201') { $now | Out-file $storedtimepath } 48 | Remove-Item $upload 49 | return $StatusCode 50 | } 51 | 52 | Try { 53 | #Get the eventlog$eventlog and then resolve the actual path to pass to the upload function 54 | $eventlog = Get-WinEvent -LogName "Microsoft-Windows-Windows Defender/Operational" | Where-Object { (($_.message -like "*AccessByUnallowedApp*") -or ($_.message -like "*FileCopiedToRemovableMedia*") -or ($_.message -like "*FileUploadedToCloud*") -or ($_.message -like "*Print*") -or ($_.message -like "*CopyToNetworkShare*") -and ($_.id -in (1134, 1133))) } 55 | foreach ($event in $eventlog) { 56 | $entry = ([Xml]$event.ToXml()).event 57 | $entry.EventData.data | ForEach-Object { $info[$_.Name] = $_."#text" } 58 | if ((get-date $event.timecreated).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") -gt $storedtime) { 59 | 60 | # Build System Assembly in order to call Kernel32:QueryDosDevice, sample from https://morgantechspace.com/2014/11/Get-Volume-Path-from-Drive-Name-using-Powershell.html 61 | $DynAssembly = New-Object System.Reflection.AssemblyName('SysUtils') 62 | $AssemblyBuilder = [AppDomain]::CurrentDomain.DefineDynamicAssembly($DynAssembly, [Reflection.Emit.AssemblyBuilderAccess]::Run) 63 | $ModuleBuilder = $AssemblyBuilder.DefineDynamicModule('SysUtils', $False) 64 | 65 | # Define [Kernel32]::QueryDosDevice method 66 | $TypeBuilder = $ModuleBuilder.DefineType('Kernel32', 'Public, Class') 67 | $PInvokeMethod = $TypeBuilder.DefinePInvokeMethod('QueryDosDevice', 'kernel32.dll', ([Reflection.MethodAttributes]::Public -bor [Reflection.MethodAttributes]::Static), [Reflection.CallingConventions]::Standard, [UInt32], [Type[]]@([String], [Text.StringBuilder], [UInt32]), [Runtime.InteropServices.CallingConvention]::Winapi, [Runtime.InteropServices.CharSet]::Auto) 68 | $DllImportConstructor = [Runtime.InteropServices.DllImportAttribute].GetConstructor(@([String])) 69 | $SetLastError = [Runtime.InteropServices.DllImportAttribute].GetField('SetLastError') 70 | $SetLastErrorCustomAttribute = New-Object Reflection.Emit.CustomAttributeBuilder($DllImportConstructor, @('kernel32.dll'), [Reflection.FieldInfo[]]@($SetLastError), @($true)) 71 | $PInvokeMethod.SetCustomAttribute($SetLastErrorCustomAttribute) 72 | $Kernel32 = $TypeBuilder.CreateType() 73 | 74 | $Max = 65536 75 | $StringBuilder = New-Object System.Text.StringBuilder($Max) 76 | 77 | $volumes = Get-WmiObject Win32_Volume | Where-Object { $_.DriveLetter } 78 | 79 | ForEach ($vol in $volumes) { 80 | $ReturnLength = $Kernel32::QueryDosDevice($vol.DriveLetter, $StringBuilder, $Max) 81 | if ($ReturnLength) { 82 | $DriveMapping = @{ 83 | DriveLetter = $vol.DriveLetter 84 | DevicePath = $StringBuilder.ToString() 85 | } 86 | New-Object PSObject -Property $DriveMapping 87 | } 88 | if ($DriveMapping.DevicePath -eq "\Device\" + $info.source.Split("\")[2]) { 89 | $replace = "\Device\" + $info.source.Split("\")[2] 90 | $copypath = $info.source.replace($replace, $DriveMapping.DriveLetter) 91 | $policy = $info.'Policy Rule Id' 92 | $policy 93 | $timestamp = (Get-Date $info.'Event Timestamp').ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss") 94 | copy-toazblob -copypath $copypath -policy $policy -computername $env:COMPUTERNAME -timestamp $timestamp -storedtimepath $storedtimepath 95 | } 96 | } 97 | } 98 | } 99 | stop-transcript 100 | } 101 | catch { 102 | $_ | Out-File "$env:temp\endpointerror.log" 103 | Stop-Transcript 104 | } 105 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/img/img1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPointDLP_preview/DocumentCopy/img/img1.png -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/img/img2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPointDLP_preview/DocumentCopy/img/img2.png -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/img/img3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPointDLP_preview/DocumentCopy/img/img3.png -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/img/img4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPointDLP_preview/DocumentCopy/img/img4.png -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/img/tst.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/DocumentCopy/readme.md: -------------------------------------------------------------------------------- 1 | [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel%2FEndPointDLP_preview%2FDocumentCopy%2FBlobtoSPO.json) 2 | 3 | 4 | # Copy document from endpoint to SharePoint repository to reference from Sentinel 5 | Please note that this is early proof of concept code so test and expand as needed for your scenario. 6 | If you have more than one workspace and Geo, you will have to set this up separately if there is a requirement to keep information in Geo at rest. 7 | 8 | ## Prerequisites 9 | - Complete setting up the other functions for the EndPoint event collection. 10 | - Change he code of the function storing the endpoint events. 11 | - Add a blob container to your existing function and retreive a SAS token for upload. 12 | 13 | ## Setup 14 | 15 | 1. Use [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) to add an additional container. 16 | 17 | 2. Create the container 18 | ![Create Container](./img/img1.png) 19 | 20 | 3. Create the Virtual Directory 21 | ![Create Virtual Directory](./img/img2.png) 22 | 23 | 4. Generate the SAS token 24 | ![Generate SAS Token](./img/img3.png) 25 | 26 | 5. The SAS token should only have Write permissions nothing elese extend the life time as appropriate in this case it is until 2022 ![Generate SAS Token](./img/img4.png) 27 | 28 | 6. Update the endpointscr.ps1 with the SAS token url, remember to modify the url to contain /endpoint/documents/$($name) default will be someblob.blob.core.windows.net/endpoint?.... 29 | 30 | 7. Test the script on a single computer, note that only AccessByUnallowedApp, Print, FileCopiedToRemovableMedia, AccessByUnallowedApp, FileCopiedToNetworkShare will generate a copy. You can add or remove events by modifying line 34. 31 | 32 | 8. Deploy the script to run on a schedule with Task Scheduler or similar. See the endpointdeploy.ps1 as example. https://docs.microsoft.com/en-us/archive/blogs/wincat/trigger-a-powershell-script-from-a-windows-event 33 | 34 | 9. Create a SharePoint Site Collection in Region with the appropriate retention time. Use a Records center template if you need to treat the information as records. See the SharePoint Online limits to determine if you need more than one collection per region. This will depend on your expected load and retention period. You can change the ingestion code to ingest information based on the Policy Name as an example to scale this out. **Create a Library named "Records"** in the newly created Site Collection. 35 | 36 | 10. [Click] Deploy to Azure above to deploy the logic app to copy from the Blob store to SharePoint. 37 | 38 | 11. Provide the right Resource Group, where the function app resides. You can move the function later. 39 | 40 | 12. When deployed change the connections used for blob store as well as for SharePoint. 41 | 42 | 13. Update the function Store endpointDLPevents after line 129 add. Note that you can add the SPO site as a variable 43 | 44 | $origpath = "https://tenant.sharepoint.com/sites/DLPArchive/" + $user.PolicyMatchInfo.RuleId + "/" + $user.PolicyMatchInfo.policyid + "_" + $user.PolicyMatchInfo.RuleId + "--" + $user.devicename + "-" + (get-date $user.creationtime).tostring("yyyy-MM-ddTHH_mm_ss") + "-" + $user.DocumentName 45 | $user | Add-Member -MemberType NoteProperty -Name "originialContent" -value $origpath 46 | 47 | ### More information 48 | 49 | + **The scheduled event should monitor for event ID 1133 in Microsoft-Windows-Windows Defender/Operational, when the event is triggered it should execute. This need to be rolled out on the devices as part of task scheduler. You can also run it as a scheduled task.** 50 | 51 | - This is the script https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/blob/master/Sentinel/EndPointDLP_preview/DocumentCopy/endpointscr.ps1 52 | - The script need to be signed to be trusted in the environment. 53 | - The script will pickup the events and check if they match the endpoint event format and filter. 54 | - On match it will make a copy of the document in TEMP to avoid locks 55 | - Upload the document in TEMP to the blob store defined in the Script. 56 | - Move the cursor up to date in the event log and remove the file in TEMP, if successful 57 | - **At this point the Logic App will trigger on the upload to Azure blob** 58 | - The logic app monitors for changes in the Blob 59 | - On trigger it will copy the Blob content to SharePoint Online 60 | - Remove the copy in the Azure Blob container 61 | 62 | - **The Azure function used to enrich events to Azure Sentinel will add a link to the content. The link points to SPO.** 63 | 64 | 65 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/QueueDLPEvents.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | #Enumerators and object to wrap the objects 5 | $pageArray = @() 6 | $msgarray = @() 7 | 8 | #Sign in Parameters 9 | $clientID = "$env:clientID" 10 | $clientSecret = "$env:clientSecret" 11 | $loginURL = "https://login.microsoftonline.com" 12 | $tenantdomain = "$env:tenantdomain" 13 | $tenantGUID = "$env:TenantGuid" 14 | $resource = "https://manage.office.com" 15 | 16 | #Workloads and end time default is on start 17 | $workloads = $env:contentTypes.split(",") 18 | $endTime = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 19 | 20 | Foreach ($workload in $workloads) { 21 | 22 | #Storage Account Settings 23 | if ($workload -eq "dlp.all") {$storageQueue = "$env:storageQueue"} 24 | if ($workload -eq "audit.general") {$storageQueue = "$env:endpointstorageQueue"} 25 | 26 | #Load the Storage Queue 27 | $storeAuthContext = New-AzStorageContext -ConnectionString $env:AzureWebJobsStorage 28 | $myQueue = Get-AzStorageQueue -Name $storageQueue -Context $storeAuthContext 29 | $messageSize = 10 30 | if (-not ($myQueue)) {throw 'Failed to connect to Storage Queue'} 31 | 32 | $Tracker = "D:\home\$workload.log" # change to location of choice this is the root. 33 | $storedTime = Get-content $Tracker 34 | #$StoredTime = "2020-01-27T20:00:35.464Z" 35 | 36 | #If events are longer apart than 24 hours 37 | $adjustTime = New-TimeSpan -start $storedTime -End $endTime 38 | If ($adjustTime.TotalHours -gt 24) { 39 | $hours = $adjustTime.TotalHours - 23.9 40 | $storedTime = (get-date $storedTime).AddHours($hours) 41 | } 42 | 43 | # Get an Oauth 2 access token based on client id, secret and tenant domain 44 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 45 | 46 | #oauthtoken in the header 47 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 48 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 49 | 50 | #Make the request 51 | $rawRef = Invoke-WebRequest -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/content?contenttype=$workload&startTime=$Storedtime&endTime=$endTime&PublisherIdentifier=$TenantGUID" -UseBasicParsing 52 | if (-not ($rawRef)) {throw 'Failed to retrieve the content Blob Url'} 53 | 54 | #If more than one page is returned capture and return in pageArray 55 | if ($rawRef.Headers.NextPageUri) { 56 | 57 | $pageTracker = $true 58 | $pagedReq = $rawRef.Headers.NextPageUri 59 | while ($pageTracker -ne $false) 60 | { 61 | $pageuri = $pagedReq + "?PublisherIdentifier=" + $TenantGUID 62 | $CurrentPage = Invoke-WebRequest -Headers $headerParams -Uri $pageuri -UseBasicParsing 63 | $pageArray += $CurrentPage 64 | 65 | if ($CurrentPage.Headers.NextPageUri) 66 | { 67 | $pageTracker = $true 68 | } 69 | Else 70 | { 71 | $pageTracker = $false 72 | } 73 | 74 | $pagedReq = $CurrentPage.Headers.NextPageUri 75 | } 76 | 77 | } 78 | 79 | $pageArray += $rawref 80 | 81 | if ($pagearray.RawContentLength -gt 3) { 82 | foreach ($page in $pageArray) 83 | { 84 | $request = $page.content | convertfrom-json 85 | 86 | $request 87 | # Setting up the paging of the Message queue adding +1 to avoid misconfiguration 88 | $runs = $request.Count/($messageSize +1) 89 | if (($runs -gt 0) -and ($runs -le "1") ) {$runs=1} 90 | $writeSize = $messageSize 91 | $i = 0 92 | while ($runs -ge 1) { 93 | 94 | if ($request.count -eq "1") {$rawmessage += $request.contenturi} 95 | Else { $rawmessage = $request[$i..$writeSize].contenturi } 96 | 97 | foreach ($msg in $rawmessage){ 98 | $msgarray += @($msg) 99 | } 100 | $message = $msgarray | convertto-json 101 | $queueMessage = [Microsoft.Azure.Storage.Queue.CloudQueueMessage]::new("$message") 102 | $myqueue.CloudQueue.AddMessage($queuemessage) 103 | 104 | $runs -= 1 105 | $i+= $messageSize +1 106 | $writeSize += $messageSize + 1 107 | 108 | Clear-Variable msgarray 109 | Clear-Variable message 110 | Clear-Variable rawMessage 111 | } 112 | 113 | } 114 | #Updating timers on success, registering the date from the latest entry returned from the API and adding 1 millisecond to avoid overlap 115 | $time = $pagearray[0].Content | convertfrom-json 116 | $Lastentry = (get-date ($time[$Time.contentcreated.Count -1].contentCreated)).AddMilliseconds(1).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") 117 | if ($Lastentry -ge $storedTime) {out-file -FilePath $Tracker -NoNewline -InputObject $Lastentry} 118 | 119 | } 120 | 121 | Clear-Variable pagearray 122 | Clear-Variable rawref -ErrorAction Ignore 123 | Clear-Variable page -ErrorAction Ignore 124 | 125 | } 126 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/Report/readme.md: -------------------------------------------------------------------------------- 1 | # Until we have published the Workbooks to the official GitHub for Sentinel 2 | 3 | ## Workbooks - how to Import and Export: 4 | 5 | Please follow the instruction below to setup the workbooks in your environment. There is one Workbook for Exchange & Teams, 6 | a separate workbook for OneDrive and SharePoint. 7 | 8 | ### Installation Instructions: 9 | 10 | 1. [Open] Workbooks from the Sentinel Workspace where you intend to install the workbooks / portal.azure.com 11 | 2. [Click] "Add workbook" 12 | 3. When the New Workbook window open select edit, then select the Advanced Editor ([click] the icon ) 13 | 4. Copy the text of the json template you are installing from this repository [paste] over any json that exists. 14 | 5. [Click] save, select the appropriate location and name for the workbook. 15 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/SensitiveInfoType.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param([string] $Timer) 3 | 4 | # Replace with your Log Analytics Workspace ID 5 | $CustomerId = $env:workspaceId 6 | 7 | # Replace with your Log Analytics Primary Key 8 | $SharedKey = $env:workspaceKey 9 | 10 | # Specify the name of the record type that you'll be creating 11 | $LogType = "DLPSensitivity" 12 | 13 | # You can use an optional field to specify the timestamp from the data. If the time field is not specified, Azure Monitor assumes the time is the message ingestion time 14 | $TimeStampField = (Get-Date) 15 | 16 | # Create the function to create the authorization signature 17 | Function Build-Signature ($customerId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) 18 | { 19 | $xHeaders = "x-ms-date:" + $date 20 | $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource 21 | 22 | $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) 23 | $keyBytes = [Convert]::FromBase64String($sharedKey) 24 | 25 | $sha256 = New-Object System.Security.Cryptography.HMACSHA256 26 | $sha256.Key = $keyBytes 27 | $calculatedHash = $sha256.ComputeHash($bytesToHash) 28 | $encodedHash = [Convert]::ToBase64String($calculatedHash) 29 | $authorization = 'SharedKey {0}:{1}' -f $customerId,$encodedHash 30 | return $authorization 31 | } 32 | 33 | # Create the function to create and post the request 34 | Function Post-LogAnalyticsData($customerId, $sharedKey, $body, $logType) 35 | { 36 | $method = "POST" 37 | $contentType = "application/json" 38 | $resource = "/api/logs" 39 | $rfc1123date = [DateTime]::UtcNow.ToString("r") 40 | $contentLength = $body.Length 41 | $signature = Build-Signature ` 42 | -customerId $customerId ` 43 | -sharedKey $sharedKey ` 44 | -date $rfc1123date ` 45 | -contentLength $contentLength ` 46 | -method $method ` 47 | -contentType $contentType ` 48 | -resource $resource 49 | $uri = "https://" + $customerId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" 50 | 51 | $headers = @{ 52 | "Authorization" = $signature; 53 | "Log-Type" = $logType; 54 | "x-ms-date" = $rfc1123date; 55 | "time-generated-field" = $TimeStampField; 56 | # "x-ms-AzureResourceId" = $resourceId; 57 | } 58 | 59 | $response = Invoke-WebRequest -Uri $uri -Method $method -ContentType $contentType -Headers $headers -Body $body -UseBasicParsing 60 | return $response.StatusCode 61 | 62 | } 63 | 64 | 65 | #This is the Exchange extraction portion of the code 66 | 67 | $expass = $env:expass 68 | $exuser = $env:exuser 69 | $password = ConvertTo-SecureString $expass -AsPlainText -Force 70 | $credentials=New-Object -TypeName System.Management.Automation.PSCredential ($exuser, $password) 71 | 72 | if ($credentials) { 73 | $session = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri https://ps.compliance.protection.outlook.com/powershell-liveid?BasicAuthToOAuthConversion=true -Credential $Credentials -Authentication Basic -AllowRedirection 74 | } 75 | 76 | if ($session) {Import-PSSession $session -CommandName Get-DlpSensitiveInformationType -AllowClobber -DisableNameChecking} 77 | 78 | $mapping = Get-DlpSensitiveInformationType | select Id,name,Publisher | ConvertTo-Json 79 | 80 | Post-LogAnalyticsData -customerId $customerId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($mapping)) -logType $logType 81 | 82 | #Update stored time and remove session 83 | remove-PSSession $session 84 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/StoreEndpointDLPEvents.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($QueueItem, $TriggerMetadata) 3 | 4 | # Replace with your Log Analytics Workspace ID 5 | $CustomerId = $env:workspaceId 6 | 7 | # Replace with your Log Analytics Primary Key 8 | $SharedKey = $env:workspaceKey 9 | 10 | # Specify the name of the record type that you'll be creating 11 | $LogType = "endpointdlp" 12 | 13 | #SharePoint Site US 14 | $SPUS = $env:SPUS 15 | 16 | 17 | #Retry logic primarily for AF429 where there is more than 60k requests per minute, much reused from https://stackoverflow.com/questions/45470999/powershell-try-catch-and-retry 18 | function Test-Command { 19 | [CmdletBinding()] 20 | Param( 21 | [Parameter(Position=0, Mandatory=$true)] 22 | [scriptblock]$ScriptBlock, 23 | 24 | [Parameter(Position=1, Mandatory=$false)] 25 | [int]$Maximum = 5, 26 | 27 | [Parameter(Position=2, Mandatory=$false)] 28 | [int]$Delay = 100 29 | ) 30 | 31 | Begin { 32 | $cnt = 0 33 | } 34 | 35 | Process { 36 | do { 37 | $cnt++ 38 | try { 39 | $ScriptBlock.Invoke() 40 | return 41 | } catch { 42 | $fault = $_.Exception.InnerException.Message | convertfrom-json 43 | Write-Error $_.Exception.InnerException.Message -ErrorAction Continue 44 | if ($fault.error.code -eq "AF429") {Start-Sleep -Milliseconds $Delay} 45 | else {$cnt = $Maximum} 46 | } 47 | } while ($cnt -lt $Maximum) 48 | 49 | throw 'Execution failed.' 50 | } 51 | } 52 | 53 | 54 | 55 | # You can use an optional field to specify the timestamp from the data. If the time field is not specified, Azure Monitor assumes the time is the message ingestion time 56 | $TimeStampField = (Get-Date) 57 | 58 | #Initiate Arrays used by the function 59 | $records = @() 60 | $endpointupload = @() 61 | $usWorkspace = @() 62 | 63 | 64 | # Create the function to create the authorization signature 65 | Function Build-Signature ($customerId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) 66 | { 67 | $xHeaders = "x-ms-date:" + $date 68 | $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource 69 | 70 | $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) 71 | $keyBytes = [Convert]::FromBase64String($sharedKey) 72 | 73 | $sha256 = New-Object System.Security.Cryptography.HMACSHA256 74 | $sha256.Key = $keyBytes 75 | $calculatedHash = $sha256.ComputeHash($bytesToHash) 76 | $encodedHash = [Convert]::ToBase64String($calculatedHash) 77 | $authorization = 'SharedKey {0}:{1}' -f $customerId,$encodedHash 78 | return $authorization 79 | } 80 | 81 | # Create the function to create and post the request 82 | Function Post-LogAnalyticsData($customerId, $sharedKey, $body, $logType) 83 | { 84 | $method = "POST" 85 | $contentType = "application/json" 86 | $resource = "/api/logs" 87 | $rfc1123date = [DateTime]::UtcNow.ToString("r") 88 | $contentLength = $body.Length 89 | $signature = Build-Signature ` 90 | -customerId $customerId ` 91 | -sharedKey $sharedKey ` 92 | -date $rfc1123date ` 93 | -contentLength $contentLength ` 94 | -method $method ` 95 | -contentType $contentType ` 96 | -resource $resource 97 | $uri = "https://" + $customerId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" 98 | 99 | $headers = @{ 100 | "Authorization" = $signature; 101 | "Log-Type" = $logType; 102 | "x-ms-date" = $rfc1123date; 103 | "time-generated-field" = $TimeStampField; 104 | # "x-ms-AzureResourceId" = $resourceId; 105 | } 106 | 107 | $response = Invoke-WebRequest -Uri $uri -Method $method -ContentType $contentType -Headers $headers -Body $body -UseBasicParsing 108 | return $response.StatusCode 109 | 110 | } 111 | 112 | 113 | $clientID = "$env:clientID" 114 | $clientSecret = "$env:clientSecret" 115 | $loginURL = "https://login.microsoftonline.com" 116 | $tenantdomain = "$env:tenantdomain" 117 | $tenantGUID = "$env:TenantGuid" 118 | $resource = "https://manage.office.com" 119 | 120 | 121 | # Get an Oauth 2 access token based on client id, secret and tenant domain 122 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 123 | 124 | #oauthtoken in the header 125 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 126 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 127 | 128 | #$message = $queueitem | convertfrom-json 129 | $content = $queueitem 130 | 131 | 132 | if ($queueitem.count -eq 1) {$content = $queueitem | convertfrom-json} 133 | 134 | foreach ( $url in $content) 135 | { 136 | $uri = $url + "?PublisherIdentifier=" + $TenantGUID 137 | $record = Test-Command -ScriptBlock { 138 | Invoke-RestMethod -UseBasicParsing -Headers $headerParams -Uri $uri 139 | } -Delay 10000 140 | $records += $record 141 | } 142 | 143 | $records.count 144 | 145 | 146 | #Here starts the enrichment functionality and routing function. 147 | 148 | #Make the GRAPH Call to get additional information, require different audience tag. 149 | $resourceG = "https://graph.microsoft.com" 150 | $bodyG = @{grant_type="client_credentials";resource=$resourceG;client_id=$ClientID;client_secret=$ClientSecret} 151 | $oauthG = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $bodyG 152 | $headerParamsG = @{'Authorization'="$($oauthG.token_type) $($oauthG.access_token)"} 153 | 154 | Foreach ($user in $records) { 155 | 156 | #EndpointDLP upload 157 | if (($user.Workload -eq "EndPoint") -and (($user.PolicyMatchInfo))) { 158 | 159 | #Add the additional attributes needed to enrich the event stored 160 | $queryString = $user.UserKey + "?$" + "select=usageLocation,Manager,department,state" 161 | $info = Invoke-RestMethod -Headers $headerParamsG -Uri "https://graph.microsoft.com/v1.0/users/$queryString" -Method GET 162 | $user | Add-Member -MemberType NoteProperty -Name "usageLocation" -Value $info.usageLocation 163 | if ($info) {$user | Add-Member -MemberType NoteProperty -Name "department" -Value $info.department} 164 | if ($user.objectId) { 165 | $document = Split-Path $user.objectId -leaf 166 | $user | Add-Member -MemberType NoteProperty -Name "DocumentName" -Value $document 167 | } 168 | 169 | $endpointupload += $user 170 | Clear-Variable -name info 171 | } 172 | } 173 | 174 | 175 | #Determine which Sentinel Workspace to route the information, remember to define the variable for each workspace as an array. 176 | foreach ($entry in $endpointupload) { 177 | $usWorkspace += $entry 178 | #if ($entry.usageLocation -eq "US") { $usWorkspace += $entry } 179 | #if ($entry.usageLocation -ne "US") { $usWorkspace += $entry } 180 | } 181 | 182 | #Upload US Workspace, to add addtional workspaces add the WorkspaceID and Workspacekey and make a new post based on those parameters 183 | if ($usWorkspace) { 184 | $jsonus = $usWorkspace | convertTo-Json -depth 20 185 | Post-LogAnalyticsData -customerId $env:workspaceId -sharedKey $env:workspaceKey -body ([System.Text.Encoding]::UTF8.GetBytes($jsonus)) -logType $logType 186 | } 187 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/enablesubscription.ps1: -------------------------------------------------------------------------------- 1 | param($Enablement) 2 | 3 | #Enable the Activity API Subscriptions 4 | $clientID = "$env:clientID" 5 | $clientSecret = "$env:clientSecret" 6 | $loginURL = "https://login.microsoftonline.com" 7 | $tenantdomain = "$env:tenantdomain" 8 | $tenantGUID = "$env:TenantGuid" 9 | $resource = "https://manage.office.com" 10 | $date = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 11 | 12 | #Adding the yaml files used as templates for the analytic rules, added to d:\home to match the analytics synch scripts 13 | $officedlp1 =@{"URL" = "https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/master/Sentinel/EndPointDLP_preview/Analytics/endpointruletemplate.yaml"; "file" = "endpointruletemplate.yaml"} 14 | $officedlp = @{"URL"= "https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/master/Sentinel/AnalyticsRule/ruletemplate.yaml"; "file" = "ruletemplate.yaml"} 15 | $Dlpdepend = $($officedlp,$officedlp1) 16 | 17 | foreach ($template in $Dlpdepend) { 18 | $webclient = New-Object System.Net.WebClient 19 | $filepath = "d:\home\" + $template.file.ToString() 20 | $filepath 21 | $template.url.ToString() 22 | $webclient.DownloadFile($template.url.ToString(),$filepath) 23 | } 24 | 25 | 26 | # Get an Oauth 2 access token based on client id, secret and tenant domain 27 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 28 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 29 | #Let's put the oauth token in the header 30 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 31 | #Start Subscriptions 32 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=DLP.All" 33 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=audit.general" 34 | #List the active subscriptions 35 | Invoke-RestMethod -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/list" 36 | 37 | #Sets up the Message Queue 38 | $storeAuthContext = New-AzStorageContext -ConnectionString $env:AzureWebJobsStorage 39 | New-AzStorageQueue -Name $env:storageQueue -context $storeAuthContext 40 | New-AzStorageQueue -Name $env:endpointstorageQueue -context $storeAuthContext 41 | 42 | $distantdate = "2005-08-18T15:32:04.000Z" 43 | 44 | #Generates the time stamp for the ingestion 45 | out-file d:\home\dlp.All.log -InputObject $date 46 | out-file d:\home\audit.general.log -InputObject $date 47 | out-file d:\home\lastofficepolicy.log -InputObject $distantdate -NoNewline 48 | out-file d:\home\lastendpointpolicy.log -InputObject $distantdate -NoNewline 49 | 50 | -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/endpointdlpservice.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/EndPointDLP_preview/endpointdlpservice.zip -------------------------------------------------------------------------------- /Sentinel/EndPointDLP_preview/readme.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | products: 4 | - office-365 5 | - Sentinel 6 | languages: 7 | - powershellcore 8 | extensions: 9 | contentType: samples 10 | createdDate: 4/21/2020 3:00:56 PM 11 | description: "This sample can be used to create a function that ingest DLP.All logs and Audit.General Endpoint DLP events to Sentinel. This is early preview code and contains some workarounds to solve current limitations in the system" 12 | --- 13 | 14 | [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel%2FEndPointDLP_preview%2Fdeploysentinelfunction.json) 15 | 16 | # Ingesting Office 365 and Endpoint DLP events to Sentinel 17 | 18 | By clicking deploy above you will deploy an Azure Function App with the functions needed to run this project. You will have to copy the code manually to the functions or use the script option below for deployment. The reason being that we want you to manage the code distribution yourself. There currently is a bug in the API that may cause duplicates of a single endpoint event during low load conditions. 19 | 20 | ### Prerequisites 21 | 22 | - You need to have an Azure Subscription 23 | - Ability to create an Azure Function App. 24 | - A Sentinel Workspace and access to the Keys 25 | - Part of the Endpoint DLP preview 26 | - Exchange credentials to get sensitive info types 27 | - You need permissions to make a new App registration. 28 | - SharePoint Library if you want to utilize the ability to store full email content in SharePoint. 29 | 30 | ### Installing 31 | 32 | * 1. Register a new application in Azure AD https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app 33 | -Microsoft GRAPH (Application permissions) 34 | - Group.Read.All 35 | - User.Read.All 36 | - Office 365 Management APIs (Application permissions) 37 | - ActivityFeed.Read 38 | - ActivityFeed.ReadDlp (Needed for detailed events) 39 | 40 | * 2. Collect the identity and secret for the new App created in step 1. For production. Store the secret in Azure Key vault https://docs.microsoft.com/en-us/azure/app-service/app-service-key-vault-references 41 | - clientID 42 | - clientSecret 43 | - TenantGuid 44 | - exuser (User account to allow for mapping to sensitive info types) 45 | - Azure Sentinel Workspace Name 46 | 47 | * 3. Get the WorkSpace ID and Workspace Key for your Sentinel Workspace. 48 | 49 | * 4. Click on Deploy to Azure Above to start the deployment. 50 | * Fill in the values for your environment. If you have an Azure Keyvault use the string something like this instead of the actual value @Microsoft.KeyVault(SecretUri=https://Myvault.vault.azure.net/secrets/MySecretKey/bd2a5f8b0f944b528af2b66da20645d4) 51 | 52 | * SPUS is only used if you are going to deploy ingestion of SharePoint. (https://myTenant.sharepoint.com/sites/DLPDetectionsFinance/Records/) 53 | **These values can be changed later, by going to configuration of the Azure Function App.** 54 | 55 | - **Please Observe, there may be a timing issue causing an error when deploying the logic apps. If it is one of the functions it can be safely ignored.** 56 | 57 | * 5. **Deployment of the code to the function** 58 | * Download the endpointdlpservice.zip from this repo 59 | * Start to connect to Azure PowerShell Connect-AzAccout 60 | * Run Publish-AzWebApp -ResourceGroupName REPLACEWITHYOURRG -Name REPLACEWITHYOURAPPNAME -ArchivePath C:\YOURPATH\endpointdlpservice.zip **Note:The names are case sensitive** 61 | 62 | * 6. To enable the app to automatically synch DLP policies to Sentinel run the following commands it will allow the APP to fully manage Sentinel 63 | * $id = (Get-AzADServicePrincipal -DisplayNameBeginsWith YourAPP).id 64 | * New-AzRoleAssignment -ResourceGroupName YOURRGWITHSENTINEL -RoleDefinitionName "Azure Sentinel Contributor" -ObjectId $id 65 | **You can use the UI as well under Identity of the function, the same process can be used granting access to your key vault** 66 | 67 | * 7. To initialize the variables in the app 68 | * Navigate to the Enablement function in your Function App, open the function under functions, open "Code + Test" , click Test/Run, click Run 69 | * Note if there are any errors generated in this run, you will see it in the logging window. If there is a typo or similar in your configuration files. Go back to the main window for the App and click Configuration to update. 70 | 71 | - **The Analytics Functions will not be successful until you have ingested both SharePoint, Exchange events and in the case of Endpoint you need Endpoint events** 72 | * If the Log Analytic rules that corresponds to DLP Policies aren't created after data ingestion, run the Enablement function again. It will reset the time scope of the functions. 73 | * To make a manual import follow the steps outlined here https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/EndPointDLP_preview/AnalyticsRule, for Exchange and SharePoint https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/AnalyticsRule 74 | **They will fail if you haven't ingested events first.** 75 | 76 | - If you want to ingest original email content to SharePoint please see https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/logicapp. 77 | 78 | - To setup reporting please see https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/EndPointDLP_preview/Report, https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/Report 79 | 80 | 81 | ## Multiple workspaces for Multi Geo and Microsoft Graph enrichment 82 | The StoreEvents.ps1 has the basic enrichment functionality. You will find it from row 110 and onward. There is a high likelihood that you want to customize this code to meet your organizations requirements. 83 | 84 | Right now it is based on usageLocation and below usageLocation US. You will likely have another attribute that we should use for event routing. 85 | 86 | - The GRAPH queryString dictates which attributes we bring back from the Azure GRAPH. What you get back can then be used to enrich the data. You can make additional calls to the Security Graph as well. 87 | 88 | - As part of the code we are adding the SPO location so that once that component is in place you can easily access the original content through that link. Storing it in SharePoint allow for granular permissions. 89 | 90 | - When preparing the Arrays for upload to Log Analytics we simply push the data to the appropriate Workspace based on the usageLocation in this sample. 91 | 92 | There is an issue returning the manager with the v1.0 which works with the Beta endpoint. 93 | To get the manager with v1.0 amend the code with 94 | $querymanager = "https://graph.microsoft.com/v1.0/users/" + $user.ExchangeMetaData.From + "/manager" 95 | $manager = Invoke-RestMethod -Headers $headerParamsG -Uri $querymanager 96 | 97 | ## Important Additional Customization 98 | 99 | For production increase the FUNCTIONS_WORKER_PROCESS_COUNT https://docs.microsoft.com/en-us/azure/azure-functions/functions-app-settings 100 | Specifies the maximum number of language worker processes, with a default value of 1. The maximum value allowed is 10. Function invocations are evenly distributed among language worker processes. Language worker processes are spawned every 10 seconds until the count set by FUNCTIONS_WORKER_PROCESS_COUNT is reached. 101 | 102 | The default function timeout is 5 minutes in the consumption plan, consider to increase it to 10 minutes depending on load. https://docs.microsoft.com/en-us/azure/azure-functions/functions-host-json#functiontimeout 103 | 104 | ## Contributing 105 | 106 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 107 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 108 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 109 | 110 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 111 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 112 | provided by the bot. You will only need to do this once across all repos using our CLA. 113 | 114 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 115 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 116 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 117 | 118 | -------------------------------------------------------------------------------- /Sentinel/QueueEvents.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | #Enumerators and object to wrap the objects 5 | $pageArray = @() 6 | $msgarray = @() 7 | 8 | #Sign in Parameters 9 | $clientID = "$env:clientID" 10 | $clientSecret = "$env:clientSecret" 11 | $loginURL = "https://login.microsoftonline.com" 12 | $tenantdomain = "$env:tenantdomain" 13 | $tenantGUID = "$env:TenantGuid" 14 | $resource = "https://manage.office.com" 15 | 16 | #Workloads and end time default is on start 17 | $workloads = $env:contentTypes.split(",") 18 | $endTime = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 19 | 20 | #Storage Account Settings 21 | $storageQueue = "$env:storageQueue" 22 | 23 | #Load the Storage Queue 24 | $storeAuthContext = New-AzStorageContext -ConnectionString $env:AzureWebJobsStorage 25 | $myQueue = Get-AzStorageQueue -Name $storageQueue -Context $storeAuthContext 26 | $messageSize = 10 27 | 28 | Foreach ($workload in $workloads) { 29 | 30 | $Tracker = "D:\home\$workload.log" # change to location of choise this is the root. 31 | $storedTime = Get-content $Tracker 32 | #$StoredTime = "2020-01-27T20:00:35.464Z" 33 | 34 | #If events are longer apart than 24 hours 35 | $adjustTime = New-TimeSpan -start $storedtime -End $endTime 36 | If ($adjustTime.TotalHours -gt 24) { 37 | $hours = $adjustTime.TotalHours - 23.9 38 | $storedTime = (get-date $storedTime).AddHours($hours) 39 | } 40 | 41 | # Get an Oauth 2 access token based on client id, secret and tenant domain 42 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 43 | 44 | #oauthtoken in the header 45 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 46 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 47 | 48 | #Make the request 49 | $rawRef = Invoke-WebRequest -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/content?contenttype=$workload&startTime=$Storedtime&endTime=$endTime&PublisherIdentifier=$TenantGUID" -UseBasicParsing 50 | 51 | #If more than one page is returned capture and return in pageArray 52 | if ($rawRef.Headers.NextPageUri) { 53 | 54 | $pageTracker = $true 55 | $pagedReq = $rawRef.Headers.NextPageUri 56 | while ($pageTracker -ne $false) 57 | { 58 | $pageuri = "$pagedReq&PublisherIdentifier=$TenantGUID" 59 | $CurrentPage = Invoke-WebRequest -Headers $headerParams -Uri $pageuri -UseBasicParsing 60 | $pageArray += $CurrentPage 61 | 62 | if ($CurrentPage.Headers.NextPageUri) 63 | { 64 | $pageTracker = $true 65 | } 66 | Else 67 | { 68 | $pageTracker = $false 69 | } 70 | 71 | $pagedReq = $CurrentPage.Headers.NextPageUri 72 | } 73 | 74 | } 75 | 76 | $pageArray += $rawref 77 | 78 | if ($pagearray.RawContentLength -gt 3) { 79 | foreach ($page in $pageArray) 80 | { 81 | $request = $page.content | convertfrom-json 82 | 83 | # Setting up the paging of the Message queue adding +1 to avoid misconfiguration 84 | $runs = $request.Count/($messageSize +1) 85 | if (($runs -gt 0) -and ($runs -le "1") ) {$runs=1} 86 | $writeSize = $messageSize 87 | $i = 0 88 | while ($runs -ge 1) { 89 | 90 | $rawmessage = $request[$i..$writeSize].contenturi 91 | 92 | foreach ($msg in $rawmessage){ 93 | $msgarray += @($msg) 94 | $message = $msgarray | convertto-json 95 | } 96 | 97 | $queueMessage = New-Object -TypeName Microsoft.Azure.Storage.Queue.CloudQueueMessage -ArgumentList "$message" 98 | $myqueue.CloudQueue.AddMessage($queuemessage) 99 | 100 | $runs -= 1 101 | $i+= $messageSize +1 102 | $writeSize += $messageSize + 1 103 | 104 | Clear-Variable msgarray 105 | Clear-Variable message 106 | Clear-Variable rawMessage 107 | } 108 | 109 | } 110 | #Updating timers on success, registering the date from the latest entry returned from the API and adding 1 millisecond to avoid overlap 111 | $time = $pagearray[0].Content | convertfrom-json 112 | $Lastentry = (get-date ($time[$Time.contentcreated.Count -1].contentCreated)).AddMilliseconds(1).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") 113 | if ($Lastentry -ge $storedTime) {out-file -FilePath $Tracker -NoNewline -InputObject $Lastentry} 114 | 115 | } 116 | 117 | Clear-Variable pagearray 118 | Clear-Variable rawref -ErrorAction Ignore 119 | Clear-Variable page -ErrorAction Ignore 120 | 121 | } 122 | -------------------------------------------------------------------------------- /Sentinel/Readme.md: -------------------------------------------------------------------------------- 1 | [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel%2FdeploySentinelfunction.json) 2 | 3 | --- 4 | page_type: sample 5 | products: 6 | - office-365 7 | - Sentinel 8 | languages: 9 | - powershellcore 10 | extensions: 11 | contentType: samples 12 | createdDate: 4/21/2020 3:00:56 PM 13 | description: "This sample can be used to create a function that ingest DLP.All logs to Sentinel." 14 | --- 15 | 16 | 17 | # Ingesting Office 365 DLP.ALL events to Sentinel 18 | 19 | By clicking deploy above you will deploy an Azure Function App with the functions needed to run this project. This version will deploy the functions for endpointdlp but will not ingest any endpoint data. See the preview for ingesting endpoint dlp data. 20 | 21 | ### Prerequisites 22 | 23 | - You need to have an Azure Subscription 24 | - Ability to create an Azure Function App. 25 | - A Sentinel Workspace and access to the Keys 26 | - You need permissions to make a new App registration. 27 | - SharePoint Library if you want to utilize the ability to store full email content in SharePoint. 28 | 29 | ### Installing 30 | 31 | * 1. Register a new application in Azure AD https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app 32 | -Microsoft GRAPH (Application permissions) 33 | - Group.Read.All 34 | - User.Read.All 35 | - Office 365 Management APIs (Application permissions) 36 | - ActivityFeed.Read 37 | - ActivityFeed.ReadDlp (Needed for detailed events) 38 | 39 | * 2. Collect the identity and secret for the new App created in step 1. For production. Store the secret in Azure Key vault https://docs.microsoft.com/en-us/azure/app-service/app-service-key-vault-references 40 | - clientID 41 | - clientSecret 42 | - TenantGuid 43 | - exuser (User account to allow for mapping to sensitive info types) 44 | - Azure Sentinel Workspace Name 45 | 46 | * 3. Get the WorkSpace ID and Workspace Key for your Sentinel Workspace. 47 | 48 | * 4. Click on Deploy to Azure Above to start the deployment. 49 | * Fill in the values for your environment. If you have an Azure Keyvault use the string something like this instead of the actual value @Microsoft.KeyVault(SecretUri=https://Myvault.vault.azure.net/secrets/MySecretKey/bd2a5f8b0f944b528af2b66da20645d4) 50 | 51 | * SPUS is only used if you are going to deploy ingestion of SharePoint. (https://myTenant.sharepoint.com/sites/DLPDetectionsFinance/Records/) 52 | **These values can be changed later, by going to configuration of the Azure Function App.** 53 | 54 | - **Please Observe, there may be a timing issue causing an error when deploying the logic apps. If it is one of the functions it can be safely ignored.** 55 | 56 | * 5. **Deployment of the code to the function** 57 | * Download the dlpservice.zip from this repo 58 | * Start to connect to Azure PowerShell Connect-AzAccout 59 | * Run Publish-AzWebApp -ResourceGroupName REPLACEWITHYOURRG -Name REPLACEWITHYOURAPPNAME -ArchivePath C:\YOURPATH\dlpservice.zip **Note:The names are case sensitive** 60 | 61 | * 6. To enable the app to automatically synch DLP policies to Sentinel run the following commands it will allow the APP to fully manage Sentinel 62 | * $id = (Get-AzADServicePrincipal -DisplayNameBeginsWith YourAPP).id 63 | * New-AzRoleAssignment -ResourceGroupName YOURRGWITHSENTINEL -RoleDefinitionName "Azure Sentinel Contributor" -ObjectId $id 64 | **You can use the UI as well under Identity of the function, the same process can be used granting access to your key vault** 65 | 66 | * 7. To initialize the variables in the app 67 | * Navigate to the Enablement function in your Function App, open the function under functions, open "Code + Test" , click Test/Run, click Run 68 | * Note if there are any errors generated in this run, you will see it in the logging window. If there is a typo or similar in your configuration files. Go back to the main window for the App and click Configuration to update. 69 | 70 | - **The Analytics Functions will not be successful until you have ingested both SharePoint, Exchange events and in the case of Endpoint you need Endpoint events** 71 | * If the Log Analytic rules that corresponds to DLP Policies aren't created after data ingestion, run the Enablement function again. It will reset the time scope of the functions. 72 | * To make a manual import follow the steps outlined here https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/EndPointDLP_preview/AnalyticsRule, for Exchange and SharePoint https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/AnalyticsRule 73 | **They will fail if you haven't ingested events first.** 74 | 75 | - If you want to ingest original email content to SharePoint please see https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/logicapp. 76 | 77 | - To setup reporting please see https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/EndPointDLP_preview/Report, https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/Report 78 | 79 | ## Multiple workspaces for Multi Geo and Microsoft Graph enrichment 80 | The StoreEvents.ps1 has the basic enrichment functionality. You will find it from row 110 and onward. There is a high likelihood that you want to customize this code to meet your organizations requirements. 81 | 82 | Right now it is based on usageLocation and below usageLocation US. You will likely have another attribute that we should use for event routing. 83 | 84 | - The GRAPH queryString dictates which attributes we bring back from the Azure GRAPH. What you get back can then be used to enrich the data. You can make additional calls to the Security Graph as well. 85 | 86 | - As part of the code we are adding the SPO location so that once that component is in place you can easily access the original content through that link. Storing it in SharePoint allow for granular permissions. 87 | 88 | - When preparing the Arrays for upload to Log Analytics we simply push the data to the appropriate Workspace based on the usageLocation in this sample. 89 | 90 | There is an issue returning the manager with the v1.0 which works with the Beta endpoint. 91 | To get the manager with v1.0 amend the code with 92 | $querymanager = "https://graph.microsoft.com/v1.0/users/" + $user.ExchangeMetaData.From + "/manager" 93 | $manager = Invoke-RestMethod -Headers $headerParamsG -Uri $querymanager 94 | 95 | ## Additional Customization 96 | 97 | For production increase the FUNCTIONS_WORKER_PROCESS_COUNT https://docs.microsoft.com/en-us/azure/azure-functions/functions-app-settings 98 | Specifies the maximum number of language worker processes, with a default value of 1. The maximum value allowed is 10. Function invocations are evenly distributed among language worker processes. Language worker processes are spawned every 10 seconds until the count set by FUNCTIONS_WORKER_PROCESS_COUNT is reached. 99 | 100 | ## Contributing 101 | 102 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 103 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 104 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 105 | 106 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 107 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 108 | provided by the bot. You will only need to do this once across all repos using our CLA. 109 | 110 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 111 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 112 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 113 | -------------------------------------------------------------------------------- /Sentinel/Report/img/report1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/Report/img/report1.png -------------------------------------------------------------------------------- /Sentinel/Report/img/report2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/Report/img/report2.png -------------------------------------------------------------------------------- /Sentinel/Report/img/report3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/Report/img/report3.png -------------------------------------------------------------------------------- /Sentinel/Report/img/report4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/Report/img/report4.png -------------------------------------------------------------------------------- /Sentinel/Report/img/test.md: -------------------------------------------------------------------------------- 1 | # blank 2 | -------------------------------------------------------------------------------- /Sentinel/Report/readme.md: -------------------------------------------------------------------------------- 1 | # Until we have published the Workbooks to the official GitHub for Sentinel 2 | 3 | ## Workbooks - how to Import and Export: 4 | 5 | Please follow the instruction below to setup the workbooks in your environment. There is one Workbook for Exchange & Teams, 6 | a separate workbook for OneDrive and SharePoint. 7 | 8 | ### Installation Instructions: 9 | 10 | 1. [Open] Workbooks from the Sentinel Workspace where you intend to install the workbooks / portal.azure.com 11 | 2. [Click] "Add workbook" 12 | 3. When the New Workbook window open select edit, then select the Advanced Editor ([click] the icon ) 13 | 4. Copy the text of the json template you are installing from this repository [paste] over any json that exists. 14 | 5. [Click] save, select the appropriate location and name for the workbook. 15 | 16 | ### The report will look something like this. 17 | The first section provides a view of the number of DLP incidents per workload. It also allows for quickly viewing the users that have breached certain Policies. If a user is selected it will filter the rest of the screen. 18 | ![Invocation Log](./img/report1.png) 19 | 20 | The next portion provides a view over time and the specific rules that triggered the DLP Policy. There is also a list over the policies where users have requested an exception to the rule. 21 | ![Invocation Log](./img/report2.png) 22 | 23 | The last section of this report provides some details to DLP exceptions as well as the details on the e-mails. Right now we are only adding subject to the emails triggering the alert. 24 | ![Invocation Log](./img/report3.png) 25 | ![Invocation Log](./img/report4.png) 26 | -------------------------------------------------------------------------------- /Sentinel/deploySentinelfunction.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "appName": { 6 | "value": null 7 | }, 8 | "storageAccountType": { 9 | "value": "Standard_LRS" 10 | }, 11 | "location": { 12 | "value": "[resourceGroup().location]" 13 | }, 14 | "runtime": { 15 | "value": "powershell" 16 | }, 17 | "ClientID": { 18 | "value": "Provide the client ID" 19 | }, 20 | "ClientSecret": { 21 | "value": "Provide the Client Secret" 22 | }, 23 | "ContentTypes": { 24 | "value": "DLP.ALL" 25 | }, 26 | "customLogName": { 27 | "value": "O365DLP" 28 | }, 29 | "domains": { 30 | "value": "youradditionaldomain.com,yourdomain.com,yourtenant.onmicrosoft.com" 31 | }, 32 | "SPUS": { 33 | "value": "SharePoint location for original emails if not exist use a temporary path" 34 | }, 35 | "storageQueue": { 36 | "value": "dlpqueue" 37 | }, 38 | "tenantDomain": { 39 | "value": "Yourtenant.onmicrosoft.com" 40 | }, 41 | "TenantGuid": { 42 | "value": "Your Tenant GUID" 43 | }, 44 | "workspaceId": { 45 | "value": "LogAnalytics Workspace Id" 46 | }, 47 | "workspaceKey": { 48 | "value": "LogAnalytics WorkspaceKey" 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Sentinel/dlpservice.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/dlpservice.zip -------------------------------------------------------------------------------- /Sentinel/enablesubscription.ps1: -------------------------------------------------------------------------------- 1 | param($Enablement) 2 | 3 | #Enable the Activity API Subscriptions 4 | $clientID = "$env:clientID" 5 | $clientSecret = "$env:clientSecret" 6 | $loginURL = "https://login.microsoftonline.com" 7 | $tenantdomain = "$env:tenantdomain" 8 | $tenantGUID = "$env:TenantGuid" 9 | $resource = "https://manage.office.com" 10 | $date = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 11 | # Get an Oauth 2 access token based on client id, secret and tenant domain 12 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 13 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 14 | #Let's put the oauth token in the header 15 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 16 | #Start Subscriptions 17 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=DLP.All" 18 | #List the active subscriptions 19 | Invoke-WebRequest -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/list" 20 | 21 | #Sets up the Message Queue 22 | $storeAuthContext = New-AzStorageContext -ConnectionString $env:AzureWebJobsStorage 23 | New-AzStorageQueue -Name $env:storageQueue -context $storeAuthContext 24 | 25 | #Generates the time stamp for the ingestion 26 | out-file d:\home\dlp.All.log -InputObject $date 27 | -------------------------------------------------------------------------------- /Sentinel/logicapp/img/blank.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /Sentinel/logicapp/img/incident1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/logicapp/img/incident1.png -------------------------------------------------------------------------------- /Sentinel/logicapp/messageid.ps1: -------------------------------------------------------------------------------- 1 | using namespace System.Net 2 | 3 | # Input bindings are passed in via param block. 4 | param($Request, $TriggerMetadata) 5 | 6 | $name = "email" 7 | 8 | #This code uses a regex pattern that identifies the MessageID in the message body. 9 | if ($name) { 10 | $status = [HttpStatusCode]::OK 11 | 12 | $regex = "(?:<(?:(?:[a-zA-Z0-9!#$%&\*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&\*+/=?^_{|}~-]+)*)|(?:(?:(?:[\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21\x23-\x5B\x5D-\x7E])|(?:\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*))@(?:(?:[a-zA-Z0-9!#$%&\*+/=?^_{|}~-]+(?:\.[a-zA-Z0-9!#$%&\'*+/=?^_`{|}~-]+)*)|(?:\[(?:(?:[\x01-\x08\x0B\x0C\x0E-\x1F\x7F]|[\x21-\x5A\x5E-\x7E])|(?:\\[\x01-\x09\x0B\x0C\x0E-\x7F]))*\]))>)" 13 | $request.body.emailbody -match $regex 14 | 15 | $body = [string]::Format($Matches.Values) 16 | $body 17 | 18 | } 19 | 20 | else { 21 | $status = [HttpStatusCode]::BadRequest 22 | $body = "Please pass a name on the query string or in the request body." 23 | } 24 | 25 | 26 | # Associate values to output bindings by calling 'Push-OutputBinding'. 27 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 28 | StatusCode = $status 29 | Body = $body 30 | }) 31 | -------------------------------------------------------------------------------- /Sentinel/logicapp/readme.md: -------------------------------------------------------------------------------- 1 | [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel%2Flogicapp%2Fdlpaction.json) 2 | 3 | # Copy Original Message from Incident Mailbox to SharePoint Site 4 | If you have more than one workspace and Geo, you will have to set this up separately if there is a requirement to keep information in Geo at rest. 5 | 6 | ## Prerequisites 7 | - Complete the steps to setup the ActualID function as part of the main package https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel 8 | 9 | ## Setup 10 | 11 | 1. Ensure to setup your DLP rule to forward the full details of incidents to the mailbox used for extraction by the function. For this sample we are using DLPAlertsEU. 12 | 13 | ![Invocation Log](./img/incident1.png) 14 | 15 | 2. Create a SharePoint Site Collection in Region with the appropriate retention time. Use a Records center template if you need to treat the information as records. See the SharePoint Online limits to determine if you need more than one collection per region. This will depend on your expected load and retention period. You can change the ingestion code to ingest information based on the Policy Name as an example to scale this out. **Create a Library named "Records"** in the newly created Site Collection. 16 | 17 | 3. [Click] Deploy to Azure above 18 | 19 | 4. Provide the right Resource Group, where the function app resides. You can move the function later. Provide the function app name used for ActualID (Name of the function created in the first step). If you want to change the Workflow name do so from here. Don't touch the connections for SharePoint and Exchange they can be changed later. 20 | 21 | 5. The SharePoint connection should go to the Site Collection itself don't specify "Records". https://tenant.sharepoint.com/sites/DLPArchive/ 22 | 23 | 6. When deployed change the connections used for when email arrives and for Export email as well as for SharePoint. 24 | 25 | 26 | ### More information 27 | The information transferred to the ActualID function from the Logic App is limited to the Body preview. It doesn't contain sensitive information. 28 | 29 | { 30 | "emailbody": "A match of one or more of your organization’s policy rules has been detected.\r\n\r\nService: Exchange\r\nMatched item: \r\nTitle: Person\r\nDocument owner:\r\nPerson who last modified do" 31 | } 32 | -------------------------------------------------------------------------------- /Sentinel/mipservice/MIPService.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel/mipservice/MIPService.zip -------------------------------------------------------------------------------- /Sentinel/mipservice/MIPmap.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Value 2 | None,LabelEventType,0 3 | LabelUpgraded,LabelEventType,1 4 | LabelDowngraded,LabelEventType,2 5 | LabelRemoved,LabelEventType,3 6 | LabelChangedSameOrder,LabelEventType,4 7 | None,ActionSource,0 8 | Default,ActionSource,1 9 | Auto,ActionSource,2 10 | Manual,ActionSource,3 11 | Recommended,ActionSource,4 12 | None,ActionSourceDetail,0 13 | AutoByPolicyMatch,ActionSourceDetail,1 14 | AutoByReplyOrForward,ActionSourceDetail,2 15 | AutoByInheritance,ActionSourceDetail,3 16 | AutoByDeploymentPipeline,ActionSourceDetail,4 17 | PublicAPI,ActionSourceDetail,5 18 | AutoByLibraryDefault,ActionSourceDetail,6 19 | Unknown,LocationType,0 20 | Local,LocationType,1 21 | Remote,LocationType,2 22 | Removable,LocationType,3 23 | Cloud,LocationType,4 24 | FileShare,LocationType,5 25 | Unknown,Platform,0 26 | Windows,Platform,1 27 | Mac,Platform,2 28 | iOS,Platform,3 29 | Android,Platform,4 30 | WebBrowser,Platform,5 31 | -------------------------------------------------------------------------------- /Sentinel/mipservice/importwatch.ps1: -------------------------------------------------------------------------------- 1 | param([string]$csvfile,[string]$Watchlist,[string]$Workspace,[string]$errlog) 2 | 3 | if ($csvfile -eq "") {$csvfile = Read-Host "Provide path to csv import" -asString} 4 | if ($errlog -eq "") {$errlog = Read-Host "Provide path to csv import" -asString} 5 | if ($Watchlist -eq "") {$Watchlist = Read-Host "Provide Watchlist" -asString} 6 | if ($Worksapce -eq "") {$Workspace = Read-Host "Provide Watchlist" -asString} 7 | 8 | $csv = Import-Csv $csvfile 9 | 10 | Connect-AzAccount 11 | $context = Get-AzContext 12 | $profileR = [Microsoft.Azure.Commands.Common.Authentication.Abstractions.AzureRmProfileProvider]::Instance.Profile 13 | $profileClient = New-Object -TypeName Microsoft.Azure.Commands.ResourceManager.Common.RMProfileClient -ArgumentList ($profileR) 14 | $token = $profileClient.AcquireAccessToken($context.Subscription.TenantId) 15 | $authHeader = @{ 16 | 'Content-Type' = 'application/json' 17 | 'Authorization' = 'Bearer ' + $token.AccessToken 18 | } 19 | $workspace.value 20 | $instance = Get-AzResource -Name $Workspace -ResourceType Microsoft.OperationalInsights/workspaces 21 | 22 | 23 | #Retreiving the current watchlist 24 | $path = $instance.ResourceId 25 | $wlists = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/watchlists?api-version=2021-09-01-preview" 26 | $watchlists = Invoke-RestMethod -Method "Get" -Uri $wlists -Headers $authHeader 27 | $watchlistname = $watchlists.value.properties | where watchlistAlias -like $Watchlist 28 | $listwitems = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/watchlists/$($watchlistname.displayName)/watchlistitems?api-version=2021-09-01-preview" 29 | $witems = Invoke-RestMethod -Method "Get" -Uri $listwitems -Headers $authHeader 30 | if (-not ($witems)) {throw 'Failed to connect to Sentinel Workspace'} 31 | 32 | # Looping through the policies and create Analytic Rules in Sentinel 33 | $errcnt = $error.count 34 | $errors = @() 35 | foreach ($item in $csv) { 36 | 37 | $matchexisting = $witems.value | where-object {$_.properties.itemsKeyValue.userPrincipalName -contains $item.userPrincipalName } | select-object 38 | 39 | if ($matchexisting) { 40 | $etag = $matchexisting[0].name 41 | 42 | $a= @{ 43 | 'etag'= $etag 44 | 'properties'= @{itemsKeyValue = @()} 45 | } 46 | $a.properties.itemsKeyValue = $item 47 | $update = $a | convertto-json 48 | 49 | $urlupdate = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/watchlists/UserAccounts/watchlistitems/$($matchexisting[0].name)?api-version=2021-03-01-preview" 50 | Invoke-RestMethod -Method "Put" -Uri $urlupdate -Headers $authHeader -body $update 51 | } 52 | 53 | if (-not $matchexisting) { 54 | $etag = New-Guid 55 | $a= @{ 56 | 'etag'= $etag 57 | 'properties'= @{itemsKeyValue = @()} 58 | } 59 | 60 | $a.properties.itemsKeyValue = $item 61 | $update = $a | convertto-json 62 | 63 | $urlupdate = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/watchlists/UserAccounts/watchlistitems/$($etag)?api-version=2021-03-01-preview" 64 | Invoke-RestMethod -Method "Put" -Uri $urlupdate -Headers $authHeader -body $update 65 | 66 | } 67 | if ($error.count -gt $errcnt) {$errors += $item.userPrincipalName.ToString(), $error[$errcnt-1].ToString()} 68 | Clear-Variable matchexisting 69 | Clear-Variable etag 70 | Clear-Variable a 71 | Clear-Variable update 72 | } 73 | 74 | if ($errors) {$errors | Out-File -FilePath $errlog} 75 | -------------------------------------------------------------------------------- /Sentinel/mipservice/readme.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | products: 4 | - m365 5 | - microsoft-sentinel 6 | languages: 7 | - powershell 8 | extensions: 9 | contentType: samples 10 | createdDate: 12/17/2021 3:00:56 PM 11 | description: "This sample can be used to create MIP events in Sentinel." 12 | --- 13 | 14 | 15 | # Ingesting Micrsoft MIP events to Sentinel 16 | 17 | Use the endpointdlp preview steps to deploy the code. https://github.com/OfficeDev/O365-ActivityFeed-AzureFunction/tree/master/Sentinel/EndPointDLP_preview 18 | 19 | **During the deployment specify all content types in the dialog, deploy to Azure** 20 | - DLP.ALL,Audit.General,Audit.Exchange,Audit.SharePoint (Alt. after deployment under configuration of the Function App) 21 | 22 | **Replace STEP 5. by using the zip file in this repo.** 23 | 24 | ### Prerequisites 25 | 26 | - You need to have an Azure Subscription 27 | - Ability to create an Azure Function App. 28 | - A Sentinel Workspace and access to the Keys 29 | - You need permissions to make a new App registration. 30 | 31 | ### Installing 32 | Permissions needed for the app 33 | - Office 365 Management APIs (Application permissions) 34 | - ActivityFeed.Read 35 | - ActivityFeed.ReadDlp (Needed for detailed events) 36 | 37 | **Deployment of the code to the function** 38 | * Download mipservice.zip from this repo, the SHA256 hash is 9A8E886C9996157FFAFC21AF3661B1C243CAA1776B45B65FB914929C625FBCF3 39 | * Start to connect to Azure PowerShell Connect-AzAccout 40 | * Run Publish-AzWebApp -ResourceGroupName REPLACEWITHYOURRG -Name REPLACEWITHYOURAPPNAME -ArchivePath C:\YOURPATH\mipservice.zip **Note:The names are case sensitive** 41 | 42 | ### Creating the Watchlists 43 | Documentation for Watchlists https://docs.microsoft.com/en-us/azure/sentinel/watchlists 44 | 45 | 1. Export the MIP labels using SCC Powershell, sample Get-Label | select ImmutableId,DisplayName,LabelActions | Export-Csv c:\tmp\slabels.csv -NoTypeInformation 46 | If you happen to get hyphens in the csv header fields, remove the hyphens since the WL engine cannot process. 47 | 2. Create a new Microsoft Sentinel Watchlist call it **Sensitive**, set the ImmutableId as the index field. 48 | 3. Create a new Microsoft Sentinel Watchlist call it **MipMap**, set the Value field as the index field, import the mipmap.csv file in this repo. (File to translate MIP operations) 49 | 4. Create a new Microsoft Sentinel Watchlist call it UserAccounts, Import your account list, **for reporting to work well you need to include, userprincipalname,department,FullName,Title (The more detail you add the cooler you can make the report dashboard or any alerts)** 50 | - The Indexing field should be the UserPrincipalName, we use it as a key to enrich the items 51 | - You can start with a small csv file, for bulk uploading a lot of data please see importwatch.ps1 in this repo it works in PS and PS Core. It supports incremental uploads as well as updating existing objects in the list. 52 | - .\importwatch.ps1 -csv C:\tmp\UserAccounts.csv -Watchlist UserAccounts -Workspace usinstance -errlog c:\wlupload.log 53 | 54 | ### Deploy the Label Statistics Workbook 55 | Deploy the workbook Sensitivitylabels.json in this repo by simply copying the code across to a new Azure Workbook. 56 | 1. [Open] Workbooks from the Sentinel Workspace where you intend to install the workbooks / portal.azure.com 57 | 2. [Click] "Add workbook" 58 | 3. When the New Workbook window open select edit, then select the Advanced Editor ([click] the icon ) 59 | 4. Copy the text of the json template you are installing from this repository [paste] over any json that exists. 60 | 5. [Click] save, select the appropriate location and name for the workbook. 61 | -------------------------------------------------------------------------------- /Sentinel/msgtrace/ingestmsgtrace.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param([string] $Timer) 3 | 4 | # Replace with your Log Analytics Workspace ID 5 | $CustomerId = $env:workspaceId 6 | 7 | # Replace with your Log Analytics Primary Key 8 | $SharedKey = $env:workspaceKey 9 | 10 | # Specify the name of the record type that you'll be creating 11 | $LogType = $env:customLogName 12 | 13 | # You can use an optional field to specify the timestamp from the data. If the time field is not specified, Azure Monitor assumes the time is the message ingestion time 14 | $TimeStampField = (Get-Date) 15 | 16 | # Create the function to create the authorization signature 17 | Function Build-Signature ($customerId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) 18 | { 19 | $xHeaders = "x-ms-date:" + $date 20 | $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource 21 | 22 | $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) 23 | $keyBytes = [Convert]::FromBase64String($sharedKey) 24 | 25 | $sha256 = New-Object System.Security.Cryptography.HMACSHA256 26 | $sha256.Key = $keyBytes 27 | $calculatedHash = $sha256.ComputeHash($bytesToHash) 28 | $encodedHash = [Convert]::ToBase64String($calculatedHash) 29 | $authorization = 'SharedKey {0}:{1}' -f $customerId,$encodedHash 30 | return $authorization 31 | } 32 | 33 | # Create the function to create and post the request 34 | Function Post-LogAnalyticsData($customerId, $sharedKey, $body, $logType) 35 | { 36 | $method = "POST" 37 | $contentType = "application/json" 38 | $resource = "/api/logs" 39 | $rfc1123date = [DateTime]::UtcNow.ToString("r") 40 | $contentLength = $body.Length 41 | $signature = Build-Signature ` 42 | -customerId $customerId ` 43 | -sharedKey $sharedKey ` 44 | -date $rfc1123date ` 45 | -contentLength $contentLength ` 46 | -method $method ` 47 | -contentType $contentType ` 48 | -resource $resource 49 | $uri = "https://" + $customerId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" 50 | 51 | $headers = @{ 52 | "Authorization" = $signature; 53 | "Log-Type" = $logType; 54 | "x-ms-date" = $rfc1123date; 55 | "time-generated-field" = $TimeStampField; 56 | # "x-ms-AzureResourceId" = $resourceId; 57 | } 58 | 59 | $response = Invoke-WebRequest -Uri $uri -Method $method -ContentType $contentType -Headers $headers -Body $body -UseBasicParsing 60 | return $response.StatusCode 61 | 62 | } 63 | 64 | 65 | #This is the Exchange extraction portion of the code 66 | 67 | $expass = $env:expass 68 | $exuser = $env:exuser 69 | $password = ConvertTo-SecureString $expass -AsPlainText -Force 70 | $credentials=New-Object -TypeName System.Management.Automation.PSCredential ($exuser, $password) 71 | 72 | if ($credentials) { 73 | $session = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri https://outlook.office365.com/PowerShell-LiveId?BasicAuthToOAuthConversion=true -Credential $Credentials -Authentication Basic -AllowRedirection 74 | } 75 | 76 | if ($session) {Import-PSSession $session -CommandName Get-MessageTrace -AllowClobber -DisableNameChecking} 77 | 78 | #This sets the start and stop time, $tracker is read from the last time the function was run. (It will fail on the first run.) 79 | $tracker = "D:\home\timetracker.log" # change to location of choise this is the root. 80 | $startTime = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 81 | 82 | #After first run remark the configured date 83 | $storedTime = Get-content $Tracker 84 | #$storedTime = "2020-03-01T11:20:35.464Z" 85 | 86 | #Run the message trace 87 | 88 | #Store the information in loganalytics 89 | $pageSize = 5000 90 | $page = 1 91 | $runs = 1 92 | 93 | while ($runs -ge 1) { 94 | $runs 95 | $messagetrace = Get-MessageTrace -EndDate $startTime -startdate $storedTime -page $page -pagesize $pagesize 96 | 97 | if (($runs -eq 1) -and($messagetrace)) {$storedtime = $messagetrace[0].received} 98 | 99 | if ($messagetrace.count -gt 0) { 100 | $pagedjson = $messagetrace | convertTo-Json 101 | Post-LogAnalyticsData -customerId $customerId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($pagedjson)) -logType $logType 102 | } 103 | $runs ++ 104 | $page ++ 105 | if ($messagetrace.count -ne $pageSize) { $runs = 0 } 106 | 107 | Clear-Variable messagetrace 108 | 109 | } 110 | 111 | #Update stored time and remove session 112 | out-file -FilePath $Tracker -NoNewline -InputObject (get-date $startTime).AddMilliseconds(1).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") 113 | remove-PSSession $session 114 | -------------------------------------------------------------------------------- /Sentinel/msgtrace/ingestmsgtrace_with_detail.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param([string] $Timer) 3 | 4 | # Replace with your Log Analytics Workspace ID 5 | $CustomerId = $env:workspaceId 6 | 7 | # Replace with your Log Analytics Primary Key 8 | $SharedKey = $env:workspaceKey 9 | 10 | # Specify the name of the record type that you'll be creating 11 | $LogType = $env:customLogName 12 | 13 | # You can use an optional field to specify the timestamp from the data. If the time field is not specified, Azure Monitor assumes the time is the message ingestion time 14 | $TimeStampField = (Get-Date) 15 | 16 | # Create the function to create the authorization signature 17 | Function Build-Signature ($customerId, $sharedKey, $date, $contentLength, $method, $contentType, $resource) { 18 | $xHeaders = "x-ms-date:" + $date 19 | $stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource 20 | 21 | $bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash) 22 | $keyBytes = [Convert]::FromBase64String($sharedKey) 23 | 24 | $sha256 = New-Object System.Security.Cryptography.HMACSHA256 25 | $sha256.Key = $keyBytes 26 | $calculatedHash = $sha256.ComputeHash($bytesToHash) 27 | $encodedHash = [Convert]::ToBase64String($calculatedHash) 28 | $authorization = 'SharedKey {0}:{1}' -f $customerId, $encodedHash 29 | return $authorization 30 | } 31 | 32 | # Create the function to create and post the request 33 | Function Post-LogAnalyticsData($customerId, $sharedKey, $body, $logType) { 34 | $method = "POST" 35 | $contentType = "application/json" 36 | $resource = "/api/logs" 37 | $rfc1123date = [DateTime]::UtcNow.ToString("r") 38 | $contentLength = $body.Length 39 | $signature = Build-Signature ` 40 | -customerId $customerId ` 41 | -sharedKey $sharedKey ` 42 | -date $rfc1123date ` 43 | -contentLength $contentLength ` 44 | -method $method ` 45 | -contentType $contentType ` 46 | -resource $resource 47 | $uri = "https://" + $customerId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01" 48 | 49 | $headers = @{ 50 | "Authorization" = $signature; 51 | "Log-Type" = $logType; 52 | "x-ms-date" = $rfc1123date; 53 | "time-generated-field" = $TimeStampField; 54 | # "x-ms-AzureResourceId" = $resourceId; 55 | } 56 | 57 | $response = Invoke-WebRequest -Uri $uri -Method $method -ContentType $contentType -Headers $headers -Body $body -UseBasicParsing 58 | return $response.StatusCode 59 | 60 | } 61 | 62 | 63 | #This is the Exchange extraction portion of the code 64 | 65 | $expass = $env:expass 66 | $exuser = $env:exuser 67 | $password = ConvertTo-SecureString $expass -AsPlainText -Force 68 | $credentials = New-Object -TypeName System.Management.Automation.PSCredential ($exuser, $password) 69 | 70 | if ($credentials) { 71 | $session = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri https://outlook.office365.com/PowerShell-LiveId?BasicAuthToOAuthConversion=true -Credential $Credentials -Authentication Basic -AllowRedirection 72 | } 73 | 74 | if ($session) { Import-PSSession $session -CommandName Get-MessageTrace, Get-MessageTraceDetail -AllowClobber -DisableNameChecking } 75 | 76 | #This sets the start and stop time, $tracker is read from the last time the function was run. (It will fail on the first run.) 77 | $tracker = "D:\home\timetracker.log" # change to location of choise this is the root. 78 | $startTime = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 79 | 80 | #After first run remark the configured date 81 | $storedTime = Get-content $Tracker 82 | #$storedTime = "2020-03-01T11:20:35.464Z" 83 | 84 | #Run the message trace 85 | 86 | #Store the information in loganalytics 87 | $pageSize = 5000 88 | $page = 1 89 | $runs = 1 90 | 91 | while ($runs -ge 1) { 92 | $runs 93 | $messagetrace = Get-MessageTrace -EndDate $startTime -startdate $storedTime -page $page -pagesize $pagesize 94 | 95 | if (($runs -eq 1) -and ($messagetrace)) { $storedtime = $messagetrace[0].received } 96 | 97 | if ($messagetrace.count -gt 0) { 98 | 99 | $messagetrace_detail_array = @(); 100 | $messagetrace | ForEach-Object -Parallel { 101 | $elem = $_ 102 | $messagetrace_detail = Get-MessageTraceDetail -MessageTraceId $elem.MessageTraceId -RecipientAddress $elem.RecipientAddress 103 | $messagetrace_detail | ForEach-Object -Parallel { 104 | $detail_elem = $_ 105 | $messagetrace_detail_array = $using:messagetrace_detail_array 106 | $elem | Add-Member -MemberType NoteProperty -Name Event -Value $detail_elem.Event -Force 107 | $elem | Add-Member -MemberType NoteProperty -Name Action -Value $detail_elem.Action -Force 108 | $elem | Add-Member -MemberType NoteProperty -Name Detail -Value $detail_elem.Detail -Force 109 | $elem | Add-Member -MemberType NoteProperty -Name "Data" -Value $detail_elem.Data -Force 110 | $messagetrace_detail_array += $elem 111 | } 112 | } 113 | 114 | $pagedjson = $messagetrace_detail_array | convertTo-Json 115 | Post-LogAnalyticsData -customerId $customerId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($pagedjson)) -logType $logType 116 | 117 | $runs ++ 118 | $page ++ 119 | if ($messagetrace.count -ne $pageSize) { $runs = 0 } 120 | 121 | Clear-Variable messagetrace 122 | 123 | } 124 | } 125 | 126 | #Update stored time and remove session 127 | out-file -FilePath $Tracker -NoNewline -InputObject (get-date $storedTime).AddMilliseconds(1).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") 128 | remove-PSSession $session 129 | -------------------------------------------------------------------------------- /Sentinel/msgtrace/readme.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | products: 4 | - office-365 5 | - microsoft-sentinel 6 | languages: 7 | - powershell 8 | extensions: 9 | contentType: samples 10 | createdDate: 4/2/2020 3:00:56 AM 11 | description: "This sample can be used to process message traces from an Azure Function." 12 | --- 13 | 14 | 15 | # Ingesting Office 365 Message Traces to Sentinel 16 | 17 | Implementing this script as part of an Azure Function will allow you to ingest Office 365 Message traces to Log Analytics. 18 | 19 | ### Prerequisites 20 | 21 | You need to have an Azure Subscription, ability to create an Azure Function App. You need to have an account with permissions to run get-messagetrace in Office 365. 22 | Use a dedicated account with a complex pwd stored in Azure Key Vault. 23 | 24 | ### Installing 25 | 26 | 1. Create the Azure Function App with PowerShell. It works well with a consumption plan in most scenarios. The runtime stack should be PowerShell Core. 27 | 2. Create a new function that is timer based. Depending on your need, set it to run on a schedule like every 5 minutes. (For high load consider more often) 28 | 3. Paste the code from ingestmsgtrace.ps1 to the code window 29 | 4. Select Platform features, by clicking on the Function App name and click the Platform features tab at the top. Click Configuration under General Settings. 30 | 5. Provide the following values, if you want to add further protection store the pwd and key in Azure Key Vault. https://docs.microsoft.com/en-us/azure/app-service/app-service-key-vault-references 31 | - expass (Exchange password) 32 | - exuser (User account with the right to run Get-messagetrace) 33 | - workspaceId (log analytics workspace) 34 | - workspaceKey (log analytics Key) 35 | - customLogName (table name in log analytics) 36 | 6. From the platform features open Console (CMD / Powershell), run the following command to initiate the file that will keep track of the runs (customize time). out-file d:\home\timetracker.log -InputObject "2020-04-02T10:22:13.962Z" 37 | 38 | ## Running the tests 39 | 40 | Verify if the code is generating any errors when running. Verify by clicking the Function and click on Logs at the bottom of the screen. 41 | Review on the Sentinel side if the logs are being ingested. 42 | 43 | ## Contributing 44 | 45 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 46 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 47 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 48 | 49 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 50 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 51 | provided by the bot. You will only need to do this once across all repos using our CLA. 52 | 53 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 54 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 55 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 56 | -------------------------------------------------------------------------------- /Sentinel_CloudApp/MIPmap.csv: -------------------------------------------------------------------------------- 1 | Name,Type,Value 2 | None,LabelEventType,0 3 | LabelUpgraded,LabelEventType,1 4 | LabelDowngraded,LabelEventType,2 5 | LabelRemoved,LabelEventType,3 6 | LabelChangedSameOrder,LabelEventType,4 7 | None,ActionSource,0 8 | Default,ActionSource,1 9 | Auto,ActionSource,2 10 | Manual,ActionSource,3 11 | Recommended,ActionSource,4 12 | None,ActionSourceDetail,0 13 | AutoByPolicyMatch,ActionSourceDetail,1 14 | AutoByReplyOrForward,ActionSourceDetail,2 15 | AutoByInheritance,ActionSourceDetail,3 16 | AutoByDeploymentPipeline,ActionSourceDetail,4 17 | PublicAPI,ActionSourceDetail,5 18 | AutoByLibraryDefault,ActionSourceDetail,6 19 | Unknown,LocationType,0 20 | Local,LocationType,1 21 | Remote,LocationType,2 22 | Removable,LocationType,3 23 | Cloud,LocationType,4 24 | FileShare,LocationType,5 25 | Unknown,Platform,0 26 | Windows,Platform,1 27 | Mac,Platform,2 28 | iOS,Platform,3 29 | Android,Platform,4 30 | WebBrowser,Platform,5 31 | -------------------------------------------------------------------------------- /Sentinel_CloudApp/readme.md: -------------------------------------------------------------------------------- 1 | ### MIP Label report [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel_CloudApp%2fLabel%20Statistics.json) 2 | ### Endpoint sensitive with risk [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel_CloudApp%2fEndpointSensitiveInfo.json) 3 | ### Sample analytic rules [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FOfficeDev%2FO365-ActivityFeed-AzureFunction%2Fmaster%2FSentinel_CloudApp%2fAzure_Sentinel_analytics_rules.json) 4 | 5 | --- 6 | page_type: sample 7 | products: 8 | - Microsoft 365 9 | - Sentinel 10 | languages: 11 | - powershellcore 12 | extensions: 13 | contentType: samples 14 | createdDate: 04/06/2022 3:00:56 PM 15 | description: "This sample can be used to create MIP events in Sentinel." 16 | --- 17 | 18 | 19 | # Utilizing MIP events in CloudAppEvents to visualise usage 20 | 21 | ### Prerequisites 22 | 23 | - Microsoft Sentinel 24 | - Microsoft Defender for Cloud 25 | - Microsoft Information Protection 26 | - Microsoft Endpoint Protection or Endpoint DLP 27 | 28 | ### Installing 29 | Permissions needed for the app 30 | - Sentinel Workspace 31 | - CloudAppEvents 32 | 33 | **Deployment of the code to the function** 34 | * 35 | ### Creating the Watchlists 36 | Documentation for Watchlists https://docs.microsoft.com/en-us/azure/sentinel/watchlists 37 | 38 | 1. Export the MIP labels using SCC Powershell, sample Get-Label | select ImmutableId,DisplayName,LabelActions | Export-Csv c:\tmp\slabels.csv -NoTypeInformation 39 | If you happen to get hyphens in the csv header fields, remove the hyphens since the WL engine cannot process. 40 | 2. Create a new Microsoft Sentinel Watchlist call it **Sensitive**, set the ImmutableId as the index field. 41 | 3. Create a new Microsoft Sentinel Watchlist call it **MipMap**, set the Value field as the index field, import the mipmap.csv file in this repo. (File to translate MIP operations) 42 | 4. Create a new Microsoft Sentinel Watchlist call it UserAccounts, Import your account list, **for reporting to work well you need to include, Userprincipalname,Department,FullName,Title,Country (The more detail you add the cooler you can make the report dashboard or any alerts), there is no normalization of the header so please capitalize the first letter as above or change the template.** 43 | - The Indexing field should be the UserPrincipalName, we use it as a key to enrich the items 44 | - Use the Microsoft Sentinel Large Watchlist to upload the list of users it scales well to 100's of thousands of users. 45 | 46 | ### To manually deploy the workbooks 47 | Deploy the workbook Sensitivitylabels.json in this repo by simply copying the code across to a new Azure Workbook. 48 | 1. [Open] Workbooks from the Sentinel Workspace where you intend to install the workbooks / portal.azure.com 49 | 2. [Click] "Add workbook" 50 | 3. When the New Workbook window open select edit, then select the Advanced Editor ([click] the icon ) 51 | 4. Copy the text of the json template you are installing from this repository [paste] over any json that exists. 52 | 5. [Click] save, select the appropriate location and name for the workbook. 53 | -------------------------------------------------------------------------------- /Sentinel_Deployment/deploymentScript.ps1: -------------------------------------------------------------------------------- 1 | param([string] $PackageUri, [string] $SubscriptionId, [string] $ResourceGroupName, [string] $FunctionAppName, [string] $FAScope, [string] $VnetScope, [string] $UAMIPrincipalId, [string] $RestrictedIPs) 2 | 3 | Set-AzContext -Subscription $SubscriptionId 4 | 5 | $tenantId = $env:TenantId 6 | $clientId = $env:ClientId 7 | $clientSecret = $env:ClientSecret 8 | $loginURL = "https://login.microsoftonline.com" 9 | $resource = "https://manage.office.com" 10 | 11 | #Get an Oauth 2 access token based on client id, secret and tenant domain 12 | $body = @{grant_type = "client_credentials"; resource = $resource; client_id = $clientId; client_secret = $clientSecret } 13 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantId/oauth2/token?api-version=1.0 -Body $body 14 | $token = $oauth.access_token | ConvertTo-SecureString -AsPlainText 15 | 16 | #Enable auditing subscriptions if needed. 17 | try { 18 | $subs = Invoke-RestMethod -Authentication Bearer -Token $token -Uri "https://manage.office.com/api/v1.0/$tenantId/activity/feed/subscriptions/list" -RetryIntervalSec 2 -MaximumRetryCount 5 19 | if (($subs | Where-Object contentType -eq DLP.All).status -ne 'enabled') { 20 | Invoke-RestMethod -Method Post -Authentication Bearer -Token $token -Uri "https://manage.office.com/api/v1.0/$tenantId/activity/feed/subscriptions/start?contentType=DLP.All" -RetryIntervalSec 2 -MaximumRetryCount 5 21 | Write-Host "Enabled DLP.ALL subscription." 22 | } 23 | else { 24 | Write-Host "DLP.ALL subscription already enabled." 25 | } 26 | } 27 | catch { Write-Error ("Error calling Office 365 Management API. " + $_.Exception) -ErrorAction Continue } 28 | 29 | #Open up public access if enabled so we can deploy code. 30 | if ($RestrictedIPs -ne '') { 31 | $resource = Get-AzResource -ResourceType Microsoft.Web/sites -ResourceGroupName $ResourceGroupName -ResourceName $FunctionAppName 32 | $resource.Properties.publicNetworkAccess = 'Enabled' 33 | $resource | Set-AzResource -Force 34 | #Give some time for access changes to kick in. 35 | Start-Sleep -Seconds 30 36 | } 37 | 38 | #Download Function App package and publish. 39 | Invoke-WebRequest -Uri $PackageUri -OutFile functionPackage.zip 40 | Publish-AzWebapp -ResourceGroupName $ResourceGroupName -Name $FunctionAppName -ArchivePath functionPackage.zip -Force 41 | 42 | #Add IP restrictions on Function App if specified. 43 | if ($RestrictedIPs -eq 'None') { 44 | $resource = Get-AzResource -ResourceType Microsoft.Web/sites -ResourceGroupName $ResourceGroupName -ResourceName $FunctionAppName 45 | $resource.Properties.publicNetworkAccess = 'Disabled' 46 | $resource | Set-AzResource -Force 47 | } 48 | elseif ($RestrictedIPs -ne '') { 49 | Add-AzWebAppAccessRestrictionRule -ResourceGroupName $ResourceGroupName -WebAppName $FunctionAppName ` 50 | -Name "Allowed" -IpAddress $RestrictedIPs.Replace(' ', ',') -Priority 100 -Action Allow 51 | 52 | Add-AzWebAppAccessRestrictionRule -ResourceGroupName $ResourceGroupName -WebAppName $FunctionAppName ` 53 | -Name "Allowed" -IpAddress $RestrictedIPs.Replace(' ', ',') -Priority 100 -Action Allow -TargetScmSite 54 | } 55 | 56 | #Cleanup the Service Principal Owner role assignments now that access is no longer needed. 57 | Remove-AzRoleAssignment -ObjectId $UAMIPrincipalId -RoleDefinitionName Owner -Scope $FAScope 58 | if ($VnetScope -ne '') { Remove-AzRoleAssignment -ObjectId $UAMIPrincipalId -RoleDefinitionName Owner -Scope $VnetScope } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage.zip -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/AzMon.Ingestion.psd1: -------------------------------------------------------------------------------- 1 | @{ 2 | 3 | # Script module or binary module file associated with this manifest. 4 | RootModule = 'AzMon.Ingestion' 5 | 6 | # Version number of this module. 7 | ModuleVersion = '0.0.2' 8 | 9 | # Supported PSEditions 10 | # CompatiblePSEditions = @() 11 | 12 | # ID used to uniquely identify this module 13 | GUID = '7d5541c1-76cb-425d-9b68-24167c1087b2' 14 | 15 | # Author of this module 16 | #Author = '' 17 | 18 | # Company or vendor of this module 19 | #CompanyName = '' 20 | 21 | # Copyright statement for this module 22 | #Copyright = '' 23 | 24 | # Description of the functionality provided by this module 25 | # Description = '' 26 | 27 | # Minimum version of the PowerShell engine required by this module 28 | # PowerShellVersion = '' 29 | 30 | # Name of the PowerShell host required by this module 31 | # PowerShellHostName = '' 32 | 33 | # Minimum version of the PowerShell host required by this module 34 | # PowerShellHostVersion = '' 35 | 36 | # Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. 37 | # DotNetFrameworkVersion = '' 38 | 39 | # Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. 40 | # CLRVersion = '' 41 | 42 | # Processor architecture (None, X86, Amd64) required by this module 43 | # ProcessorArchitecture = '' 44 | 45 | # Modules that must be imported into the global environment prior to importing this module 46 | # RequiredModules = @() 47 | 48 | # Assemblies that must be loaded prior to importing this module 49 | RequiredAssemblies = @("Azure.Monitor.Ingestion.dll", "Azure.Identity.dll") 50 | 51 | # Script files (.ps1) that are run in the caller's environment prior to importing this module. 52 | # ScriptsToProcess = @() 53 | 54 | # Type files (.ps1xml) to be loaded when importing this module 55 | # TypesToProcess = @() 56 | 57 | # Format files (.ps1xml) to be loaded when importing this module 58 | # FormatsToProcess = @() 59 | 60 | # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess 61 | # NestedModules = @() 62 | 63 | # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. 64 | FunctionsToExport = @("Get-AzMonCredential", "Get-AzMonLogsIngestionClient", "Send-AzMonData") 65 | 66 | # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. 67 | CmdletsToExport = @() 68 | 69 | # Variables to export from this module 70 | VariablesToExport = '' 71 | 72 | # Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. 73 | AliasesToExport = @() 74 | 75 | # DSC resources to export from this module 76 | # DscResourcesToExport = @() 77 | 78 | # List of all modules packaged with this module 79 | # ModuleList = @() 80 | 81 | # List of all files packaged with this module 82 | # FileList = @() 83 | 84 | # Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. 85 | PrivateData = @{ 86 | 87 | PSData = @{ 88 | 89 | # Tags applied to this module. These help with module discovery in online galleries. 90 | # Tags = @() 91 | 92 | # A URL to the license for this module. 93 | # LicenseUri = '' 94 | 95 | # A URL to the main website for this project. 96 | # ProjectUri = '' 97 | 98 | # A URL to an icon representing this module. 99 | # IconUri = '' 100 | 101 | # ReleaseNotes of this module 102 | # ReleaseNotes = '' 103 | 104 | # Prerelease string of this module 105 | # Prerelease = '' 106 | 107 | # Flag to indicate whether the module requires explicit user acceptance for install/update/save 108 | RequireLicenseAcceptance = $false 109 | 110 | # External dependent modules of this module 111 | # ExternalModuleDependencies = @() 112 | 113 | } # End of PSData hashtable 114 | 115 | } # End of PrivateData hashtable 116 | 117 | # HelpInfo URI of this module 118 | # HelpInfoURI = '' 119 | 120 | # Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. 121 | # DefaultCommandPrefix = '' 122 | 123 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/AzMon.Ingestion.psm1: -------------------------------------------------------------------------------- 1 | <# 2 | .Synopsis 3 | Gets Entra ID credential to authenticate to Azure Monitor. 4 | 5 | .Description 6 | Gets Entra ID credential to authenticate to Azure Monitor. Leverages the Azure.Identity.ManagedIdentityCredential .Net class. 7 | 8 | .Parameter UamiClientId 9 | The user-assigned managed identity client ID that will be used to authenticate to Azure Monitor. 10 | 11 | .Example 12 | # Send an array of objects to Azure Monitor Logs. 13 | Get-AzMonCredential -UamiClientId [User-Assigned Managed Identity ID]' 14 | #> 15 | function Get-AzMonCredential { 16 | param ( 17 | [string] $UamiClientId 18 | ) 19 | #Create Azure.Identity credential via User Assigned Managed Identity. 20 | return New-Object Azure.Identity.ManagedIdentityCredential($UamiClientId) 21 | } 22 | 23 | <# 24 | .Synopsis 25 | Gets new LogsIngestionClient object to be used to send data to Azure Monitor. 26 | 27 | .Description 28 | Gets new LogsIngestionClient object to be used to send data to Azure Monitor. This leverages the Azure.Monitor.Ingestion.LogsIngestionClient .Net class. 29 | 30 | .Parameter DceUri 31 | The Azure Monitor data collection endpoint URI that will be used to send the data to. 32 | 33 | .Parameter AzMonCredential 34 | The Azure.Identity.ManagedIdentityCredential object obtained via the Get-AzMonCredential cmdlet. 35 | 36 | .Example 37 | # Send an array of objects to Azure Monitor Logs. 38 | Get-AzMonCredential -UamiClientId [User-Assigned Managed Identity ID]' 39 | #> 40 | function Get-AzMonLogsIngestionClient { 41 | param ( 42 | [string] $DceUri, 43 | [object] $AzMonCredential 44 | ) 45 | return New-Object Azure.Monitor.Ingestion.LogsIngestionClient($DceUri, $AzMonCredential) 46 | } 47 | 48 | <# 49 | .Synopsis 50 | Sends mutiple events/objects to Azure Monitor Logs. 51 | 52 | .Description 53 | Leveraging the Azure Monitor Ingestion client library for .Net, this module sends an array of objects to Azure Monitor Logs via the Logs Ingestion API and a Data Collection Rule (DCR). This function includes logic to properly split the array of data into chunks that are accepted by Azure Monitor. 54 | 55 | .Parameter Data 56 | The data to be sent to Azure Monitor. This can be a PowerShell object or an array of objects. 57 | 58 | .Parameter BatchSize 59 | (Optional) The number of objects within the arrary to send to the Azure Monitor API in a single request. If not specified, the entire array will be sent in a single request. 60 | 61 | .Parameter JsonDepth 62 | (Optional) Specifies how many levels of contained objects are included in the JSON. Default is 100. 63 | 64 | .Parameter LogsIngestionClient 65 | The Azure.Monitor.Ingestion.LogsIngestionClient object obtained via the Get-AzMonLogsIngestionClient cmdlet. 66 | 67 | .Parameter TableName 68 | Name of Azure Monitor Logs table that the data will be sent to. The name needs to include the proper prefix suffix as specified in the DCR (e.g., Custom-TableName_CL). 69 | 70 | .Parameter DcrImmutableId 71 | The Azure monitor data collection rule immutable ID. 72 | 73 | .Parameter DataAlreadyGzipEncoded 74 | (Optional) Specifies if the data to be sent is already Gzip encoded (compressed). Default value is false, in which the data will be compressed via Gzip before sending to Azure Monitor. 75 | 76 | .Parameter SortBySize 77 | (Optional) If set to true, the objects in the array will be sorted from smallest to largest before sending to Azure Monitor. This can increase throughput for arrays that have objects that vary greatly in size. Default is true. 78 | 79 | .Parameter MaxRetries 80 | (Optional) Specified how many times the request will be retried in the event an error occurs during transmission. Default value is 5. 81 | 82 | .Parameter EventIdPropertyName 83 | (Optional) The property within the object that represents the unique id of the object/event. If specified, this property will be logged in the event an object is too large to send to Azure Monitor. 84 | 85 | .Example 86 | # Send an array of objects to Azure Monitor Logs. 87 | Send-AzMonData -Data $array -TableName "Custom-TableName_CL" -LogsIngestionClient $logsIngestionClient -JsonDepth 100 -dcrImmutableId $dcrImmutableId -DataAlreadyGZipEncoded $false -SortBySize $true -DelayInMilliseconds 0 -BatchSize 10000 -EventIdPropertyName 'Identifier' 88 | #> 89 | function Send-AzMonData { 90 | param ( 91 | $Data, 92 | [int] $BatchSize = 0, 93 | [string] $TableName, 94 | [int] $JsonDepth = 100, 95 | [object]$LogsIngestionClient, 96 | [string] $DcrImmutableId, 97 | [boolean] $DataAlreadyGZipEncoded = $false, 98 | [boolean] $SortBySize = $false, 99 | [int] $Delay = 0, 100 | [int] $MaxRetries = 5, 101 | [string] $EventIdPropertyName 102 | ) 103 | $skip = 0 104 | $errorCount = 0 105 | if ($BatchSize -eq 0) { $BatchSize = $Data.Count } 106 | #Sort data by size, smallest to largest to get optimal batching. 107 | if ($SortBySize -eq $true) { 108 | Write-Host "Sorting data..." 109 | $getSize = { ($_ | ConvertTo-Json -Depth $JsonDepth).Length } 110 | $Data = $Data | Sort-Object -Property $getSize 111 | } 112 | #Enter error handling loop to send data. 113 | Write-Host ("Sending " + $Data.Count + " events/objects to Azure Monitor...") 114 | do { 115 | try { 116 | do { 117 | if ($Data.Count -lt $BatchSize) { $BatchSize = $Data.Count } 118 | $batchedData = $Data | Select-Object -Skip $skip -First $BatchSize 119 | if ($batchedData.Count -eq 0) { return } 120 | #Send data to Azure Monitor 121 | if ($DataAlreadyGZipEncoded -eq $false) { $LogsIngestionClient.Upload($DcrImmutableId, $TableName, ($batchedData | ConvertTo-Json -Depth $JsonDepth -AsArray)) | Out-Null } 122 | else { $LogsIngestionClient.Upload($DcrImmutableId, $TableName, ($batchedData | ConvertTo-Json -Depth $JsonDepth -AsArray), 'gzip') | Out-Null } 123 | $skip += $BatchSize 124 | Start-Sleep -Milliseconds $Delay 125 | } until ($skip -ge $Data.Count) 126 | Write-Host "Completed sending data to Azure Monitor." 127 | return 128 | } 129 | catch { 130 | if ($_.Exception.InnerException.Message -like "*ErrorCode: ContentLengthLimitExceeded*") { 131 | if ($BatchSize -eq 1) { 132 | Write-Error ("Event ID: " + $batchedData[0].$EventIdPropertyName + " is too large to submit to Azure Monitor. JSON Length: " + ($batchedData[0] | ConvertTo-Json -Depth $JsonDepth).Length + ". $_") -ErrorAction Continue 133 | if ($skip -lt ($Data.Count - 1 )) { 134 | $skip++ 135 | } 136 | else { 137 | $errorCount = $MaxRetries + 1 138 | } 139 | } 140 | else { 141 | $BatchSize = [math]::Round($BatchSize / 2) 142 | if ($BatchSize -lt 1) { $BatchSize = 1 } 143 | Write-Host ("Data too large, reducing batch size to: $BatchSize.") 144 | } 145 | } 146 | else { 147 | Write-Error $_ -ErrorAction Continue 148 | $errorCount++ 149 | if ($errorCount -gt $MaxRetries) { Write-Error "Max number of retries reached, aborting." -ErrorAction Continue } 150 | } 151 | } 152 | } until ($errorCount -gt $MaxRetries) 153 | } 154 | 155 | Export-ModuleMember -Function Get-AzMonCredential 156 | Export-ModuleMember -Function Get-AzMonLogsIngestionClient 157 | Export-ModuleMember -Function Send-AzMonData -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Azure.Core.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Azure.Core.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Azure.Identity.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Azure.Identity.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Azure.Monitor.Ingestion.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Azure.Monitor.Ingestion.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.Bcl.AsyncInterfaces.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.Bcl.AsyncInterfaces.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.Identity.Client.Extensions.Msal.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.Identity.Client.Extensions.Msal.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.Identity.Client.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.Identity.Client.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.IdentityModel.Abstractions.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/Microsoft.IdentityModel.Abstractions.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/System.ClientModel.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/System.ClientModel.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/System.Memory.Data.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/System.Memory.Data.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/System.Security.Cryptography.ProtectedData.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/functionPackage/Modules/AzMon.Ingestion/System.Security.Cryptography.ProtectedData.dll -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/QueueDLPEvents/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "name": "Timer", 5 | "type": "timerTrigger", 6 | "direction": "in", 7 | "schedule": "0 */1 * * * *" 8 | } 9 | ], 10 | "disabled": false 11 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/QueueDLPEvents/run.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | #Enumerators and object to wrap the objects 5 | $pageArray = @() 6 | $msgarray = @() 7 | 8 | #Sign in Parameters 9 | $clientID = "$env:clientID" 10 | $clientSecret = "$env:clientSecret" 11 | $loginURL = "https://login.microsoftonline.com" 12 | $tenantGUID = "$env:TenantGuid" 13 | $resource = "https://manage.office.com" 14 | 15 | #Workloads and end time default is on start 16 | $workloads = $env:contentTypes.split(",") 17 | $endTime = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 18 | 19 | Foreach ($workload in $workloads) { 20 | #Storage Account Settings 21 | if ($workload -eq "dlp.all") { $storageQueue = "$env:storageQueue" } 22 | 23 | #Load the Storage Queue 24 | $storeAuthContext = New-AzStorageContext -ConnectionString $env:AzureWebJobsStorage 25 | $myQueue = Get-AzStorageQueue -Name $storageQueue -Context $storeAuthContext 26 | $messageSize = 10 27 | if (-not ($myQueue)) { throw 'Failed to connect to Storage Queue' } 28 | 29 | $Tracker = "D:\home\$workload.log" # change to location of choice this is the root. 30 | if ((Test-Path -Path $Tracker) -eq $true) { 31 | $storedTime = Get-content $Tracker 32 | } 33 | else { 34 | Write-Error "Time tracker log file not found, creating new file and using 1 minute lookback. If this is the very first run, this error can be ignored." -ErrorAction Continue 35 | $date = (Get-date).AddMinutes(-1).ToString('yyyy-MM-ddTHH:mm:ss.fffZ') 36 | out-file d:\home\$workload.log -InputObject $date 37 | $storedTime = Get-content $Tracker 38 | } 39 | 40 | try { 41 | $adjustTime = New-TimeSpan -start $storedTime -End $endTime 42 | } 43 | catch { 44 | throw "Unable to calculate start time. Ensure valid timestamp is in [workload].log file." 45 | } 46 | 47 | #If events are longer apart than 24 hours 48 | If ($adjustTime.TotalHours -gt 24) { 49 | $hours = $adjustTime.TotalHours - 23.9 50 | $storedTime = (get-date $storedTime).AddHours($hours) 51 | } 52 | 53 | # Get an Oauth 2 access token based on client id, secret and tenant domain 54 | $body = @{grant_type = "client_credentials"; resource = $resource; client_id = $ClientID; client_secret = $ClientSecret } 55 | 56 | #oauthtoken in the header 57 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantGUID/oauth2/token?api-version=1.0 -Body $body 58 | $token = $oauth.access_token | ConvertTo-SecureString -AsPlainText 59 | 60 | #Make the request 61 | try { $rawRef = Invoke-WebRequest -Authentication Bearer -Token $token -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/content?contenttype=$workload&startTime=$Storedtime&endTime=$endTime&PublisherIdentifier=$tenantGUID" -UseBasicParsing -RetryIntervalSec 2 -MaximumRetryCount 5 } 62 | catch { throw ("Error calling Office 365 Management API. " + $_.Exception) } 63 | 64 | if (-not ($rawRef)) { throw 'Failed to retrieve the content Blob Url' } 65 | 66 | #If more than one page is returned capture and return in pageArray 67 | if ($rawRef.Headers.NextPageUri) { 68 | $pageTracker = $true 69 | $pagedReq = $rawRef.Headers.NextPageUri 70 | while ($pageTracker -ne $false) { 71 | $pageuri = "$pagedReq&PublisherIdentifier=$tenantGUID" 72 | try { $CurrentPage = Invoke-WebRequest -Authentication Bearer -Token $token -Uri $pageuri -UseBasicParsing -RetryIntervalSec 2 -MaximumRetryCount 5 } 73 | catch { throw ("Error calling Office 365 Management API. " + $_.Exception) } 74 | $pageArray += $CurrentPage 75 | if ($CurrentPage.Headers.NextPageUri) { 76 | $pageTracker = $true 77 | } 78 | Else { 79 | $pageTracker = $false 80 | } 81 | $pagedReq = $CurrentPage.Headers.NextPageUri 82 | } 83 | } 84 | 85 | $pageArray += $rawref 86 | 87 | if ($pagearray.RawContentLength -gt 3) { 88 | foreach ($page in $pageArray) { 89 | $request = $page.content | convertfrom-json 90 | $request 91 | # Setting up the paging of the Message queue adding +1 to avoid misconfiguration 92 | $runs = $request.Count / ($messageSize + 1) 93 | if (($runs -gt 0) -and ($runs -le "1") ) { $runs = 1 } 94 | $writeSize = $messageSize 95 | $i = 0 96 | while ($runs -ge 1) { 97 | 98 | if ($request.count -eq "1") { $rawmessage += $request.contenturi } 99 | Else { $rawmessage = $request[$i..$writeSize].contenturi } 100 | 101 | foreach ($msg in $rawmessage) { 102 | $msgarray += @($msg) 103 | } 104 | $message = $msgarray | convertto-json 105 | $bytes = [System.Text.Encoding]::ASCII.GetBytes($message) 106 | $messageBase64 =[Convert]::ToBase64String($bytes) 107 | $myQueue.QueueClient.SendMessage($messageBase64) 108 | 109 | $runs -= 1 110 | $i += $messageSize + 1 111 | $writeSize += $messageSize + 1 112 | 113 | Clear-Variable msgarray 114 | Clear-Variable message 115 | Clear-Variable rawMessage 116 | } 117 | } 118 | #Updating timers on success, registering the date from the latest entry returned from the API and adding 1 millisecond to avoid overlap 119 | $time = $pagearray[0].Content | convertfrom-json 120 | 121 | try { 122 | $Lastentry = (get-date ($time[$Time.contentcreated.Count - 1].contentCreated)).AddMilliseconds(1).ToString("yyyy-MM-ddTHH:mm:ss.fffZ") 123 | } 124 | catch { 125 | throw "Unable to get date from last entry." 126 | } 127 | 128 | if ($Lastentry -ge $storedTime) { out-file -FilePath $Tracker -NoNewline -InputObject $Lastentry } 129 | 130 | } 131 | 132 | Clear-Variable pagearray 133 | Clear-Variable rawref -ErrorAction Ignore 134 | Clear-Variable page -ErrorAction Ignore 135 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/StoreDLPEvents/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "name": "QueueItem", 5 | "type": "queueTrigger", 6 | "direction": "in", 7 | "queueName": "dlpqueue", 8 | "connection": "AzureWebJobsStorage" 9 | } 10 | ], 11 | "disabled": false 12 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/SyncDLPAnalyticsRules/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "name": "Timer", 5 | "schedule": "0 0 * * * *", 6 | "direction": "in", 7 | "type": "timerTrigger" 8 | } 9 | ] 10 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/SyncDLPAnalyticsRules/readme.md: -------------------------------------------------------------------------------- 1 | # TimerTrigger - PowerShell 2 | 3 | The `TimerTrigger` makes it incredibly easy to have your functions executed on a schedule. This sample demonstrates a simple use case of calling your function every 5 minutes. 4 | 5 | ## How it works 6 | 7 | For a `TimerTrigger` to work, you provide a schedule in the form of a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression)(See the link for full details). A cron expression is a string with 6 separate expressions which represent a given schedule via patterns. The pattern we use to represent every 5 minutes is `0 */5 * * * *`. This, in plain text, means: "When seconds is equal to 0, minutes is divisible by 5, for any hour, day of the month, month, and day of the week". 8 | 9 | ## Learn more 10 | 11 | Documentation 12 | -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/SyncDLPAnalyticsRules/workloads.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Alias": "SPOD", 4 | "Names": ["SharePoint", "OneDrive"] 5 | }, 6 | { 7 | "Alias": "EXOT", 8 | "Names": ["Exchange", "MicrosoftTeams"] 9 | }, 10 | { 11 | "Alias": "Endpoint", 12 | "Names": ["Endpoint"] 13 | } 14 | ] -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/SyncSensitivityLabels/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "name": "Timer", 5 | "schedule": "0 0 * * * *", 6 | "direction": "in", 7 | "type": "timerTrigger" 8 | } 9 | ] 10 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/SyncSensitivityLabels/readme.md: -------------------------------------------------------------------------------- 1 | # TimerTrigger - PowerShell 2 | 3 | The `TimerTrigger` makes it incredibly easy to have your functions executed on a schedule. This sample demonstrates a simple use case of calling your function every 5 minutes. 4 | 5 | ## How it works 6 | 7 | For a `TimerTrigger` to work, you provide a schedule in the form of a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression)(See the link for full details). A cron expression is a string with 6 separate expressions which represent a given schedule via patterns. The pattern we use to represent every 5 minutes is `0 */5 * * * *`. This, in plain text, means: "When seconds is equal to 0, minutes is divisible by 5, for any hour, day of the month, month, and day of the week". 8 | 9 | ## Learn more 10 | 11 | Documentation 12 | -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/SyncSensitivityLabels/run.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | #Sign in Parameters 5 | $clientID = "$env:clientID" 6 | $clientSecret = "$env:clientSecret" 7 | $loginURL = "https://login.microsoftonline.com" 8 | $tenantGUID = "$env:TenantGuid" 9 | $resource = "https://graph.microsoft.com" 10 | 11 | # Get an Oauth 2 access token based on client id, secret and tenant domain 12 | $body = @{grant_type = "client_credentials"; resource = $resource; client_id = $ClientID; client_secret = $ClientSecret } 13 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantGUID/oauth2/token?api-version=1.0 -Body $body 14 | $tokenG = $oauth.access_token | ConvertTo-SecureString -AsPlainText 15 | 16 | #Code to sign-in to Sentinel 17 | $context = Get-AzContext 18 | $profileR = [Microsoft.Azure.Commands.Common.Authentication.Abstractions.AzureRmProfileProvider]::Instance.Profile 19 | $profileClient = New-Object -TypeName Microsoft.Azure.Commands.ResourceManager.Common.RMProfileClient -ArgumentList ($profileR) 20 | $token = ($profileClient.AcquireAccessToken($context.Subscription.TenantId)).AccessToken | ConvertTo-SecureString -AsPlainText 21 | $headers = @{ 22 | 'Content-Type' = 'application/json' 23 | } 24 | 25 | Set-AzContext $context.Subscription.name 26 | $instance = Get-AzResource -Name $env:SentinelWorkspace -ResourceType Microsoft.OperationalInsights/workspaces 27 | $WorkspaceID = (Get-AzOperationalInsightsWorkspace -Name $instance.Name -ResourceGroupName $Instance.ResourceGroupName).CustomerID 28 | 29 | #Get the Watchlist so that we don't store duplicates 30 | $q2 = '(_GetWatchlist("SensitivityLabels") | project SearchKey)' 31 | try { $watchlist = Invoke-AzOperationalInsightsQuery -WorkspaceId $WorkspaceID -Query $q2 } 32 | catch { throw ("Error getting watchlist. " + $_.Exception) } 33 | 34 | #Fetch the labels and prepare for export 35 | try { $labels = Invoke-RestMethod -Authentication Bearer -Token $tokenG -Uri "https://graph.microsoft.com/beta/security/informationProtection/sensitivityLabels" -Method Get -ContentType "application/json" -MaximumRetryCount 5 -RetryIntervalSec 2 } 36 | catch { throw ("Error calling Graph API. " + $_.Exception) } 37 | 38 | $sLabels = $labels.value | select id, name, @{N = 'parent'; E = { $_.parent.name } } 39 | 40 | # Watchlist update 41 | $path = $instance.ResourceId 42 | $csv = $sLabels 43 | 44 | foreach ($item in $csv) { 45 | if ($item.id -notin $watchlist.results.SearchKey) { 46 | $etag = New-Guid 47 | $a = @{ 48 | 'etag' = $etag.guid 49 | 'properties' = @{itemsKeyValue = @() } 50 | } 51 | $a.properties.itemsKeyValue = $item 52 | $update = $a | convertto-json 53 | $urlupdate = "https://management.azure.com$path/providers/Microsoft.SecurityInsights/watchlists/SensitivityLabels/watchlistitems/$($etag)?api-version=2023-04-01-preview" 54 | 55 | try { Invoke-RestMethod -Method "Put" -Uri $urlupdate -Headers $headers -Authentication Bearer -Token $token -Body $update -MaximumRetryCount 5 -RetryIntervalSec 2 } 56 | catch { throw ("Error calling Azure Management API. " + $_.Exception) } 57 | } 58 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "managedDependency": { 4 | "Enabled": true 5 | }, 6 | "extensionBundle": { 7 | "id": "Microsoft.Azure.Functions.ExtensionBundle", 8 | "version": "[3.3.0, 4.0.0)" 9 | }, 10 | "functionTimeout": "00:05:00" 11 | } -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/profile.ps1: -------------------------------------------------------------------------------- 1 | # Azure Functions profile.ps1 2 | # 3 | # This profile.ps1 will get executed every "cold start" of your Function App. 4 | # "cold start" occurs when: 5 | # 6 | # * A Function App starts up for the very first time 7 | # * A Function App starts up after being de-allocated due to inactivity 8 | # 9 | # You can define helper functions, run commands, or specify environment variables 10 | # NOTE: any variables defined that are not environment variables will get reset after the first execution 11 | 12 | # Authenticate with Azure PowerShell using MSI. 13 | # Remove this if you are not planning on using MSI or Azure PowerShell. 14 | if ($env:MSI_SECRET) { 15 | Disable-AzContextAutosave -scope process | Out-Null 16 | Connect-AzAccount -Identity -AccountId ($env:UamiClientId) 17 | } 18 | 19 | # Uncomment the next line to enable legacy AzureRm alias in Azure PowerShell. 20 | # Enable-AzureRmAlias 21 | 22 | # You can also define functions or aliases that can be referenced in any of your PowerShell functions. -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/requirements.psd1: -------------------------------------------------------------------------------- 1 | # This file enables modules to be automatically managed by the Functions service. 2 | # See https://aka.ms/functionsmanageddependency for additional information. 3 | # 4 | @{ 5 | # For latest supported version, go to 'https://www.powershellgallery.com/packages/Az'. 6 | 'Az.Accounts' = '4.*' 7 | 'Az.Storage' = '8.*' 8 | 'Az.OperationalInsights' = '3.*' 9 | 'Az.Resources' = '7.*' 10 | } 11 | -------------------------------------------------------------------------------- /Sentinel_Deployment/functionPackage/version.info: -------------------------------------------------------------------------------- 1 | 1.1.2 2 | -------------------------------------------------------------------------------- /Sentinel_Deployment/images/incident.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/images/incident.png -------------------------------------------------------------------------------- /Sentinel_Deployment/images/incidentManagement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/images/incidentManagement.png -------------------------------------------------------------------------------- /Sentinel_Deployment/images/lawid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OfficeDev/O365-ActivityFeed-AzureFunction/5845e7f6ad191af76e2862e61057d43b850583e6/Sentinel_Deployment/images/lawid.png -------------------------------------------------------------------------------- /Sentinel_Deployment/modules/functionApp.bicep: -------------------------------------------------------------------------------- 1 | param FunctionAppName string 2 | param Location string 3 | param UserAssignedMiId string 4 | param HostingPlanId string 5 | param EnablePrivateNetworking bool 6 | param FunctionAppSubnetId string = '' 7 | param AppSettings array 8 | param AlwaysOn bool 9 | 10 | resource functionApp 'Microsoft.Web/sites@2024-04-01' = { 11 | name: FunctionAppName 12 | location: Location 13 | identity: { 14 | type: 'UserAssigned' 15 | userAssignedIdentities: { 16 | '${UserAssignedMiId}': {} 17 | } 18 | } 19 | kind: 'functionapp' 20 | properties: { 21 | serverFarmId: HostingPlanId 22 | keyVaultReferenceIdentity: UserAssignedMiId 23 | httpsOnly: true 24 | clientCertEnabled: true 25 | clientCertMode: 'OptionalInteractiveUser' 26 | virtualNetworkSubnetId: EnablePrivateNetworking == true ? FunctionAppSubnetId : (null) 27 | vnetContentShareEnabled: EnablePrivateNetworking == true ? true : false 28 | vnetRouteAllEnabled: EnablePrivateNetworking == true ? true : false 29 | siteConfig: { 30 | appSettings: AppSettings 31 | powerShellVersion: '7.4' 32 | minTlsVersion: '1.2' 33 | ftpsState: 'Disabled' 34 | http20Enabled: true 35 | alwaysOn: AlwaysOn 36 | publicNetworkAccess: 'Enabled' 37 | cors: { 38 | allowedOrigins: [ 39 | 'https://portal.azure.com' 40 | ] 41 | } 42 | } 43 | } 44 | } 45 | 46 | output functionAppName string = functionApp.name 47 | output functionAppId string = functionApp.id 48 | -------------------------------------------------------------------------------- /Sentinel_Deployment/modules/functionAppPE.bicep: -------------------------------------------------------------------------------- 1 | param FunctionAppName string 2 | param FunctionAppId string 3 | param PrivateEndpointSubnetId string 4 | param location string 5 | param VnetId string 6 | 7 | resource peFunctionApp 'Microsoft.Network/privateEndpoints@2022-07-01' = { 8 | name: 'pe-${FunctionAppName}' 9 | location: location 10 | properties: { 11 | subnet: { 12 | id: PrivateEndpointSubnetId 13 | } 14 | privateLinkServiceConnections: [ 15 | { 16 | name: 'pe-${FunctionAppName}' 17 | properties: { 18 | privateLinkServiceId: FunctionAppId 19 | groupIds: [ 20 | 'sites' 21 | ] 22 | } 23 | } 24 | ] 25 | } 26 | } 27 | 28 | resource privateDnsZoneFunctionApp 'Microsoft.Network/privateDnsZones@2020-06-01' = { 29 | name: 'privatelink.azurewebsites.net' 30 | location: 'global' 31 | } 32 | 33 | resource privateDnsZoneLinkFunctionApp 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { 34 | name: '${privateDnsZoneFunctionApp.name}-link' 35 | parent: privateDnsZoneFunctionApp 36 | location: 'global' 37 | properties: { 38 | registrationEnabled: false 39 | virtualNetwork: { 40 | id: VnetId 41 | } 42 | } 43 | } 44 | 45 | resource peDnsGroupFunctionApp 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2022-07-01' = { 46 | name: '${peFunctionApp.name}/dnsGroup' 47 | properties: { 48 | privateDnsZoneConfigs: [ 49 | { 50 | name: 'config1' 51 | properties: { 52 | privateDnsZoneId: privateDnsZoneFunctionApp.id 53 | } 54 | } 55 | ] 56 | } 57 | } 58 | 59 | output functionAppName string = FunctionAppName 60 | -------------------------------------------------------------------------------- /Sentinel_Deployment/modules/keyVault.bicep: -------------------------------------------------------------------------------- 1 | param kvName string 2 | param location string 3 | param skuFamily string 4 | param skuName string 5 | param principalId string 6 | param secretPermissions array 7 | param aclIpRules string = '' 8 | param aclBypass string = 'None' 9 | param aclDefaultAction string = 'AzureServices' 10 | 11 | resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { 12 | name: kvName 13 | location: location 14 | properties: { 15 | sku: { 16 | family: skuFamily 17 | name: skuName 18 | } 19 | tenantId: subscription().tenantId 20 | accessPolicies: [ 21 | { 22 | objectId: principalId 23 | permissions: { 24 | secrets: secretPermissions 25 | } 26 | tenantId: subscription().tenantId 27 | } 28 | ] 29 | networkAcls: { 30 | bypass: aclBypass 31 | defaultAction: aclDefaultAction 32 | ipRules: aclIpRules == '' ? [] : json('${'[{"value": "'}${replace(aclIpRules, ',', '"},{"value": "')}${'"}]'}') 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Sentinel_Deployment/modules/lawCustomTable.bicep: -------------------------------------------------------------------------------- 1 | param lawName string 2 | param tableName string 3 | param plan string 4 | param columns array 5 | param retention int = -1 6 | 7 | resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { 8 | name: lawName 9 | } 10 | 11 | resource table 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' = { 12 | parent: logAnalyticsWorkspace 13 | name: tableName 14 | properties: { 15 | schema: { 16 | name: tableName 17 | columns: columns 18 | } 19 | plan: plan 20 | retentionInDays: retention != -1 ? retention : null 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Sentinel_Deployment/modules/lawFunction.bicep: -------------------------------------------------------------------------------- 1 | param lawName string 2 | param functionName string 3 | param category string 4 | param displayName string 5 | param query string 6 | param functionParams string = '' 7 | param functionAlias string 8 | 9 | resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { 10 | name: lawName 11 | } 12 | 13 | resource function 'Microsoft.OperationalInsights/workspaces/savedSearches@2020-08-01' = { 14 | parent: logAnalyticsWorkspace 15 | name: functionName 16 | properties: { 17 | category: category 18 | displayName: displayName 19 | query: query 20 | functionParameters: functionParams 21 | functionAlias: functionAlias 22 | etag: '*' 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Sentinel_Deployment/modules/lawRoleAssignment.bicep: -------------------------------------------------------------------------------- 1 | param lawName string 2 | param principalId string 3 | param roleName string = 'Custom Role - Sentinel DLP Contributor' 4 | 5 | resource law 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { 6 | name: lawName 7 | } 8 | 9 | resource tablePurviewDLP 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' existing = { 10 | parent: law 11 | name: 'PurviewDLP_CL' 12 | } 13 | 14 | resource tableWatchlist 'Microsoft.OperationalInsights/workspaces/tables@2022-10-01' existing = { 15 | parent: law 16 | name: 'Watchlist' 17 | } 18 | 19 | @description('Array of actions for the roleDefinition') 20 | param actions array = [ 21 | 'Microsoft.SecurityInsights/alertRules/write' 22 | 'Microsoft.SecurityInsights/alertRules/read' 23 | 'Microsoft.OperationalInsights/workspaces/read' 24 | 'Microsoft.OperationalInsights/workspaces/query/read' 25 | 'Microsoft.OperationalInsights/workspaces/analytics/query/action' 26 | 'Microsoft.OperationalInsights/workspaces/search/action' 27 | ] 28 | 29 | @description('Array of notActions for the roleDefinition') 30 | param notActions array = [ 31 | 'Microsoft.OperationalInsights/workspaces/sharedKeys/read' 32 | ] 33 | 34 | var roleDescription = 'Provides access to query Sentinel Watchlists and alert rules. Also provides limited permissions to read workspace details and run a query in the workspace, but not to read data from any tables.' 35 | 36 | var roleDefName = guid(resourceGroup().id, string(actions), string(notActions)) 37 | 38 | resource roleDef 'Microsoft.Authorization/roleDefinitions@2022-04-01' = { 39 | name: roleDefName 40 | properties: { 41 | roleName: roleName 42 | description: roleDescription 43 | type: 'customRole' 44 | permissions: [ 45 | { 46 | actions: actions 47 | notActions: notActions 48 | } 49 | ] 50 | assignableScopes: [ 51 | resourceGroup().id 52 | ] 53 | } 54 | } 55 | 56 | resource roleAssignmentWorkspace 'Microsoft.Authorization/roleAssignments@2022-04-01' = { 57 | name: guid(law.id, roleDef.id, principalId) 58 | scope: law 59 | properties: { 60 | roleDefinitionId: roleDef.id 61 | principalId: principalId 62 | principalType: 'ServicePrincipal' 63 | } 64 | } 65 | 66 | var roleIdReader = '/providers/Microsoft.Authorization/roleDefinitions/acdd72a7-3385-48ef-bd42-f606fba81ae7' 67 | 68 | resource roleAssignmentPurviewDLP 'Microsoft.Authorization/roleAssignments@2022-04-01' = { 69 | name: guid(tablePurviewDLP.id, roleIdReader, principalId) 70 | scope: tablePurviewDLP 71 | properties: { 72 | roleDefinitionId: roleIdReader 73 | principalId: principalId 74 | principalType: 'ServicePrincipal' 75 | } 76 | } 77 | 78 | resource roleAssignmentWatchlist 'Microsoft.Authorization/roleAssignments@2022-04-01' = { 79 | name: guid(tableWatchlist.id, roleIdReader, principalId) 80 | scope: tableWatchlist 81 | properties: { 82 | roleDefinitionId: roleIdReader 83 | principalId: principalId 84 | principalType: 'ServicePrincipal' 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Sentinel_Deployment/modules/sentinelWatchlists.bicep: -------------------------------------------------------------------------------- 1 | param lawName string 2 | param policySync bool = false 3 | param labelSync bool = true 4 | param principalId string 5 | 6 | resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { 7 | name: lawName 8 | } 9 | 10 | resource watchlistPolicy 'Microsoft.SecurityInsights/watchlists@2023-02-01' = if(policySync == true) { 11 | name: 'Policy' 12 | scope: workspace 13 | properties: { 14 | displayName: 'Policy' 15 | itemsSearchKey: 'Name' 16 | provider: 'DLP' 17 | source: 'DLP' 18 | contentType: 'text/csv' 19 | numberOfLinesToSkip: 0 20 | description: 'DLP Policies' 21 | rawContent: ''' 22 | Name,Workload 23 | 40489b3c-b060-4122-af94-5dbe51996729,40489b3c-b060-4122-af94-5dbe51996729 24 | ''' 25 | } 26 | } 27 | 28 | resource watchlistSL 'Microsoft.SecurityInsights/watchlists@2023-02-01' = if(labelSync == true) { 29 | name: 'SensitivityLabels' 30 | scope: workspace 31 | properties: { 32 | displayName: 'SensitivityLabels' 33 | itemsSearchKey: 'id' 34 | provider: 'DLP' 35 | source: 'DLP' 36 | contentType: 'text/csv' 37 | numberOfLinesToSkip: 0 38 | description: 'Sensitivity Labels' 39 | rawContent: ''' 40 | id,name,parent 41 | 40489b3c-b060-4122-af94-5dbe51996729,40489b3c-b060-4122-af94-5dbe51996729,40489b3c-b060-4122-af94-5dbe51996729 42 | ''' 43 | } 44 | } 45 | 46 | var roleIdContributor = '/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c' 47 | 48 | resource roleAssignmentPolicy 'Microsoft.Authorization/roleAssignments@2022-04-01' = if(policySync == true) { 49 | name: guid(watchlistPolicy.id, roleIdContributor, principalId) 50 | scope: watchlistPolicy 51 | properties: { 52 | principalId: principalId 53 | roleDefinitionId: roleIdContributor 54 | principalType: 'ServicePrincipal' 55 | } 56 | } 57 | 58 | resource roleAssignmentSL 'Microsoft.Authorization/roleAssignments@2022-04-01' = if(labelSync == true) { 59 | name: guid(watchlistSL.id, roleIdContributor, principalId) 60 | scope: watchlistSL 61 | properties: { 62 | principalId: principalId 63 | roleDefinitionId: roleIdContributor 64 | principalType: 'ServicePrincipal' 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Sentinel_Deployment/nuget.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Sentinel_Deployment/releaseNotes.md: -------------------------------------------------------------------------------- 1 | # Release Notes 2 | ## 1.1.2 (1/4/2025) 3 | ### Changes/Fixes 4 | - Function App Code 5 | - Updated .Net libraries and PowerShell modules to latest versions. 6 | - Updated QueueDLPEvents Function to use QueueClient.SendMessage method as required by new PS module. 7 | - Deployment 8 | - Fixed issue where previous method to deploy Azure Files Share (required for Consumption and Elastic Premium plans) no longer worked and was causing depoyments to fail. 9 | - Updated Function App PowerShell to version 7.4. 10 | - Updated post deployment script to use Az module 12.3. 11 | 12 | ## 1.1.1 (11/20/2023) 13 | ### Changes/Fixes 14 | - Function App Code 15 | - Configured Function timeout for 5 min. 16 | - Updated Az.Storage to 6.*. 17 | - Suppressed output when running Disable-AzContextAutosave. 18 | - Addded error handling to sync functions to prevent duplicates. 19 | - Deployment 20 | - Set storage account to minimum TLS 1.2 and set trusted network exceptions where needed. 21 | - Reduced permissions needed on Watchlists. 22 | - Bicep Log Analytics Workspace Id reference cleanup. 23 | 24 | ## 1.1.0 (11/15/2023) 25 | ### Changes/Fixes 26 | - Function App Code 27 | - Disabled the sending of Power BI SIT information by default as the core event was not being sent. To enable this workload (Private Preview), set the "EnablePBIWorkload" Application Setting to a value of "1" on the Function App. 28 | - Updated .Net libraries to latest versions. Added .csproj file to repo so GitHub Dependabot can monitor for updates. 29 | - Optimized Azure Monitor ingestion PowerShell function to make less authentication calls. Renamed to AzMon.Ingestion. 30 | - Resolved intermittent Azure Monitor HTTP 400 error during high/concurrent loads. 31 | - Deployment 32 | - Added new configuration values to Function App and ARM parameters to make future updates more seamless. 33 | - Updated scope to create workbooks in the same resource group as the Sentinel workspace so they appear in the Sentinel workbooks interface. 34 | - Updated Function App to use 32 bit instead of 64 bit. 35 | - Removed network access rules on Key Vault as apparently [Function App does not always access Key Vault from the designated outboud IP addresses](https://learn.microsoft.com/en-us/azure/azure-functions/ip-addresses?tabs=portal#find-outbound-ip-addresses). 36 | - Updated Key Vault reference to dynamically populate the DNS suffix to make the deployment more cross-environment friendly. 37 | - Added parameter to specify GitHub content location to make testing new code easier. 38 | - Updated Azure Monitor function to account for events that don't have any SIT info and to account for potential duplicate sensitivity label entries in the Watchlist. 39 | - Added new "ShowDetections" parameter to Azure Monitor function to control if sensitive info type detection values are returned in the query/alerts. 40 | - Added custom role to reduce access needed to Sentinel workspace. 41 | - Added Private Networking (Private Endpoints) option to deployment. 42 | - Updated Readme. 43 | 44 | ## 1.0.0 (10/25/2023) 45 | ### New Features 46 | - Fully packaged into a single ARM/Bicep deployment for easy installation and setup. 47 | - Leverages new Data Collection Rule (DCR) custom tables to ingest and store DLP data. This provides fine grained security and unlocks new capabilities such as data transformations. 48 | - Provides option to hash, remove, or retain the detected sensitive information values. 49 | - Includes "PurviewDLP" Azure Monitor Function to normalize the DLP event data across all of the different workload types (Endpoint, Teams/Exchange, and SharePoint/OneDrive). 50 | - Separates DLP data into the below three separate tables to allow for all sensitive information data to be ingested (some events would exceed the max field size when trying to store everything in a single row). This also allows for more flexible queries and restricting access to the sensitive information data if desired. 51 | - PurviewDLP: Core DLP event information, including violating user, impacted files/messages, etc. 52 | - PurviewDLPSIT: Contains the sensitive information types that were detected. 53 | - PurviewDLPDetections: Contains the sensitive information type detected values (evidence). 54 | - For Endpoint DLP events, the severity of the alert/event is not currently included in the API, so by default the severity is derived from DLP policy rule name. The rule name must have a "Low", "Medium", or "High" suffix value with a space as the delimiter. For example, "DLP rule name Medium" or "DLP rule name High". 55 | - Includes 3 built-in Sentinel workbooks to provide advanced incident management and reporting: 56 | - Microsoft DLP Incident Management 57 | - Microsoft DLP Activity 58 | - Microsoft DLP Organizational Context 59 | - Includes two options for automatically deploying the built-in Sentinel analytics rules: 60 | - A single rule to create alerts and incidents across all DLP workload types. This will work for most environments where the 150 events per 5 min. limit is not being exceeded. 61 | - A rule for each Purview DLP policy and workload (DLP Policy Sync). This is to be used in scenarios where the 150 events per 5 min. limit is being exceeded or where more customization is desired based on workload. 62 | - The syncing of the sensitivity label information and analytics rules now uses modern authentication mechanisms. 63 | - Better error handling has been introduced to the code along with a more hardened configuration for the Azure components. For example, secrets are now stored in a Key Vault with restricted access from the Function App. 64 | -------------------------------------------------------------------------------- /V2Function_Investigate/EnableSubscription.ps1: -------------------------------------------------------------------------------- 1 | #Enable the Activity API Subscriptions 2 | 3 | $clientID = "YOUR CLIENT ID” 4 | $clientSecret = "YOUR CLIENT SECRET" 5 | $loginURL = "https://login.windows.net" 6 | $tenantdomain = "YOUR TENANT" 7 | $tenantGUID = "YOUR TENANT GUID" 8 | $resource = "https://manage.office.com" 9 | 10 | # Get an Oauth 2 access token based on client id, secret and tenant domain 11 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 12 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 13 | 14 | #Let's put the oauth token in the header 15 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 16 | 17 | #Start Subscriptions 18 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.AzureActiveDirectory" 19 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.Exchange" 20 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.SharePoint" 21 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.General" 22 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=DLP.All" 23 | 24 | #List the active subscriptions 25 | Invoke-WebRequest -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/list" 26 | -------------------------------------------------------------------------------- /V2Function_Investigate/QueueEvents.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | #Enumerators and object to wrap the objects 5 | $pageArray = @() 6 | $output= @() 7 | $msgarray = @() 8 | 9 | 10 | #Sign in Parameters 11 | $clientID = "YOUR CLIENT ID” 12 | $clientSecret = "YOUR CLIENT SECRET" 13 | $loginURL = "https://login.windows.net" 14 | $tenantdomain = "YOUR TENANT" 15 | $tenantGUID = "YOUR TENANT GUID" 16 | $resource = "https://manage.office.com" 17 | 18 | #Workloads and end time default is on start 19 | $workloads = @("Audit.AzureActiveDirectory","Audit.SharePoint","Audit.Exchange","Audit.General","DLP.All") 20 | $endTime = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 21 | 22 | 23 | #Storage Account Settings, this is needed to access the storage queue 24 | $storageAccountName = "STORAGE ACCOUNTNAME" 25 | $storageAccountKey = "STORAGE ACCOUNT KEY" 26 | $storageQueue = 'STORAGE QUEUE NAME' 27 | 28 | #Load the Storage Queue 29 | $storeAuthContext = New-AzStorageContext $StorageAccountName -StorageAccountKey $StorageAccountKey 30 | $myQueue = Get-AzStorageQueue -Name $storageQueue -Context $storeAuthContext 31 | $messageSize = 10 32 | 33 | 34 | Foreach ($workload in $workloads) { 35 | 36 | $Tracker = "D:\home\$workload.log" # change to location of choise this is the root. 37 | $StoredTime = Get-content $Tracker 38 | #$StoredTime = "2019-11-04T11:20:35.464Z" 39 | 40 | # Get an Oauth 2 access token based on client id, secret and tenant domain 41 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 42 | 43 | #oauthtoken in the header 44 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 45 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 46 | 47 | #Make the request 48 | $rawRef = Invoke-WebRequest -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/content?contenttype=$workload&startTime=$Storedtime&endTime=$endTime&&PublisherIdentifier=$TenantGUID" -UseBasicParsing 49 | 50 | 51 | #If more than one page is returned capture and return in pageArray 52 | 53 | if ($rawRef.Headers.NextPageUri) { 54 | 55 | 56 | $pageTracker = $true 57 | 58 | $pagedReq = $rawRef.Headers.NextPageUri 59 | 60 | 61 | 62 | while ($pageTracker -ne $false) 63 | 64 | 65 | 66 | { 67 | $pageuri = $pagedReq + "?PublisherIdentifier=" + $TenantGUID 68 | $CurrentPage = Invoke-WebRequest -Headers $headerParams -Uri $pageuri -UseBasicParsing 69 | 70 | $pageArray += $CurrentPage 71 | 72 | 73 | if ($CurrentPage.Headers.NextPageUri) 74 | 75 | { 76 | 77 | $pageTracker = $true 78 | 79 | } 80 | 81 | Else 82 | 83 | { 84 | 85 | $pageTracker = $false 86 | 87 | } 88 | 89 | $pagedReq = $CurrentPage.Headers.NextPageUri 90 | 91 | } 92 | 93 | } 94 | 95 | $pageArray += $rawref 96 | 97 | if ($pagearray.RawContentLength -gt 3) { 98 | 99 | foreach ($page in $pageArray) 100 | 101 | { 102 | 103 | $request = $page.content | convertfrom-json 104 | 105 | $runs = $request.Count/($messageSize +1) 106 | $writeSize = $messageSize 107 | $i = 0 108 | 109 | while ($runs -ge 1) { 110 | 111 | $rawmessage = $request[$i..$writeSize].contenturi 112 | 113 | foreach ($msg in $rawmessage){ 114 | $msgarray += @($msg) 115 | $message = $msgarray | convertto-json 116 | } 117 | 118 | $queueMessage = New-Object -TypeName Microsoft.Azure.Storage.Queue.CloudQueueMessage -ArgumentList "$message" 119 | $myqueue.CloudQueue.AddMessage($queuemessage) 120 | 121 | $runs -= 1 122 | $i+= $messageSize +1 123 | $writeSize += $messageSize + 1 124 | 125 | 126 | Clear-Variable msgarray 127 | Clear-Variable message 128 | Clear-Variable rawMessage 129 | } 130 | 131 | if ($runs -gt 0) { 132 | 133 | $rawMessage = $request[$i..$writeSize].contenturi 134 | foreach ($msg in $rawMessage){ 135 | $msgarray += @($msg) 136 | $message = $msgarray | convertto-json 137 | } 138 | 139 | $runs -=1 140 | 141 | $queueMessage = New-Object -TypeName Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage -ArgumentList "$message" 142 | $myqueue.CloudQueue.AddMessage($queuemessage) 143 | 144 | } 145 | 146 | Clear-Variable rawMessage 147 | Clear-Variable message 148 | Clear-Variable msgarray 149 | } 150 | $time = $pagearray[0].Content | convertfrom-json 151 | $Lastentry = $time[$Time.contentcreated.Count -1].contentCreated 152 | if ($Lastentry -ge $storedTime) {out-file -FilePath $Tracker -NoNewline -InputObject (get-date $lastentry).Addseconds(1).ToString("yyyy-MM-ddTHH:mm:ss.fffZ")} 153 | 154 | } 155 | 156 | Clear-Variable pagearray 157 | Clear-Variable rawref -ErrorAction Ignore 158 | Clear-Variable page -ErrorAction Ignore 159 | 160 | } 161 | -------------------------------------------------------------------------------- /V2Function_Investigate/StoreEvents.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($QueueItem, $TriggerMetadata) 3 | 4 | #Sign in Parameters 5 | $clientID = "YOUR CLIENT ID” 6 | $clientSecret = "YOUR CLIENT SECRET" 7 | $loginURL = "https://login.windows.net" 8 | $tenantdomain = "YOUR TENANT" 9 | $tenantGUID = "YOUR TENANT GUID" 10 | $resource = "https://manage.office.com" 11 | 12 | 13 | # Get an Oauth 2 access token based on client id, secret and tenant domain 14 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 15 | 16 | #oauthtoken in the header 17 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 18 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 19 | 20 | $queueitem.count 21 | 22 | if ($queueitem.count -eq "1") 23 | 24 | { 25 | $item = $queueitem | convertfrom-json 26 | $uri = $item + "?PublisherIdentifier=" + $TenantGUID 27 | $records = Invoke-WebRequest -UseBasicParsing -Headers $headerParams -Uri $uri 28 | Push-OutputBinding -Name outputDocument -value $records.content -clobber 29 | } 30 | 31 | elseif ($queueitem.count -gt "1") { 32 | 33 | foreach ( $content in $queueitem) 34 | { 35 | $uri = $content + "?PublisherIdentifier=" + $TenantGUID 36 | $records = Invoke-WebRequest -UseBasicParsing -Headers $headerParams -Uri $uri 37 | Push-OutputBinding -Name outputDocument -value $records.content -clobber 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /investigation/Enablesubscription.ps1: -------------------------------------------------------------------------------- 1 | #Enable the Activity API Subscriptions 2 | 3 | $clientID = "YOUR CLIENT ID” 4 | $clientSecret = "YOUR CLIENT SECRET" 5 | $loginURL = "https://login.windows.net" 6 | $tenantdomain = "YOUR TENANT" 7 | $tenantGUID = "YOUR TENANT GUID" 8 | $resource = "https://manage.office.com" 9 | 10 | # Get an Oauth 2 access token based on client id, secret and tenant domain 11 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 12 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 13 | 14 | #Let's put the oauth token in the header 15 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 16 | 17 | 18 | #Start Subscriptions 19 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.AzureActiveDirectory" 20 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.Exchange" 21 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.SharePoint" 22 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=Audit.General" 23 | Invoke-RestMethod -Method Post -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/start?contentType=DLP.All" 24 | 25 | #List the active subscriptions 26 | Invoke-WebRequest -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/list" 27 | -------------------------------------------------------------------------------- /investigation/QueueEvents.ps1: -------------------------------------------------------------------------------- 1 | #Enumerators and objects 2 | $pageArray = @() 3 | $output= @() 4 | $msgarray = @() 5 | 6 | #Sign in Parameters 7 | $clientID = "YOUR CLIENT ID” 8 | $clientSecret = "YOUR CLIENT SECRET" 9 | $loginURL = "https://login.windows.net" 10 | $tenantdomain = "YOUR TENANT" 11 | $tenantGUID = "YOUR TENANT GUID" 12 | $resource = "https://manage.office.com" 13 | 14 | #Workloads and end time default is on start, change to the workloads you need 15 | $workloads = @("Audit.AzureActiveDirectory","Audit.SharePoint","Audit.Exchange","Audit.General","DLP.All") 16 | $endTime = Get-date -format "yyyy-MM-ddTHH:mm:ss.fffZ" 17 | 18 | #Storage Account Settings, this is needed to access the storage queue 19 | $storageAccountName = "STORAGE ACCOUNTNAME" 20 | $storageAccountKey = "STORAGE ACCOUNT KEY" 21 | $storageQueue = 'STORAGE QUEUE NAME' 22 | 23 | #Load the Storage Queue 24 | $storeAuthContext = New-AzureStorageContext $storageAccountName -StorageAccountKey $storageAccountKey 25 | $myQueue = Get-AzureStorageQueue -Name $storageQueue -Context $storeAuthContext 26 | $messageSize = 10 27 | 28 | 29 | Foreach ($workload in $workloads) { 30 | 31 | $tracker = "D:\home\$workload.log" # change to location of choise this is the root. 32 | $storedTime = Get-content $Tracker 33 | 34 | # Get an Oauth 2 access token based on client id, secret and tenant domain 35 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$ClientID;client_secret=$ClientSecret} 36 | 37 | 38 | #oauthtoken in the header 39 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 40 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 41 | 42 | #Make the request 43 | $rawRef = Invoke-WebRequest -Headers $headerParams -Uri "https://manage.office.com/api/v1.0/$tenantGUID/activity/feed/subscriptions/content?contenttype=$workload&startTime=$Storedtime&endTime=$endTime&PublisherIdentifier=$TenantGUID" -UseBasicParsing 44 | 45 | 46 | #If more than one page is returned capture and return in pageArray 47 | 48 | if ($rawRef.Headers.NextPageUri) { 49 | 50 | 51 | $pageTracker = $true 52 | 53 | $pagedReq = $rawRef.Headers.NextPageUri 54 | 55 | 56 | 57 | while ($pageTracker -ne $false) 58 | 59 | 60 | 61 | { 62 | $pageuri = $pagedReq + "?PublisherIdentifier=" + $TenantGUID 63 | $currentPage = Invoke-WebRequest -Headers $headerParams -Uri $pageuri -UseBasicParsing 64 | 65 | $pageArray += $currentPage 66 | 67 | 68 | if ($currentPage.Headers.NextPageUri) 69 | 70 | { 71 | 72 | $pageTracker = $true 73 | 74 | } 75 | 76 | Else 77 | 78 | { 79 | 80 | $pageTracker = $false 81 | 82 | } 83 | 84 | $pagedReq = $currentPage.Headers.NextPageUri 85 | 86 | } 87 | 88 | } 89 | 90 | $pageArray += $rawref 91 | 92 | if ($pageArray.RawContentLength -gt 3) { 93 | 94 | foreach ($page in $pageArray) 95 | 96 | { 97 | 98 | $request = $page.content | convertfrom-json 99 | 100 | $runs = $request.Count/($messageSize +1) 101 | $writeSize = $messageSize 102 | $i = 0 103 | 104 | while ($runs -ge 1) { 105 | 106 | $rawmessage = $request[$i..$writeSize].contenturi 107 | 108 | foreach ($msg in $rawmessage){ 109 | $msgarray += @($msg) 110 | $message = $msgarray | convertto-json 111 | } 112 | 113 | $queueMessage = New-Object -TypeName Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage -ArgumentList "$message" 114 | $myQueue.CloudQueue.AddMessage($queuemessage) 115 | 116 | $runs -= 1 117 | $i+= $messageSize +1 118 | $writeSize += $messageSize + 1 119 | 120 | 121 | Clear-Variable msgarray 122 | Clear-Variable message 123 | Clear-Variable rawMessage 124 | } 125 | 126 | if ($runs -gt 0) { 127 | 128 | $rawMessage = $request[$i..$writeSize].contenturi 129 | foreach ($msg in $rawMessage){ 130 | $msgarray += @($msg) 131 | $message = $msgarray | convertto-json 132 | } 133 | $runs -=1 134 | $queueMessage = New-Object -TypeName Microsoft.WindowsAzure.Storage.Queue.CloudQueueMessage -ArgumentList "$message" 135 | $myQueue.CloudQueue.AddMessage($queueMessage) 136 | 137 | } 138 | 139 | 140 | Clear-Variable rawMessage 141 | Clear-Variable message 142 | Clear-Variable msgarray 143 | } 144 | $time = $pagearray[0].Content | convertfrom-json 145 | $Lastentry = $time[$Time.contentcreated.Count -1].contentCreated 146 | if ($Lastentry -ge $storedTime) {out-file -FilePath $Tracker -NoNewline -InputObject (get-date $lastentry).Addseconds(1).ToString("yyyy-MM-ddTHH:mm:ss.fffZ")} 147 | 148 | } 149 | 150 | 151 | Clear-Variable pagearray 152 | Clear-Variable rawref -ErrorAction Ignore 153 | Clear-Variable page -ErrorAction Ignore 154 | 155 | } 156 | -------------------------------------------------------------------------------- /investigation/StoreEvents.ps1: -------------------------------------------------------------------------------- 1 | #Input from the Message queue on trigger 2 | $rawRequest = Get-Content $triggerInput 3 | 4 | #Sign in Parameters 5 | $clientID = "YOUR CLIENT ID” 6 | $clientSecret = "YOUR CLIENT SECRET" 7 | $loginURL = "https://login.windows.net" 8 | $tenantdomain = "YOUR TENANT" 9 | $tenantGUID = "YOUR TENANT GUID" 10 | $resource = "https://manage.office.com" 11 | 12 | # Get an Oauth 2 access token based on client id, secret and tenant domain 13 | $body = @{grant_type="client_credentials";resource=$resource;client_id=$clientID;client_secret=$clientSecret} 14 | 15 | #oauthtoken in the header 16 | $oauth = Invoke-RestMethod -Method Post -Uri $loginURL/$tenantdomain/oauth2/token?api-version=1.0 -Body $body 17 | $headerParams = @{'Authorization'="$($oauth.token_type) $($oauth.access_token)"} 18 | 19 | $request = $rawRequest | ConvertFrom-Json 20 | 21 | foreach ( $content in $request) 22 | { 23 | $uri = $content + "?PublisherIdentifier=" + $tenantGUID 24 | Invoke-WebRequest -UseBasicParsing -Headers $headerParams -Uri $uri -PassThru -OutFile $outputdocument 25 | } 26 | --------------------------------------------------------------------------------