├── .gitattributes ├── .gitignore ├── Dockerfile ├── README.md ├── Src ├── Contoso.UnitTests │ ├── CodeAnalysis.ruleset │ ├── Contoso.UnitTests.csproj │ └── Services │ │ ├── CosmosDBServiceTests.cs │ │ └── SampleServiceTests.cs ├── Contoso.sln ├── Contoso │ ├── CodeAnalysis.ruleset │ ├── Contoso.csproj │ ├── Controllers │ │ ├── ISumComputationAPI.cs │ │ └── SampleController.cs │ ├── Models │ │ └── ComputedSum.cs │ ├── Observability │ │ ├── Histogram.cs │ │ ├── MetricsService.cs │ │ ├── PrometheusSerilogSink.cs │ │ └── TelemetryInitializer.cs │ ├── Program.cs │ ├── Properties │ │ └── launchSettings.json │ ├── README.md │ ├── Services │ │ ├── AzureService.cs │ │ ├── CosmosDBService.cs │ │ ├── ICosmosDBService.cs │ │ ├── ISampleService.cs │ │ ├── SampleService.cs │ │ └── SumComputationException.cs │ ├── Startup.cs │ ├── appsettings.Development.json │ └── appsettings.json └── stylecop.json ├── azure-pipelines.yml ├── charts └── contoso │ ├── Chart.yaml │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── configmap.yaml │ ├── deployment.yaml │ ├── ingress.yaml │ ├── secrets.yaml │ ├── service.yaml │ └── tests │ │ └── test-connection.yaml │ └── values.yaml ├── docs ├── images │ ├── Architecture.png │ ├── Build pipeline.png │ ├── Data flows.png │ ├── IaC layers.png │ ├── Jobs.png │ ├── Live Metrics.png │ ├── Metrics.png │ ├── PAT_Userprofile.png │ ├── Prometheus custom.png │ ├── Prometheus.png │ └── Structured logging AI.png ├── installation.md └── walkthrough.md └── infrastructure ├── ci-cd-pipeline.yml ├── terraform-app ├── backend.tf ├── main.tf ├── outputs.tf ├── providers.tf └── variables.tf ├── terraform-destroy ├── backend.tf └── providers.tf ├── terraform-shared ├── acr │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── aks │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── app-insights │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── backend.tf ├── cosmosdb │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── devops-agent │ ├── devops_agent_init.sh │ ├── install_software.sh │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── main.tf ├── outputs.tf ├── providers.tf ├── variables.tf └── vnet │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── terraform-template.yml └── terraform ├── backend.tf ├── main.tf ├── outputs.tf ├── providers.tf └── variables.tf /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.tfstate* 2 | .terraform* 3 | .vscode/ 4 | 5 | ## Ignore Visual Studio temporary files, build results, and 6 | ## files generated by popular Visual Studio add-ons. 7 | ## 8 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 9 | 10 | # User-specific files 11 | *.rsuser 12 | *.suo 13 | *.user 14 | *.userosscache 15 | *.sln.docstates 16 | 17 | # User-specific files (MonoDevelop/Xamarin Studio) 18 | *.userprefs 19 | 20 | # Mono auto generated files 21 | mono_crash.* 22 | 23 | # Build results 24 | [Dd]ebug/ 25 | [Dd]ebugPublic/ 26 | [Rr]elease/ 27 | [Rr]eleases/ 28 | x64/ 29 | x86/ 30 | [Ww][Ii][Nn]32/ 31 | [Aa][Rr][Mm]/ 32 | [Aa][Rr][Mm]64/ 33 | bld/ 34 | [Bb]in/ 35 | [Oo]bj/ 36 | [Ll]og/ 37 | [Ll]ogs/ 38 | 39 | # Visual Studio 2015/2017 cache/options directory 40 | .vs/ 41 | # Uncomment if you have tasks that create the project's static files in wwwroot 42 | #wwwroot/ 43 | 44 | # Visual Studio 2017 auto generated files 45 | Generated\ Files/ 46 | 47 | # MSTest test Results 48 | [Tt]est[Rr]esult*/ 49 | [Bb]uild[Ll]og.* 50 | 51 | # NUnit 52 | *.VisualState.xml 53 | TestResult.xml 54 | nunit-*.xml 55 | 56 | # Build Results of an ATL Project 57 | [Dd]ebugPS/ 58 | [Rr]eleasePS/ 59 | dlldata.c 60 | 61 | # Benchmark Results 62 | BenchmarkDotNet.Artifacts/ 63 | 64 | # .NET Core 65 | project.lock.json 66 | project.fragment.lock.json 67 | artifacts/ 68 | 69 | # ASP.NET Scaffolding 70 | ScaffoldingReadMe.txt 71 | 72 | # StyleCop 73 | StyleCopReport.xml 74 | 75 | # Files built by Visual Studio 76 | *_i.c 77 | *_p.c 78 | *_h.h 79 | *.ilk 80 | *.meta 81 | *.obj 82 | *.iobj 83 | *.pch 84 | *.pdb 85 | *.ipdb 86 | *.pgc 87 | *.pgd 88 | *.rsp 89 | *.sbr 90 | *.tlb 91 | *.tli 92 | *.tlh 93 | *.tmp 94 | *.tmp_proj 95 | *_wpftmp.csproj 96 | *.log 97 | *.vspscc 98 | *.vssscc 99 | .builds 100 | *.pidb 101 | *.svclog 102 | *.scc 103 | 104 | # Chutzpah Test files 105 | _Chutzpah* 106 | 107 | # Visual C++ cache files 108 | ipch/ 109 | *.aps 110 | *.ncb 111 | *.opendb 112 | *.opensdf 113 | *.sdf 114 | *.cachefile 115 | *.VC.db 116 | *.VC.VC.opendb 117 | 118 | # Visual Studio profiler 119 | *.psess 120 | *.vsp 121 | *.vspx 122 | *.sap 123 | 124 | # Visual Studio Trace Files 125 | *.e2e 126 | 127 | # TFS 2012 Local Workspace 128 | $tf/ 129 | 130 | # Guidance Automation Toolkit 131 | *.gpState 132 | 133 | # ReSharper is a .NET coding add-in 134 | _ReSharper*/ 135 | *.[Rr]e[Ss]harper 136 | *.DotSettings.user 137 | 138 | # TeamCity is a build add-in 139 | _TeamCity* 140 | 141 | # DotCover is a Code Coverage Tool 142 | *.dotCover 143 | 144 | # AxoCover is a Code Coverage Tool 145 | .axoCover/* 146 | !.axoCover/settings.json 147 | 148 | # Coverlet is a free, cross platform Code Coverage Tool 149 | coverage*[.json, .xml, .info] 150 | 151 | # Visual Studio code coverage results 152 | *.coverage 153 | *.coveragexml 154 | 155 | # NCrunch 156 | _NCrunch_* 157 | .*crunch*.local.xml 158 | nCrunchTemp_* 159 | 160 | # MightyMoose 161 | *.mm.* 162 | AutoTest.Net/ 163 | 164 | # Web workbench (sass) 165 | .sass-cache/ 166 | 167 | # Installshield output folder 168 | [Ee]xpress/ 169 | 170 | # DocProject is a documentation generator add-in 171 | DocProject/buildhelp/ 172 | DocProject/Help/*.HxT 173 | DocProject/Help/*.HxC 174 | DocProject/Help/*.hhc 175 | DocProject/Help/*.hhk 176 | DocProject/Help/*.hhp 177 | DocProject/Help/Html2 178 | DocProject/Help/html 179 | 180 | # Click-Once directory 181 | publish/ 182 | 183 | # Publish Web Output 184 | *.[Pp]ublish.xml 185 | *.azurePubxml 186 | # Note: Comment the next line if you want to checkin your web deploy settings, 187 | # but database connection strings (with potential passwords) will be unencrypted 188 | *.pubxml 189 | *.publishproj 190 | 191 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 192 | # checkin your Azure Web App publish settings, but sensitive information contained 193 | # in these scripts will be unencrypted 194 | PublishScripts/ 195 | 196 | # NuGet Packages 197 | *.nupkg 198 | # NuGet Symbol Packages 199 | *.snupkg 200 | # The packages folder can be ignored because of Package Restore 201 | **/[Pp]ackages/* 202 | # except build/, which is used as an MSBuild target. 203 | !**/[Pp]ackages/build/ 204 | # Uncomment if necessary however generally it will be regenerated when needed 205 | #!**/[Pp]ackages/repositories.config 206 | # NuGet v3's project.json files produces more ignorable files 207 | *.nuget.props 208 | *.nuget.targets 209 | 210 | # Microsoft Azure Build Output 211 | csx/ 212 | *.build.csdef 213 | 214 | # Microsoft Azure Emulator 215 | ecf/ 216 | rcf/ 217 | 218 | # Windows Store app package directories and files 219 | AppPackages/ 220 | BundleArtifacts/ 221 | Package.StoreAssociation.xml 222 | _pkginfo.txt 223 | *.appx 224 | *.appxbundle 225 | *.appxupload 226 | 227 | # Visual Studio cache files 228 | # files ending in .cache can be ignored 229 | *.[Cc]ache 230 | # but keep track of directories ending in .cache 231 | !?*.[Cc]ache/ 232 | 233 | # Others 234 | ClientBin/ 235 | ~$* 236 | *~ 237 | *.dbmdl 238 | *.dbproj.schemaview 239 | *.jfm 240 | *.pfx 241 | *.publishsettings 242 | orleans.codegen.cs 243 | 244 | # Including strong name files can present a security risk 245 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 246 | #*.snk 247 | 248 | # Since there are multiple workflows, uncomment next line to ignore bower_components 249 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 250 | #bower_components/ 251 | 252 | # RIA/Silverlight projects 253 | Generated_Code/ 254 | 255 | # Backup & report files from converting an old project file 256 | # to a newer Visual Studio version. Backup files are not needed, 257 | # because we have git ;-) 258 | _UpgradeReport_Files/ 259 | Backup*/ 260 | UpgradeLog*.XML 261 | UpgradeLog*.htm 262 | ServiceFabricBackup/ 263 | *.rptproj.bak 264 | 265 | # SQL Server files 266 | *.mdf 267 | *.ldf 268 | *.ndf 269 | 270 | # Business Intelligence projects 271 | *.rdl.data 272 | *.bim.layout 273 | *.bim_*.settings 274 | *.rptproj.rsuser 275 | *- [Bb]ackup.rdl 276 | *- [Bb]ackup ([0-9]).rdl 277 | *- [Bb]ackup ([0-9][0-9]).rdl 278 | 279 | # Microsoft Fakes 280 | FakesAssemblies/ 281 | 282 | # GhostDoc plugin setting file 283 | *.GhostDoc.xml 284 | 285 | # Node.js Tools for Visual Studio 286 | .ntvs_analysis.dat 287 | node_modules/ 288 | 289 | # Visual Studio 6 build log 290 | *.plg 291 | 292 | # Visual Studio 6 workspace options file 293 | *.opt 294 | 295 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 296 | *.vbw 297 | 298 | # Visual Studio LightSwitch build output 299 | **/*.HTMLClient/GeneratedArtifacts 300 | **/*.DesktopClient/GeneratedArtifacts 301 | **/*.DesktopClient/ModelManifest.xml 302 | **/*.Server/GeneratedArtifacts 303 | **/*.Server/ModelManifest.xml 304 | _Pvt_Extensions 305 | 306 | # Paket dependency manager 307 | .paket/paket.exe 308 | paket-files/ 309 | 310 | # FAKE - F# Make 311 | .fake/ 312 | 313 | # CodeRush personal settings 314 | .cr/personal 315 | 316 | # Python Tools for Visual Studio (PTVS) 317 | __pycache__/ 318 | *.pyc 319 | 320 | # Cake - Uncomment if you are using it 321 | # tools/** 322 | # !tools/packages.config 323 | 324 | # Tabs Studio 325 | *.tss 326 | 327 | # Telerik's JustMock configuration file 328 | *.jmconfig 329 | 330 | # BizTalk build output 331 | *.btp.cs 332 | *.btm.cs 333 | *.odx.cs 334 | *.xsd.cs 335 | 336 | # OpenCover UI analysis results 337 | OpenCover/ 338 | 339 | # Azure Stream Analytics local run output 340 | ASALocalRun/ 341 | 342 | # MSBuild Binary and Structured Log 343 | *.binlog 344 | 345 | # NVidia Nsight GPU debugger configuration file 346 | *.nvuser 347 | 348 | # MFractors (Xamarin productivity tool) working folder 349 | .mfractor/ 350 | 351 | # Local History for Visual Studio 352 | .localhistory/ 353 | 354 | # BeatPulse healthcheck temp database 355 | healthchecksdb 356 | 357 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 358 | MigrationBackup/ 359 | 360 | # Ionide (cross platform F# VS Code tools) working folder 361 | .ionide/ 362 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging. 2 | 3 | ARG ASPNET_VERSION=3.1 4 | 5 | FROM mcr.microsoft.com/dotnet/core/sdk:$ASPNET_VERSION AS build 6 | 7 | ARG VersionPrefix=0.0.0 8 | 9 | WORKDIR /app 10 | 11 | # copy csproj and restore as distinct layers 12 | COPY Src/*.sln . 13 | COPY Src/Contoso/*.csproj ./Contoso/ 14 | COPY Src/Contoso.UnitTests/*.csproj ./Contoso.UnitTests/ 15 | RUN dotnet restore 16 | 17 | # copy everything else and build app 18 | COPY Src/. . 19 | WORKDIR /app/Contoso 20 | RUN dotnet build -c Release /p:VersionPrefix=${VersionPrefix} /p:TreatWarningsAsErrors=true -warnaserror 21 | 22 | 23 | FROM build AS testrunner 24 | WORKDIR /app/Contoso.UnitTests 25 | ENTRYPOINT ["dotnet", "test", "--logger:trx", "--collect:XPlat Code Coverage"] 26 | 27 | 28 | FROM build AS test 29 | WORKDIR /app/Contoso.UnitTests 30 | RUN dotnet test --collect:"XPlat Code Coverage" 31 | 32 | FROM build AS publish 33 | WORKDIR /app/Contoso 34 | RUN dotnet publish -c Release -o /out 35 | 36 | 37 | FROM mcr.microsoft.com/dotnet/core/aspnet:$ASPNET_VERSION AS runtime 38 | 39 | WORKDIR /app 40 | COPY --from=publish /out . 41 | ENTRYPOINT ["dotnet", "Contoso.dll"] 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Terraform ASP.NET Core Kubernetes DevOps template 2 | 3 | This sample demonstrates how to build and deploy ASP.NET Core microservice applications using a CI/CD pipeline that includes IaC deployment jobs with Terraform. The pipeline is designed for fully automated end-to-end integration testing with 4 | high deployment speed and high branch concurrency. 5 | 6 | ![Jobs.png](docs/images/Jobs.png) 7 | *Build that completed in 8 minutes with parallel jobs, building and deploying application and transient cloud infrastructure in 4 minutes, running integration/load tests for 2 minutes, and producing unit test, coverage and integration test reports.* 8 | 9 | This sample is used by the Microsoft [Commercial Software Engineering](https://microsoft.github.io/code-with-engineering-playbook/CSE.html) teams to bootstrap agile DevOps projects. It enables entire teams of developers to submit multiple Pull Requests (PRs) 10 | per day, maintaining the integrity of the master branch by ensuring full deployment and integration tests are performed 11 | before code is merged. It has for example been used to build the [K2Bridge](https://github.com/microsoft/K2Bridge) open-source project. 12 | 13 | - ASP.NET Core modules and custom code for application observability, including exposing rich metrics to Application Insights or third-party tools. 14 | - Managing the entire application lifecycle including infrastructure deployment, application build and installation, and automated testing with a single multi-job YAML pipeline. 15 | - Building a very fast CI/CD pipeline with parallel jobs and self-hosted agents. 16 | - Effective, declarative integration testing using Taurus/JMeter running on self-hosted agents in the same VNET as your Kubernetes cluster, thus having "line of sight" into the cluster also when no external ingress is provided. 17 | 18 | Sample integration test testing the microservice endpoint summing numbers from 1 to 100, running 1 rps for 20 seconds, 19 | and verifying the response: 20 | 21 | ```yaml 22 | - task: AlexandreGattiker.jmeter-tasks.custom-taurus-runner-task.TaurusRunner@0 23 | displayName: 'Run Taurus' 24 | inputs: 25 | reportName: 'Test Sum Computation Endpoint' 26 | taurusConfig: | 27 | execution: 28 | - scenario: 29 | requests: 30 | - url: $(SERVICE_URL)/sample/sumNumbersUpTo?value=100 31 | assert: 32 | - contains: 33 | - 5050 34 | subject: body 35 | hold-for: 20s 36 | throughput: 1 37 | reporting: 38 | - module: junit-xml 39 | filename: taurus-output/TEST-Taurus.xml 40 | ``` 41 | 42 | # Walkthrough 43 | 44 | [Walkthrough](docs/walkthrough.md) 45 | 46 | # Installation 47 | 48 | [Installation](docs/installation.md) 49 | -------------------------------------------------------------------------------- /Src/Contoso.UnitTests/CodeAnalysis.ruleset: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Src/Contoso.UnitTests/Contoso.UnitTests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 8.0 6 | enable 7 | enable 8 | CodeAnalysis.ruleset 9 | true 10 | 11 | 12 | 13 | 14 | all 15 | runtime; build; native; contentfiles; analyzers; buildtransitive 16 | 17 | 18 | 19 | all 20 | runtime; build; native; contentfiles; analyzers; buildtransitive 21 | 22 | 23 | runtime; build; native; contentfiles; analyzers; buildtransitive 24 | all 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /Src/Contoso.UnitTests/Services/CosmosDBServiceTests.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso.UnitTests 6 | { 7 | using Contoso; 8 | using Xunit; 9 | 10 | public class CosmosDBServiceTests 11 | { 12 | [Fact] 13 | public void ParseCosmosDBContainerResourceId() 14 | { 15 | var cosmosDBContainerResourceId = "/subscriptions/a4ed7b9a-0000-49b4-a6ee-fd07ff6e296d/resourceGroups/aspnetmplt/providers/Microsoft.DocumentDB/databaseAccounts/cosmos-aspnetmplt-cd/apis/sql/databases/ComputedSums/containers/ComputedSumsCtr"; 16 | 17 | CosmosDBService.ParseCosmosDBContainerResourceId( 18 | cosmosDBContainerResourceId, 19 | out string cosmosDBResourceId, 20 | out string cosmosDBDatabase, 21 | out string cosmosDBContainer); 22 | Assert.Equal("/subscriptions/a4ed7b9a-0000-49b4-a6ee-fd07ff6e296d/resourceGroups/aspnetmplt/providers/Microsoft.DocumentDB/databaseAccounts/cosmos-aspnetmplt-cd", cosmosDBResourceId); 23 | Assert.Equal("ComputedSums", cosmosDBDatabase); 24 | Assert.Equal("ComputedSumsCtr", cosmosDBContainer); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Src/Contoso.UnitTests/Services/SampleServiceTests.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso.UnitTests 6 | { 7 | using System.Threading.Tasks; 8 | using Contoso; 9 | using Microsoft.Extensions.Logging; 10 | using Moq; 11 | using Xunit; 12 | 13 | public class SampleServiceTests 14 | { 15 | [Fact] 16 | public async void AddTwoNumbers() 17 | { 18 | var controller = new Mock(); 19 | 20 | controller.Setup(foo => foo.SumNumbersUpTo(2)).Returns(Task.FromResult(3)); 21 | 22 | var service = new SampleService( 23 | controller.Object, 24 | new Mock>().Object, 25 | new Mock(null).Object, 26 | new Mock().Object); 27 | var returnedString = await service.SumNumbersUpToAsync(3); 28 | Assert.Equal(6, returnedString); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Src/Contoso.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Version 16 3 | VisualStudioVersion = 16.0.29020.237 4 | MinimumVisualStudioVersion = 10.0.40219.1 5 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Contoso", "Contoso\Contoso.csproj", "{F1641EEA-CA25-4FF3-9786-F823202EF7B1}" 6 | EndProject 7 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{44B06D40-F904-4106-ABAC-106726BBDA8C}" 8 | EndProject 9 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Contoso.UnitTests", "Contoso.UnitTests\Contoso.UnitTests.csproj", "{B219CDAD-7641-4ECB-8B96-EB47E25DE5AC}" 10 | EndProject 11 | Global 12 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 13 | Debug|Any CPU = Debug|Any CPU 14 | Release|Any CPU = Release|Any CPU 15 | EndGlobalSection 16 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 17 | {F1641EEA-CA25-4FF3-9786-F823202EF7B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 18 | {F1641EEA-CA25-4FF3-9786-F823202EF7B1}.Debug|Any CPU.Build.0 = Debug|Any CPU 19 | {F1641EEA-CA25-4FF3-9786-F823202EF7B1}.Release|Any CPU.ActiveCfg = Release|Any CPU 20 | {F1641EEA-CA25-4FF3-9786-F823202EF7B1}.Release|Any CPU.Build.0 = Release|Any CPU 21 | {B219CDAD-7641-4ECB-8B96-EB47E25DE5AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 22 | {B219CDAD-7641-4ECB-8B96-EB47E25DE5AC}.Debug|Any CPU.Build.0 = Debug|Any CPU 23 | {B219CDAD-7641-4ECB-8B96-EB47E25DE5AC}.Release|Any CPU.ActiveCfg = Release|Any CPU 24 | {B219CDAD-7641-4ECB-8B96-EB47E25DE5AC}.Release|Any CPU.Build.0 = Release|Any CPU 25 | EndGlobalSection 26 | GlobalSection(SolutionProperties) = preSolution 27 | HideSolutionNode = FALSE 28 | EndGlobalSection 29 | GlobalSection(ExtensibilityGlobals) = postSolution 30 | SolutionGuid = {E55E11EA-AB0F-4280-943D-F6F1EC519DFB} 31 | EndGlobalSection 32 | EndGlobal 33 | -------------------------------------------------------------------------------- /Src/Contoso/CodeAnalysis.ruleset: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Src/Contoso/Contoso.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 8.0 6 | enable 7 | enable 8 | CodeAnalysis.ruleset 9 | true 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | all 19 | runtime; build; native; contentfiles; analyzers; buildtransitive 20 | 21 | 22 | 23 | 24 | all 25 | runtime; build; native; contentfiles; analyzers; buildtransitive 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /Src/Contoso/Controllers/ISumComputationAPI.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System.Threading.Tasks; 8 | using Refit; 9 | 10 | /// 11 | /// API providing number summation services. 12 | /// 13 | public interface ISumComputationAPI 14 | { 15 | /// 16 | /// Add two numbers and return their sum. 17 | /// 18 | /// Number to add values up to. 19 | /// Sum of integer numbers from 0 to value. 20 | [Get("/sample/sumNumbersUpTo")] 21 | Task SumNumbersUpTo(int value); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Src/Contoso/Controllers/SampleController.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System.Threading.Tasks; 8 | using Microsoft.AspNetCore.Mvc; 9 | using Microsoft.Extensions.Logging; 10 | 11 | /// 12 | /// Sample controller. 13 | /// 14 | [ApiController] 15 | [Route("[controller]/[action]")] 16 | public class SampleController : ControllerBase, ISumComputationAPI 17 | { 18 | /// 19 | /// Initializes a new instance of the class. 20 | /// 21 | /// Service to add numbers. 22 | /// Logger. 23 | public SampleController(ISampleService sampleService, ILogger logger) 24 | { 25 | this.Logger = logger; 26 | this.SampleService = sampleService; 27 | } 28 | 29 | private ISampleService SampleService { get; set; } 30 | 31 | private ILogger Logger { get; set; } 32 | 33 | /// 34 | /// Add two numbers and return their sum. 35 | /// 36 | /// Number to add values up to. 37 | /// Sum of integer numbers from 0 to value. 38 | [HttpGet] 39 | public Task SumNumbersUpTo(int value) 40 | { 41 | var response = this.SampleService.SumNumbersUpToAsync(value); 42 | 43 | return response; 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Src/Contoso/Models/ComputedSum.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System; 8 | using System.Diagnostics.CodeAnalysis; 9 | using System.Globalization; 10 | using System.Threading.Tasks; 11 | using Microsoft.Azure.Cosmos; 12 | using Microsoft.Azure.Cosmos.Fluent; 13 | using Microsoft.Extensions.Configuration; 14 | using Microsoft.Extensions.Logging; 15 | 16 | /// 17 | /// Computed sum of numbers. 18 | /// 19 | public class ComputedSum 20 | { 21 | /// 22 | /// Gets or sets the value up to which the sum is computed. 23 | /// 24 | [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Format required by Cosmos DB.")] 25 | public string? id { get; set; } 26 | 27 | /// 28 | /// Gets or sets the sum of numbers from 0 to Id. 29 | /// 30 | public long? Sum { get; set; } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Src/Contoso/Observability/Histogram.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using Microsoft.ApplicationInsights; 8 | using Prometheus; 9 | 10 | /// 11 | /// Encapsulation for Metric Histogram. 12 | /// 13 | public class Histogram 14 | { 15 | private readonly IHistogram histogram; 16 | private readonly Metric? appInsightsMetric; 17 | 18 | /// 19 | /// Initializes a new instance of the class. 20 | /// 21 | /// A histogram. 22 | /// A name. 23 | /// Help text. 24 | /// The ApplicationInsights object. 25 | public Histogram(IHistogram histogram, string name, string help, Metric? appInsightsMetric) 26 | { 27 | this.histogram = histogram; 28 | this.Name = name; 29 | this.Help = help; 30 | this.appInsightsMetric = appInsightsMetric; 31 | } 32 | 33 | /// 34 | /// Gets Name. 35 | /// 36 | public string Name { get; private set; } 37 | 38 | /// 39 | /// Gets help. 40 | /// 41 | public string Help { get; private set; } 42 | 43 | /// 44 | /// Observes a single event with the given value. 45 | /// 46 | /// The Value. 47 | public void Observe(double val) 48 | { 49 | this.histogram.Observe(val); 50 | 51 | // AppInsights might not be on and the metric could be null 52 | this.appInsightsMetric?.TrackValue(val); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Src/Contoso/Observability/MetricsService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using Microsoft.ApplicationInsights; 8 | using Prometheus; 9 | 10 | /// 11 | /// Prometheus Histograms to collect query performance data. 12 | /// 13 | public class MetricsService 14 | { 15 | /// 16 | /// Initializes a new instance of the class. 17 | /// 18 | /// The ApplicationInsights telemetry client (null if AppInsights is off). 19 | public MetricsService(TelemetryClient? telemetryClient) 20 | { 21 | var name = "sum_computation_api_call_duration_s"; 22 | var help = "Duration in seconds of calls to the SumComputationAPI."; 23 | this.SumComputationAPICallDuration = new Histogram( 24 | Prometheus.Metrics.CreateHistogram(name, help, new HistogramConfiguration 25 | { 26 | Buckets = Prometheus.Histogram.ExponentialBuckets(start: 0.001, factor: 2, count: 15), 27 | }), 28 | name, 29 | help, 30 | telemetryClient?.GetMetric(name)); 31 | } 32 | 33 | /// 34 | /// Gets the metric for the duration in seconds of calls to the SumComputationAPI. 35 | /// 36 | public Histogram? SumComputationAPICallDuration { get; } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Src/Contoso/Observability/PrometheusSerilogSink.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System.IO; 8 | using Prometheus; 9 | using Serilog.Core; 10 | using Serilog.Events; 11 | 12 | /// 13 | /// Logging custom sink that increments Prometheus exception counters every time 14 | /// exceptions are logged. 15 | /// 16 | /// Exposes a Prometheus counter called "exceptions" which counts all logged 17 | /// exceptions, and a labeled "exceptions_by_type" counter that counts exceptions 18 | /// grouped by type and context, e.g.: 19 | /// exceptions_by_type{ExceptionType="JsonReaderException",SourceContext="MyNs.MyClass",ActionName="MyNs.MyController.MyAction (MyApp)"}. 20 | /// 21 | public class PrometheusSerilogSink : ILogEventSink 22 | { 23 | /// 24 | /// Internal Serilog format code to render properties without extra double quotes. 25 | /// 26 | private const string SerilogRawFormat = "l"; 27 | 28 | /// 29 | /// Prometheus counter for all logged exceptions. 30 | /// 31 | private static readonly Counter ExceptionsCounter = Prometheus.Metrics 32 | .CreateCounter("exceptions", "Exceptions logged"); 33 | 34 | /// 35 | /// Prometheus counter for all exceptions grouped by type, context and action. 36 | /// 37 | private static readonly Counter ExceptionsByTypeCounter = Prometheus.Metrics 38 | .CreateCounter("exceptions_by_type", "Exceptions, by type", new CounterConfiguration 39 | { 40 | LabelNames = new[] { "ExceptionType", "SourceContext", "ActionName" }, 41 | }); 42 | 43 | /// 44 | /// Emit event. 45 | /// 46 | /// The log event. 47 | public void Emit(LogEvent logEvent) 48 | { 49 | if (logEvent == null || logEvent.Exception == null) 50 | { 51 | return; 52 | } 53 | 54 | ExceptionsCounter.Inc(); 55 | 56 | var typeName = logEvent.Exception.GetType().Name; 57 | using var sourceContext = new StringWriter(); 58 | if (logEvent.Properties.TryGetValue("SourceContext", out LogEventPropertyValue? prop)) 59 | { 60 | prop.Render(sourceContext, SerilogRawFormat); 61 | } 62 | 63 | using var actionName = new StringWriter(); 64 | if (logEvent.Properties.TryGetValue("ActionName", out prop)) 65 | { 66 | prop.Render(actionName, SerilogRawFormat); 67 | } 68 | 69 | ExceptionsByTypeCounter.WithLabels( 70 | typeName, 71 | sourceContext.ToString(), 72 | actionName.ToString()) 73 | .Inc(); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /Src/Contoso/Observability/TelemetryInitializer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System; 8 | using Microsoft.ApplicationInsights.AspNetCore.TelemetryInitializers; 9 | using Microsoft.ApplicationInsights.Channel; 10 | using Microsoft.ApplicationInsights.DataContracts; 11 | using Microsoft.AspNetCore.Http; 12 | 13 | /// 14 | /// This class adds identifiable information to each AppInsights telemetry. 15 | /// 16 | internal class TelemetryInitializer : TelemetryInitializerBase 17 | { 18 | private const string SyntheticSourceHeaderValue = "Availability Monitoring"; 19 | 20 | private const string AppNamePropertyName = "app-name"; 21 | private readonly string telemetryAppId; 22 | private readonly string healthCheckRoute; 23 | 24 | /// 25 | /// Initializes a new instance of the class. 26 | /// 27 | /// The class. 28 | /// The id representing this application deployment. 29 | /// The route used to check if the application is alive (to marked as synthetic). 30 | public TelemetryInitializer(IHttpContextAccessor httpContextAccessor, string telemetryAppId, string healthCheckRoute) 31 | : base(httpContextAccessor) 32 | { 33 | this.telemetryAppId = telemetryAppId; 34 | this.healthCheckRoute = healthCheckRoute; 35 | } 36 | 37 | /// 38 | /// A method with the telemetry and http context where we can update what is going to be reported. 39 | /// 40 | /// The HttpContext. 41 | /// The RequestTelemetry. 42 | /// The ApplicationInsights Telemetry. 43 | protected override void OnInitializeTelemetry(HttpContext platformContext, RequestTelemetry requestTelemetry, ITelemetry telemetry) 44 | { 45 | if (telemetry is ISupportProperties itemProperties) 46 | { 47 | itemProperties.Properties[AppNamePropertyName] = this.telemetryAppId; 48 | } 49 | 50 | if (platformContext != null && string.IsNullOrEmpty(telemetry?.Context?.Operation?.SyntheticSource)) 51 | { 52 | var path = platformContext.Request.Path; 53 | 54 | if (path.StartsWithSegments(this.healthCheckRoute, StringComparison.OrdinalIgnoreCase)) 55 | { 56 | var operation = telemetry?.Context?.Operation; 57 | if (operation != null) 58 | { 59 | operation.SyntheticSource = SyntheticSourceHeaderValue; 60 | } 61 | } 62 | } 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Src/Contoso/Program.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System; 8 | using System.Diagnostics.CodeAnalysis; 9 | using System.IO; 10 | using Microsoft.AspNetCore.Builder; 11 | using Microsoft.AspNetCore.Hosting; 12 | using Microsoft.Extensions.Configuration; 13 | using Microsoft.Extensions.Hosting; 14 | using Serilog; 15 | 16 | /// 17 | /// ASP.NET Core entry point. 18 | /// 19 | [ExcludeFromCodeCoverage] 20 | public static class Program 21 | { 22 | private static readonly string? AssemblyVersion = typeof(Program).Assembly.GetName().Version?.ToString(); 23 | 24 | /// 25 | /// Gets the Configuration for the app. 26 | /// The config is stored in appsettings.json. 27 | /// It can also be found on appsettings.Development.json (in local env). 28 | /// 29 | public static IConfiguration Configuration { get; } = new ConfigurationBuilder() 30 | .SetBasePath(Directory.GetCurrentDirectory()) 31 | .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) 32 | .AddJsonFile("settings/appsettings.json", optional: true) 33 | .AddJsonFile($"appsettings.{Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Production"}.json", optional: true) 34 | .AddEnvironmentVariables() 35 | .Build(); 36 | 37 | /// 38 | /// Entry point. 39 | /// 40 | /// Program args. 41 | public static void Main(string[] args) 42 | { 43 | Log.Logger = new LoggerConfiguration() 44 | .ReadFrom.Configuration(Configuration) 45 | .Enrich.FromLogContext() 46 | .WriteTo.Sink(new PrometheusSerilogSink()) 47 | .CreateLogger(); 48 | 49 | try 50 | { 51 | Log.Information($"***** Starting Contoso {AssemblyVersion} *****"); 52 | 53 | CreateHostBuilder(args).Build().Run(); 54 | } 55 | catch (Exception ex) 56 | { 57 | Log.Fatal(ex, "Host terminated unexpectedly"); 58 | throw; 59 | } 60 | finally 61 | { 62 | Log.CloseAndFlush(); 63 | } 64 | } 65 | 66 | private static IHostBuilder CreateHostBuilder(string[] args) 67 | { 68 | return Host.CreateDefaultBuilder(args) 69 | .ConfigureWebHostDefaults(webBuilder => 70 | { 71 | webBuilder.UseStartup(); 72 | }) 73 | .UseSerilog(); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /Src/Contoso/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true 5 | }, 6 | "profiles": { 7 | "Contoso": { 8 | "commandName": "Project", 9 | "launchBrowser": true, 10 | "environmentVariables": { 11 | "ASPNETCORE_ENVIRONMENT": "Development" 12 | }, 13 | "applicationUrl": "http://localhost:5000" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /Src/Contoso/README.md: -------------------------------------------------------------------------------- 1 | Generated with: 2 | 3 | dotnet new web --framework netcoreapp3.1 --no-restore 4 | -------------------------------------------------------------------------------- /Src/Contoso/Services/AzureService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System; 8 | using Microsoft.Azure.Management.Fluent; 9 | using Microsoft.Azure.Management.ResourceManager.Fluent; 10 | using Microsoft.Extensions.Configuration; 11 | using Microsoft.Extensions.Logging; 12 | 13 | /// 14 | /// Interface to Azure management operations. 15 | /// 16 | public class AzureService 17 | { 18 | /// 19 | /// Initializes a new instance of the class. 20 | /// 21 | /// Configuration for service. 22 | /// Logger. 23 | public AzureService(IConfiguration configuration, ILogger logger) 24 | { 25 | if (configuration == null) 26 | { 27 | throw new ArgumentNullException(nameof(configuration)); 28 | } 29 | 30 | var clientId = configuration["aadClientId"]; 31 | var clientSecret = configuration["aadClientSecret"]; 32 | var tenantId = configuration["aadTenantId"]; 33 | 34 | var credentials = SdkContext.AzureCredentialsFactory 35 | .FromServicePrincipal( 36 | clientId, 37 | clientSecret, 38 | tenantId, 39 | AzureEnvironment.AzureGlobalCloud); 40 | 41 | try 42 | { 43 | this.Azure = Microsoft.Azure.Management.Fluent.Azure 44 | .Configure() 45 | .Authenticate(credentials) 46 | .WithDefaultSubscription(); 47 | } 48 | catch (Exception e) 49 | { 50 | logger.LogCritical( 51 | e, 52 | "Couldn't authenticate to Azure with client {clientId} in tenant {tenantId}", 53 | clientId, 54 | tenantId); 55 | throw; 56 | } 57 | } 58 | 59 | /// 60 | /// Gets Azure interface. 61 | /// 62 | public IAzure Azure { get; } 63 | } 64 | } -------------------------------------------------------------------------------- /Src/Contoso/Services/CosmosDBService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System; 8 | using System.Globalization; 9 | using System.Linq; 10 | using System.Threading.Tasks; 11 | using Microsoft.Azure.Cosmos; 12 | using Microsoft.Azure.Cosmos.Fluent; 13 | using Microsoft.Azure.Management.CosmosDB.Fluent; 14 | using Microsoft.Extensions.Configuration; 15 | using Microsoft.Extensions.Logging; 16 | 17 | /// 18 | /// Service wrapping Confluent.Kafka.IProducer. 19 | /// 20 | /// The methods in this class are thread-safe. 21 | /// 22 | public class CosmosDBService : ICosmosDBService 23 | { 24 | private readonly Container container; 25 | 26 | private readonly ILogger logger; 27 | 28 | /// 29 | /// Initializes a new instance of the class. 30 | /// 31 | /// Configuration. 32 | /// Azure management connection. 33 | /// Logger. 34 | public CosmosDBService(IConfiguration configuration, AzureService azure, ILogger logger) 35 | { 36 | if (configuration == null) 37 | { 38 | throw new ArgumentNullException(nameof(configuration)); 39 | } 40 | 41 | if (azure == null) 42 | { 43 | throw new ArgumentNullException(nameof(azure)); 44 | } 45 | 46 | this.logger = logger; 47 | 48 | var cosmosDBContainerResourceId = configuration["cosmosDBContainer"]; 49 | 50 | logger.LogInformation( 51 | "Initializing connection to Cosmos DB container {cosmosDBContainerResourceId}", 52 | cosmosDBContainerResourceId); 53 | 54 | ParseCosmosDBContainerResourceId( 55 | cosmosDBContainerResourceId, 56 | out string cosmosDBResourceId, 57 | out string cosmosDBDatabase, 58 | out string cosmosDBContainer); 59 | 60 | var cosmosDB = this.ConnectToCosmosDBAccount(azure, cosmosDBResourceId); 61 | this.container = this.ConnectToCosmosDBContainer(cosmosDB, cosmosDBDatabase, cosmosDBContainer); 62 | } 63 | 64 | /// 65 | /// Parse a Cosmos DB container ARM Resource ID into its component parts. 66 | /// 67 | /// Cosmos DB container ARM Resource ID. 68 | /// Cosmos DB account ARM Resource ID. 69 | /// Cosmos DB database name. 70 | /// Cosmos DB container name. 71 | public static void ParseCosmosDBContainerResourceId( 72 | string cosmosDBContainerResourceId, 73 | out string cosmosDBResourceId, 74 | out string cosmosDBDatabase, 75 | out string cosmosDBContainer) 76 | { 77 | if (cosmosDBContainerResourceId == null) 78 | { 79 | throw new ArgumentNullException(nameof(cosmosDBContainerResourceId)); 80 | } 81 | 82 | var items = cosmosDBContainerResourceId.Split('/'); 83 | cosmosDBResourceId = string.Join('/', items.Take(9)); 84 | cosmosDBDatabase = items[12]; 85 | cosmosDBContainer = items[14]; 86 | } 87 | 88 | /// 89 | /// Persist a message to Cosmos DB. 90 | /// 91 | /// Value up to which the sum is computed. 92 | /// Sum of numbers from 0 to value. 93 | /// Cosmos DB operation result. 94 | public async Task> PersistSum(long value, long sum) 95 | { 96 | var e = new ComputedSum 97 | { 98 | id = value.ToString(CultureInfo.InvariantCulture), 99 | Sum = sum, 100 | }; 101 | return await this.container.UpsertItemAsync(e); 102 | } 103 | 104 | private ICosmosDBAccount ConnectToCosmosDBAccount(AzureService azure, string cosmosDBResourceId) 105 | { 106 | try 107 | { 108 | return azure 109 | .Azure 110 | .CosmosDBAccounts 111 | .GetById(cosmosDBResourceId); 112 | } 113 | catch (Exception e) 114 | { 115 | this.logger.LogCritical( 116 | e, 117 | "Couldn't retrieve Cosmos DB account keys for rule {cosmosDBResourceId}", 118 | cosmosDBResourceId); 119 | throw; 120 | } 121 | } 122 | 123 | private Container ConnectToCosmosDBContainer(ICosmosDBAccount cosmosDB, string cosmosDBDatabase, string cosmosDBContainer) 124 | { 125 | try 126 | { 127 | var client = new CosmosClientBuilder( 128 | cosmosDB.DocumentEndpoint, 129 | cosmosDB.ListKeys().PrimaryMasterKey) 130 | .Build(); 131 | return client 132 | .GetDatabase(cosmosDBDatabase) 133 | .GetContainer(cosmosDBContainer); 134 | } 135 | catch (Exception e) 136 | { 137 | this.logger.LogCritical( 138 | e, 139 | "Couldn't retrieve Cosmos DB container {cosmosDBContainer}", 140 | cosmosDBContainer); 141 | throw; 142 | } 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /Src/Contoso/Services/ICosmosDBService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System.Threading.Tasks; 8 | using Microsoft.Azure.Cosmos; 9 | 10 | /// 11 | /// Service wrapping Confluent.Kafka.IProducer. 12 | /// 13 | /// The methods in this class are thread-safe. 14 | /// 15 | public interface ICosmosDBService 16 | { 17 | /// 18 | /// Persist a message to Cosmos DB. 19 | /// 20 | /// Value up to which the sum is computed. 21 | /// Sum of numbers from 0 to value. 22 | /// Cosmos DB operation result. 23 | public Task> PersistSum(long value, long sum); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Src/Contoso/Services/ISampleService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System.Threading.Tasks; 8 | 9 | /// 10 | /// Sample service. 11 | /// 12 | public interface ISampleService 13 | { 14 | /// 15 | /// Add two numbers and return their sum. 16 | /// 17 | /// Number to add values up to. 18 | /// Sum of integer numbers from 0 to value. 19 | Task SumNumbersUpToAsync(int value); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Src/Contoso/Services/SampleService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System.Diagnostics; 8 | using System.Threading.Tasks; 9 | using Microsoft.Extensions.Logging; 10 | 11 | /// 12 | /// Sample service. 13 | /// 14 | public class SampleService : ISampleService 15 | { 16 | private readonly ISumComputationAPI client; 17 | private readonly ILogger logger; 18 | private readonly MetricsService metrics; 19 | private readonly ICosmosDBService cosmosDB; 20 | 21 | /// 22 | /// Initializes a new instance of the class. 23 | /// 24 | /// Remote controller. 25 | /// Logger. 26 | /// Metrics service. 27 | /// Cosmos DB persistence service. 28 | public SampleService(ISumComputationAPI client, ILogger logger, MetricsService metrics, ICosmosDBService cosmosDB) 29 | { 30 | this.client = client; 31 | this.logger = logger; 32 | this.metrics = metrics; 33 | this.cosmosDB = cosmosDB; 34 | } 35 | 36 | /// 37 | /// Add two numbers and return their sum. 38 | /// 39 | /// Number to add values up to. 40 | /// Sum of integer numbers from 0 to value. 41 | public async Task SumNumbersUpToAsync(int value) 42 | { 43 | if (value < 0) 44 | { 45 | throw new SumComputationException("Can't sum numbers up to a negative value"); 46 | } 47 | 48 | // Timer to be used to report the duration of a query to. 49 | var stopwatch = new Stopwatch(); 50 | stopwatch.Start(); 51 | var result = await this.SumNumbersUpToInternalAsync(value); 52 | stopwatch.Stop(); 53 | var duration = stopwatch.Elapsed; 54 | 55 | this.logger.LogInformation("Sum of numbers from 0 to {value} was {result}, computed in {duration}s", value, result, duration); 56 | 57 | await this.cosmosDB.PersistSum(value, result); 58 | 59 | this.metrics?.SumComputationAPICallDuration?.Observe(duration.TotalSeconds); 60 | 61 | return result; 62 | } 63 | 64 | private async Task SumNumbersUpToInternalAsync(int value) 65 | { 66 | if (value <= 1) 67 | { 68 | return value; 69 | } 70 | else 71 | { 72 | var sumUpToValueMinusOne = await this.client.SumNumbersUpTo(value - 1); 73 | return value + sumUpToValueMinusOne; 74 | } 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /Src/Contoso/Services/SumComputationException.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System; 8 | 9 | /// 10 | /// Thrown when trying to sum number up to a negative number. 11 | /// 12 | public class SumComputationException : Exception 13 | { 14 | /// 15 | /// Initializes a new instance of the class. 16 | /// 17 | public SumComputationException() 18 | : base() 19 | { 20 | } 21 | 22 | /// 23 | /// Initializes a new instance of the class. 24 | /// 25 | /// The message that describes the error. 26 | public SumComputationException(string message) 27 | : base(message) 28 | { 29 | } 30 | 31 | /// 32 | /// Initializes a new instance of the class. 33 | /// 34 | /// The message that describes the error. 35 | /// The exception that is the cause of the current exception, 36 | /// or a null reference if no inner exception is specified. 37 | public SumComputationException(string message, Exception innerException) 38 | : base(message, innerException) 39 | { 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Src/Contoso/Startup.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | // See LICENSE file in the project root for full license information. 4 | 5 | namespace Contoso 6 | { 7 | using System; 8 | using System.Diagnostics.CodeAnalysis; 9 | using Microsoft.ApplicationInsights; 10 | using Microsoft.ApplicationInsights.Channel; 11 | using Microsoft.ApplicationInsights.Extensibility; 12 | using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel; 13 | using Microsoft.AspNetCore.Builder; 14 | using Microsoft.AspNetCore.Hosting; 15 | using Microsoft.AspNetCore.Http; 16 | using Microsoft.Extensions.Configuration; 17 | using Microsoft.Extensions.DependencyInjection; 18 | using Microsoft.Extensions.Hosting; 19 | using Microsoft.Extensions.Logging; 20 | using Prometheus; 21 | using Serilog; 22 | 23 | /// 24 | /// Startup ASP.NET Core class. 25 | /// 26 | [ExcludeFromCodeCoverage] 27 | public class Startup 28 | { 29 | private const string HealthCheckRoute = "/health"; 30 | 31 | /// 32 | /// Initializes a new instance of the class. 33 | /// 34 | /// The configuration for the app. 35 | public Startup(IConfiguration configuration) 36 | { 37 | this.Configuration = configuration; 38 | } 39 | 40 | /// 41 | /// Gets application configuration. 42 | /// 43 | public IConfiguration Configuration { get; } 44 | 45 | private ServerTelemetryChannel? TelemetryChannel { get; set; } 46 | 47 | /// 48 | /// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 49 | /// 50 | /// Application builder. 51 | /// Host environment. 52 | /// Application lifetime. 53 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env, IHostApplicationLifetime lt) 54 | { 55 | lt?.ApplicationStopping.Register(this.OnShutdown); 56 | 57 | if (env.IsDevelopment()) 58 | { 59 | app.UseDeveloperExceptionPage(); 60 | } 61 | 62 | // detailed request logging 63 | app.UseSerilogRequestLogging(); 64 | 65 | app.UseRouting(); 66 | 67 | app.UseHeaderPropagation(); 68 | 69 | // Expose HTTP Metrics to Prometheus: 70 | // Number of HTTP requests in progress. 71 | // Total number of received HTTP requests. 72 | // Duration of HTTP requests. 73 | app.UseHttpMetrics(); 74 | 75 | app.UseEndpoints(endpoints => 76 | { 77 | // Starts a Prometheus metrics exporter using endpoint routing. 78 | // Using The default URL: /metrics. 79 | endpoints.MapMetrics(); 80 | 81 | endpoints.MapControllers(); 82 | 83 | // Enable middleware to serve from health endpoint 84 | endpoints.MapHealthChecks(HealthCheckRoute); 85 | }); 86 | } 87 | 88 | /// 89 | /// This method gets called by the runtime. Use this method to add services to the container. 90 | /// For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940. 91 | /// 92 | /// Service collection to be configured. 93 | public void ConfigureServices(IServiceCollection services) 94 | { 95 | this.ConfigureTelemetryServices(services); 96 | 97 | services.AddSingleton( 98 | s => new AzureService( 99 | this.Configuration, 100 | s.GetService>())); 101 | 102 | services.AddSingleton( 103 | s => new CosmosDBService( 104 | this.Configuration, 105 | s.GetService(), 106 | s.GetService>())); 107 | 108 | services.AddControllers(); 109 | 110 | services.AddTransient( 111 | s => new SampleService( 112 | s.GetRequiredService(), 113 | s.GetRequiredService>(), 114 | s.GetRequiredService(), 115 | s.GetRequiredService())); 116 | 117 | services.AddHeaderPropagation(options => 118 | { 119 | options.Headers.Add("X-TraceId"); 120 | }); 121 | 122 | // use this http client factory to issue requests to the compute service 123 | services.AddHttpClient("ComputeServiceClient", c => 124 | { 125 | var computeServiceAddress = this.Configuration["computeServiceAddress"]; 126 | c.BaseAddress = new Uri(computeServiceAddress); 127 | }) 128 | .AddHeaderPropagation() 129 | .AddTypedClient(c => Refit.RestService.For(c)); 130 | 131 | // Add a health/liveness service 132 | services.AddHealthChecks(); 133 | } 134 | 135 | private void OnShutdown() 136 | { 137 | if (this.TelemetryChannel == null) 138 | { 139 | return; 140 | } 141 | 142 | Log.Information("***** Flushing telemetry *****"); 143 | 144 | this.TelemetryChannel?.Flush(); 145 | 146 | // Wait while the data is flushed 147 | System.Threading.Thread.Sleep(1000); 148 | } 149 | 150 | /// 151 | /// Configures all telemetry services - internal (like Prometheus endpoint) and external (Application Insights). 152 | /// 153 | private void ConfigureTelemetryServices(IServiceCollection services) 154 | { 155 | // using GetService since TelemetryClient won't exist if AppInsights is turned off. 156 | services.AddSingleton( 157 | s => new MetricsService(s.GetService())); 158 | 159 | // complete the config for AppInsights. 160 | this.ConfigureApplicationInsights(services); 161 | } 162 | 163 | /// 164 | /// Configures the ApplicationInsights telemetry. 165 | /// 166 | private void ConfigureApplicationInsights(IServiceCollection services) 167 | { 168 | // verify we got a valid instrumentation key, if we didn't, we just skip AppInsights 169 | // we do not log this, as at this point we still don't have a logger 170 | var hasKey = Guid.TryParse(this.Configuration["instrumentationKey"], out Guid instrumentationKey); 171 | if (hasKey) 172 | { 173 | var telemetryAppId = this.Configuration["instrumentationAppId"] ?? "App"; 174 | Log.Information("***** Configuring Application Insights telemetry with Id {identifier} *****", telemetryAppId); 175 | 176 | // This sets up ServerTelemetryChannel with StorageFolder set to a custom location. 177 | this.TelemetryChannel = new ServerTelemetryChannel() { StorageFolder = this.Configuration["appInsightsStorageFolder"] }; 178 | services.AddSingleton(typeof(ITelemetryChannel), this.TelemetryChannel); 179 | 180 | services.AddApplicationInsightsTelemetry(instrumentationKey.ToString()); 181 | 182 | services.AddSingleton(s => 183 | new TelemetryInitializer(s.GetRequiredService(), telemetryAppId, HealthCheckRoute)); 184 | } 185 | else 186 | { 187 | Log.Information("***** Not configuring Application Insights telemetry *****"); 188 | } 189 | } 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /Src/Contoso/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "computeServiceAddress": "http://localhost:5000" 3 | } 4 | -------------------------------------------------------------------------------- /Src/Contoso/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Serilog": { 3 | "Using": [ 4 | "Serilog.Sinks.Console" 5 | ], 6 | "MinimumLevel": { 7 | "Default": "Debug", 8 | "Override": { 9 | "Microsoft": "Warning", 10 | "Microsoft.AspNetCore": "Warning", 11 | "Serilog.AspNetCore.RequestLoggingMiddleware": "Warning", 12 | "System": "Warning" 13 | } 14 | }, 15 | "WriteTo": [ 16 | { 17 | "Name": "Console", 18 | "Args": { 19 | "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact" 20 | } 21 | } 22 | ], 23 | "Enrich": [ 24 | "FromLogContext", 25 | "WithCorrelationId", 26 | "WithCorrelationIdHeader" 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Src/stylecop.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", 3 | "settings": { 4 | "documentationRules": { 5 | "xmlHeader": false, 6 | "companyName": "Microsoft Corporation", 7 | "copyrightText": "Copyright (c) {companyName}. All rights reserved.\nLicensed under the {licenseName} license.\nSee {licenseFile} file in the project root for full license information.", 8 | "variables": { 9 | "licenseName": "MIT", 10 | "licenseFile": "LICENSE" 11 | } 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | branches: 3 | include: 4 | - master 5 | 6 | pr: 7 | branches: 8 | include: 9 | - master 10 | paths: 11 | exclude: 12 | - '*.md' 13 | 14 | name: $(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) 15 | 16 | variables: 17 | - group: terraform-aspnet-devops-template 18 | # should contain variables 19 | # - APP_NAME (short globally unique name, e.g. "starterterraform") 20 | # - AGENT_POOL_MANAGEMENT_TOKEN 21 | # - AKS_SP_CLIENT_SECRET 22 | # - TERRAFORM_SP_CLIENT_SECRET 23 | - name: MAJOR_MINOR_VERSION 24 | value: "0.1" # update between releases 25 | - name: SEMANTIC_VERSION 26 | value: "$(MAJOR_MINOR_VERSION).$(Build.BuildId)" 27 | - name: ACR_NAME 28 | value: $(APP_NAME)aspnetmplt 29 | - name: TERRAFORM_SP_CLIENT_ID 30 | value: df9df564-e889-4852-9bb2-aa912d990c93 31 | - name: AKS_SP_CLIENT_ID 32 | value: cd0ad856-af8e-424e-b935-ce7ad6da5c9d 33 | - name: AKS_SP_OBJECT_ID 34 | value: ddf63018-48fb-4738-a3f5-0062fb16dbb0 35 | - name: APP_SP_CLIENT_ID 36 | value: cd0ad856-af8e-424e-b935-ce7ad6da5c9d 37 | - name: APP_SP_OBJECT_ID 38 | value: ddf63018-48fb-4738-a3f5-0062fb16dbb0 39 | - name: AKS_VERSION 40 | value: 1.18.1 41 | - name: HELM_RELEASE_NAME 42 | value: contoso 43 | - name: RESOURCE_GROUP 44 | value: $(APP_NAME) 45 | - name: SUBSCRIPTION_ID 46 | value: a4ed7b9a-b128-49b4-a6ee-fd07ff6e296d 47 | - name: TENANT_ID 48 | value: 72f988bf-86f1-41af-91ab-2d7cd011db47 49 | - name: TERRAFORM_SERVICE_CONNECTION 50 | value: Terraform 51 | - name: TERRAFORM_STORAGE_ACCOUNT 52 | value: $(APP_NAME)terraform 53 | - name: AGENT_POOL_NAME 54 | value: aspnetmplt 55 | - name: JMETER_VERSION 56 | value: 5.1.1 57 | # Enable buildkit to speed up multi-stage pipeline builds (avoid running unit tests twice) 58 | - name: DOCKER_BUILDKIT 59 | value: 1 60 | # Variables for Terraform Azure storage backend provider 61 | - name: ARM_SUBSCRIPTION_ID 62 | value: $(SUBSCRIPTION_ID) 63 | - name: ARM_TENANT_ID 64 | value: $(TENANT_ID) 65 | - name: ARM_CLIENT_ID 66 | value: $(TERRAFORM_SP_CLIENT_ID) 67 | - name: ARM_CLIENT_SECRET 68 | value: $(TERRAFORM_SP_CLIENT_SECRET) 69 | 70 | pool: 71 | vmImage: ubuntu-latest 72 | 73 | 74 | jobs: 75 | - template: infrastructure/ci-cd-pipeline.yml 76 | -------------------------------------------------------------------------------- /charts/contoso/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "1.0" 3 | description: A sample app 4 | name: contoso 5 | version: 0.0.1 6 | -------------------------------------------------------------------------------- /charts/contoso/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Wait for the pods to be fully started. 2 | kubectl get pods --namespace {{ .Release.Namespace }} 3 | 4 | 2. Endpoint to connect to, is: 5 | {{- if .Values.ingress.enabled }} 6 | {{- range $host := .Values.ingress.hosts }} 7 | {{- range .paths }} 8 | http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} 9 | {{- end }} 10 | {{- end }} 11 | {{- else if contains "NodePort" .Values.service.type }} 12 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "contoso.fullname" . }}) 13 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 14 | http://$NODE_IP:$NODE_PORT 15 | {{- else if contains "LoadBalancer" .Values.service.type }} 16 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 17 | You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "contoso.fullname" . }}' 18 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "contoso.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 19 | http://$SERVICE_IP:{{ .Values.service.port }} 20 | {{- else if contains "ClusterIP" .Values.service.type }} 21 | http://127.0.0.1:8080. This is an internal address in the cluster and you should run this to access it locally: 22 | kubectl port-forward service/{{ include "contoso.fullname" .}} 8080 --namespace {{ .Release.Namespace }} 23 | {{- end }} 24 | -------------------------------------------------------------------------------- /charts/contoso/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "contoso.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "contoso.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "contoso.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | -------------------------------------------------------------------------------- /charts/contoso/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "contoso.fullname" . }}-settings 5 | labels: 6 | app.kubernetes.io/name: {{ include "contoso.name" . }} 7 | helm.sh/chart: {{ include "contoso.chart" . }} 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | data: 11 | computeServiceAddress: "{{ .Values.settings.computeServiceAddress }}" 12 | appInsightsStorageFolder: "{{ .Values.settings.appInsightsStorageFolder }}" 13 | aadClientId: "{{ .Values.settings.aadClientId }}" 14 | aadTenantId: "{{ .Values.settings.aadTenantId }}" 15 | instrumentationAppId: "{{ .Values.settings.instrumentationAppId }}" 16 | instrumentationKey: "{{ .Values.settings.instrumentationKey }}" 17 | cosmosDBContainer: "{{ .Values.settings.cosmosDBContainer }}" 18 | 19 | --- 20 | 21 | apiVersion: v1 22 | kind: ConfigMap 23 | metadata: 24 | name: {{ include "contoso.fullname" . }}-appsettings 25 | labels: 26 | app.kubernetes.io/name: {{ include "contoso.name" . }} 27 | helm.sh/chart: {{ include "contoso.chart" . }} 28 | app.kubernetes.io/instance: {{ .Release.Name }} 29 | app.kubernetes.io/managed-by: {{ .Release.Service }} 30 | data: 31 | appsettings.json: | 32 | { 33 | "Serilog": { 34 | "Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.ApplicationInsights" ], 35 | "MinimumLevel": { 36 | "Default": "Debug", 37 | "Override": { 38 | "Microsoft": "Warning", 39 | "Microsoft.AspNetCore": "Warning", 40 | "Serilog.AspNetCore.RequestLoggingMiddleware": "Warning", 41 | "System": "Warning" 42 | } 43 | }, 44 | "WriteTo": [ 45 | { 46 | "Name": "Console", 47 | "Args": { 48 | "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact" 49 | } 50 | }, 51 | { 52 | "Name": "ApplicationInsights", 53 | "Args": { 54 | "restrictedToMinimumLevel": "Debug", 55 | "telemetryConverter": "Serilog.Sinks.ApplicationInsights.Sinks.ApplicationInsights.TelemetryConverters.TraceTelemetryConverter, Serilog.Sinks.ApplicationInsights" 56 | } 57 | } 58 | ], 59 | "Enrich": [ "FromLogContext", "WithCorrelationId", "WithCorrelationIdHeader" ] 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /charts/contoso/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "contoso.fullname" . }} 5 | labels: 6 | app.kubernetes.io/name: {{ include "contoso.name" . }} 7 | helm.sh/chart: {{ include "contoso.chart" . }} 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | spec: 11 | replicas: {{ .Values.replicaCount }} 12 | selector: 13 | matchLabels: 14 | app.kubernetes.io/name: {{ include "contoso.name" . }} 15 | app.kubernetes.io/instance: {{ .Release.Name }} 16 | template: 17 | metadata: 18 | labels: 19 | app.kubernetes.io/name: {{ include "contoso.name" . }} 20 | app.kubernetes.io/instance: {{ .Release.Name }} 21 | spec: 22 | containers: 23 | - name: {{ .Chart.Name }} 24 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 25 | imagePullPolicy: {{ .Values.image.pullPolicy }} 26 | ports: 27 | - name: http 28 | containerPort: 80 29 | protocol: TCP 30 | readinessProbe: 31 | httpGet: 32 | path: /health 33 | port: 80 34 | initialDelaySeconds: 5 35 | periodSeconds: 10 36 | livenessProbe: 37 | httpGet: 38 | path: /health 39 | port: 80 40 | initialDelaySeconds: 5 41 | periodSeconds: 20 42 | resources: 43 | requests: 44 | memory: "500Mi" 45 | cpu: "500m" 46 | limits: 47 | memory: "2000Mi" 48 | cpu: "2" 49 | envFrom: 50 | - configMapRef: 51 | name: {{ include "contoso.fullname" . }}-settings 52 | env: 53 | - name: aadClientSecret 54 | valueFrom: 55 | secretKeyRef: 56 | name: {{ include "contoso.fullname" . }}-credentials 57 | key: aadClientSecret 58 | volumeMounts: 59 | - name: appsettings-volume 60 | mountPath: /app/settings 61 | volumes: 62 | - name: appsettings-volume 63 | configMap: 64 | # Provide the name of the ConfigMap containing the files you want 65 | # to add to the container 66 | name: {{ include "contoso.fullname" . }}-appsettings 67 | -------------------------------------------------------------------------------- /charts/contoso/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "contoso.fullname" . -}} 3 | apiVersion: extensions/v1beta1 4 | kind: Ingress 5 | metadata: 6 | name: {{ $fullName }} 7 | labels: 8 | app.kubernetes.io/name: {{ include "contoso.name" . }} 9 | helm.sh/chart: {{ include "contoso.chart" . }} 10 | app.kubernetes.io/instance: {{ .Release.Name }} 11 | app.kubernetes.io/managed-by: {{ .Release.Service }} 12 | {{- with .Values.ingress.annotations }} 13 | annotations: 14 | {{- toYaml . | nindent 4 }} 15 | {{- end }} 16 | spec: 17 | {{- if .Values.ingress.tls }} 18 | tls: 19 | {{- range .Values.ingress.tls }} 20 | - hosts: 21 | {{- range .hosts }} 22 | - {{ . | quote }} 23 | {{- end }} 24 | secretName: {{ .secretName }} 25 | {{- end }} 26 | {{- end }} 27 | rules: 28 | {{- range .Values.ingress.hosts }} 29 | - host: {{ .host | quote }} 30 | http: 31 | paths: 32 | {{- range .paths }} 33 | - path: {{ . }} 34 | backend: 35 | serviceName: {{ $fullName }} 36 | servicePort: http 37 | {{- end }} 38 | {{- end }} 39 | {{- end }} 40 | -------------------------------------------------------------------------------- /charts/contoso/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: {{ include "contoso.fullname" . }}-credentials 5 | type: Opaque 6 | data: 7 | aadClientSecret: {{ .Values.settings.aadClientSecret | b64enc | quote }} 8 | -------------------------------------------------------------------------------- /charts/contoso/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "contoso.fullname" . }} 5 | labels: 6 | app.kubernetes.io/name: {{ include "contoso.name" . }} 7 | helm.sh/chart: {{ include "contoso.chart" . }} 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | spec: 11 | type: {{ .Values.service.type }} 12 | ports: 13 | - port: {{ .Values.service.port }} 14 | targetPort: {{ .Values.service.targetPort }} 15 | protocol: TCP 16 | name: http 17 | selector: 18 | app.kubernetes.io/name: {{ include "contoso.name" . }} 19 | app.kubernetes.io/instance: {{ .Release.Name }} 20 | -------------------------------------------------------------------------------- /charts/contoso/templates/tests/test-connection.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: "{{ include "contoso.fullname" . }}-test-connection" 5 | labels: 6 | app.kubernetes.io/name: {{ include "contoso.name" . }} 7 | helm.sh/chart: {{ include "contoso.chart" . }} 8 | app.kubernetes.io/instance: {{ .Release.Name }} 9 | app.kubernetes.io/managed-by: {{ .Release.Service }} 10 | annotations: 11 | "helm.sh/hook": test-success 12 | spec: 13 | containers: 14 | - name: wget 15 | image: busybox 16 | command: ['wget'] 17 | args: ['{{ include "contoso.fullname" . }}:{{ .Values.service.port }}'] 18 | restartPolicy: Never 19 | -------------------------------------------------------------------------------- /charts/contoso/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for contoso. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 2 6 | 7 | image: 8 | repository: 9 | tag: latest 10 | pullPolicy: Always 11 | 12 | settings: 13 | computeServiceAddress: "http://contoso:80/" 14 | appInsightsStorageFolder: "/tmp" 15 | aadClientId: "00000000-0000-0000-0000-000000000000" 16 | aadClientSecret: "secret/pa$$w0rd" 17 | aadTenantId: "00000000-0000-0000-0000-000000000000" 18 | instrumentationAppId: "contoso" 19 | instrumentationKey: "00000000-0000-0000-0000-000000000000" 20 | cosmosDBContainer: "/subscriptions/0000/resourceGroups/rg/providers/Microsoft.DocumentDB/databaseAccounts/ACC/apis/sql/databases/DB/containers/CONTAINER" 21 | 22 | service: 23 | type: ClusterIP 24 | port: 80 25 | targetPort: 80 26 | 27 | ingress: 28 | enabled: false 29 | annotations: {} 30 | -------------------------------------------------------------------------------- /docs/images/Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Architecture.png -------------------------------------------------------------------------------- /docs/images/Build pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Build pipeline.png -------------------------------------------------------------------------------- /docs/images/Data flows.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Data flows.png -------------------------------------------------------------------------------- /docs/images/IaC layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/IaC layers.png -------------------------------------------------------------------------------- /docs/images/Jobs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Jobs.png -------------------------------------------------------------------------------- /docs/images/Live Metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Live Metrics.png -------------------------------------------------------------------------------- /docs/images/Metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Metrics.png -------------------------------------------------------------------------------- /docs/images/PAT_Userprofile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/PAT_Userprofile.png -------------------------------------------------------------------------------- /docs/images/Prometheus custom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Prometheus custom.png -------------------------------------------------------------------------------- /docs/images/Prometheus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Prometheus.png -------------------------------------------------------------------------------- /docs/images/Structured logging AI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/algattik/terraform-aspnet-devops-template/2c8d204720926c14c52eb379529d0feb7447b8f9/docs/images/Structured logging AI.png -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | The variables referenced in the following guide refer to the [azure-pipelines.yml](../azure-pipelines.yml) file. To edit them, change the value there, e.g.: 4 | 5 | ```yml 6 | - name: TERRAFORM_SP_CLIENT_ID 7 | value: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX 8 | ``` 9 | 10 | ## Azure Active Directory configuration 11 | 12 | 1. Create service principals for Terraform & AKS: 13 | - Info: An Azure service principal is a security identity used by user-created apps, services, and automation tools to access specific Azure resources. Think of it as a 'user identity' (login and password or certificate) with a specific role, and tightly controlled permissions to access your resources. We use these to grant access to Azure DevOps and Terraform to deploy and manage resources within our Azure subscription. 14 | 15 | - Navigate to the Azure Portal & open the [Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview). You need to create two service principals (Azure Active Directory App registrations) for Terraform and AKS by entering the following commands: 16 | 17 | ``` 18 | az ad sp create-for-rbac --name "PROJECT_Azure_DevOps_Terraform_SP" --role owner 19 | 20 | az ad sp create-for-rbac --name "PROJECT_AKS_SP" --skip-assignment 21 | ``` 22 | 23 | - Save the outputs of these commands as you'll need these details during the installation. The output comes in the following format: 24 | 25 | ```json 26 | { 27 | "appId": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", 28 | "displayName": "PROJECT_Azure_DevOps_Terraform_SP", 29 | "name": "http://PROJECT_Azure_DevOps_Terraform_SP", 30 | "password": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", 31 | "tenant": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" 32 | } 33 | ``` 34 | 35 | 2. Retrieve and save the object_id for each of the Service principals by using the Cloud Shell: 36 | 37 | - Replace XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX with the appID each of the Service Principals 38 | 39 | ```bash 40 | az ad sp show --id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX | jq -r .objectId 41 | ``` 42 | 43 | 3. Replace the values of the following variables in the [azure-pipelines.yml](../azure-pipelines.yml): 44 | 45 | | name | value | 46 | |--|--| 47 | | TERRAFORM_SP_CLIENT_ID | The appId of the Terraform Service Principal | 48 | | AKS_SP_CLIENT_ID | The appId of the AKS Service Principal | 49 | | AKS_SP_OBJECT_ID| The objectId of the AKS Service Principal - from the previous step | 50 | 51 | ## Azure configuration 52 | 53 | 1. Create the resource group for your project in Azure by running the following command: 54 | 55 | ```bash 56 | az group create --name PROJECT_RG --location westeurope 57 | ``` 58 | 59 | 1. Grant TERRAFORM_SP_CLIENT_ID *Owner* permission on the RG. To do this, replace the values in assignee and scope parameters accordingly. The scope parameter is the id parameter from your previously created resource group. 60 | 61 | ```bash 62 | az role assignment create --assignee TERRAFORM_SP_CLIENT_ID --role Owner --scope /subscriptions/YourSubscriptionId/resourceGroups/PROJECT_RG 63 | ``` 64 | 65 | 1. Create the storage account $(TERRAFORM_STORAGE_ACCOUNT) within the RG. Info: Storage account name must be between 3 and 24 characters in length and use numbers and lower-case letters only. 66 | 67 | ```bash 68 | az storage account create --name projectstorageaccount --resource-group PROJECT_RG 69 | ``` 70 | 71 | 1. Create the container "terraformstate" within the storage account 72 | 73 | ```bash 74 | az storage container create --name terraformstate --account-name projectstorageaccount 75 | ``` 76 | 77 | 1. Replace the values of the following variables in the [azure-pipelines.yml](../azure-pipelines.yml): 78 | 79 | | name | value | 80 | |---|---| 81 | | RESOURCE_GROUP | The name of the resource group | 82 | | TERRAFORM_STORAGE_ACCOUNT | The name of the storage account | 83 | 84 | 85 | ## Azure DevOps configuration 86 | 87 | ### 1. Create an ADO agent pool named $(AGENT_POOL_NAME) 88 | - To create an agent pool, go to the Azure DevOps web UI and navigate to your current organization 89 | - On the bottom left, go to "Organization Settings" 90 | - In the left pane, under Pipelines, click on Agent pools 91 | - Click on "Add Pool" at the top right of the window 92 | - Type in the desired name for the agent pool - make note of that name for variable assignment later 93 | 94 | ### 2. Create an Azure DevOps Variable Group named `terraform-aspnet-devops-template`: 95 | 96 | - Go to the Azure DevOps UI and in your project, navigate to Pipelines/Library 97 | - Click the "+ Variable Group" button 98 | - Add these variables: 99 | 100 | | name | value | 101 | | --- | --- | 102 | | APP_NAME | Pick a short, globally unique name, e.g. "aspnetmplt000010" | 103 | | AGENT_POOL_MANAGEMENT_TOKEN | Create an ADO PAT with Agent Manage permission (See 2.1) | 104 | | AKS_SP_CLIENT_SECRET | Secret for AKS_SP_CLIENT_ID | 105 | | TERRAFORM_SP_CLIENT_SECRET | Secret for TERRAFORM_SP_CLIENT_ID | 106 | 107 | ### 2.1. Create an ADO Personal Access Token (PAT) with Agent manage permission 108 | 109 | - On the top left of the Azure DevOps UI, click on the "User settings" icon and select "Profile". 110 | 111 | ![User settings icon](images/PAT_Userprofile.png) 112 | 113 | - In the left pane, select "Personal Access Tokens" 114 | 115 | - Click "New Token" 116 | 117 | - Select the "Agent manage permission" 118 | 119 | - Copy the secret to a secure place and then into the Variable Group 120 | 121 | ### 2.2. Get the secret for the AKS Service Principal 122 | 123 | When creating the AKS Service principal, a password was created which is needed here. To retrieve that secret, you can go to the Azure portal and navigate to: `Azure Active Directory/App registrations/` and click on **Certificates and secrets** on the left pane. 124 | 125 | ### 2.3. Get the secret for the Terraform Service principal 126 | 127 | When creating the Terraform Service principal, a password was created which is needed here. To retrieve that secret, you can go to the Azure portal and navigate to: `Azure Active Directory/App registrations/` and click on **Certificates and secrets** on the left pane. 128 | 129 | 130 | ### 3. Install ADO extensions: 131 | 132 | | Extension | Url | 133 | |---|---| 134 | | Secure dev tools | https://marketplace.visualstudio.com/acquisition?itemName=securedevelopmentteam.vss-secure-development-tools | 135 | | Terraform | https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks | 136 | | JMeter | https://marketplace.visualstudio.com/items?itemName=AlexandreGattiker.jmeter-tasks | 137 | 138 | 139 | ### 4. Create a new Azure Resource Manager Service Connection $(TERRAFORM_SERVICE_CONNECTION) with access to the $(RESOURCE_GROUP) resource group. 140 | 141 | - Go to Azure DevOps Project settings 142 | - Select Service Connections (if you want to learn more about this, got to [Service connections in Azure docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml)) 143 | - Create a new Service connection 144 | - Pick Azure Resource Manager (which gives access to Azure Resources commands) 145 | - Create a new Azure service connection: 146 | - Select Service Principal (automatic) 147 | - Don't select a resource group, as we don't have one yet - this allows the SP to access all Resource Groups 148 | - Give the Service connection a name - we will need that name for Terraform later (e.g. Terraform_SP) 149 | - Check the "Grant access permission to all pipelines" box 150 | 151 | ### 5. Replace the values of the following variables in the [azure-pipelines.yml](../azure-pipelines.yml): 152 | 153 | | name | value | 154 | |---|---| 155 | | AGENT_POOL_NAME | The name of the created Agent Pool in step 1 of the Azure DevOps configuration | 156 | | TERRAFORM_SERVICE_CONNECTION | The name of the service connection | 157 | | ACR_NAME | Pick the name of the Azure Container Registry, e.g. yourprojectacr | 158 | | HELM_RELEASE_NAME | Pick a name for the Helm release | 159 | | SUBSCRIPTION_ID | The id of your Azure Subscription. See 5.1. | 160 | | TENANT_ID | Tenant id of your Azure subscription. See 5.2. | 161 | 162 | ### 5.1. Get Azure Subscription id 163 | 164 | Run the following command to get a list of all your subscriptions and pick your subscriptions id: 165 | 166 | ```bash 167 | az account list --output tab file 168 | ``` 169 | 170 | ### 5.2. Get Azure Subscription tenant id 171 | 172 | Run the following command and replace the sub-id with the subscription id of your subscription: 173 | 174 | ```bash 175 | az account show -s XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX | jq -r .tenantId 176 | ``` 177 | 178 | ### 6. Run the pipeline azure-pipelines.yml on master 179 | 180 | - Hint: On the first run some of the jobs might fail with the error message: 181 | 182 | ```##[error]No agents were found in pool $AGENT_POOL_NAME. Configure an agent for the pool and try again.``` 183 | 184 | which can happen due to a delay in creation of agents in the agent pool. 185 | 186 | - if you don't run on master, make sure to include the variable RUN_FLAG_TERRAFORM to 1 in the Azure DevOps portal before running the pipeline. See the [ci-cd-pipeline.yml](../infrastructure/ci-cd-pipeline.yml) for more information. 187 | -------------------------------------------------------------------------------- /docs/walkthrough.md: -------------------------------------------------------------------------------- 1 | # Walkthrough 2 | 3 | ## Introduction 4 | 5 | This sample demonstrates how to build and deploy ASP.NET Core microservice applications using a CI/CD pipeline that includes IaC deployment jobs with Terraform. The pipeline is designed for fully automated end-to-end integration testing with 6 | high deployment speed and high branch concurrency. 7 | 8 | The solution aims to maximize developer productivity. A build pipeline (Azure Pipelines) deploys shared infrastructure, retained between builds, as well as transient infrastructure on which the application is deployed for integration testing. At the end of the build the application and underlying transient infrastructure are automatically destroyed. 9 | 10 | The sample application shows how a microservice application can integrate with the Azure ecosystem to provide observability with Azure Monitor (Application Insights and Log Analytics). 11 | 12 | The CI/CD pipeline is heavily optimized for speed and full automation, with the deployment of managed build agents to quickly build docker images and run multiple parallel jobs. The sample solution can be adapted to almost any stack by switching the Azure components and implementing application logic as required. 13 | 14 | ## Overview 15 | 16 | The architecture contains an Virtual Network with a subnet containing Azure DevOps self-hosted build agents managed as a 17 | Virtual Machine Scale Set, and a subnet containing an Azure Kubernetes Service cluster. A Cosmos DB Database serves as backend 18 | for the application. 19 | 20 | ![Architecture.png](images/Architecture.png) 21 | 22 | An Azure DevOps pipeline creates 23 | the entire infrastructure, builds the application Docker container on the build agents, and pushes the image to Azure Container 24 | Registry. The pipeline then deploys an "area" for the build including a separate Cosmos DB collection and Kubernetes namespace, 25 | runs integration tests within the area, and destroys the area infrastructure. This allows multiple pipeline runs to execute 26 | concurrently, reusing core infrastructure while largely avoiding testability issues with shared application-level resources. 27 | 28 | ## CI/CD pipeline 29 | 30 | The [build pipeline](../infrastructure/ci-cd-pipeline.yml) is highly optimized for speed, and achieves high task parallelism by breaking down the process into 10 jobs 31 | that can partly run concurrently. 32 | 33 | ![Build pipeline.png](images/Build%20pipeline.png) 34 | 35 | - **Deploy shared infrastructure**: Uses Terraform to deploy core infrastructure shared between builds. This job is skipped by 36 | default on PR builds, to maximize build speed. The sample solution deploys: 37 | - [Azure Virtual Network](../infrastructure/terraform-shared/vnet/main.tf): 38 | for hosting VMs and Kubernetes 39 | - [Virtual Machine Scale Set with self-hosted build agents](../infrastructure/terraform-shared/devops-agent/main.tf): 40 | for running subsequent build jobs. Multiple Azure DevOps 41 | build agents are deployed per VM, to provide high concurrency at low cost. 42 | - [Azure Kubernetes Service](../infrastructure/terraform-shared/aks/main.tf): 43 | for hosting the microservice application 44 | - [Azure Container Registry](../infrastructure/terraform-shared/acr/main.tf): 45 | for storing the built application Docker container 46 | - [Application Insights](../infrastructure/terraform-shared/app-insights/main.tf): 47 | for application monitoring and metrics 48 | - [Azure Log Analytics](../infrastructure/terraform-shared/aks/main.tf): 49 | for Kubernetes and application logs 50 | - Cosmos DB Database: 51 | as an application backend 52 | - **Start agents**: Scales up the Virtual Machine Scale Set to ensure agents are available. This enables users to set up 53 | jobs that scale down the scale set (even to zero) to save costs overnight, while making those VMs available for the next 54 | build that requires them, without additional work. This job runs concurrently with other jobs, so has no impact if the VMs are already running. 55 | - **Read shared infrastructure**: Uses Terraform to populate job variables with the deployed configuration such as names 56 | and credentials to resources. 57 | - **Build and unit tests**: Runs the application dockerized build and unit testing. Using a self-hosted agent allows reusing cached Docker layers for significant speed gain. 58 | - **Deploy infrastructure**: Uses Terraform to deploy infrastructure specific to each individual build. This step is run in parallel with the application build, to maximize pipeline speed. The sample solution deploys: 59 | - Kubernetes namespace: as a container for the application Helm release 60 | - Cosmos DB Collection: as an application backend 61 | - **Deploy application**: Uses Terraform to deploy the application container, packaged in a Helm chart. 62 | - **Integration tests**: Uses Taurus and JMeter to run integration tests by hitting the application API endpoint and verifying assertions on the response. 63 | - **Destroy infrastructure**: Uses Terraform to destroy the per-build infrastructure provisioned by the **Deploy infrastructure** step. Destroying the Kubernetes namespace causes the application to also be destroyed, so this steps also destroys the components provisioned by the **Deploy application** job. This job retain the shared infrastructure deployed by 64 | the **Deploy shared infrastructure** job. 65 | - **Security analysis**: Runs static code analysis on the source code. To speed up the end-to-end pipeline, this job is run 66 | concurrently with build & integration testing. 67 | - **Promote latest image**: This step tags images that have passed unit and integration tests as well as security analysis. 68 | You can be customized it for your particular needs, such as triggering a production deployment, and/or pushing the image 69 | to a separate container registry or repository. 70 | 71 | ## Multi-layer IaC approach 72 | 73 | This image shows the three layers corresponding to three distinct Terraform jobs that each deploy part of the infrastructure. 74 | Separating the deployment into such layers enable: 75 | - high concurrency of builds, without deadlocks on Terraform state (as the *shared infrastructure* is not run on PR builds). 76 | Multiple "areas" deployed for each build can co-exist at any point in time, and can be automatically deleted after builds, 77 | or retained e.g. for demos or manual testing. 78 | - high speed of builds, as the *infrastructure* components are deployed while the app is built. 79 | 80 | ![IaC layers.png](images/IaC%20layers.png) 81 | 82 | ## Sample ASP.NET Core application 83 | 84 | The [sample ASP.NET Core application](../Src) is a dummy Web API application exposing a single service. The service accepts a positive 85 | number N and returns the sum of integers from 1 to N. It is implemented recursively, and calls itself with the value N-1 and sums the result with N. 86 | 87 | ![Data flows.png](images/Data%20flows.png) 88 | 89 | The application demonstrates observability patterns: 90 | - Health and liveness probes using [ASP.NET Core Health Checks Middleware](https://docs.microsoft.com/en-us/aspnet/core/host-and-deploy/health-checks?view=aspnetcore-3.1) 91 | - Kubernetes monitoring using [Azure Monitor for containers](https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-overview) 92 | - Distributed tracing with header propagation across HTTP calls using [ASP.NET Core IHttpClientFactory](https://docs.microsoft.com/en-us/dotnet/architecture/microservices/implement-resilient-applications/use-httpclientfactory-to-implement-resilient-http-requests) 93 | - Application telemetry using [Application Insights for ASP.NET Core applications](https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-core) 94 | - Structured application log capture using Serilog: 95 | - [Serilog sink for Azure Application Insights](https://github.com/serilog/serilog-sinks-applicationinsights) 96 | - Logging to console and using [Azure Monitor for containers](https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-overview) to forward logs to Azure Log Analytics 97 | 98 | - Application metrics collection: 99 | 100 | - Standard host and platform metrics using [Application Insights for ASP.NET Core applications](https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-core) 101 | - [Custom application metrics]((../Src/Contoso/Observability/MetricsService.cs) and gauges, e.g. tracking timing of external calls 102 | - Exception counters, [using Serilog](../charts/contoso/templates/configmap.yaml) 103 | 104 | Metrics are exposed in two ways: 105 | 106 | - [Pushing metrics](../Src/Contoso/Observability/TelemetryInitializer.cs) to Application Insights 107 | - [Exposing metrics](../Src/Contoso/Observability/PrometheusSerilogSink.cs) through a Prometheus-compatible endpoint. Prometheus is a popular open-source monitoring package, and 108 | many vendor monitoring package support metrics collection from Prometheus endpoint as a de facto standard. 109 | 110 | ## Screenshots 111 | 112 | Live metrics generated in Application Insights during integration (load) testing. 113 | 114 | ![Live Metrics.png](images/Live%20Metrics.png) 115 | 116 | Application metrics query interface in Application Insights. 117 | 118 | ![Metrics.png](images/Metrics.png) 119 | 120 | 121 | Prometheus metrics exposed by the application out of the box. 122 | 123 | ![Prometheus.png](images/Prometheus.png) 124 | 125 | Code to [generate custom metrics](../Src/Contoso/Observability/MetricsService.cs), exposed to the Prometheus endpoint and Application Insights. 126 | 127 | ![Prometheus custom.png](images/Prometheus%20custom.png) 128 | 129 | Structured logs generated by the C# code. Logs contain both the formatted string and the values of the individual 130 | placeholders, allowing [rich analytical queries](https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/log-query-overview). This log line was generated by this [line of code](../Src/Contoso/Services/SampleService.cs): 131 | 132 | ```c# 133 | this.logger.LogInformation("Sum of numbers from 0 to {value} was {result}, computed in {duration}s", value, result, duration); 134 | ``` 135 | 136 | ![Structured logging.png](images/Structured%20logging%20AI.png) 137 | -------------------------------------------------------------------------------- /infrastructure/ci-cd-pipeline.yml: -------------------------------------------------------------------------------- 1 | # CI/CD Azure DevOps deployment pipeline. 2 | # The following variables can be optionally set for each pipeline run: 3 | # - RUN_FLAG_TERRAFORM: Set to 1 to deploy shared infrastructure with Terraform. 4 | # By default this step only runs on the master branch. 5 | # - RUN_FLAG_PROMOTE: Set to 1 to promote the Docker image to `latest` tag if 6 | # tests are successful. By default this is only done on the master branch. 7 | # - RUN_SET_NAMEBASE: Set to a string to deploy to the given AKS namespace, 8 | # and not delete the namespace after the build. By default the build deploys to 9 | # the `master` AKS namespace if run on the master branch, and otherwise to a 10 | # temporary AKS namespace that is deleted at the end of the build. 11 | 12 | jobs: 13 | 14 | - job: build 15 | displayName: Build and unit tests 16 | pool: $(AGENT_POOL_NAME) 17 | steps: 18 | 19 | - bash: | 20 | set -eux # fail on error 21 | # Only build first stage of Dockerfile (build and unit test) 22 | docker build --pull --target testrunner --build-arg VersionPrefix="$(SEMANTIC_VERSION)" -t contoso-build-$(Build.BuildId):test . 23 | docker run --rm -v $PWD/TestResults:/app/Contoso.UnitTests/TestResults contoso-build-$(Build.BuildId):test 24 | displayName: Docker build & test 25 | 26 | - task: PublishTestResults@2 27 | displayName: Publish test results 28 | condition: succeededOrFailed() 29 | inputs: 30 | testRunner: VSTest 31 | testResultsFiles: '**/*.trx' 32 | failTaskOnFailedTests: true 33 | testRunTitle: 'Unit Tests' 34 | 35 | # Publish the code coverage result (summary and web site) 36 | # The summary allows to view the coverage percentage in the summary tab 37 | # The web site allows to view which lines are covered directly in Azure Pipeline 38 | - task: PublishCodeCoverageResults@1 39 | displayName: 'Publish code coverage' 40 | inputs: 41 | codeCoverageTool: 'Cobertura' 42 | summaryFileLocation: '**/coverage.cobertura.xml' 43 | pathToSources: '$(Build.SourcesDirectory)/Src' 44 | failIfCoverageEmpty: true 45 | 46 | - task: AzureCLI@1 47 | displayName: Build runtime image 48 | inputs: 49 | azureSubscription: $(TERRAFORM_SERVICE_CONNECTION) 50 | scriptLocation: inlineScript 51 | inlineScript: | 52 | set -eux # fail on error 53 | 54 | az configure --defaults acr="$ACR_NAME" 55 | az acr login 56 | 57 | # Build runtime Docker image 58 | # Reuses the cached build stage from the previous docker build task 59 | docker build --build-arg VersionPrefix="$(SEMANTIC_VERSION)" \ 60 | -t "$ACR_NAME.azurecr.io/contoso:$(SEMANTIC_VERSION)" \ 61 | . 62 | 63 | # Push Docker image to ACR 64 | docker push "$ACR_NAME.azurecr.io/contoso:$SEMANTIC_VERSION" 65 | 66 | 67 | - job: Terraform_shared 68 | displayName: Deploy shared infrastructure 69 | # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file. 70 | condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM'])) 71 | variables: 72 | TERRAFORM_DIRECTORY: infrastructure/terraform-shared 73 | steps: 74 | 75 | - bash: | 76 | set -euo pipefail 77 | curl -sfu ":$(AGENT_POOL_MANAGEMENT_TOKEN)" '$(System.CollectionUri)_apis/distributedtask/pools?poolName=$(AGENT_POOL_NAME)&actionFilter=manage&api-version=5.1' \ 78 | | jq -e '.count>0' 79 | displayName: Verify agent pool token 80 | 81 | - template: terraform-template.yml 82 | parameters: 83 | TerraformApply: true 84 | TerraformStateKey: cd 85 | TerraformVariables: 86 | ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET) 87 | TF_VAR_appname: $(APP_NAME) 88 | TF_VAR_environment: cd 89 | TF_VAR_resource_group: $(RESOURCE_GROUP) 90 | TF_VAR_acr_name: $(ACR_NAME) 91 | TF_VAR_aks_version: $(AKS_VERSION) 92 | TF_VAR_aks_sp_client_id: $(AKS_SP_CLIENT_ID) 93 | TF_VAR_aks_sp_client_secret: $(AKS_SP_CLIENT_SECRET) 94 | TF_VAR_aks_sp_object_id: $(AKS_SP_OBJECT_ID) 95 | TF_VAR_app_sp_object_id: $(APP_SP_OBJECT_ID) 96 | TF_VAR_az_devops_agent_pool: $(AGENT_POOL_NAME) 97 | TF_VAR_az_devops_url: $(System.CollectionUri) 98 | TF_VAR_az_devops_pat: $(AGENT_POOL_MANAGEMENT_TOKEN) 99 | 100 | - job: Terraform_shared_outputs 101 | displayName: Read shared infrastructure 102 | dependsOn: 103 | - Terraform_shared 104 | condition: | 105 | in(dependencies.Terraform_shared.result, 'Succeeded', 'SucceededWithIssues', 'Skipped') 106 | variables: 107 | TERRAFORM_DIRECTORY: infrastructure/terraform-shared 108 | steps: 109 | 110 | - template: terraform-template.yml 111 | parameters: 112 | TerraformStateKey: cd 113 | TerraformVariables: 114 | ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET) 115 | 116 | - job: Terraform 117 | displayName: Deploy infrastructure 118 | pool: $(AGENT_POOL_NAME) 119 | dependsOn: 120 | - Terraform_shared_outputs 121 | variables: 122 | KUBE_CONFIG_BASE64: $[ dependencies.Terraform_shared_outputs.outputs['Outputs.kube_config_base64'] ] 123 | COSMOSDB_ACCOUNT_NAME: $[ dependencies.Terraform_shared_outputs.outputs['Outputs.cosmosdb_account_name'] ] 124 | TERRAFORM_DIRECTORY: infrastructure/terraform 125 | steps: 126 | 127 | - bash: | 128 | set -eu # fail on error 129 | 130 | AREA_NAME="build$(Build.BuildId)" 131 | if [ "$(Build.SourceBranch)" = "refs/heads/master" ]; then 132 | AREA_NAME="master" 133 | fi 134 | if [ "${RUN_SET_NAMEBASE:-}" != "" ]; then 135 | AREA_NAME="$RUN_SET_NAMEBASE" 136 | fi 137 | 138 | echo "Area name: $AREA_NAME" 139 | 140 | echo "##vso[task.setvariable variable=AREA_NAME;isOutput=true]$AREA_NAME" 141 | 142 | displayName: Define Deployment area 143 | name: area 144 | 145 | - template: terraform-template.yml 146 | parameters: 147 | TerraformDestroy: true 148 | TerraformApply: true 149 | TerraformStateKey: infra/$(area.AREA_NAME) 150 | TerraformVariables: 151 | ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET) 152 | TF_VAR_appname: $(APP_NAME) 153 | TF_VAR_resource_group: $(RESOURCE_GROUP) 154 | TF_VAR_area_name: $(area.AREA_NAME) 155 | TF_VAR_cosmosdb_account_name: $(COSMOSDB_ACCOUNT_NAME) 156 | 157 | - job: Terraform_app 158 | displayName: Deploy application 159 | pool: $(AGENT_POOL_NAME) 160 | dependsOn: 161 | - build 162 | - Terraform_shared_outputs 163 | - Terraform 164 | variables: 165 | KUBERNETES_NAMESPACE: $[ dependencies.Terraform.outputs['Outputs.kubernetes_namespace'] ] 166 | KUBE_CONFIG_BASE64: $[ dependencies.Terraform_shared_outputs.outputs['Outputs.kube_config_base64'] ] 167 | INSTRUMENTATION_KEY: $[ dependencies.Terraform_shared_outputs.outputs['Outputs.instrumentation_key'] ] 168 | COSMOS_DB_CONTAINER_ID: $[ dependencies.Terraform.outputs['Outputs.cosmosdb_container_id'] ] 169 | TERRAFORM_DIRECTORY: infrastructure/terraform-app 170 | steps: 171 | 172 | - template: terraform-template.yml 173 | parameters: 174 | TerraformApply: true 175 | TerraformStateKey: app/build-$(Build.BuildId) 176 | TerraformVariables: 177 | ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET) 178 | TF_VAR_kubernetes_namespace: $(KUBERNETES_NAMESPACE) 179 | TF_VAR_release_name: $(HELM_RELEASE_NAME) 180 | TF_VAR_image_repository: $(ACR_NAME).azurecr.io/contoso 181 | TF_VAR_image_tag: $(SEMANTIC_VERSION) 182 | TF_VAR_client_id: $(APP_SP_CLIENT_ID) 183 | TF_VAR_tenant_id: $(ARM_TENANT_ID) 184 | TF_VAR_client_secret: $(APP_SP_CLIENT_SECRET) 185 | TF_VAR_instrumentation_key: $(INSTRUMENTATION_KEY) 186 | TF_VAR_cosmosdb_container_id: $(COSMOS_DB_CONTAINER_ID) 187 | 188 | - job: Start_agents 189 | displayName: Start agents 190 | dependsOn: 191 | - Terraform_shared_outputs 192 | variables: 193 | AGENT_VMSS_NAME: $[ dependencies.Terraform_shared_outputs.outputs['Outputs.agent_vmss_name'] ] 194 | TERRAFORM_DIRECTORY: infrastructure/terraform-shared 195 | steps: 196 | 197 | - task: AzureCLI@1 198 | displayName: Start agents 199 | inputs: 200 | azureSubscription: $(TERRAFORM_SERVICE_CONNECTION) 201 | scriptLocation: inlineScript 202 | inlineScript: | 203 | set -eux # fail on error 204 | # Trigger rerun of provisioning script if it has changed (based on CustomScript timestamp attribute) 205 | az vmss update-instances --instance-ids '*' --name $AGENT_VMSS_NAME --resource-group $RESOURCE_GROUP --no-wait 206 | az vmss scale --new-capacity 2 -o table --name $AGENT_VMSS_NAME --resource-group $RESOURCE_GROUP 207 | 208 | - job: integration_tests 209 | displayName: Integration tests 210 | pool: $(AGENT_POOL_NAME) 211 | dependsOn: 212 | - Terraform 213 | - Terraform_shared_outputs 214 | - Terraform_app 215 | variables: 216 | KUBE_CONFIG_BASE64: $[ dependencies.Terraform_shared_outputs.outputs['Outputs.kube_config_base64'] ] 217 | KUBERNETES_NAMESPACE: $[ dependencies.Terraform.outputs['Outputs.kubernetes_namespace'] ] 218 | steps: 219 | 220 | - task: KubectlInstaller@0 221 | displayName: Install kubectl 222 | inputs: 223 | kubectlVersion: $(AKS_VERSION) 224 | 225 | - task: AlexandreGattiker.jmeter-tasks.custom-jmeter-installer-task.JMeterInstaller@0 226 | displayName: 'Install JMeter' 227 | inputs: 228 | jmeterVersion: $(JMETER_VERSION) 229 | plugins: jpgc-casutg,jpgc-dummy,jpgc-ffw,jpgc-fifo,jpgc-functions,jpgc-json,jpgc-perfmon,jpgc-prmctl,jpgc-tst,jmeter.backendlistener.azure 230 | 231 | - task: AlexandreGattiker.jmeter-tasks.custom-taurus-installer-task.TaurusInstaller@0 232 | displayName: 'Install Taurus' 233 | inputs: 234 | taurusVersion: 1.14.1 235 | pythonCommand: python3 236 | 237 | - bash: | 238 | set -eu # fail on error 239 | base64 -d <<< $KUBE_CONFIG_BASE64 > kube_config 240 | echo "##vso[task.setvariable variable=KUBECONFIG]$PWD/kube_config" 241 | displayName: Save kubeconfig 242 | env: 243 | KUBE_CONFIG_BASE64: $(KUBE_CONFIG_BASE64) 244 | 245 | - bash: | 246 | set -eux # fail on error 247 | read -d, firstNodeIP < <(kubectl -n "$KUBERNETES_NAMESPACE" get nodes -o jsonpath="{.items[0].status.addresses[?(@.type=='InternalIP')].address},") 248 | read -d, nodePort < <(kubectl -n "$KUBERNETES_NAMESPACE" get svc "$(HELM_RELEASE_NAME)" -o jsonpath="{.spec.ports[0].nodePort},") 249 | url="http://$firstNodeIP:$nodePort" 250 | echo "##vso[task.setvariable variable=SERVICE_URL]$url" 251 | echo "$url" 252 | displayName: Get Service URL 253 | 254 | # FIXME: JUnit report generation fails if merging multiple scenarios in a single Task, so splitting across multiple Taurus runs. 255 | 256 | - task: AlexandreGattiker.jmeter-tasks.custom-taurus-runner-task.TaurusRunner@0 257 | displayName: 'Run Taurus' 258 | inputs: 259 | outputDir: 'taurus-output' 260 | reportName: 'Test Prometheus Endpoint' 261 | taurusConfig: | 262 | modules: 263 | jmeter: 264 | properties: 265 | jmeter.reportgenerator.overall_granularity: 5000 266 | execution: 267 | - scenario: 268 | requests: 269 | - url: $(SERVICE_URL)/metrics 270 | assert: 271 | - contains: 272 | - process_virtual_memory_bytes 273 | subject: body 274 | concurrency: 10 275 | ramp-up: 30s 276 | hold-for: 1m 277 | throughput: 20 278 | reporting: 279 | - module: junit-xml 280 | filename: taurus-output/TEST-Taurus.xml 281 | 282 | - task: AlexandreGattiker.jmeter-tasks.custom-taurus-runner-task.TaurusRunner@0 283 | displayName: 'Run Taurus' 284 | inputs: 285 | outputDir: 'taurus-output2' 286 | reportName: 'Test Sum Computation Endpoint' 287 | taurusConfig: | 288 | modules: 289 | jmeter: 290 | properties: 291 | jmeter.reportgenerator.overall_granularity: 5000 292 | execution: 293 | - scenario: 294 | requests: 295 | - url: $(SERVICE_URL)/sample/sumNumbersUpTo?value=100 296 | assert: 297 | - contains: 298 | - 5050 299 | subject: body 300 | hold-for: 20s 301 | throughput: 1 302 | #- scenario: 303 | # requests: 304 | # - url: $(SERVICE_URL)/sample/sumNumbersUpTo?value=-10 305 | # assert: 306 | # - contains: 307 | # - 500 308 | # subject: status-code 309 | # iterations: 5 310 | reporting: 311 | - module: junit-xml 312 | filename: taurus-output2/TEST-Taurus.xml 313 | 314 | - task: PublishTestResults@2 315 | displayName: Publish test results 316 | inputs: 317 | testRunTitle: Integration tests 318 | failTaskOnFailedTests: true 319 | 320 | - job: Cleanup 321 | displayName: Destroy infrastructure 322 | dependsOn: 323 | - integration_tests 324 | - Terraform_shared_outputs 325 | - Terraform 326 | pool: $(AGENT_POOL_NAME) 327 | variables: 328 | AREA_NAME: $[ dependencies.Terraform.outputs['area.AREA_NAME'] ] 329 | KUBE_CONFIG_BASE64: $[ dependencies.Terraform_shared_outputs.outputs['Outputs.kube_config_base64'] ] 330 | TERRAFORM_DIRECTORY: infrastructure/terraform-destroy 331 | # Destroy build-specific infrastructure unless either: 332 | # - deploying on master branch 333 | # - namespace was manually set with RUN_SET_NAMEBASE 334 | condition: and(always(), not(eq(variables['Build.SourceBranch'], 'refs/heads/master')), not(variables['RUN_SET_NAMEBASE'])) 335 | steps: 336 | 337 | - template: terraform-template.yml 338 | parameters: 339 | TerraformDestroy: true 340 | TerraformStateKey: infra/$(AREA_NAME) 341 | TerraformVariables: 342 | ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET) 343 | 344 | - job: Promote 345 | displayName: Promote latest image 346 | dependsOn: integration_tests 347 | condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_PROMOTE'])) 348 | steps: 349 | 350 | - task: AzureCLI@1 351 | displayName: Tag Docker image as latest 352 | inputs: 353 | azureSubscription: $(TERRAFORM_SERVICE_CONNECTION) 354 | scriptLocation: inlineScript 355 | inlineScript: | 356 | set -eux # fail on error 357 | az configure --defaults acr="$ACR_NAME" 358 | az acr login 359 | docker pull "$ACR_NAME.azurecr.io/contoso:$SEMANTIC_VERSION" 360 | docker tag \ 361 | "$ACR_NAME.azurecr.io/contoso:$SEMANTIC_VERSION" \ 362 | "$ACR_NAME.azurecr.io/contoso:latest" 363 | docker push "$ACR_NAME.azurecr.io/contoso:latest" 364 | -------------------------------------------------------------------------------- /infrastructure/terraform-app/backend.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform backend 2 | terraform { 3 | # Backend variables are initialized by Azure DevOps 4 | backend "azurerm" {} 5 | } 6 | -------------------------------------------------------------------------------- /infrastructure/terraform-app/main.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "build" { 2 | name = var.release_name 3 | chart = "../../charts/contoso" 4 | namespace = var.kubernetes_namespace 5 | 6 | wait = true 7 | timeout = 300 8 | 9 | set { 10 | name = "image.repository" 11 | value = var.image_repository 12 | } 13 | set { 14 | name = "image.tag" 15 | value = var.image_tag 16 | } 17 | set { 18 | name = "replicaCount" 19 | value = 2 20 | } 21 | set { 22 | name = "service.type" 23 | value = "NodePort" 24 | } 25 | set { 26 | name = "settings.aadClientId" 27 | value = var.client_id 28 | } 29 | set_sensitive { 30 | name = "settings.aadClientSecret" 31 | value = var.client_secret 32 | } 33 | set { 34 | name = "settings.aadTenantId" 35 | value = var.tenant_id 36 | } 37 | set { 38 | name = "settings.instrumentationAppId" 39 | value = var.release_name 40 | } 41 | set { 42 | name = "settings.instrumentationKey" 43 | value = var.instrumentation_key 44 | } 45 | set { 46 | name = "settings.cosmosDBContainer" 47 | value = var.cosmosdb_container_id 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /infrastructure/terraform-app/outputs.tf: -------------------------------------------------------------------------------- 1 | output "release_name" { 2 | value = helm_release.build.name 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure/terraform-app/providers.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform required version 2 | 3 | terraform { 4 | required_version = ">= 0.12.6" 5 | } 6 | 7 | # Configure the Azure Provider 8 | 9 | provider "azurerm" { 10 | # It is recommended to pin to a given version of the Provider 11 | version = "=2.2.0" 12 | skip_provider_registration = true 13 | features {} 14 | } 15 | 16 | provider "helm" { 17 | } 18 | -------------------------------------------------------------------------------- /infrastructure/terraform-app/variables.tf: -------------------------------------------------------------------------------- 1 | variable "kubernetes_namespace" { 2 | type = string 3 | description = "Kubernetes namespace to create." 4 | } 5 | 6 | variable "release_name" { 7 | type = string 8 | } 9 | 10 | variable "image_repository" { 11 | type = string 12 | } 13 | 14 | variable "image_tag" { 15 | type = string 16 | } 17 | 18 | variable "client_id" { 19 | type = string 20 | } 21 | 22 | variable "client_secret" { 23 | type = string 24 | } 25 | 26 | variable "tenant_id" { 27 | type = string 28 | } 29 | 30 | variable "instrumentation_key" { 31 | type = string 32 | description = "App Insights instrumentation key to send metrics to." 33 | } 34 | 35 | variable "cosmosdb_container_id" { 36 | type = string 37 | description = "Cosmos DB account in which to save data." 38 | } 39 | -------------------------------------------------------------------------------- /infrastructure/terraform-destroy/backend.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform backend 2 | terraform { 3 | # Backend variables are initialized by Azure DevOps 4 | backend "azurerm" {} 5 | } 6 | -------------------------------------------------------------------------------- /infrastructure/terraform-destroy/providers.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform required version 2 | 3 | terraform { 4 | required_version = ">= 0.12.6" 5 | } 6 | 7 | # Configure the Azure Provider 8 | 9 | provider "azurerm" { 10 | # It is recommended to pin to a given version of the Provider 11 | version = "=2.2.0" 12 | skip_provider_registration = true 13 | features {} 14 | } 15 | 16 | provider "kubernetes" { 17 | } 18 | 19 | provider "helm" { 20 | } 21 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/acr/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_container_registry" "acr" { 2 | name = var.acr_name 3 | resource_group_name = var.resource_group_name 4 | location = var.location 5 | sku = "Standard" 6 | admin_enabled = false 7 | } 8 | 9 | resource "azurerm_role_assignment" "aks_sp_container_registry" { 10 | scope = azurerm_container_registry.acr.id 11 | role_definition_name = "AcrPull" 12 | principal_id = var.aks_sp_object_id 13 | } 14 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/acr/outputs.tf: -------------------------------------------------------------------------------- 1 | output "acr_name" { 2 | value = azurerm_container_registry.acr.name 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/acr/variables.tf: -------------------------------------------------------------------------------- 1 | variable "acr_name" { 2 | type = string 3 | } 4 | 5 | variable "resource_group_name" { 6 | type = string 7 | } 8 | 9 | variable "location" { 10 | type = string 11 | } 12 | 13 | variable "aks_sp_object_id" { 14 | type = string 15 | } 16 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/aks/main.tf: -------------------------------------------------------------------------------- 1 | # Application Insights 2 | 3 | resource "random_id" "workspace" { 4 | keepers = { 5 | # Generate a new id each time we switch to a new resource group 6 | group_name = var.resource_group_name 7 | } 8 | 9 | byte_length = 8 10 | } 11 | 12 | resource "azurerm_log_analytics_workspace" "aks" { 13 | name = "log-${var.appname}-${var.environment}-${random_id.workspace.hex}" 14 | location = var.location 15 | resource_group_name = var.resource_group_name 16 | sku = "PerGB2018" 17 | } 18 | 19 | resource "azurerm_log_analytics_solution" "aks" { 20 | solution_name = "ContainerInsights" 21 | location = var.location 22 | resource_group_name = var.resource_group_name 23 | workspace_resource_id = azurerm_log_analytics_workspace.aks.id 24 | workspace_name = azurerm_log_analytics_workspace.aks.name 25 | 26 | plan { 27 | publisher = "Microsoft" 28 | product = "OMSGallery/ContainerInsights" 29 | } 30 | } 31 | 32 | # Subnet permission 33 | 34 | resource "azurerm_role_assignment" "aks_subnet" { 35 | scope = var.subnet_id 36 | role_definition_name = "Network Contributor" 37 | principal_id = var.aks_sp_object_id 38 | } 39 | 40 | # Kubernetes Service 41 | 42 | resource "azurerm_kubernetes_cluster" "aks" { 43 | name = "aks-${var.appname}-${var.environment}" 44 | location = var.location 45 | resource_group_name = var.resource_group_name 46 | dns_prefix = "aks-${var.appname}-${var.environment}" 47 | kubernetes_version = var.aks_version 48 | 49 | default_node_pool { 50 | type = "VirtualMachineScaleSets" 51 | name = "default" 52 | node_count = 4 53 | vm_size = "Standard_D2s_v3" 54 | os_disk_size_gb = 30 55 | vnet_subnet_id = var.subnet_id 56 | enable_auto_scaling = true 57 | max_count = 15 58 | min_count = 3 59 | } 60 | 61 | lifecycle { 62 | ignore_changes = [ 63 | default_node_pool.0.node_count, 64 | ] 65 | } 66 | 67 | addon_profile { 68 | oms_agent { 69 | enabled = true 70 | log_analytics_workspace_id = azurerm_log_analytics_workspace.aks.id 71 | } 72 | } 73 | 74 | service_principal { 75 | client_id = var.aks_sp_client_id 76 | client_secret = var.aks_sp_client_secret 77 | } 78 | 79 | depends_on = [ 80 | azurerm_role_assignment.aks_subnet 81 | ] 82 | } 83 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/aks/outputs.tf: -------------------------------------------------------------------------------- 1 | output "kube_config_raw" { 2 | value = azurerm_kubernetes_cluster.aks.kube_config_raw 3 | } 4 | 5 | output "kubernetes_version" { 6 | value = azurerm_kubernetes_cluster.aks.kubernetes_version 7 | } 8 | 9 | output "name" { 10 | value = azurerm_kubernetes_cluster.aks.name 11 | } 12 | 13 | output "id" { 14 | value = azurerm_kubernetes_cluster.aks.id 15 | } 16 | 17 | output "location" { 18 | value = azurerm_kubernetes_cluster.aks.location 19 | } 20 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/aks/variables.tf: -------------------------------------------------------------------------------- 1 | variable "appname" { 2 | type = string 3 | } 4 | 5 | variable "environment" { 6 | type = string 7 | } 8 | 9 | variable "resource_group_name" { 10 | type = string 11 | } 12 | 13 | variable "location" { 14 | type = string 15 | } 16 | 17 | variable "aks_version" { 18 | type = string 19 | } 20 | 21 | variable "subnet_id" { 22 | type = string 23 | } 24 | 25 | variable "aks_sp_client_id" { 26 | type = string 27 | } 28 | 29 | variable "aks_sp_object_id" { 30 | type = string 31 | } 32 | 33 | variable "aks_sp_client_secret" { 34 | type = string 35 | } 36 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/app-insights/main.tf: -------------------------------------------------------------------------------- 1 | # Application Insights 2 | 3 | resource "azurerm_application_insights" "appi" { 4 | name = "appi-${var.appname}-${var.environment}" 5 | location = var.location 6 | resource_group_name = var.resource_group_name 7 | application_type = "other" 8 | } 9 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/app-insights/outputs.tf: -------------------------------------------------------------------------------- 1 | output "instrumentation_key" { 2 | value = azurerm_application_insights.appi.instrumentation_key 3 | } 4 | 5 | output "app_id" { 6 | value = azurerm_application_insights.appi.app_id 7 | } 8 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/app-insights/variables.tf: -------------------------------------------------------------------------------- 1 | variable "appname" { 2 | type = string 3 | } 4 | 5 | variable "environment" { 6 | type = string 7 | } 8 | 9 | variable "resource_group_name" { 10 | type = string 11 | } 12 | 13 | variable "location" { 14 | type = string 15 | } 16 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/backend.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform backend 2 | terraform { 3 | # Backend variables are initialized by Azure DevOps 4 | backend "azurerm" {} 5 | } 6 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/cosmosdb/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_cosmosdb_account" "db" { 2 | name = "cosmos-${var.appname}-${var.environment}" 3 | location = var.location 4 | resource_group_name = var.resource_group_name 5 | offer_type = "Standard" 6 | 7 | consistency_policy { 8 | consistency_level = "BoundedStaleness" 9 | } 10 | 11 | geo_location { 12 | location = var.location 13 | failover_priority = 0 14 | } 15 | } 16 | 17 | resource "azurerm_role_assignment" "app_cosmosdb" { 18 | scope = azurerm_cosmosdb_account.db.id 19 | role_definition_name = "DocumentDB Account Contributor" 20 | principal_id = var.app_sp_object_id 21 | } 22 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/cosmosdb/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cosmosdb_account_name" { 2 | value = azurerm_cosmosdb_account.db.name 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/cosmosdb/variables.tf: -------------------------------------------------------------------------------- 1 | variable "appname" { 2 | type = string 3 | } 4 | 5 | variable "environment" { 6 | type = string 7 | } 8 | 9 | variable "resource_group_name" { 10 | type = string 11 | } 12 | 13 | variable "location" { 14 | type = string 15 | } 16 | 17 | variable "app_sp_object_id" { 18 | type = string 19 | } 20 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/devops-agent/devops_agent_init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | test -n "$1" || { echo "The argument az_devops_url must be provided"; exit 1; } 5 | az_devops_url="$1" 6 | [[ "$az_devops_url" == */ ]] || { echo "The argument az_devops_url must end with /"; exit 1; } 7 | test -n "$2" || { echo "The argument az_devops_pat must be provided"; exit 1; } 8 | az_devops_pat="$2" 9 | test -n "$3" || { echo "The argument az_devops_agent_pool must be provided"; exit 1; } 10 | az_devops_agent_pool="$3" 11 | test -n "$4" || { echo "The argument az_devops_agents_per_vm must be provided"; exit 1; } 12 | az_devops_agents_per_vm="$4" 13 | 14 | 15 | #strict mode, fail on error 16 | set -euo pipefail 17 | 18 | 19 | echo "start" 20 | 21 | if test -e /dev/sdc && ! test -f /var/lib/docker-mounted-data-disk; then 22 | 23 | echo "Configure data disk /dev/sdc as Docker data directory" 24 | 25 | if test ! -e /dev/sdc1; then 26 | 27 | echo "Formatting data disk" 28 | 29 | ( 30 | echo n # Add a new partition 31 | echo p # Primary partition 32 | echo 1 # Partition number 33 | echo # First sector (Accept default) 34 | echo # Last sector (Accept default) 35 | echo p # Print partition table 36 | echo w # Write changes 37 | ) | fdisk /dev/sdc 38 | 39 | partprobe 40 | fi 41 | 42 | mkfs -t xfs /dev/sdc1 43 | 44 | mkdir /var/lib/docker 45 | mount /dev/sdc1 /var/lib/docker 46 | 47 | var=$(blkid /dev/sdc1 -s UUID | awk -F'UUID="|"' '{print $2}') 48 | echo >> /etc/fstab "UUID=$var /var/lib/docker xfs defaults,nofail 1 2" 49 | touch /var/lib/docker-mounted-data-disk 50 | 51 | fi 52 | 53 | echo "install Ubuntu packages" 54 | 55 | # To make it easier for build and release pipelines to run apt-get, 56 | # configure apt to not require confirmation (assume the -y argument by default) 57 | export DEBIAN_FRONTEND=noninteractive 58 | echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90assumeyes 59 | echo 'Dpkg::Use-Pty "0";' > /etc/apt/apt.conf.d/00usepty 60 | 61 | 62 | apt-get update 63 | apt-get install -y --no-install-recommends \ 64 | ca-certificates \ 65 | jq \ 66 | apt-transport-https \ 67 | docker.io \ 68 | curl \ 69 | unzip 70 | 71 | echo "Configuring regular cleanup of cached Docker images and data" 72 | 73 | cat > /etc/cron.d/delete_old_docker_images << EOF 74 | 01 * * * * root docker system prune --all --force --filter "until=24h" 75 | EOF 76 | 77 | echo "Allowing agent to run docker" 78 | 79 | usermod -aG docker azuredevopsuser 80 | 81 | echo "Installing Azure CLI" 82 | 83 | curl -sL https://aka.ms/InstallAzureCLIDeb | bash 84 | 85 | echo "install VSTS Agent" 86 | 87 | cd /home/azuredevopsuser 88 | mkdir -p agent 89 | cd agent 90 | 91 | AGENTRELEASE="$(curl -s https://api.github.com/repos/Microsoft/azure-pipelines-agent/releases/latest | grep -oP '"tag_name": "v\K(.*)(?=")')" 92 | AGENTURL="https://vstsagentpackage.azureedge.net/agent/${AGENTRELEASE}/vsts-agent-linux-x64-${AGENTRELEASE}.tar.gz" 93 | echo "Release "${AGENTRELEASE}" appears to be latest" 94 | echo "Downloading..." 95 | wget -q -O agent_package.tar.gz ${AGENTURL} 96 | 97 | for agent_num in $(seq 1 $az_devops_agents_per_vm); do 98 | agent_dir="agent-$agent_num" 99 | mkdir -p "$agent_dir" 100 | pushd "$agent_dir" 101 | agent_id="${HOSTNAME}_${agent_num}" 102 | echo "installing agent $agent_id" 103 | tar zxf ../agent_package.tar.gz 104 | chmod -R 777 . 105 | echo "extracted" 106 | ./bin/installdependencies.sh 107 | echo "dependencies installed" 108 | 109 | if test -e .agent; then 110 | echo "attempting to uninstall agent" 111 | ./svc.sh stop || true 112 | ./svc.sh uninstall || true 113 | sudo -u azuredevopsuser ./config.sh remove --unattended --auth pat --token "$az_devops_pat" || true 114 | fi 115 | 116 | echo "running installation" 117 | sudo -u azuredevopsuser ./config.sh --unattended --url "$az_devops_url" --auth pat --token "$az_devops_pat" --pool "$az_devops_agent_pool" --agent "$agent_id" --acceptTeeEula --work ./_work --runAsService 118 | echo "configuration done" 119 | ./svc.sh install 120 | echo "service installed" 121 | ./svc.sh start 122 | echo "service started" 123 | echo "config done" 124 | popd 125 | done 126 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/devops-agent/install_software.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #strict mode, fail on error 4 | set -euo pipefail 5 | 6 | # Install Python 3 packages for installing Taurus 7 | apt-get install python3-pip python3-dev 8 | 9 | # Install JDK for running JMeter 10 | apt-get install openjdk-8-jdk 11 | 12 | # Install the .NET Core SDK, for test coverage report generation 13 | wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb 14 | dpkg -i packages-microsoft-prod.deb 15 | add-apt-repository universe 16 | apt-get update 17 | apt-get install apt-transport-https 18 | apt-get update 19 | apt-get install dotnet-sdk-3.1 20 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/devops-agent/main.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_user_assigned_identity" "devops" { 2 | resource_group_name = var.resource_group_name 3 | location = var.location 4 | name = "agent${var.appname}${var.environment}" 5 | } 6 | 7 | resource "azurerm_storage_account" "devops" { 8 | name = "stado${var.appname}${var.environment}" 9 | resource_group_name = var.resource_group_name 10 | location = var.location 11 | account_tier = "Standard" 12 | account_replication_type = "LRS" 13 | } 14 | 15 | resource "azurerm_storage_container" "devops" { 16 | name = "content" 17 | storage_account_name = azurerm_storage_account.devops.name 18 | container_access_type = "private" 19 | } 20 | 21 | locals { 22 | init_script = join("\n\n", [file("${path.module}/devops_agent_init.sh"), file("${path.module}/install_software.sh")]) 23 | max_int32_value = 2147483647 24 | } 25 | 26 | resource "azurerm_storage_blob" "devops_agent_init" { 27 | name = "provision_agent.sh" 28 | storage_account_name = azurerm_storage_account.devops.name 29 | storage_container_name = azurerm_storage_container.devops.name 30 | type = "Block" 31 | source_content = local.init_script 32 | } 33 | 34 | data "azurerm_storage_account_blob_container_sas" "devops_agent_init" { 35 | connection_string = azurerm_storage_account.devops.primary_connection_string 36 | container_name = azurerm_storage_container.devops.name 37 | https_only = true 38 | 39 | start = "2000-01-01" 40 | expiry = "2099-01-01" 41 | 42 | permissions { 43 | read = true 44 | add = false 45 | create = false 46 | write = false 47 | delete = false 48 | list = false 49 | } 50 | } 51 | 52 | 53 | # Create virtual machine 54 | 55 | resource "random_password" "agent_vms" { 56 | length = 24 57 | special = true 58 | override_special = "!@#$%&*()-_=+[]:?" 59 | min_upper = 1 60 | min_lower = 1 61 | min_numeric = 1 62 | min_special = 1 63 | } 64 | 65 | resource "azurerm_linux_virtual_machine_scale_set" "devops" { 66 | # limit name length to avoid conflict in truncated service names on the VMs 67 | name = format("%.24s", "vm${var.appname}devops${var.environment}") 68 | location = var.location 69 | resource_group_name = var.resource_group_name 70 | network_interface { 71 | name = "nic" 72 | primary = true 73 | 74 | ip_configuration { 75 | name = "AzureDevOpsNicConfiguration" 76 | subnet_id = var.subnet_id 77 | primary = true 78 | } 79 | } 80 | sku = var.az_devops_agent_vm_size 81 | 82 | os_disk { 83 | caching = "ReadWrite" 84 | storage_account_type = "Premium_LRS" 85 | } 86 | 87 | data_disk { 88 | caching = "ReadWrite" 89 | disk_size_gb = 128 90 | lun = 0 91 | storage_account_type = "Premium_LRS" 92 | } 93 | 94 | source_image_reference { 95 | publisher = "Canonical" 96 | offer = "UbuntuServer" 97 | sku = "18.04-LTS" 98 | version = "latest" 99 | } 100 | 101 | admin_username = var.az_devops_agent_admin_user 102 | admin_password = random_password.agent_vms.result 103 | 104 | disable_password_authentication = false 105 | 106 | dynamic "admin_ssh_key" { 107 | for_each = var.az_devops_agent_sshkeys 108 | content { 109 | username = "azuredevopsuser" 110 | public_key = each.key 111 | } 112 | } 113 | 114 | boot_diagnostics { 115 | storage_account_uri = azurerm_storage_account.devops.primary_blob_endpoint 116 | } 117 | 118 | identity { 119 | type = "UserAssigned" 120 | identity_ids = [azurerm_user_assigned_identity.devops.id] 121 | } 122 | 123 | # must scale up instances after azurerm_virtual_machine_scale_set_extension has been applied 124 | instances = 0 125 | scale_in_policy = "NewestVM" 126 | 127 | lifecycle { 128 | ignore_changes = [ 129 | instances 130 | ] 131 | } 132 | } 133 | 134 | resource "azurerm_virtual_machine_scale_set_extension" "devops" { 135 | name = "install_azure_devops_agent" 136 | virtual_machine_scale_set_id = azurerm_linux_virtual_machine_scale_set.devops.id 137 | publisher = "Microsoft.Azure.Extensions" 138 | type = "CustomScript" 139 | type_handler_version = "2.0" 140 | 141 | #timestamp: use this field only to trigger a re-run of the script by changing value of this field. 142 | # Any int32 value is acceptable; it must only be different than the previous value. 143 | settings = jsonencode({ 144 | "timestamp" : parseint(sha1(local.init_script), 16) % local.max_int32_value 145 | }) 146 | protected_settings = jsonencode({ 147 | "fileUris": ["${azurerm_storage_blob.devops_agent_init.url}${data.azurerm_storage_account_blob_container_sas.devops_agent_init.sas}"], 148 | "commandToExecute": "bash ${azurerm_storage_blob.devops_agent_init.name} '${var.az_devops_url}' '${var.az_devops_pat}' '${var.az_devops_agent_pool}' '${var.az_devops_agents_per_vm}'" 149 | }) 150 | #output goes to /var/lib/waagent/custom-script 151 | } 152 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/devops-agent/outputs.tf: -------------------------------------------------------------------------------- 1 | output "agent_vmss_id" { 2 | value = azurerm_linux_virtual_machine_scale_set.devops.id 3 | } 4 | 5 | output "agent_vmss_name" { 6 | value = azurerm_linux_virtual_machine_scale_set.devops.name 7 | } 8 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/devops-agent/variables.tf: -------------------------------------------------------------------------------- 1 | variable "appname" { 2 | type = string 3 | } 4 | 5 | variable "environment" { 6 | type = string 7 | } 8 | 9 | variable "resource_group_name" { 10 | type = string 11 | } 12 | 13 | variable "location" { 14 | type = string 15 | } 16 | 17 | variable "subnet_id" { 18 | type = string 19 | } 20 | 21 | variable "az_devops_url" { 22 | type = string 23 | description = "Specify the Azure DevOps url e.g. https://dev.azure.com/myorg" 24 | } 25 | 26 | variable "az_devops_pat" { 27 | type = string 28 | description = "Provide a Personal Access Token (PAT) for Azure DevOps. Create it at https://dev.azure.com/[Organization]/_usersSettings/tokens with permission Agent Pools > Read & manage" 29 | } 30 | 31 | variable "az_devops_agent_pool" { 32 | type = string 33 | description = "Specify the name of the agent pool - must exist before. Create it at https://dev.azure.com/[Organization]/_settings/agentpools" 34 | } 35 | 36 | variable "az_devops_agent_admin_user" { 37 | type = string 38 | default = "azuredevopsuser" 39 | description = "Username of the admin user on the agent VMs" 40 | } 41 | 42 | variable "az_devops_agent_sshkeys" { 43 | type = list(string) 44 | description = "Optionally provide ssh public key(s) to logon to the VM" 45 | } 46 | 47 | variable "az_devops_agent_vm_size" { 48 | type = string 49 | description = "Specify the size of the VM" 50 | } 51 | 52 | variable "az_devops_agents_per_vm" { 53 | type = number 54 | description = "Number of Azure DevOps agents spawned per VM. Agents will be named with a random prefix." 55 | default = 4 56 | } 57 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/main.tf: -------------------------------------------------------------------------------- 1 | # For suggested naming conventions, refer to: 2 | # https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/ready/azure-best-practices/naming-and-tagging 3 | 4 | module "acr" { 5 | source = "./acr" 6 | acr_name = var.acr_name 7 | resource_group_name = var.resource_group 8 | location = var.location 9 | aks_sp_object_id = var.aks_sp_object_id 10 | } 11 | 12 | module "vnet" { 13 | source = "./vnet" 14 | appname = var.appname 15 | environment = var.environment 16 | resource_group_name = var.resource_group 17 | location = var.location 18 | } 19 | 20 | module "devops-agent" { 21 | source = "./devops-agent" 22 | appname = var.appname 23 | environment = var.environment 24 | location = var.location 25 | resource_group_name = var.resource_group 26 | subnet_id = module.vnet.agents_subnet_id 27 | az_devops_url = var.az_devops_url 28 | az_devops_pat = var.az_devops_pat 29 | az_devops_agent_pool = var.az_devops_agent_pool 30 | az_devops_agents_per_vm = var.az_devops_agents_per_vm 31 | az_devops_agent_sshkeys = var.az_devops_agent_sshkeys 32 | az_devops_agent_vm_size = var.az_devops_agent_vm_size 33 | } 34 | 35 | module "aks" { 36 | source = "./aks" 37 | appname = var.appname 38 | environment = var.environment 39 | aks_version = var.aks_version 40 | resource_group_name = var.resource_group 41 | location = var.location 42 | subnet_id = module.vnet.aks_subnet_id 43 | aks_sp_client_id = var.aks_sp_client_id 44 | aks_sp_object_id = var.aks_sp_object_id 45 | aks_sp_client_secret = var.aks_sp_client_secret 46 | } 47 | 48 | module "cosmosdb" { 49 | source = "./cosmosdb" 50 | appname = var.appname 51 | environment = var.environment 52 | resource_group_name = var.resource_group 53 | location = var.location 54 | app_sp_object_id = var.app_sp_object_id 55 | } 56 | 57 | module "app-insights" { 58 | source = "./app-insights" 59 | appname = var.appname 60 | environment = var.environment 61 | resource_group_name = var.resource_group 62 | location = var.location 63 | } 64 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/outputs.tf: -------------------------------------------------------------------------------- 1 | output "subscription_id" { 2 | value = data.azurerm_client_config.current.subscription_id 3 | } 4 | 5 | output "vnet_name" { 6 | value = module.vnet.name 7 | } 8 | 9 | output "vnet_id" { 10 | value = module.vnet.id 11 | } 12 | 13 | output "agent_pool_name" { 14 | value = var.az_devops_agent_pool 15 | } 16 | 17 | output "agent_vmss_name" { 18 | value = module.devops-agent.agent_vmss_name 19 | } 20 | 21 | output "aks_name" { 22 | value = module.aks.name 23 | } 24 | 25 | output "kube_config_base64" { 26 | value = base64encode(module.aks.kube_config_raw) 27 | sensitive = true 28 | } 29 | 30 | output "kubernetes_version" { 31 | value = module.aks.kubernetes_version 32 | } 33 | 34 | output "cosmosdb_account_name" { 35 | value = module.cosmosdb.cosmosdb_account_name 36 | } 37 | 38 | output "instrumentation_key" { 39 | value = module.app-insights.instrumentation_key 40 | } 41 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/providers.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform required version 2 | 3 | terraform { 4 | required_version = ">= 0.12.6" 5 | } 6 | 7 | # Configure the Azure Provider 8 | 9 | provider "azurerm" { 10 | # It is recommended to pin to a given version of the Provider 11 | version = "=2.2.0" 12 | skip_provider_registration = true 13 | features {} 14 | } 15 | 16 | # Configure other Providers 17 | 18 | provider "random" { 19 | version = "~> 2.2" 20 | } 21 | 22 | # Data 23 | 24 | # Provides client_id, tenant_id, subscription_id and object_id variables 25 | data "azurerm_client_config" "current" {} 26 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/variables.tf: -------------------------------------------------------------------------------- 1 | variable "appname" { 2 | type = string 3 | description = "Application name. Use only lowercase letters and numbers" 4 | } 5 | 6 | variable "environment" { 7 | type = string 8 | description = "Environment name, e.g. 'dev' or 'cd'" 9 | default = "dev" 10 | } 11 | 12 | variable "location" { 13 | type = string 14 | description = "Azure region where to create resources." 15 | default = "East US" 16 | } 17 | 18 | variable "resource_group" { 19 | type = string 20 | description = "Resource group to deploy in." 21 | } 22 | 23 | variable "az_devops_url" { 24 | type = string 25 | description = "Specify the Azure DevOps url e.g. https://dev.azure.com/myorg" 26 | } 27 | 28 | variable "az_devops_pat" { 29 | type = string 30 | description = "Provide a Personal Access Token (PAT) for Azure DevOps. Create it at https://dev.azure.com/[Organization]/_usersSettings/tokens with permission Agent Pools > Read & manage" 31 | } 32 | 33 | variable "az_devops_agent_pool" { 34 | type = string 35 | description = "Specify the name of the agent pool - must exist before. Create it at https://dev.azure.com/[Organization]/_settings/agentpools" 36 | default = "pool001" 37 | } 38 | 39 | variable "az_devops_agent_sshkeys" { 40 | type = list(string) 41 | description = "Optionally provide ssh public key(s) to logon to the VM" 42 | default = [] 43 | } 44 | 45 | variable "az_devops_agent_vm_size" { 46 | type = string 47 | description = "Specify the size of the VM" 48 | default = "Standard_B2ms" 49 | } 50 | 51 | variable "az_devops_agents_per_vm" { 52 | type = number 53 | description = "Number of Azure DevOps agents spawned per VM. Agents will be named with a random prefix." 54 | default = 4 55 | } 56 | 57 | variable "acr_name" { 58 | type = string 59 | description = "Name of the generated Azure Container Registry instance." 60 | } 61 | 62 | variable "aks_version" { 63 | type = string 64 | description = "Kubernetes version of the AKS cluster." 65 | } 66 | 67 | variable "aks_sp_client_id" { 68 | type = string 69 | description = "Service principal client ID for the Azure Kubernetes Service cluster identity." 70 | } 71 | 72 | variable "aks_sp_object_id" { 73 | type = string 74 | description = "Service principal object ID for the Azure Kubernetes Service cluster identity. Should be object IDs of service principals, not object IDs of the application nor application IDs. To retrieve, navigate in the AAD portal from an App registration to 'Managed application in local directory'." 75 | } 76 | 77 | variable "aks_sp_client_secret" { 78 | type = string 79 | description = "Service principal client secret for the Azure Kubernetes Service cluster identity." 80 | } 81 | 82 | variable "app_sp_object_id" { 83 | type = string 84 | description = "Service principal object ID for the application principal to be granted permissions on Aure resources." 85 | } 86 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/vnet/main.tf: -------------------------------------------------------------------------------- 1 | # Create virtual network 2 | resource "azurerm_virtual_network" "main" { 3 | name = "vnet-${var.appname}-${var.environment}" 4 | address_space = ["10.100.0.0/16"] 5 | location = var.location 6 | resource_group_name = var.resource_group_name 7 | } 8 | 9 | # Create subnets 10 | 11 | resource "azurerm_subnet" "aks" { 12 | name = "aks-subnet" 13 | resource_group_name = var.resource_group_name 14 | virtual_network_name = azurerm_virtual_network.main.name 15 | address_prefix = "10.100.1.0/24" 16 | } 17 | 18 | resource "azurerm_subnet" "agents" { 19 | name = "agents-subnet" 20 | resource_group_name = var.resource_group_name 21 | virtual_network_name = azurerm_virtual_network.main.name 22 | address_prefix = "10.100.2.0/24" 23 | } 24 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/vnet/outputs.tf: -------------------------------------------------------------------------------- 1 | output "name" { 2 | value = azurerm_virtual_network.main.name 3 | } 4 | 5 | output "id" { 6 | value = azurerm_virtual_network.main.id 7 | } 8 | 9 | output "aks_subnet_name" { 10 | value = azurerm_subnet.aks.name 11 | } 12 | 13 | output "aks_subnet_id" { 14 | value = azurerm_subnet.aks.id 15 | } 16 | 17 | output "agents_subnet_id" { 18 | value = azurerm_subnet.agents.id 19 | } 20 | -------------------------------------------------------------------------------- /infrastructure/terraform-shared/vnet/variables.tf: -------------------------------------------------------------------------------- 1 | variable "appname" { 2 | type = string 3 | } 4 | 5 | variable "environment" { 6 | type = string 7 | } 8 | 9 | 10 | variable "resource_group_name" { 11 | type = string 12 | } 13 | 14 | variable "location" { 15 | type = string 16 | } 17 | -------------------------------------------------------------------------------- /infrastructure/terraform-template.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | - name: TerraformApply 3 | type: boolean 4 | default: false 5 | - name: TerraformDestroy 6 | type: boolean 7 | default: false 8 | - name: TerraformStateKey 9 | type: string 10 | - name: TerraformVariables 11 | type: object 12 | - name: TerraformDirectory 13 | default: $(TERRAFORM_DIRECTORY) 14 | - name: TerraformVersion 15 | default: 0.12.24 16 | 17 | steps: 18 | 19 | - bash: | 20 | set -eu # fail on error 21 | base64 -d <<< $KUBE_CONFIG_BASE64 > kube_config 22 | echo "##vso[task.setvariable variable=KUBECONFIG]$PWD/kube_config" 23 | displayName: Save kubeconfig 24 | condition: ne(variables.KUBE_CONFIG_BASE64, '') 25 | env: 26 | KUBE_CONFIG_BASE64: $(KUBE_CONFIG_BASE64) 27 | 28 | - task: ms-devlabs.custom-terraform-tasks.custom-terraform-installer-task.TerraformInstaller@0 29 | displayName: Install Terraform 30 | inputs: 31 | terraformVersion: ${{ parameters.TerraformVersion }} 32 | 33 | - bash: | 34 | set -eux # fail on error 35 | terraform init \ 36 | -input=false \ 37 | -backend-config=storage_account_name=$(TERRAFORM_STORAGE_ACCOUNT) \ 38 | -backend-config=container_name=terraformstate \ 39 | -backend-config=key=${{ parameters.TerraformStateKey }}.tfstate \ 40 | -backend-config=resource_group_name=$(RESOURCE_GROUP) \ 41 | workingDirectory: $(TERRAFORM_DIRECTORY) 42 | displayName: Terraform init 43 | env: 44 | ${{ parameters.TerraformVariables }} 45 | 46 | - bash: | 47 | set -eu 48 | terraform destroy -input=false -auto-approve 49 | displayName: Terraform destroy 50 | condition: ${{ parameters.TerraformDestroy }} 51 | workingDirectory: $(TERRAFORM_DIRECTORY) 52 | env: 53 | ${{ parameters.TerraformVariables }} 54 | 55 | - bash: | 56 | set -eu 57 | terraform plan -out=tfplan -input=false 58 | terraform apply -input=false -auto-approve tfplan 59 | displayName: Terraform apply 60 | condition: ${{ parameters.TerraformApply }} 61 | workingDirectory: $(TERRAFORM_DIRECTORY) 62 | env: 63 | ${{ parameters.TerraformVariables }} 64 | 65 | - bash: | 66 | set -euo pipefail 67 | 68 | echo "Setting job variables from Terraform outputs:" 69 | 70 | terraform output -json | jq -r ' 71 | . as $in 72 | | keys[] 73 | | ($in[.].value | tostring | gsub("\\\\"; "\\\\") | gsub("\n"; "\\n")) as $value 74 | | ($in[.].sensitive | tostring) as $sensitive 75 | | [ 76 | "- " + . + ": " + if $in[.].sensitive then "(sensitive)" else $value end, # output name to console 77 | "##vso[task.setvariable variable=" + . + ";isSecret=" + $sensitive + "]" + $value, # set as ADO task variable 78 | "##vso[task.setvariable variable=" + . + ";isOutput=true;isSecret=" + $sensitive + "]" + $value # also set as ADO job variable 79 | ] 80 | | .[]' 81 | 82 | name: Outputs 83 | displayName: Read Terraform outputs 84 | workingDirectory: ${{ parameters.TerraformDirectory }} 85 | env: 86 | ${{ parameters.TerraformVariables }} 87 | -------------------------------------------------------------------------------- /infrastructure/terraform/backend.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform backend 2 | terraform { 3 | # Backend variables are initialized by Azure DevOps 4 | backend "azurerm" {} 5 | } 6 | -------------------------------------------------------------------------------- /infrastructure/terraform/main.tf: -------------------------------------------------------------------------------- 1 | # For suggested naming conventions, refer to: 2 | # https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/ready/azure-best-practices/naming-and-tagging 3 | 4 | resource "kubernetes_namespace" "build" { 5 | metadata { 6 | name = var.area_name 7 | } 8 | } 9 | 10 | resource "azurerm_cosmosdb_sql_database" "sums" { 11 | name = var.area_name 12 | resource_group_name = var.resource_group 13 | account_name = var.cosmosdb_account_name 14 | throughput = 400 15 | } 16 | 17 | resource "azurerm_cosmosdb_sql_container" "sums" { 18 | name = "ComputedSums" 19 | resource_group_name = var.resource_group 20 | account_name = var.cosmosdb_account_name 21 | database_name = azurerm_cosmosdb_sql_database.sums.name 22 | partition_key_path = "/Id" 23 | throughput = 400 24 | } 25 | -------------------------------------------------------------------------------- /infrastructure/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "kubernetes_namespace" { 2 | value = kubernetes_namespace.build.metadata[0].name 3 | } 4 | 5 | output "cosmosdb_container_id" { 6 | value = azurerm_cosmosdb_sql_container.sums.id 7 | } 8 | -------------------------------------------------------------------------------- /infrastructure/terraform/providers.tf: -------------------------------------------------------------------------------- 1 | #Set the terraform required version 2 | 3 | terraform { 4 | required_version = ">= 0.12.6" 5 | } 6 | 7 | # Configure the Azure Provider 8 | 9 | provider "azurerm" { 10 | # It is recommended to pin to a given version of the Provider 11 | version = "=2.2.0" 12 | skip_provider_registration = true 13 | features {} 14 | } 15 | 16 | provider "kubernetes" { 17 | } 18 | -------------------------------------------------------------------------------- /infrastructure/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "location" { 2 | type = string 3 | description = "Azure region where to create resources." 4 | default = "North Europe" 5 | } 6 | 7 | variable "resource_group" { 8 | type = string 9 | description = "Resource group to deploy in." 10 | } 11 | 12 | variable "appname" { 13 | type = string 14 | description = "Application name. Use only lowercase letters and numbers" 15 | } 16 | 17 | variable "area_name" { 18 | type = string 19 | description = "'Area' name to create, name from which resource names and Kubernetes namespace are derived." 20 | } 21 | 22 | variable "cosmosdb_account_name" { 23 | type = string 24 | description = "The Cosmos DB Account name in which to save results." 25 | } 26 | --------------------------------------------------------------------------------