├── .github └── workflows │ ├── main.yaml │ └── patch.yaml ├── .gitignore ├── Deployment.md ├── LICENSE ├── README.md ├── apps ├── bookstore │ ├── bookinfo-gateway.yaml │ ├── bookinfo.yaml │ ├── kustomization.yaml │ └── namespace.yaml ├── chat │ └── chat.yaml ├── eShopOnDapr │ └── eshop.yaml ├── httpbin │ ├── deployment.yaml │ └── kustomization.yaml └── whatos-api │ ├── knative-service.yaml │ ├── kustomization.yaml │ └── namespace.yaml ├── assets ├── namespaces.png └── resources.png ├── cluster-manifests ├── apps.yaml ├── common │ ├── customresourcedefinitions │ │ ├── istio-operator-crd.yaml │ │ ├── knative-serving-crds.yaml │ │ └── kustomization.yaml │ ├── istio │ │ ├── configuration │ │ │ ├── istio-cluster-roles.yaml │ │ │ ├── istio-configuration.yaml │ │ │ └── kustomization.yaml │ │ └── gateway │ │ │ ├── default-ingress.yaml │ │ │ ├── istio-ingress-gateway.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── secretsprovider.yaml │ │ │ └── serviceaccount.yaml │ ├── knative │ │ ├── kustomization.yaml │ │ ├── net-istio.yaml │ │ ├── serving-config.yaml │ │ └── serving-core.yaml │ ├── kubecost.yaml │ ├── kured.yaml │ ├── kustomization.yaml │ ├── otel │ │ ├── deployment.yaml │ │ ├── kustomization.yaml │ │ ├── namespace.yaml │ │ ├── secretsprovider.yaml │ │ └── serviceaccount.yaml │ └── prometheus │ │ ├── ama-metrics-settings-configmap.yaml │ │ ├── container-azm-ms-agentconfig.yaml │ │ ├── kustomization.yaml │ │ └── prometheus-config └── kustomization.yaml ├── infrastructure ├── acr.tf ├── aks-cluster.tf ├── aks-diag-setup.tf ├── aks-nodepools.tf ├── aks-post-setup-config.tf ├── cluster.tfvars ├── dapr.tf ├── flux.tf ├── github-actions.tf ├── identities.tf ├── keyvault.tf ├── logging.tf ├── main.tf ├── network.tf ├── otel-config.tpl ├── prereqs │ ├── azuredeploy.central.json │ ├── azuredeploy.northcentral.json │ ├── azuredeploy.nsg.json │ ├── azuredeploy.southcentral.json │ └── azuredeploy.template.json ├── prometheus-recording-rules.tf ├── prometheus.tf ├── providers.tf ├── roles.tf └── variables.tf └── scripts ├── aks-application-bootstrap.sh ├── aks-certificate-rotation.sh ├── aks-preview-features.sh └── aks-repave.sh /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Creates K8s with a Mesh installed 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | cluster_name: 6 | description: 'The name of the cluster to create - default_cluster_name|f1dd32' 7 | required: true 8 | default: 'default_cluster_name' 9 | 10 | jobs: 11 | deploy: 12 | name: Deploy Cluster 13 | runs-on: self-hosted 14 | env: 15 | ARM_USE_MSI: true 16 | ARM_CLIENT_ID: ${{ secrets.MSI_CLIENT_ID }} 17 | ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} 18 | CORE_SUBSCRIPTION_ID: ${{ secrets.CORE_SUBSCRIPTION_ID }} 19 | ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} 20 | GITHUB_TOKEN: ${{ secrets.PAT_TOKEN }} 21 | CLUSTER_RG: Apps02_K8S_${{ github.event.inputs.cluster_name }}_RG 22 | CLUSTER_NAME: ${{ github.event.inputs.cluster_name }} 23 | GITHUB_ACCOUNT: briandenicola 24 | GITHUB_REPO: kubernetes-cluster-setup 25 | REPO_BRANCH: cluster/default_cluster_name 26 | 27 | defaults: 28 | run: 29 | working-directory: './infrastructure' 30 | 31 | steps: 32 | - name: "Checkout" 33 | uses: actions/checkout@master 34 | 35 | - name: Login to Azure 36 | run: | 37 | az login --identity -u ${{ env.ARM_CLIENT_ID }} 38 | 39 | - name: "Terraform Setup" 40 | uses: hashicorp/setup-terraform@v1 41 | 42 | - name: "Terraform Init" 43 | id: init 44 | run: terraform init -backend=true -backend-config="tenant_id=${{ env.ARM_TENANT_ID }}" -backend-config="subscription_id=${{ env.CORE_SUBSCRIPTION_ID }}" -backend-config="key=${{ env.CLUSTER_NAME }}.terraform.tfstate" 45 | 46 | - name: "Terraform Plan" 47 | id: plan 48 | run: terraform plan -out="${{ env.CLUSTER_NAME }}.plan" -var "cluster_name=${{ env.CLUSTER_NAME }}" -var "resource_group_name=${{ env.CLUSTER_RG }}" -var "certificate_base64_encoded=${{ secrets.CERTIFICATE }}" -var "certificate_password=${{ secrets.CERT_PASSWORD }}" -var-file="cluster.tfvars" 49 | continue-on-error: false 50 | 51 | - name: "Terraform Apply" 52 | id: apply 53 | run: terraform apply -auto-approve ${{ env.CLUSTER_NAME }}.plan -------------------------------------------------------------------------------- /.github/workflows/patch.yaml: -------------------------------------------------------------------------------- 1 | name: Perform a Node Upgrade on an AKS cluster 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | cluster_name: 6 | description: 'The name of the cluster to create - default_cluster_name|f1dd32' 7 | required: true 8 | default: 'default_cluster_name' 9 | mesh_type: 10 | description: 'Type of service mesh to include in the cluster - istio|osm' 11 | required: true 12 | default: 'istio' 13 | 14 | jobs: 15 | upgrade-node: 16 | name: Update Cluster Node Image 17 | runs-on: ubuntu-latest 18 | 19 | env: 20 | ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }} 21 | ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }} 22 | ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} 23 | CLUSTER_RG: AApps02_K8S_${{ github.event.inputs.cluster_name }}_RG 24 | CLUSTER_NAME: ${{ github.event.inputs.cluster_name }} 25 | 26 | defaults: 27 | run: 28 | working-directory: './infrastructure' 29 | 30 | steps: 31 | - name: Login to Azure 32 | run: | 33 | az login --identity -u ${{ env.ARM_CLIENT_ID }} 34 | 35 | - name: Upgrade default node images 36 | uses: Azure/cli@v1.0.0 37 | with: 38 | inlineScript: az aks nodepool upgrade -n default -g ${{ env.CLUSTER_RG }} --cluster-name ${{ env.CLUSTER_NAME }} --node-image-only --max-surge 25% --yes 39 | 40 | - name: Upgrade eshop node images 41 | uses: Azure/cli@v1.0.0 42 | with: 43 | inlineScript: az aks nodepool upgrade -n eshop -g ${{ env.CLUSTER_RG }} --cluster-name ${{ env.CLUSTER_NAME }} --node-image-only --max-surge 25% --yes -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | .terraform/ 7 | .terraform* 8 | terraform.tfstate 9 | terraform.tfstate.backup 10 | 11 | # User-specific files 12 | *.rsuser 13 | *.suo 14 | *.user 15 | *.userosscache 16 | *.sln.docstates 17 | 18 | # User-specific files (MonoDevelop/Xamarin Studio) 19 | *.userprefs 20 | 21 | # Mono auto generated files 22 | mono_crash.* 23 | 24 | # Build results 25 | [Dd]ebug/ 26 | [Dd]ebugPublic/ 27 | [Rr]elease/ 28 | [Rr]eleases/ 29 | x64/ 30 | x86/ 31 | [Aa][Rr][Mm]/ 32 | [Aa][Rr][Mm]64/ 33 | bld/ 34 | [Bb]in/ 35 | [Oo]bj/ 36 | [Ll]og/ 37 | [Ll]ogs/ 38 | 39 | # Visual Studio 2015/2017 cache/options directory 40 | .vs/ 41 | # Uncomment if you have tasks that create the project's static files in wwwroot 42 | #wwwroot/ 43 | 44 | # Visual Studio 2017 auto generated files 45 | Generated\ Files/ 46 | 47 | # MSTest test Results 48 | [Tt]est[Rr]esult*/ 49 | [Bb]uild[Ll]og.* 50 | 51 | # NUnit 52 | *.VisualState.xml 53 | TestResult.xml 54 | nunit-*.xml 55 | 56 | # Build Results of an ATL Project 57 | [Dd]ebugPS/ 58 | [Rr]eleasePS/ 59 | dlldata.c 60 | 61 | # Benchmark Results 62 | BenchmarkDotNet.Artifacts/ 63 | 64 | # .NET Core 65 | project.lock.json 66 | project.fragment.lock.json 67 | artifacts/ 68 | 69 | # StyleCop 70 | StyleCopReport.xml 71 | 72 | # Files built by Visual Studio 73 | *_i.c 74 | *_p.c 75 | *_h.h 76 | *.ilk 77 | *.meta 78 | *.obj 79 | *.iobj 80 | *.pch 81 | *.pdb 82 | *.ipdb 83 | *.pgc 84 | *.pgd 85 | *.rsp 86 | *.sbr 87 | *.tlb 88 | *.tli 89 | *.tlh 90 | *.tmp 91 | *.tmp_proj 92 | *_wpftmp.csproj 93 | *.log 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Visual Studio code coverage results 146 | *.coverage 147 | *.coveragexml 148 | 149 | # NCrunch 150 | _NCrunch_* 151 | .*crunch*.local.xml 152 | nCrunchTemp_* 153 | 154 | # MightyMoose 155 | *.mm.* 156 | AutoTest.Net/ 157 | 158 | # Web workbench (sass) 159 | .sass-cache/ 160 | 161 | # Installshield output folder 162 | [Ee]xpress/ 163 | 164 | # DocProject is a documentation generator add-in 165 | DocProject/buildhelp/ 166 | DocProject/Help/*.HxT 167 | DocProject/Help/*.HxC 168 | DocProject/Help/*.hhc 169 | DocProject/Help/*.hhk 170 | DocProject/Help/*.hhp 171 | DocProject/Help/Html2 172 | DocProject/Help/html 173 | 174 | # Click-Once directory 175 | publish/ 176 | 177 | # Publish Web Output 178 | *.[Pp]ublish.xml 179 | *.azurePubxml 180 | # Note: Comment the next line if you want to checkin your web deploy settings, 181 | # but database connection strings (with potential passwords) will be unencrypted 182 | *.pubxml 183 | *.publishproj 184 | 185 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 186 | # checkin your Azure Web App publish settings, but sensitive information contained 187 | # in these scripts will be unencrypted 188 | PublishScripts/ 189 | 190 | # NuGet Packages 191 | *.nupkg 192 | # NuGet Symbol Packages 193 | *.snupkg 194 | # The packages folder can be ignored because of Package Restore 195 | **/[Pp]ackages/* 196 | # except build/, which is used as an MSBuild target. 197 | !**/[Pp]ackages/build/ 198 | # Uncomment if necessary however generally it will be regenerated when needed 199 | #!**/[Pp]ackages/repositories.config 200 | # NuGet v3's project.json files produces more ignorable files 201 | *.nuget.props 202 | *.nuget.targets 203 | 204 | # Microsoft Azure Build Output 205 | csx/ 206 | *.build.csdef 207 | 208 | # Microsoft Azure Emulator 209 | ecf/ 210 | rcf/ 211 | 212 | # Windows Store app package directories and files 213 | AppPackages/ 214 | BundleArtifacts/ 215 | Package.StoreAssociation.xml 216 | _pkginfo.txt 217 | *.appx 218 | *.appxbundle 219 | *.appxupload 220 | 221 | # Visual Studio cache files 222 | # files ending in .cache can be ignored 223 | *.[Cc]ache 224 | # but keep track of directories ending in .cache 225 | !?*.[Cc]ache/ 226 | 227 | # Others 228 | ClientBin/ 229 | ~$* 230 | *~ 231 | *.dbmdl 232 | *.dbproj.schemaview 233 | *.jfm 234 | *.pfx 235 | *.publishsettings 236 | orleans.codegen.cs 237 | 238 | # Including strong name files can present a security risk 239 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 240 | #*.snk 241 | 242 | # Since there are multiple workflows, uncomment next line to ignore bower_components 243 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 244 | #bower_components/ 245 | 246 | # RIA/Silverlight projects 247 | Generated_Code/ 248 | 249 | # Backup & report files from converting an old project file 250 | # to a newer Visual Studio version. Backup files are not needed, 251 | # because we have git ;-) 252 | _UpgradeReport_Files/ 253 | Backup*/ 254 | UpgradeLog*.XML 255 | UpgradeLog*.htm 256 | ServiceFabricBackup/ 257 | *.rptproj.bak 258 | 259 | # SQL Server files 260 | *.mdf 261 | *.ldf 262 | *.ndf 263 | 264 | # Business Intelligence projects 265 | *.rdl.data 266 | *.bim.layout 267 | *.bim_*.settings 268 | *.rptproj.rsuser 269 | *- [Bb]ackup.rdl 270 | *- [Bb]ackup ([0-9]).rdl 271 | *- [Bb]ackup ([0-9][0-9]).rdl 272 | 273 | # Microsoft Fakes 274 | FakesAssemblies/ 275 | 276 | # GhostDoc plugin setting file 277 | *.GhostDoc.xml 278 | 279 | # Node.js Tools for Visual Studio 280 | .ntvs_analysis.dat 281 | node_modules/ 282 | 283 | # Visual Studio 6 build log 284 | *.plg 285 | 286 | # Visual Studio 6 workspace options file 287 | *.opt 288 | 289 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 290 | *.vbw 291 | 292 | # Visual Studio LightSwitch build output 293 | **/*.HTMLClient/GeneratedArtifacts 294 | **/*.DesktopClient/GeneratedArtifacts 295 | **/*.DesktopClient/ModelManifest.xml 296 | **/*.Server/GeneratedArtifacts 297 | **/*.Server/ModelManifest.xml 298 | _Pvt_Extensions 299 | 300 | # Paket dependency manager 301 | .paket/paket.exe 302 | paket-files/ 303 | 304 | # FAKE - F# Make 305 | .fake/ 306 | 307 | # CodeRush personal settings 308 | .cr/personal 309 | 310 | # Python Tools for Visual Studio (PTVS) 311 | __pycache__/ 312 | *.pyc 313 | 314 | # Cake - Uncomment if you are using it 315 | # tools/** 316 | # !tools/packages.config 317 | 318 | # Tabs Studio 319 | *.tss 320 | 321 | # Telerik's JustMock configuration file 322 | *.jmconfig 323 | 324 | # BizTalk build output 325 | *.btp.cs 326 | *.btm.cs 327 | *.odx.cs 328 | *.xsd.cs 329 | 330 | # OpenCover UI analysis results 331 | OpenCover/ 332 | 333 | # Azure Stream Analytics local run output 334 | ASALocalRun/ 335 | 336 | # MSBuild Binary and Structured Log 337 | *.binlog 338 | 339 | # NVidia Nsight GPU debugger configuration file 340 | *.nvuser 341 | 342 | # MFractors (Xamarin productivity tool) working folder 343 | .mfractor/ 344 | 345 | # Local History for Visual Studio 346 | .localhistory/ 347 | 348 | # BeatPulse healthcheck temp database 349 | healthchecksdb 350 | 351 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 352 | MigrationBackup/ 353 | 354 | # Ionide (cross platform F# VS Code tools) working folder 355 | .ionide/ 356 | infrastructure/terraform/terraform.tfstate 357 | infrastructure/terraform/terraform.tfstate.backup 358 | infrastructure/terraform/variables.tfvars 359 | infrastructure/terraform/.terraform/ 360 | source/ui/.env 361 | source/ui/build 362 | source/ui/src/config.json 363 | -------------------------------------------------------------------------------- /Deployment.md: -------------------------------------------------------------------------------- 1 | # Deployment 2 | The following is a detailed guide on how to standup an AKS cluster using the code in this repository via GitHub Actions 3 | 4 | # Prerequisites 5 | ## Subscriptions and Artifacts 6 | * An Azure subscription (two for a more Enterprise Scale-like deployment) 7 | * A GitHub respository 8 | * A custom domain with a TLS certificate - the following will use bjdazure.tech and a cert from [Let's Encrypt](https://letsencrypt.org/) 9 | * Enable [AKS Preview Features](./scripts/aks-preview-features.sh) - Once time per subscription 10 | 11 | ## Required Existing Resources and Configuration 12 | | | | 13 | --------------- | --------------- 14 | | Azure Virtual Network (Core Components) | Azure Virtual Network (Kubernetes Cluster) | 15 | | A DNS server | [Private Endpoint DNS Configuration](https://docs.microsoft.com/en-us/azure/private-link/private-endpoint-dns#on-premises-workloads-using-a-dns-forwarder) | 16 | | The two Virtual Networks Peered | Vnets DNS set to DNS Server | 17 | | Subnets | Subnet for Kubernetes API Server (at least /24) | 18 | || Subnet for Kubernetes Nodes (/24) | 19 | || Subnet for Private Endpoints named private-endpoints | 20 | | A [Github Actions Runner VM](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners) with: | A User Assigned Manage Identity | 21 | || Identity granted Owner permissions over each subscription | 22 | | Azure Container Repository | Private EndPoint for ACR | 23 | | An Azure SPN for ACR Access | Granted AcrPush/AcrPull RBAC from ACR | 24 | | Azure Firewall| [AKS Egress Policies](https://docs.microsoft.com/en-us/azure/aks/limit-egress-traffic) | 25 | | Route Table | Route 0.0.0.0/0 traffic from Kubernetes subnet to Azure Firewall | 26 | | Azure Storage | Blob Container (for Terraform state files) | 27 | | Private DNS Zones (attached to Core Vnet) | privatelink.${region}.azmk8s.io | 28 | || privatelink.vaultcore.azure.net | 29 | || privatelink.azurecr.io | 30 | || Custom Domain (example - bjdazure.tech) | 31 | > **Note**: These [ARM Templates](https://github.com/briandenicola/kubernetes-cluster-setup/tree/main/infrastructure/prereqs) can help create most of the require Azure networking resources 32 | 33 | # Steps for Complete Environment with Flux and Istio Service Mesh 34 | 1. Fork this repository and [eShopOnDapr](https://github.com/briandenicola/eShopOnDapr/) into your own Github Account 35 | 1. Package eShopOnDapr and Push to your Azure Container Repository 36 | *. cd deploy/k8s/helm/ 37 | *. helm package . 38 | *. az acr helm push -n ${ACR} eshopondapr-2.0.0.tgz 39 | 1. Fork repository 40 | 1. Create a new branch for the cluster 41 | 1. Search and Replace default_cluster_name with the new name of the cluster 42 | 1. Commit branch to your repository 43 | 1. Create the follow Secrets in GitHub: 44 | | Secret Name | Purpose | 45 | --------------- | --------------- 46 | | ARM_SUBSCRIPTION_ID | The AKS Subscription ID used for Terraform access | 47 | | ARM_CLIENT_ID | The Client ID of the Github Managed Identity | 48 | | ARM_TENANT_ID | The Azure AD tenant of the Github Managed Identity | 49 | | PAT_TOKEN | A GitHub Personal Access Token with Repo permissions for flux | 50 | | CERTIFICATE | The base64 encoded string of the TLS cert in PFX format | 51 | | CERT_PASSWORD | The password for the PFX file | 52 | | ACR_SPN_ID | The client id of the ACR SPN used for Terraform access | 53 | | ACR_SPN_PASSWORD | The client secret of the ACR SPN used for Terraform access | 54 | | MSI_CLIENT_ID | The client id of the managed identity for the Github Actions Task Runner | 55 | 1. Update infrastructure/istio.tfvars with correct values 56 | | Secret Name | Purpose | Default | 57 | --------------- | --------------- | --------------- 58 | | agent_count | Number of nodes in default node pool | 3 | 59 | | location | The Azure Region for the resources| centralus | 60 | | k8s_vnet_resource_group_name | The Resource Group (RG) where the Azure Vnet for AKS is deployed | Apps02_Network_RG | 61 | | k8s_vnet | The AKS Virtual Network Name | DevSub02-Vnet-Sandbox-001 | 62 | | k8s_nodes_subnet | The subnet for the AKS nodes to deploy into | kubernetes-nodes | 63 | | k8s_apiserver_subnet | The subnet for the AKS API to deploy into | kubernetes-apiserver | 64 | | dns_service_ip | The DNS IP for CoreDNS in AKS| 192.168.0.10 | 65 | | service_cidr | The Services CIDR in AKS | 192.168.0.0/16 | 66 | | core_subscription | The Azure Subscription ID for Core Components| | 67 | | dns_resource_group_name | The RG where the Private Zone DNS resources have been deployed to | Core_Infra_DNS_RG | 68 | | acr_resource_group |The RG where the Azure Container Repository has been deployed to | Core_Infra_ContainerRepo_RG | 69 | | acr_name | The name of the Azure Container Repository || 70 | | github_actions_identity_name | The name of the managed identity for the Github Actions runner used to bootstrap flux | github-actions-identity | 71 | | github_actions_identity_resource_group |The RG for the managed identity for the Github Actions runner| Core_Infra_GithubActions_RG | 72 | | certificate_name | The name of the secret that will store the TLS wildcard certificate | wildcard_certificate | 73 | | vm_size | The VM size of the default node pool | Standard_B4ms | 74 | 1. Trigger the 'Creates K8s with a Mesh installed' Github Action to create the cluster. 75 | * Accept the default cluster name and Service Mesh 76 | 1. Search and Replace default_ingress_client_id with the client id of Istio Service Mesh Ingress managed identity 77 | 1. Search and Replace default_zipkin_client_id with the client id of Zipkin managed identity 78 | 1. Search and Replace 16b3c013-d300-468d-ac64-7eda0820b6d3 with ther proper Azure AD tenant ID 79 | 1. Commit branch to your repository 80 | 1. The pipeline calls the ./scripts/aks-flux-configuration.sh script to confiugre flux and execute the GitOps flow 81 | 82 | ## Post Creation Steps 83 | 1. Create wildcard '*' DNS record pointing to Istio Gateway Service IP for the custom domain 84 | 85 | # Steps for Basic Private Cluster 86 | 1. Fork repository 87 | 1. Create a new branch for the cluster 88 | 1. Search and Replace default_cluster_name with the new name of the cluster 89 | 1. Commit branch to your repository 90 | 1. Create the follow Secrets in GitHub: 91 | | Secret Name | Purpose | 92 | --------------- | --------------- 93 | | ARM_SUBSCRIPTION_ID | The AKS Subscription ID used for Terraform access | 94 | | ARM_CLIENT_ID | The Client ID of the Github Managed Identity | 95 | | ARM_TENANT_ID | The Azure AD tenant of the Github Managed Identity | 96 | | PAT_TOKEN | A GitHub Personal Access Token with Repo permissions for flux | 97 | | CERTIFICATE | The base64 encoded string of the TLS cert in PFX format | 98 | | CERT_PASSWORD | The password for the PFX file | 99 | | ACR_SPN_ID | The client id of the ACR SPN used for Terraform access | 100 | | ACR_SPN_PASSWORD | The client secret of the ACR SPN used for Terraform access | 101 | | MSI_CLIENT_ID | The client id of the managed identity for the Github Actions Task Runner | 102 | 1. Update infrastructure/k8s.tfvars with correct values 103 | | Secret Name | Purpose | Default | 104 | --------------- | --------------- | --------------- 105 | | agent_count | Number of nodes in default node pool | 3 | 106 | | location | The Azure Region for the resources| centralus | 107 | | k8s_vnet_resource_group_name | The Resource Group (RG) where the Azure Vnet for AKS is deployed | Apps02_Network_RG | 108 | | k8s_vnet | The AKS Virtual Network Name | DevSub02-Vnet-Sandbox-001 | 109 | | k8s_nodes_subnet | The subnet for the AKS nodes to deploy into | kubernetes-nodes | 110 | | k8s_apiserver_subnet | The subnet for the AKS API to deploy into | kubernetes-apiserver | 111 | | dns_service_ip | The DNS IP for CoreDNS in AKS| 192.168.0.10 | 112 | | service_cidr | The Services CIDR in AKS | 192.168.0.0/16 | 113 | | core_subscription | The Azure Subscription ID for Core Components| | 114 | | dns_resource_group_name | The RG where the Private Zone DNS resources have been deployed to | Core_Infra_DNS_RG | 115 | | acr_resource_group |The RG where the Azure Container Repository has been deployed to | Core_Infra_ContainerRepo_RG | 116 | | acr_name | The name of the Azure Container Repository || 117 | | github_actions_identity_name | The name of the managed identity for the Github Actions runner used to bootstrap flux | github-actions-identity | 118 | | github_actions_identity_resource_group |The RG for the managed identity for the Github Actions runner| Core_Infra_GithubActions_RG | 119 | | certificate_name | The name of the secret that will store the TLS wildcard certificate | wildcard_certificate | 120 | | vm_size | The VM size of the default node pool | Standard_B4ms | 121 | 1. Trigger the 'Creates K8s without a Mesh installed' Github Action to create the cluster. 122 | * Accept the new cluster name -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Brian Denicola 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | This repository is covers how to stand up a secure, private AKS/Kubernetes cluster. This is not intended to be an AKS 101 or cover all possible secure AKS designs (for example using [HTTP Proxy](https://docs.microsoft.com/en-us/azure/aks/http-prox) instead of Azure Firewall). Azure has plenty of awesome [documentation](https://docs.microsoft.com/en-us/azure/architecture/reference-architectures/containers/aks-pci/aks-pci-ra-code-assets) and [guides](https://github.com/mspnp/aks-baseline-regulated) that go into depth on the overall design and best practices. 3 | 4 | This code is an opiniated method of applying the standards into an end to end solution using Terraform, GitHub Actions, Flux and Istio. 5 | 6 | ## Detail Deployment Guide 7 | Follow this [guide](./Deployment.md) to stand up a cluster in your environment 8 | 9 | # Resources Created 10 | Every kuberernetes cluster has several moving parts - even those deployed in a managed cloud environment. More so, if the cluster is to be private with egress filtering. Several resources in Azure must in place for the code to work properly. Typically these resources are stood-up by an Enterprise Platform Team following Azure's Cloud Adoption Framework so for more secure AKS builds, these are out of scope. There is an example [Azure Firewall and RouteTable ARM Template](./infrastructure/prereqs/azuredeploy.template.json) that can elp with some of the prereqs. 11 | 12 | ## Infrastructure 13 | | | | 14 | --------------- | --------------- 15 | | Log Analytics Workspace | Application Insights | 16 | | KeyVault (Private endpoint) | TLS Certificate: *.bjdazure.tech | 17 | | | KeyVault Secret: OpenTelemetry/Zipkin Configuration | 18 | | Managed User identities | AKS Cluster Identity | 19 | | | AKS Kubelet Identity | 20 | | | Istio Pod Identity | 21 | | Private AKS cluster | Add-Ons: OMS Policy Agent, KeyVault CSI Driver, Azure Pod Identity v1, FluxCDv2 || 22 | | Azure Internal Load Balancer 23 | 24 | ### Diagram 25 | ![resources](./assets/resources.png) 26 | --- 27 | 28 | ## Kubernetes Resources 29 | | Flux | Istio | 30 | --------------- | --------------- 31 | | Istio Operator | Ingress Gateway with TLS | 32 | | Istio Service Mesh | Egress Gateway | 33 | | Open Telemetry Collector | Distributive Tracing forwarded to Zipkin/Otel Collector | 34 | | Kured Reboot Daemon | Catch-all Virtual Service || 35 | | Dapr Distributive Runtime || 36 | | Keda Autoscaler || 37 | | Azure Monitor configuration update for Prometheus || 38 | | [Istio Bookinfo](https://istio.io/latest/docs/examples/bookinfo/)|| 39 | | [eShopOnDapr](https://github.com/briandenicola/eShopOnDapr)|| 40 | 41 | ### Diagram 42 | ![namespaces|100x100](./assets/namespaces.png) 43 | 44 | 45 | -------------------------------------------------------------------------------- /apps/bookstore/bookinfo-gateway.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1beta1 2 | kind: VirtualService 3 | metadata: 4 | name: bookinfo 5 | namespace: bookstore 6 | spec: 7 | hosts: 8 | - bookstore.bjdazure.tech 9 | gateways: 10 | - default/default-bjdazure-tech-gw 11 | http: 12 | - match: 13 | - uri: 14 | exact: /productpage 15 | - uri: 16 | prefix: /static 17 | - uri: 18 | exact: /login 19 | - uri: 20 | exact: /logout 21 | - uri: 22 | prefix: /api/v1/products 23 | route: 24 | - destination: 25 | host: productpage.bookstore.svc.cluster.local 26 | port: 27 | number: 9080 28 | -------------------------------------------------------------------------------- /apps/bookstore/bookinfo.yaml: -------------------------------------------------------------------------------- 1 | # Copyright Istio Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: details 18 | namespace: bookstore 19 | labels: 20 | app: details 21 | service: details 22 | spec: 23 | ports: 24 | - port: 9080 25 | name: http 26 | selector: 27 | app: details 28 | --- 29 | apiVersion: v1 30 | kind: ServiceAccount 31 | metadata: 32 | name: bookinfo-details 33 | namespace: bookstore 34 | labels: 35 | account: details 36 | --- 37 | apiVersion: apps/v1 38 | kind: Deployment 39 | metadata: 40 | name: details-v1 41 | namespace: bookstore 42 | labels: 43 | app: details 44 | version: v1 45 | spec: 46 | replicas: 1 47 | selector: 48 | matchLabels: 49 | app: details 50 | version: v1 51 | template: 52 | metadata: 53 | labels: 54 | app: details 55 | version: v1 56 | spec: 57 | serviceAccountName: bookinfo-details 58 | containers: 59 | - name: details 60 | image: docker.io/istio/examples-bookinfo-details-v1:1.16.2 61 | imagePullPolicy: IfNotPresent 62 | ports: 63 | - containerPort: 9080 64 | securityContext: 65 | runAsUser: 1000 66 | --- 67 | ################################################################################################## 68 | # Ratings service 69 | ################################################################################################## 70 | apiVersion: v1 71 | kind: Service 72 | metadata: 73 | name: ratings 74 | namespace: bookstore 75 | labels: 76 | app: ratings 77 | service: ratings 78 | spec: 79 | ports: 80 | - port: 9080 81 | name: http 82 | selector: 83 | app: ratings 84 | --- 85 | apiVersion: v1 86 | kind: ServiceAccount 87 | metadata: 88 | name: bookinfo-ratings 89 | namespace: bookstore 90 | labels: 91 | account: ratings 92 | --- 93 | apiVersion: apps/v1 94 | kind: Deployment 95 | metadata: 96 | name: ratings-v1 97 | namespace: bookstore 98 | labels: 99 | app: ratings 100 | version: v1 101 | spec: 102 | replicas: 1 103 | selector: 104 | matchLabels: 105 | app: ratings 106 | version: v1 107 | template: 108 | metadata: 109 | labels: 110 | app: ratings 111 | version: v1 112 | spec: 113 | serviceAccountName: bookinfo-ratings 114 | containers: 115 | - name: ratings 116 | image: docker.io/istio/examples-bookinfo-ratings-v1:1.16.2 117 | imagePullPolicy: IfNotPresent 118 | ports: 119 | - containerPort: 9080 120 | securityContext: 121 | runAsUser: 1000 122 | --- 123 | ################################################################################################## 124 | # Reviews service 125 | ################################################################################################## 126 | apiVersion: v1 127 | kind: Service 128 | metadata: 129 | name: reviews 130 | namespace: bookstore 131 | labels: 132 | app: reviews 133 | service: reviews 134 | spec: 135 | ports: 136 | - port: 9080 137 | name: http 138 | selector: 139 | app: reviews 140 | --- 141 | apiVersion: v1 142 | kind: ServiceAccount 143 | metadata: 144 | name: bookinfo-reviews 145 | namespace: bookstore 146 | labels: 147 | account: reviews 148 | --- 149 | apiVersion: apps/v1 150 | kind: Deployment 151 | metadata: 152 | name: reviews-v1 153 | namespace: bookstore 154 | labels: 155 | app: reviews 156 | version: v1 157 | spec: 158 | replicas: 1 159 | selector: 160 | matchLabels: 161 | app: reviews 162 | version: v1 163 | template: 164 | metadata: 165 | labels: 166 | app: reviews 167 | version: v1 168 | spec: 169 | serviceAccountName: bookinfo-reviews 170 | containers: 171 | - name: reviews 172 | image: docker.io/istio/examples-bookinfo-reviews-v1:1.16.2 173 | imagePullPolicy: IfNotPresent 174 | env: 175 | - name: LOG_DIR 176 | value: "/tmp/logs" 177 | ports: 178 | - containerPort: 9080 179 | volumeMounts: 180 | - name: tmp 181 | mountPath: /tmp 182 | - name: wlp-output 183 | mountPath: /opt/ibm/wlp/output 184 | securityContext: 185 | runAsUser: 1000 186 | volumes: 187 | - name: wlp-output 188 | emptyDir: {} 189 | - name: tmp 190 | emptyDir: {} 191 | --- 192 | apiVersion: apps/v1 193 | kind: Deployment 194 | metadata: 195 | name: reviews-v2 196 | namespace: bookstore 197 | labels: 198 | app: reviews 199 | version: v2 200 | spec: 201 | replicas: 1 202 | selector: 203 | matchLabels: 204 | app: reviews 205 | version: v2 206 | template: 207 | metadata: 208 | labels: 209 | app: reviews 210 | version: v2 211 | spec: 212 | serviceAccountName: bookinfo-reviews 213 | containers: 214 | - name: reviews 215 | image: docker.io/istio/examples-bookinfo-reviews-v2:1.16.2 216 | imagePullPolicy: IfNotPresent 217 | env: 218 | - name: LOG_DIR 219 | value: "/tmp/logs" 220 | ports: 221 | - containerPort: 9080 222 | volumeMounts: 223 | - name: tmp 224 | mountPath: /tmp 225 | - name: wlp-output 226 | mountPath: /opt/ibm/wlp/output 227 | securityContext: 228 | runAsUser: 1000 229 | volumes: 230 | - name: wlp-output 231 | emptyDir: {} 232 | - name: tmp 233 | emptyDir: {} 234 | --- 235 | apiVersion: apps/v1 236 | kind: Deployment 237 | metadata: 238 | name: reviews-v3 239 | namespace: bookstore 240 | labels: 241 | app: reviews 242 | version: v3 243 | spec: 244 | replicas: 1 245 | selector: 246 | matchLabels: 247 | app: reviews 248 | version: v3 249 | template: 250 | metadata: 251 | labels: 252 | app: reviews 253 | version: v3 254 | spec: 255 | serviceAccountName: bookinfo-reviews 256 | containers: 257 | - name: reviews 258 | image: docker.io/istio/examples-bookinfo-reviews-v3:1.16.2 259 | imagePullPolicy: IfNotPresent 260 | env: 261 | - name: LOG_DIR 262 | value: "/tmp/logs" 263 | ports: 264 | - containerPort: 9080 265 | volumeMounts: 266 | - name: tmp 267 | mountPath: /tmp 268 | - name: wlp-output 269 | mountPath: /opt/ibm/wlp/output 270 | securityContext: 271 | runAsUser: 1000 272 | volumes: 273 | - name: wlp-output 274 | emptyDir: {} 275 | - name: tmp 276 | emptyDir: {} 277 | --- 278 | ################################################################################################## 279 | # Productpage services 280 | ################################################################################################## 281 | apiVersion: v1 282 | kind: Service 283 | metadata: 284 | name: productpage 285 | namespace: bookstore 286 | labels: 287 | app: productpage 288 | service: productpage 289 | spec: 290 | ports: 291 | - port: 9080 292 | name: http 293 | selector: 294 | app: productpage 295 | --- 296 | apiVersion: v1 297 | kind: ServiceAccount 298 | metadata: 299 | name: bookinfo-productpage 300 | namespace: bookstore 301 | labels: 302 | account: productpage 303 | --- 304 | apiVersion: apps/v1 305 | kind: Deployment 306 | metadata: 307 | name: productpage-v1 308 | namespace: bookstore 309 | labels: 310 | app: productpage 311 | version: v1 312 | spec: 313 | replicas: 1 314 | selector: 315 | matchLabels: 316 | app: productpage 317 | version: v1 318 | template: 319 | metadata: 320 | labels: 321 | app: productpage 322 | version: v1 323 | spec: 324 | serviceAccountName: bookinfo-productpage 325 | containers: 326 | - name: productpage 327 | image: docker.io/istio/examples-bookinfo-productpage-v1:1.16.2 328 | imagePullPolicy: IfNotPresent 329 | ports: 330 | - containerPort: 9080 331 | volumeMounts: 332 | - name: tmp 333 | mountPath: /tmp 334 | securityContext: 335 | runAsUser: 1000 336 | volumes: 337 | - name: tmp 338 | emptyDir: {} 339 | --- 340 | -------------------------------------------------------------------------------- /apps/bookstore/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - namespace.yaml 5 | - bookinfo.yaml 6 | - bookinfo-gateway.yaml -------------------------------------------------------------------------------- /apps/bookstore/namespace.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: bookstore 6 | labels: 7 | istio.io/rev: asm-1-17 8 | -------------------------------------------------------------------------------- /apps/chat/chat.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: chatapp 5 | labels: 6 | istio.io/rev: asm-1-17 7 | --- 8 | apiVersion: source.toolkit.fluxcd.io/v1beta2 9 | kind: HelmRepository 10 | metadata: 11 | name: chat 12 | namespace: flux-system 13 | spec: 14 | interval: 10m 15 | type: oci 16 | url: oci://bjd145.azurecr.io/helm 17 | passCredentials: true 18 | secretRef: 19 | name: https-credentials 20 | --- 21 | apiVersion: helm.toolkit.fluxcd.io/v2beta1 22 | kind: HelmRelease 23 | metadata: 24 | name: chatapp 25 | namespace: chatapp 26 | spec: 27 | interval: 5m 28 | chart: 29 | spec: 30 | chart: sentimentchat 31 | version: '0.3.3' 32 | sourceRef: 33 | kind: HelmRepository 34 | name: chat 35 | namespace: flux-system 36 | interval: 1m 37 | values: 38 | chat_image_version: '9c73d9f2fcb02a3f7fe8daa7707d0819269a6eef' 39 | keyvault_name: 'chat-e0ce7b6-kv' 40 | keyvault_secret_name: 'chat-e0ce7b6-cogs-key' 41 | pod_identity: 'chat-e0ce7b6-identity' 42 | hostname: 'chat.bjdazure.tech' 43 | tenant_id: '72f988bf-86f1-41af-91ab-2d7cd011db47' -------------------------------------------------------------------------------- /apps/eShopOnDapr/eshop.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: eshopondapr 5 | labels: 6 | istio.io/rev: asm-1-17 7 | annotations: 8 | scheduler.alpha.kubernetes.io/defaultTolerations: '[{"key": "reservedFor", "operator": "Equal", "effect": "NoSchedule", "value": "eShopOnDapr"}]' 9 | --- 10 | apiVersion: source.toolkit.fluxcd.io/v1beta2 11 | kind: HelmRepository 12 | metadata: 13 | name: eshopondapr 14 | namespace: flux-system 15 | spec: 16 | interval: 10m 17 | type: oci 18 | url: oci://bjdcsa.azurecr.io/helm 19 | passCredentials: true 20 | secretRef: 21 | name: https-credentials 22 | --- 23 | apiVersion: helm.toolkit.fluxcd.io/v2beta1 24 | kind: HelmRelease 25 | metadata: 26 | name: eshopondapr-201 27 | namespace: eshopondapr 28 | spec: 29 | interval: 5m 30 | chart: 31 | spec: 32 | chart: eshopondapr 33 | version: '2.0.2' 34 | sourceRef: 35 | kind: HelmRepository 36 | name: eshopondapr 37 | namespace: flux-system 38 | interval: 1m 39 | values: 40 | gatewayDnsNameOrIP: eshop-api.bjdazure.tech 41 | identityDnsNameOrIP: identity-api.bjdazure.tech 42 | blazorDnsNameOrIP: shop.bjdazure.tech 43 | istio_gateway: istio-gateways/default-bjdazure-tech-gw -------------------------------------------------------------------------------- /apps/httpbin/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: apps 5 | labels: 6 | istio.io/rev: asm-1-17 7 | pod-security.kubernetes.io/enforce: privileged 8 | pod-security.kubernetes.io/enforce-version: v1.25 9 | pod-security.kubernetes.io/audit: restricted 10 | pod-security.kubernetes.io/audit-version: v1.25 11 | pod-security.kubernetes.io/warn: restricted 12 | pod-security.kubernetes.io/warn-version: v1.25 13 | annotations: 14 | scheduler.alpha.kubernetes.io/defaultTolerations: '[{"key": "reservedFor", "operator": "Equal", "effect": "NoSchedule", "value": "apps"}]' 15 | --- 16 | apiVersion: apps/v1 17 | kind: Deployment 18 | metadata: 19 | name: httpbin 20 | namespace: apps 21 | spec: 22 | replicas: 3 23 | selector: 24 | matchLabels: 25 | app: httpbin 26 | strategy: 27 | type: RollingUpdate 28 | rollingUpdate: 29 | maxSurge: 1 30 | maxUnavailable: 1 31 | minReadySeconds: 5 32 | template: 33 | metadata: 34 | labels: 35 | app: httpbin 36 | spec: 37 | securityContext: 38 | runAsUser: 1000 39 | runAsGroup: 1000 40 | supplementalGroups: [2000] 41 | fsGroup: 2000 42 | seccompProfile: 43 | type: RuntimeDefault 44 | containers: 45 | - name: httpbin 46 | image: bjd145/httpbin:1087 47 | imagePullPolicy: Always 48 | securityContext: 49 | runAsNonRoot: true 50 | readOnlyRootFilesystem: true 51 | allowPrivilegeEscalation: false 52 | capabilities: 53 | drop: ["ALL"] 54 | ports: 55 | - containerPort: 8080 56 | resources: 57 | limits: 58 | memory: "512Mi" 59 | cpu: "2" 60 | requests: 61 | memory: "64Mi" 62 | cpu: "0.25" 63 | volumeMounts: 64 | - mountPath: /usr/tmp 65 | name: tmp-volume 66 | tolerations: 67 | - key: "reservedFor" 68 | operator: "Equal" 69 | value: "apps" 70 | effect: "NoSchedule" 71 | volumes: 72 | - name: tmp-volume 73 | emptyDir: {} 74 | --- 75 | apiVersion: v1 76 | kind: Service 77 | metadata: 78 | name: httpbinsvc 79 | namespace: apps 80 | spec: 81 | selector: 82 | app: httpbin 83 | ports: 84 | - port: 80 85 | targetPort: 8080 86 | type: ClusterIP 87 | --- 88 | apiVersion: networking.istio.io/v1beta1 89 | kind: VirtualService 90 | metadata: 91 | name: httpbinsvc-vs 92 | namespace: apps 93 | spec: 94 | hosts: 95 | - httpbin.bjdazure.tech 96 | gateways: 97 | - aks-istio-ingress/default-bjdazure-tech-gw 98 | http: 99 | - route: 100 | - destination: 101 | host: httpbinsvc 102 | port: 103 | number: 80 104 | 105 | -------------------------------------------------------------------------------- /apps/httpbin/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - deployment.yaml 5 | 6 | -------------------------------------------------------------------------------- /apps/whatos-api/knative-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: serving.knative.dev/v1 2 | kind: Service 3 | metadata: 4 | name: autoscale 5 | namespace: knative-example 6 | spec: 7 | template: 8 | metadata: 9 | annotations: 10 | autoscaling.knative.dev/class: kpa.autoscaling.knative.dev 11 | autoscaling.knative.dev/metric: concurrency 12 | autoscaling.knative.dev/target: "10" 13 | autoscaling.knative.dev/min-scale: "1" 14 | autoscaling.knative.dev/max-scale: "5" 15 | spec: 16 | containers: 17 | - image: bjd145/whatos:2.0 18 | ports: 19 | - name: http8081 20 | containerPort: 8081 21 | -------------------------------------------------------------------------------- /apps/whatos-api/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - namespaces.yaml 5 | - knative-service.yaml 6 | 7 | 8 | -------------------------------------------------------------------------------- /apps/whatos-api/namespace.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: knative-example 6 | labels: 7 | istio.io/rev: asm-1-17 8 | -------------------------------------------------------------------------------- /assets/namespaces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/briandenicola/private-kubernetes-cluster-setup/34bafc24b08743a327546f896c5070b499a38198/assets/namespaces.png -------------------------------------------------------------------------------- /assets/resources.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/briandenicola/private-kubernetes-cluster-setup/34bafc24b08743a327546f896c5070b499a38198/assets/resources.png -------------------------------------------------------------------------------- /cluster-manifests/apps.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 3 | kind: Kustomization 4 | metadata: 5 | name: httpbin-app 6 | namespace: flux-system 7 | spec: 8 | interval: 5m0s 9 | path: ./apps/httpbin 10 | prune: true 11 | sourceRef: 12 | kind: GitRepository 13 | name: aks-flux-extension 14 | validation: client 15 | --- 16 | apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 17 | kind: Kustomization 18 | metadata: 19 | name: bookstore-app 20 | namespace: flux-system 21 | spec: 22 | interval: 5m0s 23 | path: ./apps/bookstore 24 | prune: true 25 | sourceRef: 26 | kind: GitRepository 27 | name: aks-flux-extension 28 | validation: client 29 | # --- 30 | # apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 31 | # kind: Kustomization 32 | # metadata: 33 | # name: eshopondapr 34 | # namespace: flux-system 35 | # spec: 36 | # interval: 5m0s 37 | # sourceRef: 38 | # kind: GitRepository 39 | # name: aks-flux-extension 40 | # path: ./apps/eShopOnDapr 41 | # prune: true 42 | # validation: client -------------------------------------------------------------------------------- /cluster-manifests/common/customresourcedefinitions/istio-operator-crd.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: istiooperators.install.istio.io 6 | labels: 7 | release: istio 8 | spec: 9 | conversion: 10 | strategy: None 11 | group: install.istio.io 12 | names: 13 | kind: IstioOperator 14 | listKind: IstioOperatorList 15 | plural: istiooperators 16 | singular: istiooperator 17 | shortNames: 18 | - iop 19 | - io 20 | scope: Namespaced 21 | versions: 22 | - additionalPrinterColumns: 23 | - description: Istio control plane revision 24 | jsonPath: .spec.revision 25 | name: Revision 26 | type: string 27 | - description: IOP current state 28 | jsonPath: .status.status 29 | name: Status 30 | type: string 31 | - description: 'CreationTimestamp is a timestamp representing the server time 32 | when this object was created. It is not guaranteed to be set in happens-before 33 | order across separate operations. Clients may not set this value. It is represented 34 | in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for 35 | lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' 36 | jsonPath: .metadata.creationTimestamp 37 | name: Age 38 | type: date 39 | name: v1alpha1 40 | subresources: 41 | status: {} 42 | schema: 43 | openAPIV3Schema: 44 | type: object 45 | x-kubernetes-preserve-unknown-fields: true 46 | served: true 47 | storage: true 48 | --- 49 | # Source: istio-operator/templates/namespace.yaml 50 | apiVersion: v1 51 | kind: Namespace 52 | metadata: 53 | name: istio-operator 54 | labels: 55 | istio-operator-managed: Reconcile 56 | istio-injection: disabled 57 | --- 58 | # Source: istio-operator/templates/service_account.yaml 59 | apiVersion: v1 60 | kind: ServiceAccount 61 | metadata: 62 | namespace: istio-operator 63 | name: istio-operator 64 | --- 65 | # Source: istio-operator/templates/clusterrole.yaml 66 | apiVersion: rbac.authorization.k8s.io/v1 67 | kind: ClusterRole 68 | metadata: 69 | creationTimestamp: null 70 | name: istio-operator 71 | rules: 72 | # istio groups 73 | - apiGroups: 74 | - authentication.istio.io 75 | resources: 76 | - '*' 77 | verbs: 78 | - '*' 79 | - apiGroups: 80 | - config.istio.io 81 | resources: 82 | - '*' 83 | verbs: 84 | - '*' 85 | - apiGroups: 86 | - install.istio.io 87 | resources: 88 | - '*' 89 | verbs: 90 | - '*' 91 | - apiGroups: 92 | - networking.istio.io 93 | resources: 94 | - '*' 95 | verbs: 96 | - '*' 97 | - apiGroups: 98 | - security.istio.io 99 | resources: 100 | - '*' 101 | verbs: 102 | - '*' 103 | # k8s groups 104 | - apiGroups: 105 | - admissionregistration.k8s.io 106 | resources: 107 | - mutatingwebhookconfigurations 108 | - validatingwebhookconfigurations 109 | verbs: 110 | - '*' 111 | - apiGroups: 112 | - apiextensions.k8s.io 113 | resources: 114 | - customresourcedefinitions.apiextensions.k8s.io 115 | - customresourcedefinitions 116 | verbs: 117 | - '*' 118 | - apiGroups: 119 | - apps 120 | - extensions 121 | resources: 122 | - daemonsets 123 | - deployments 124 | - deployments/finalizers 125 | - replicasets 126 | verbs: 127 | - '*' 128 | - apiGroups: 129 | - autoscaling 130 | resources: 131 | - horizontalpodautoscalers 132 | verbs: 133 | - '*' 134 | - apiGroups: 135 | - monitoring.coreos.com 136 | resources: 137 | - servicemonitors 138 | verbs: 139 | - get 140 | - create 141 | - update 142 | - apiGroups: 143 | - policy 144 | resources: 145 | - poddisruptionbudgets 146 | verbs: 147 | - '*' 148 | - apiGroups: 149 | - rbac.authorization.k8s.io 150 | resources: 151 | - clusterrolebindings 152 | - clusterroles 153 | - roles 154 | - rolebindings 155 | verbs: 156 | - '*' 157 | - apiGroups: 158 | - coordination.k8s.io 159 | resources: 160 | - leases 161 | verbs: 162 | - get 163 | - create 164 | - update 165 | - apiGroups: 166 | - "" 167 | resources: 168 | - configmaps 169 | - endpoints 170 | - events 171 | - namespaces 172 | - pods 173 | - pods/proxy 174 | - persistentvolumeclaims 175 | - secrets 176 | - services 177 | - serviceaccounts 178 | verbs: 179 | - '*' 180 | --- 181 | # Source: istio-operator/templates/clusterrole_binding.yaml 182 | kind: ClusterRoleBinding 183 | apiVersion: rbac.authorization.k8s.io/v1 184 | metadata: 185 | name: istio-operator 186 | subjects: 187 | - kind: ServiceAccount 188 | name: istio-operator 189 | namespace: istio-operator 190 | roleRef: 191 | kind: ClusterRole 192 | name: istio-operator 193 | apiGroup: rbac.authorization.k8s.io 194 | --- 195 | # Source: istio-operator/templates/service.yaml 196 | apiVersion: v1 197 | kind: Service 198 | metadata: 199 | namespace: istio-operator 200 | labels: 201 | name: istio-operator 202 | name: istio-operator 203 | spec: 204 | ports: 205 | - name: http-metrics 206 | port: 8383 207 | targetPort: 8383 208 | protocol: TCP 209 | selector: 210 | name: istio-operator 211 | --- 212 | # Source: istio-operator/templates/deployment.yaml 213 | apiVersion: apps/v1 214 | kind: Deployment 215 | metadata: 216 | namespace: istio-operator 217 | name: istio-operator 218 | spec: 219 | replicas: 1 220 | selector: 221 | matchLabels: 222 | name: istio-operator 223 | template: 224 | metadata: 225 | labels: 226 | name: istio-operator 227 | spec: 228 | serviceAccountName: istio-operator 229 | containers: 230 | - name: istio-operator 231 | image: docker.io/istio/operator:1.15.4 232 | command: 233 | - operator 234 | - server 235 | securityContext: 236 | allowPrivilegeEscalation: false 237 | capabilities: 238 | drop: 239 | - ALL 240 | privileged: false 241 | readOnlyRootFilesystem: true 242 | runAsGroup: 1337 243 | runAsUser: 1337 244 | runAsNonRoot: true 245 | imagePullPolicy: IfNotPresent 246 | resources: 247 | limits: 248 | cpu: 200m 249 | memory: 256Mi 250 | requests: 251 | cpu: 50m 252 | memory: 128Mi 253 | env: 254 | - name: WATCH_NAMESPACE 255 | value: "istio-system" 256 | - name: LEADER_ELECTION_NAMESPACE 257 | value: "istio-operator" 258 | - name: POD_NAME 259 | valueFrom: 260 | fieldRef: 261 | fieldPath: metadata.name 262 | - name: OPERATOR_NAME 263 | value: "istio-operator" 264 | - name: WAIT_FOR_RESOURCES_TIMEOUT 265 | value: "300s" 266 | - name: REVISION 267 | value: "" 268 | -------------------------------------------------------------------------------- /cluster-manifests/common/customresourcedefinitions/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | # - istio-operator-crd.yaml 5 | - knative-serving-crds.yaml 6 | 7 | -------------------------------------------------------------------------------- /cluster-manifests/common/istio/configuration/istio-cluster-roles.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: istio-custom-creator-role 6 | rules: 7 | - apiGroups: 8 | - networking.istio.io 9 | resources: 10 | - virtualservices 11 | verbs: 12 | - create 13 | - delete 14 | - deletecollection 15 | - patch 16 | - update 17 | - get 18 | - list 19 | - watch 20 | -------------------------------------------------------------------------------- /cluster-manifests/common/istio/configuration/istio-configuration.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | metadata: 4 | name: istio-shared-configmap-asm-1-17 5 | namespace: aks-istio-system 6 | data: 7 | mesh: |- 8 | defaultConfig: 9 | tracing: 10 | zipkin: 11 | address: otel-collector.otel-system.svc.cluster.local:9411 12 | accessLogFile: /dev/stdout 13 | accessLogEncoding: JSON 14 | enableTracing: true 15 | outboundTrafficPolicy: 16 | mode: ALLOW_ANY 17 | 18 | meshNetworks: 'networks: {}' -------------------------------------------------------------------------------- /cluster-manifests/common/istio/configuration/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - istio-configuration.yaml 5 | - istio-cluster-roles.yaml 6 | -------------------------------------------------------------------------------- /cluster-manifests/common/istio/gateway/default-ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: default-backend 5 | namespace: aks-istio-ingress 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: default-ingress-backend 10 | replicas: 1 11 | strategy: 12 | type: RollingUpdate 13 | rollingUpdate: 14 | maxSurge: 1 15 | maxUnavailable: 1 16 | minReadySeconds: 5 17 | template: 18 | metadata: 19 | labels: 20 | app: default-ingress-backend 21 | spec: 22 | serviceAccountName: istio-ingress-sa-identity 23 | containers: 24 | - name: ingress 25 | image: nginx:latest 26 | ports: 27 | - containerPort: 80 28 | resources: 29 | requests: 30 | memory: "64Mi" 31 | cpu: "250m" 32 | limits: 33 | memory: "128Mi" 34 | cpu: "500m" 35 | volumeMounts: 36 | - name: tls-mount 37 | mountPath: "/mnt/secrets-store" 38 | readOnly: true 39 | volumes: 40 | - name: tls-mount 41 | csi: 42 | driver: secrets-store.csi.k8s.io 43 | readOnly: true 44 | volumeAttributes: 45 | secretProviderClass: "istio-tls" 46 | --- 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | name: default-backend-svc 51 | namespace: aks-istio-ingress 52 | spec: 53 | selector: 54 | app: default-ingress-backend 55 | ports: 56 | - port: 80 57 | targetPort: 80 58 | type: ClusterIP 59 | -------------------------------------------------------------------------------- /cluster-manifests/common/istio/gateway/istio-ingress-gateway.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1beta1 2 | kind: Gateway 3 | metadata: 4 | name: default-bjdazure-tech-gw 5 | namespace: aks-istio-ingress 6 | spec: 7 | selector: 8 | istio: aks-istio-ingressgateway-internal 9 | servers: 10 | - port: 11 | number: 80 12 | name: http 13 | protocol: HTTP 14 | hosts: 15 | - "*.bjdazure.tech" 16 | - port: 17 | number: 80 18 | name: http-serving 19 | protocol: HTTP 20 | hosts: 21 | - "*.knative.bjdazure.tech" 22 | - port: 23 | number: 443 24 | name: https 25 | protocol: HTTPS 26 | tls: 27 | mode: SIMPLE 28 | credentialName: istio-ingress-tls 29 | hosts: 30 | - "*.bjdazure.tech" 31 | --- 32 | apiVersion: networking.istio.io/v1beta1 33 | kind: VirtualService 34 | metadata: 35 | name: default-bjdazure-tech-vs 36 | namespace: aks-istio-ingress 37 | spec: 38 | hosts: 39 | - istio-default.bjdazure.tech 40 | gateways: 41 | - aks-istio-ingress/default-bjdazure-tech-gw 42 | http: 43 | - route: 44 | - destination: 45 | host: default-backend-svc 46 | port: 47 | number: 80 48 | 49 | -------------------------------------------------------------------------------- /cluster-manifests/common/istio/gateway/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - serviceaccount.yaml 5 | - secretsprovider.yaml 6 | - default-ingress.yaml 7 | - istio-ingress-gateway.yaml -------------------------------------------------------------------------------- /cluster-manifests/common/istio/gateway/secretsprovider.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 2 | kind: SecretProviderClass 3 | metadata: 4 | name: istio-tls 5 | namespace: aks-istio-ingress 6 | spec: 7 | provider: azure 8 | secretObjects: 9 | - secretName: istio-ingress-tls 10 | type: kubernetes.io/tls 11 | data: 12 | - objectName: wildcard-certificate 13 | key: tls.key 14 | - objectName: wildcard-certificate 15 | key: tls.crt 16 | parameters: 17 | usePodIdentity: "false" 18 | useVMManagedIdentity: "false" 19 | keyvaultName: default_cluster_name-kv 20 | clientID: default_ingress_client_id 21 | cloudName: "" 22 | objects: | 23 | array: 24 | - | 25 | objectName: wildcard-certificate 26 | objectType: secret 27 | objectVersion: "" # [OPTIONAL] object versions, default to latest if empty 28 | tenantId: 16b3c013-d300-468d-ac64-7eda0820b6d3 -------------------------------------------------------------------------------- /cluster-manifests/common/istio/gateway/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: istio-ingress-sa-identity 5 | namespace: aks-istio-ingress 6 | annotations: 7 | azure.workload.identity/client-id: default_ingress_client_id 8 | azure.workload.identity/tenant-id: 16b3c013-d300-468d-ac64-7eda0820b6d3 9 | labels: 10 | azure.workload.identity/use: "true" -------------------------------------------------------------------------------- /cluster-manifests/common/knative/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | # CRDs are install when Istio CRDs are deployed to the cluster under ./istio/customresources 5 | - serving-core.yaml 6 | - net-istio.yaml 7 | - serving-config.yaml -------------------------------------------------------------------------------- /cluster-manifests/common/knative/net-istio.yaml: -------------------------------------------------------------------------------- 1 | # Generated when HEAD was e0cfc603de1f6d70572bdbe6f3c5f4ceb2b63184 2 | # 3 | # Copyright 2019 The Knative Authors 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | kind: ClusterRole 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | metadata: 20 | # These are the permissions needed by the Istio Ingress implementation. 21 | name: knative-serving-istio 22 | labels: 23 | app.kubernetes.io/component: net-istio 24 | app.kubernetes.io/name: knative-serving 25 | app.kubernetes.io/version: "1.11.0" 26 | serving.knative.dev/controller: "true" 27 | networking.knative.dev/ingress-provider: istio 28 | rules: 29 | - apiGroups: ["networking.istio.io"] 30 | resources: ["virtualservices", "gateways", "destinationrules"] 31 | verbs: ["get", "list", "create", "update", "delete", "patch", "watch"] 32 | 33 | --- 34 | # Copyright 2019 The Knative Authors 35 | # 36 | # Licensed under the Apache License, Version 2.0 (the "License"); 37 | # you may not use this file except in compliance with the License. 38 | # You may obtain a copy of the License at 39 | # 40 | # https://www.apache.org/licenses/LICENSE-2.0 41 | # 42 | # Unless required by applicable law or agreed to in writing, software 43 | # distributed under the License is distributed on an "AS IS" BASIS, 44 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 45 | # See the License for the specific language governing permissions and 46 | # limitations under the License. 47 | 48 | # This is the shared Gateway for all Knative routes to use. 49 | apiVersion: networking.istio.io/v1beta1 50 | kind: Gateway 51 | metadata: 52 | name: knative-ingress-gateway 53 | namespace: knative-serving 54 | labels: 55 | app.kubernetes.io/component: net-istio 56 | app.kubernetes.io/name: knative-serving 57 | app.kubernetes.io/version: "1.11.0" 58 | networking.knative.dev/ingress-provider: istio 59 | spec: 60 | selector: 61 | istio: ingressgateway 62 | servers: 63 | - port: 64 | number: 80 65 | name: http 66 | protocol: HTTP 67 | hosts: 68 | - "*" 69 | 70 | --- 71 | # Copyright 2019 The Knative Authors 72 | # 73 | # Licensed under the Apache License, Version 2.0 (the "License"); 74 | # you may not use this file except in compliance with the License. 75 | # You may obtain a copy of the License at 76 | # 77 | # https://www.apache.org/licenses/LICENSE-2.0 78 | # 79 | # Unless required by applicable law or agreed to in writing, software 80 | # distributed under the License is distributed on an "AS IS" BASIS, 81 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 82 | # See the License for the specific language governing permissions and 83 | # limitations under the License. 84 | 85 | # A cluster local gateway to allow pods outside of the mesh to access 86 | # Services and Routes not exposing through an ingress. If the users 87 | # do have a service mesh setup, this isn't required. 88 | apiVersion: networking.istio.io/v1beta1 89 | kind: Gateway 90 | metadata: 91 | name: knative-local-gateway 92 | namespace: knative-serving 93 | labels: 94 | app.kubernetes.io/component: net-istio 95 | app.kubernetes.io/name: knative-serving 96 | app.kubernetes.io/version: "1.11.0" 97 | networking.knative.dev/ingress-provider: istio 98 | spec: 99 | selector: 100 | istio: ingressgateway 101 | servers: 102 | - port: 103 | number: 8081 104 | name: http 105 | protocol: HTTP 106 | hosts: 107 | - "*" 108 | --- 109 | apiVersion: v1 110 | kind: Service 111 | metadata: 112 | name: knative-local-gateway 113 | namespace: aks-istio-system 114 | labels: 115 | app.kubernetes.io/component: net-istio 116 | app.kubernetes.io/name: knative-serving 117 | app.kubernetes.io/version: "1.11.0" 118 | networking.knative.dev/ingress-provider: istio 119 | experimental.istio.io/disable-gateway-port-translation: "true" 120 | spec: 121 | type: ClusterIP 122 | selector: 123 | istio: ingressgateway 124 | ports: 125 | - name: http2 126 | port: 80 127 | targetPort: 8081 128 | 129 | --- 130 | # Copyright 2018 The Knative Authors 131 | # 132 | # Licensed under the Apache License, Version 2.0 (the "License"); 133 | # you may not use this file except in compliance with the License. 134 | # You may obtain a copy of the License at 135 | # 136 | # https://www.apache.org/licenses/LICENSE-2.0 137 | # 138 | # Unless required by applicable law or agreed to in writing, software 139 | # distributed under the License is distributed on an "AS IS" BASIS, 140 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 141 | # See the License for the specific language governing permissions and 142 | # limitations under the License. 143 | 144 | apiVersion: v1 145 | kind: ConfigMap 146 | metadata: 147 | name: config-istio 148 | namespace: knative-serving 149 | labels: 150 | app.kubernetes.io/component: net-istio 151 | app.kubernetes.io/name: knative-serving 152 | app.kubernetes.io/version: "1.11.0" 153 | networking.knative.dev/ingress-provider: istio 154 | data: 155 | # TODO(nghia): Extract the .svc.cluster.local suffix into its own config. 156 | _example: | 157 | ################################ 158 | # # 159 | # EXAMPLE CONFIGURATION # 160 | # # 161 | ################################ 162 | 163 | # This block is not actually functional configuration, 164 | # but serves to illustrate the available configuration 165 | # options and document them in a way that is accessible 166 | # to users that `kubectl edit` this config map. 167 | # 168 | # These sample configuration options may be copied out of 169 | # this example block and unindented to be in the data block 170 | # to actually change the configuration. 171 | 172 | # A gateway and Istio service to serve external traffic. 173 | # The configuration format should be 174 | # `gateway.{{gateway_namespace}}.{{gateway_name}}: "{{ingress_name}}.{{ingress_namespace}}.svc.cluster.local"`. 175 | # The {{gateway_namespace}} is optional; when it is omitted, the system will search for 176 | # the gateway in the serving system namespace `knative-serving` 177 | gateway.knative-serving.knative-ingress-gateway: "istio-ingressgateway.istio-system.svc.cluster.local" 178 | 179 | # A cluster local gateway to allow pods outside of the mesh to access 180 | # Services and Routes not exposing through an ingress. If the users 181 | # do have a service mesh setup, this isn't required and can be removed. 182 | # 183 | # An example use case is when users want to use Istio without any 184 | # sidecar injection (like Knative's istio-ci-no-mesh.yaml). Since every pod 185 | # is outside of the service mesh in that case, a cluster-local service 186 | # will need to be exposed to a cluster-local gateway to be accessible. 187 | # The configuration format should be `local-gateway.{{local_gateway_namespace}}. 188 | # {{local_gateway_name}}: "{{cluster_local_gateway_name}}. 189 | # {{cluster_local_gateway_namespace}}.svc.cluster.local"`. The 190 | # {{local_gateway_namespace}} is optional; when it is omitted, the system 191 | # will search for the local gateway in the serving system namespace 192 | # `knative-serving` 193 | local-gateway.knative-serving.knative-local-gateway: "knative-local-gateway.istio-system.svc.cluster.local" 194 | 195 | --- 196 | # Allows the Webhooks to be reached by kube-api with or without 197 | # sidecar injection and with mTLS PERMISSIVE and STRICT. 198 | apiVersion: "security.istio.io/v1beta1" 199 | kind: "PeerAuthentication" 200 | metadata: 201 | name: "webhook" 202 | namespace: "knative-serving" 203 | labels: 204 | app.kubernetes.io/component: net-istio 205 | app.kubernetes.io/name: knative-serving 206 | app.kubernetes.io/version: "1.11.0" 207 | networking.knative.dev/ingress-provider: istio 208 | spec: 209 | selector: 210 | matchLabels: 211 | app: webhook 212 | portLevelMtls: 213 | "8443": 214 | mode: PERMISSIVE 215 | --- 216 | apiVersion: "security.istio.io/v1beta1" 217 | kind: "PeerAuthentication" 218 | metadata: 219 | name: "net-istio-webhook" 220 | namespace: "knative-serving" 221 | labels: 222 | app.kubernetes.io/component: net-istio 223 | app.kubernetes.io/name: knative-serving 224 | app.kubernetes.io/version: "1.11.0" 225 | networking.knative.dev/ingress-provider: istio 226 | spec: 227 | selector: 228 | matchLabels: 229 | app: net-istio-webhook 230 | portLevelMtls: 231 | "8443": 232 | mode: PERMISSIVE 233 | 234 | --- 235 | # Copyright 2019 The Knative Authors 236 | # 237 | # Licensed under the Apache License, Version 2.0 (the "License"); 238 | # you may not use this file except in compliance with the License. 239 | # You may obtain a copy of the License at 240 | # 241 | # https://www.apache.org/licenses/LICENSE-2.0 242 | # 243 | # Unless required by applicable law or agreed to in writing, software 244 | # distributed under the License is distributed on an "AS IS" BASIS, 245 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 246 | # See the License for the specific language governing permissions and 247 | # limitations under the License. 248 | 249 | apiVersion: apps/v1 250 | kind: Deployment 251 | metadata: 252 | name: net-istio-controller 253 | namespace: knative-serving 254 | labels: 255 | app.kubernetes.io/component: net-istio 256 | app.kubernetes.io/name: knative-serving 257 | app.kubernetes.io/version: "1.11.0" 258 | networking.knative.dev/ingress-provider: istio 259 | spec: 260 | selector: 261 | matchLabels: 262 | app: net-istio-controller 263 | template: 264 | metadata: 265 | annotations: 266 | # This must be outside of the mesh to probe the gateways. 267 | # NOTE: this is allowed here and not elsewhere because 268 | # this is the Istio controller, and so it may be Istio-aware. 269 | sidecar.istio.io/inject: "false" 270 | labels: 271 | app: net-istio-controller 272 | app.kubernetes.io/component: net-istio 273 | app.kubernetes.io/name: knative-serving 274 | app.kubernetes.io/version: "1.11.0" 275 | spec: 276 | serviceAccountName: controller 277 | containers: 278 | - name: controller 279 | # This is the Go import path for the binary that is containerized 280 | # and substituted here. 281 | image: gcr.io/knative-releases/knative.dev/net-istio/cmd/controller@sha256:27e7beb7c62036216fc464fb2181e56b030158ad4ceb57a7de172f54b4fe43db 282 | resources: 283 | requests: 284 | cpu: 30m 285 | memory: 40Mi 286 | limits: 287 | cpu: 300m 288 | memory: 400Mi 289 | env: 290 | - name: SYSTEM_NAMESPACE 291 | valueFrom: 292 | fieldRef: 293 | fieldPath: metadata.namespace 294 | - name: CONFIG_LOGGING_NAME 295 | value: config-logging 296 | - name: CONFIG_OBSERVABILITY_NAME 297 | value: config-observability 298 | - name: ENABLE_SECRET_INFORMER_FILTERING_BY_CERT_UID 299 | value: "false" 300 | # TODO(https://github.com/knative/pkg/pull/953): Remove stackdriver specific config 301 | - name: METRICS_DOMAIN 302 | value: knative.dev/net-istio 303 | securityContext: 304 | allowPrivilegeEscalation: false 305 | readOnlyRootFilesystem: true 306 | runAsNonRoot: true 307 | capabilities: 308 | drop: 309 | - ALL 310 | seccompProfile: 311 | type: RuntimeDefault 312 | readinessProbe: 313 | httpGet: 314 | path: /readiness 315 | port: probes 316 | scheme: HTTP 317 | periodSeconds: 5 318 | failureThreshold: 3 319 | livenessProbe: 320 | httpGet: 321 | path: /health 322 | port: probes 323 | scheme: HTTP 324 | periodSeconds: 5 325 | failureThreshold: 6 326 | ports: 327 | - name: metrics 328 | containerPort: 9090 329 | - name: profiling 330 | containerPort: 8008 331 | - name: probes 332 | containerPort: 8080 333 | 334 | # Unlike other controllers, this doesn't need a Service defined for metrics and 335 | # profiling because it opts out of the mesh (see annotation above). 336 | 337 | --- 338 | # Copyright 2020 The Knative Authors 339 | # 340 | # Licensed under the Apache License, Version 2.0 (the "License"); 341 | # you may not use this file except in compliance with the License. 342 | # You may obtain a copy of the License at 343 | # 344 | # https://www.apache.org/licenses/LICENSE-2.0 345 | # 346 | # Unless required by applicable law or agreed to in writing, software 347 | # distributed under the License is distributed on an "AS IS" BASIS, 348 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 349 | # See the License for the specific language governing permissions and 350 | # limitations under the License. 351 | 352 | apiVersion: apps/v1 353 | kind: Deployment 354 | metadata: 355 | name: net-istio-webhook 356 | namespace: knative-serving 357 | labels: 358 | app.kubernetes.io/component: net-istio 359 | app.kubernetes.io/name: knative-serving 360 | app.kubernetes.io/version: "1.11.0" 361 | networking.knative.dev/ingress-provider: istio 362 | spec: 363 | selector: 364 | matchLabels: 365 | app: net-istio-webhook 366 | role: net-istio-webhook 367 | template: 368 | metadata: 369 | labels: 370 | app: net-istio-webhook 371 | role: net-istio-webhook 372 | app.kubernetes.io/component: net-istio 373 | app.kubernetes.io/name: knative-serving 374 | app.kubernetes.io/version: "1.11.0" 375 | spec: 376 | serviceAccountName: controller 377 | containers: 378 | - name: webhook 379 | # This is the Go import path for the binary that is containerized 380 | # and substituted here. 381 | image: gcr.io/knative-releases/knative.dev/net-istio/cmd/webhook@sha256:0cdef272e39c57971ce9977765f164dd8e3abb9395a4f60e7a4160d57dcc09f2 382 | resources: 383 | requests: 384 | cpu: 20m 385 | memory: 20Mi 386 | limits: 387 | cpu: 200m 388 | memory: 200Mi 389 | env: 390 | - name: SYSTEM_NAMESPACE 391 | valueFrom: 392 | fieldRef: 393 | fieldPath: metadata.namespace 394 | - name: CONFIG_LOGGING_NAME 395 | value: config-logging 396 | - name: CONFIG_OBSERVABILITY_NAME 397 | value: config-observability 398 | # TODO(https://github.com/knative/pkg/pull/953): Remove stackdriver specific config 399 | - name: METRICS_DOMAIN 400 | value: knative.dev/net-istio 401 | - name: WEBHOOK_NAME 402 | value: net-istio-webhook 403 | # If you change WEBHOOK_PORT, you will also need to change the 404 | # containerPort "https-webhook" to the same value. 405 | - name: WEBHOOK_PORT 406 | value: "8443" 407 | securityContext: 408 | runAsNonRoot: true 409 | allowPrivilegeEscalation: false 410 | capabilities: 411 | drop: 412 | - ALL 413 | seccompProfile: 414 | type: RuntimeDefault 415 | readinessProbe: 416 | periodSeconds: 1 417 | httpGet: 418 | scheme: HTTPS 419 | port: 8443 420 | httpHeaders: 421 | - name: k-kubelet-probe 422 | value: "webhook" 423 | failureThreshold: 3 424 | livenessProbe: 425 | periodSeconds: 1 426 | httpGet: 427 | scheme: HTTPS 428 | port: 8443 429 | httpHeaders: 430 | - name: k-kubelet-probe 431 | value: "webhook" 432 | failureThreshold: 6 433 | initialDelaySeconds: 20 434 | ports: 435 | - name: metrics 436 | containerPort: 9090 437 | - name: profiling 438 | containerPort: 8008 439 | - name: https-webhook 440 | containerPort: 8443 441 | 442 | --- 443 | # Copyright 2020 The Knative Authors 444 | # 445 | # Licensed under the Apache License, Version 2.0 (the "License"); 446 | # you may not use this file except in compliance with the License. 447 | # You may obtain a copy of the License at 448 | # 449 | # https://www.apache.org/licenses/LICENSE-2.0 450 | # 451 | # Unless required by applicable law or agreed to in writing, software 452 | # distributed under the License is distributed on an "AS IS" BASIS, 453 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 454 | # See the License for the specific language governing permissions and 455 | # limitations under the License. 456 | 457 | apiVersion: v1 458 | kind: Secret 459 | metadata: 460 | name: net-istio-webhook-certs 461 | namespace: knative-serving 462 | labels: 463 | app.kubernetes.io/component: net-istio 464 | app.kubernetes.io/name: knative-serving 465 | app.kubernetes.io/version: "1.11.0" 466 | networking.knative.dev/ingress-provider: istio 467 | 468 | --- 469 | # Copyright 2020 The Knative Authors 470 | # 471 | # Licensed under the Apache License, Version 2.0 (the "License"); 472 | # you may not use this file except in compliance with the License. 473 | # You may obtain a copy of the License at 474 | # 475 | # https://www.apache.org/licenses/LICENSE-2.0 476 | # 477 | # Unless required by applicable law or agreed to in writing, software 478 | # distributed under the License is distributed on an "AS IS" BASIS, 479 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 480 | # See the License for the specific language governing permissions and 481 | # limitations under the License. 482 | 483 | apiVersion: v1 484 | kind: Service 485 | metadata: 486 | name: net-istio-webhook 487 | namespace: knative-serving 488 | labels: 489 | role: net-istio-webhook 490 | app.kubernetes.io/component: net-istio 491 | app.kubernetes.io/name: knative-serving 492 | app.kubernetes.io/version: "1.11.0" 493 | networking.knative.dev/ingress-provider: istio 494 | spec: 495 | ports: 496 | # Define metrics and profiling for them to be accessible within service meshes. 497 | - name: http-metrics 498 | port: 9090 499 | targetPort: metrics 500 | - name: http-profiling 501 | port: 8008 502 | targetPort: profiling 503 | - name: https-webhook 504 | port: 443 505 | targetPort: https-webhook 506 | selector: 507 | app: net-istio-webhook 508 | 509 | --- 510 | # Copyright 2020 The Knative Authors 511 | # 512 | # Licensed under the Apache License, Version 2.0 (the "License"); 513 | # you may not use this file except in compliance with the License. 514 | # You may obtain a copy of the License at 515 | # 516 | # https://www.apache.org/licenses/LICENSE-2.0 517 | # 518 | # Unless required by applicable law or agreed to in writing, software 519 | # distributed under the License is distributed on an "AS IS" BASIS, 520 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 521 | # See the License for the specific language governing permissions and 522 | # limitations under the License. 523 | 524 | apiVersion: admissionregistration.k8s.io/v1 525 | kind: MutatingWebhookConfiguration 526 | metadata: 527 | name: webhook.istio.networking.internal.knative.dev 528 | labels: 529 | app.kubernetes.io/component: net-istio 530 | app.kubernetes.io/name: knative-serving 531 | app.kubernetes.io/version: "1.11.0" 532 | networking.knative.dev/ingress-provider: istio 533 | webhooks: 534 | - admissionReviewVersions: 535 | - v1 536 | - v1beta1 537 | clientConfig: 538 | service: 539 | name: net-istio-webhook 540 | namespace: knative-serving 541 | failurePolicy: Fail 542 | sideEffects: None 543 | objectSelector: 544 | matchExpressions: 545 | - {key: "serving.knative.dev/configuration", operator: Exists} 546 | name: webhook.istio.networking.internal.knative.dev 547 | 548 | --- 549 | # Copyright 2020 The Knative Authors 550 | # 551 | # Licensed under the Apache License, Version 2.0 (the "License"); 552 | # you may not use this file except in compliance with the License. 553 | # You may obtain a copy of the License at 554 | # 555 | # https://www.apache.org/licenses/LICENSE-2.0 556 | # 557 | # Unless required by applicable law or agreed to in writing, software 558 | # distributed under the License is distributed on an "AS IS" BASIS, 559 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 560 | # See the License for the specific language governing permissions and 561 | # limitations under the License. 562 | 563 | apiVersion: admissionregistration.k8s.io/v1 564 | kind: ValidatingWebhookConfiguration 565 | metadata: 566 | name: config.webhook.istio.networking.internal.knative.dev 567 | labels: 568 | app.kubernetes.io/component: net-istio 569 | app.kubernetes.io/name: knative-serving 570 | app.kubernetes.io/version: "1.11.0" 571 | networking.knative.dev/ingress-provider: istio 572 | webhooks: 573 | - admissionReviewVersions: 574 | - v1 575 | - v1beta1 576 | clientConfig: 577 | service: 578 | name: net-istio-webhook 579 | namespace: knative-serving 580 | failurePolicy: Fail 581 | sideEffects: None 582 | name: config.webhook.istio.networking.internal.knative.dev 583 | objectSelector: 584 | matchLabels: 585 | app.kubernetes.io/name: knative-serving 586 | app.kubernetes.io/component: net-istio 587 | 588 | --- 589 | -------------------------------------------------------------------------------- /cluster-manifests/common/knative/serving-config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: config-domain 5 | namespace: knative-serving 6 | labels: 7 | app.kubernetes.io/name: knative-serving 8 | app.kubernetes.io/component: controller 9 | app.kubernetes.io/version: "1.11.1" 10 | serving.knative.dev/release: "v1.11.1" 11 | annotations: 12 | knative.dev/bjdazure-checksum: "81552d0b" 13 | data: 14 | knative.bjdazure.tech: "" -------------------------------------------------------------------------------- /cluster-manifests/common/kubecost.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: kubecost-system 6 | --- 7 | apiVersion: source.toolkit.fluxcd.io/v1beta1 8 | kind: HelmRepository 9 | metadata: 10 | name: kubecost 11 | namespace: flux-system 12 | spec: 13 | interval: 1m0s 14 | url: https://kubecost.github.io/cost-analyzer/ 15 | --- 16 | apiVersion: helm.toolkit.fluxcd.io/v2beta1 17 | kind: HelmRelease 18 | metadata: 19 | name: kubecost-release 20 | namespace: flux-system 21 | spec: 22 | interval: 5m 23 | targetNamespace: kubecost-system 24 | chart: 25 | spec: 26 | chart: cost-analyzer 27 | version: '1.101.3' 28 | sourceRef: 29 | kind: HelmRepository 30 | name: kubecost 31 | namespace: flux-system 32 | interval: 1m -------------------------------------------------------------------------------- /cluster-manifests/common/kured.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: kured-system 6 | --- 7 | apiVersion: source.toolkit.fluxcd.io/v1beta1 8 | kind: HelmRepository 9 | metadata: 10 | name: kubereboot 11 | namespace: flux-system 12 | spec: 13 | interval: 1m0s 14 | url: https://kubereboot.github.io/charts/ 15 | --- 16 | apiVersion: helm.toolkit.fluxcd.io/v2beta1 17 | kind: HelmRelease 18 | metadata: 19 | name: kured-release 20 | namespace: flux-system 21 | spec: 22 | interval: 5m 23 | targetNamespace: kured-system 24 | chart: 25 | spec: 26 | chart: kured 27 | version: '4.0.2' 28 | sourceRef: 29 | kind: HelmRepository 30 | name: kubereboot 31 | namespace: flux-system 32 | interval: 1m 33 | values: 34 | nodeSelector: 35 | kubernetes.io/os: linux -------------------------------------------------------------------------------- /cluster-manifests/common/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ./otel 5 | - ./prometheus 6 | - ./knative 7 | - kubecost.yaml 8 | - kured.yaml -------------------------------------------------------------------------------- /cluster-manifests/common/otel/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: otel-collector 5 | namespace: otel-system 6 | labels: 7 | app: opentelemetry 8 | component: otel-collector 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: opentelemetry 14 | template: 15 | metadata: 16 | labels: 17 | app: opentelemetry 18 | component: otel-collector 19 | spec: 20 | serviceAccountName: otel-sa-identity 21 | containers: 22 | - name: otel-collector 23 | image: otel/opentelemetry-collector-contrib-dev:latest 24 | command: 25 | - "/otelcontribcol" 26 | - "--config=/conf/otel-collector-config" 27 | resources: 28 | limits: 29 | cpu: 1 30 | memory: 2Gi 31 | requests: 32 | cpu: 200m 33 | memory: 400Mi 34 | ports: 35 | - containerPort: 9411 # Default endpoint for Zipkin receiver. 36 | volumeMounts: 37 | - name: otel-collector-config-vol 38 | mountPath: /conf 39 | livenessProbe: 40 | httpGet: 41 | path: / 42 | port: 13133 43 | readinessProbe: 44 | httpGet: 45 | path: / 46 | port: 13133 47 | volumes: 48 | - name: otel-collector-config-vol 49 | csi: 50 | driver: secrets-store.csi.k8s.io 51 | readOnly: true 52 | volumeAttributes: 53 | secretProviderClass: "otel-collector-config" 54 | --- 55 | apiVersion: v1 56 | kind: Service 57 | metadata: 58 | name: otel-collector 59 | namespace: otel-system 60 | labels: 61 | app: opencesus 62 | component: otel-collector 63 | spec: 64 | ports: 65 | - name: zipkin 66 | port: 9411 67 | protocol: TCP 68 | targetPort: 9411 69 | selector: 70 | component: otel-collector -------------------------------------------------------------------------------- /cluster-manifests/common/otel/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - namespace.yaml 5 | - serviceaccount.yaml 6 | - deployment.yaml 7 | - secretsprovider.yaml -------------------------------------------------------------------------------- /cluster-manifests/common/otel/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: otel-system 5 | labels: 6 | istio.io/rev: asm-1-17 -------------------------------------------------------------------------------- /cluster-manifests/common/otel/secretsprovider.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 2 | kind: SecretProviderClass 3 | metadata: 4 | name: otel-collector-config 5 | namespace: otel-system 6 | spec: 7 | provider: azure 8 | parameters: 9 | usePodIdentity: "false" 10 | useVMManagedIdentity: "false" 11 | keyvaultName: default_cluster_name-kv 12 | clientID: default_otel_client_id 13 | cloudName: "" 14 | objects: | 15 | array: 16 | - | 17 | objectName: otel-collector-config 18 | objectType: secret 19 | objectVersion: "" # [OPTIONAL] object versions, default to latest if empty 20 | tenantId: 16b3c013-d300-468d-ac64-7eda0820b6d3 21 | --- -------------------------------------------------------------------------------- /cluster-manifests/common/otel/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: otel-sa-identity 5 | namespace: otel-system 6 | annotations: 7 | azure.workload.identity/client-id: default_otel_lient_id 8 | azure.workload.identity/tenant-id: 16b3c013-d300-468d-ac64-7eda0820b6d3 9 | labels: 10 | azure.workload.identity/use: "true" -------------------------------------------------------------------------------- /cluster-manifests/common/prometheus/ama-metrics-settings-configmap.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | data: 4 | schema-version: 5 | #string.used by agent to parse config. supported versions are {v1}. Configs with other schema versions will be rejected by the agent. 6 | v1 7 | config-version: 8 | #string.used by customer to keep track of this config file's version in their source control/repository (max allowed 10 chars, other chars will be truncated) 9 | ver1 10 | prometheus-collector-settings: |- 11 | cluster_alias = "" 12 | default-scrape-settings-enabled: |- 13 | kubelet = true 14 | coredns = false 15 | cadvisor = true 16 | kubeproxy = false 17 | apiserver = false 18 | kubestate = true 19 | nodeexporter = true 20 | windowsexporter = false 21 | windowskubeproxy = false 22 | kappiebasic = false 23 | prometheuscollectorhealth = false 24 | # Regex for which namespaces to scrape through pod annotation based scraping. 25 | # This is none by default. Use '.*' to scrape all namespaces of annotated pods. 26 | pod-annotation-based-scraping: |- 27 | podannotationnamepsaceregex = ".*" 28 | default-targets-metrics-keep-list: |- 29 | kubelet = "" 30 | coredns = "" 31 | cadvisor = "" 32 | kubeproxy = "" 33 | apiserver = "" 34 | kubestate = "" 35 | nodeexporter = "" 36 | windowsexporter = "" 37 | windowskubeproxy = "" 38 | podannotations = "" 39 | kappiebasic = "" 40 | minimalingestionprofile = true 41 | default-targets-scrape-interval-settings: |- 42 | kubelet = "30s" 43 | coredns = "30s" 44 | cadvisor = "30s" 45 | kubeproxy = "30s" 46 | apiserver = "30s" 47 | kubestate = "30s" 48 | nodeexporter = "30s" 49 | windowsexporter = "30s" 50 | windowskubeproxy = "30s" 51 | kappiebasic = "30s" 52 | prometheuscollectorhealth = "30s" 53 | podannotations = "30s" 54 | debug-mode: |- 55 | enabled = false 56 | metadata: 57 | name: ama-metrics-settings-configmap 58 | namespace: kube-system 59 | -------------------------------------------------------------------------------- /cluster-manifests/common/prometheus/container-azm-ms-agentconfig.yaml: -------------------------------------------------------------------------------- 1 | kind: ConfigMap 2 | apiVersion: v1 3 | data: 4 | schema-version: 5 | v1 6 | config-version: 7 | ver1 8 | log-data-collection-settings: |- 9 | [log_collection_settings] 10 | [log_collection_settings.stdout] 11 | enabled = true 12 | exclude_namespaces = ["kube-system","gatekeeper-system"] 13 | 14 | [log_collection_settings.stderr] 15 | enabled = true 16 | exclude_namespaces = ["kube-system","gatekeeper-system"] 17 | 18 | [log_collection_settings.env_var] 19 | enabled = true 20 | [log_collection_settings.enrich_container_logs] 21 | enabled = false 22 | [log_collection_settings.collect_all_kube_events] 23 | enabled = false 24 | 25 | 26 | prometheus-data-collection-settings: |- 27 | [prometheus_data_collection_settings.cluster] 28 | interval = "1m" 29 | monitor_kubernetes_pods = true 30 | monitor_kubernetes_pods_namespaces = ["default"] 31 | 32 | [prometheus_data_collection_settings.node] 33 | interval = "1m" 34 | 35 | 36 | metric_collection_settings: |- 37 | [metric_collection_settings.collect_kube_system_pv_metrics] 38 | enabled = false 39 | 40 | alertable-metrics-configuration-settings: |- 41 | [alertable_metrics_configuration_settings.container_resource_utilization_thresholds] 42 | container_cpu_threshold_percentage = 95.0 43 | container_memory_rss_threshold_percentage = 95.0 44 | container_memory_working_set_threshold_percentage = 95.0 45 | 46 | [alertable_metrics_configuration_settings.pv_utilization_thresholds] 47 | pv_usage_threshold_percentage = 60.0 48 | 49 | # Alertable metrics configuration settings for completed jobs count 50 | [alertable_metrics_configuration_settings.job_completion_threshold] 51 | job_completion_threshold_time_minutes = 360 52 | integrations: |- 53 | [integrations.azure_network_policy_manager] 54 | collect_basic_metrics = false 55 | collect_advanced_metrics = true 56 | [integrations.azure_subnet_ip_usage] 57 | enabled = false 58 | 59 | agent-settings: |- 60 | [agent_settings.prometheus_fbit_settings] 61 | tcp_listener_chunk_size = 10 62 | tcp_listener_buffer_size = 10 63 | tcp_listener_mem_buf_limit = 200 64 | 65 | metadata: 66 | name: container-azm-ms-agentconfig 67 | namespace: kube-system 68 | -------------------------------------------------------------------------------- /cluster-manifests/common/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - container-azm-ms-agentconfig.yaml 5 | - ama-metrics-settings-configmap.yaml 6 | #- prometheus-config.yaml -------------------------------------------------------------------------------- /cluster-manifests/common/prometheus/prometheus-config: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | 4 | scrape_configs: 5 | - job_name: 'kubernetes-pods' 6 | kubernetes_sd_configs: 7 | - role: pod 8 | 9 | relabel_configs: 10 | # Scrape only pods with the annotation: prometheus.io/scrape = true 11 | - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] 12 | action: keep 13 | regex: true 14 | 15 | # If prometheus.io/path is specified, scrape this path instead of /metrics 16 | - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] 17 | action: replace 18 | target_label: __metrics_path__ 19 | regex: (.+) 20 | 21 | # If prometheus.io/port is specified, scrape this port instead of the default 22 | - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] 23 | action: replace 24 | regex: ([^:]+)(?::\d+)?;(\d+) 25 | replacement: $1:$2 26 | target_label: __address__ 27 | 28 | # If prometheus.io/scheme is specified, scrape with this scheme instead of http 29 | - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] 30 | action: replace 31 | regex: (http|https) 32 | target_label: __scheme__ 33 | 34 | # Include the pod namespace as a label for each metric 35 | - source_labels: [__meta_kubernetes_namespace] 36 | action: replace 37 | target_label: kubernetes_namespace 38 | 39 | # Include the pod name as a label for each metric 40 | - source_labels: [__meta_kubernetes_pod_name] 41 | action: replace 42 | target_label: kubernetes_pod_name 43 | 44 | # [Optional] Include all pod labels as labels for each metric 45 | - action: labelmap 46 | regex: __meta_kubernetes_pod_label_(.+) -------------------------------------------------------------------------------- /cluster-manifests/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | resources: 4 | - ./common 5 | - apps.yaml -------------------------------------------------------------------------------- /infrastructure/acr.tf: -------------------------------------------------------------------------------- 1 | 2 | data "azurerm_container_registry" "acr_repo" { 3 | name = var.acr_name 4 | resource_group_name = var.acr_resource_group 5 | provider = azurerm.core 6 | } -------------------------------------------------------------------------------- /infrastructure/aks-cluster.tf: -------------------------------------------------------------------------------- 1 | data "azurerm_kubernetes_service_versions" "current" { 2 | location = azurerm_resource_group.k8s.location 3 | } 4 | 5 | locals { 6 | kubernetes_version = data.azurerm_kubernetes_service_versions.current.versions[length(data.azurerm_kubernetes_service_versions.current.versions) - 2] 7 | } 8 | 9 | resource "azurerm_kubernetes_cluster" "k8s" { 10 | depends_on = [ 11 | azurerm_role_assignment.aks_role_assignemnt_dns, 12 | azurerm_role_assignment.aks_role_assignemnt_msi 13 | ] 14 | 15 | lifecycle { 16 | ignore_changes = [ 17 | default_node_pool.0.node_count, 18 | ] 19 | } 20 | 21 | name = var.cluster_name 22 | location = azurerm_resource_group.k8s.location 23 | resource_group_name = azurerm_resource_group.k8s.name 24 | node_resource_group = replace(var.resource_group_name, "_RG", "_Nodes_RG") 25 | dns_prefix_private_cluster = var.cluster_name 26 | kubernetes_version = local.kubernetes_version 27 | private_cluster_enabled = true 28 | private_dns_zone_id = data.azurerm_private_dns_zone.aks_private_zone.id 29 | sku_tier = "Standard" 30 | automatic_upgrade_channel = "patch" 31 | node_os_upgrade_channel = "NodeImage" 32 | local_account_disabled = true 33 | azure_policy_enabled = true 34 | open_service_mesh_enabled = false 35 | run_command_enabled = false 36 | oidc_issuer_enabled = true 37 | workload_identity_enabled = true 38 | role_based_access_control_enabled = true 39 | image_cleaner_enabled = true 40 | image_cleaner_interval_hours = 48 41 | cost_analysis_enabled = true 42 | 43 | # api_server_access_profile { 44 | # vnet_integration_enabled = true 45 | # subnet_id = data.azurerm_subnet.k8s_apiserver_subnet.id 46 | # } 47 | 48 | azure_active_directory_role_based_access_control { 49 | azure_rbac_enabled = true 50 | tenant_id = data.azurerm_client_config.current.tenant_id 51 | } 52 | 53 | identity { 54 | type = "UserAssigned" 55 | identity_ids = [azurerm_user_assigned_identity.aks_identity.id] 56 | } 57 | 58 | kubelet_identity { 59 | client_id = azurerm_user_assigned_identity.aks_kubelet_identity.client_id 60 | object_id = azurerm_user_assigned_identity.aks_kubelet_identity.principal_id 61 | user_assigned_identity_id = azurerm_user_assigned_identity.aks_kubelet_identity.id 62 | } 63 | 64 | default_node_pool { 65 | name = "system" 66 | node_count = var.agent_count 67 | zones = var.location == "northcentralus" ? null : ["1", "2", "3"] 68 | vm_size = var.vm_size 69 | os_disk_size_gb = 127 70 | os_disk_type = "Ephemeral" 71 | os_sku = "AzureLinux" 72 | vnet_subnet_id = data.azurerm_subnet.k8s_nodes_subnet.id 73 | type = "VirtualMachineScaleSets" 74 | auto_scaling_enabled = "true" 75 | min_count = 1 76 | max_count = 5 77 | kubelet_disk_type = "Temporary" 78 | only_critical_addons_enabled = true 79 | max_pods = 250 80 | 81 | upgrade_settings { 82 | max_surge = "33%" 83 | } 84 | } 85 | 86 | network_profile { 87 | dns_service_ip = var.dns_service_ip 88 | service_cidr = var.service_cidr 89 | pod_cidr = var.pod_cidr 90 | network_plugin = "azure" 91 | network_plugin_mode = "overlay" 92 | load_balancer_sku = "standard" 93 | network_data_plane = "cilium" 94 | network_policy = "cilium" 95 | outbound_type = "userDefinedRouting" 96 | } 97 | 98 | maintenance_window_auto_upgrade { 99 | frequency = "Weekly" 100 | interval = 1 101 | duration = 4 102 | day_of_week = "Friday" 103 | utc_offset = "-06:00" 104 | start_time = "20:00" 105 | } 106 | 107 | maintenance_window_node_os { 108 | frequency = "Weekly" 109 | interval = 1 110 | duration = 4 111 | day_of_week = "Saturday" 112 | utc_offset = "-06:00" 113 | start_time = "20:00" 114 | } 115 | 116 | auto_scaler_profile { 117 | max_unready_nodes = "1" 118 | } 119 | 120 | workload_autoscaler_profile { 121 | keda_enabled = true 122 | } 123 | 124 | storage_profile { 125 | blob_driver_enabled = true 126 | disk_driver_enabled = true 127 | file_driver_enabled = true 128 | } 129 | 130 | oms_agent { 131 | log_analytics_workspace_id = azurerm_log_analytics_workspace.k8s.id 132 | msi_auth_for_monitoring_enabled = true 133 | } 134 | 135 | microsoft_defender { 136 | log_analytics_workspace_id = azurerm_log_analytics_workspace.k8s.id 137 | } 138 | 139 | monitor_metrics { 140 | } 141 | 142 | key_vault_secrets_provider { 143 | secret_rotation_enabled = true 144 | secret_rotation_interval = "2m" 145 | } 146 | 147 | service_mesh_profile { 148 | mode = "Istio" 149 | internal_ingress_gateway_enabled = true 150 | revisions = local.istio_version 151 | } 152 | 153 | tags = { 154 | ServiceMeshType = var.service_mesh_type 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /infrastructure/aks-diag-setup.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_monitor_diagnostic_setting" "aks" { 2 | name = "${var.cluster_name}-diag" 3 | target_resource_id = azurerm_kubernetes_cluster.k8s.id 4 | log_analytics_workspace_id = azurerm_log_analytics_workspace.k8s.id 5 | 6 | enabled_log { 7 | category = "kube-apiserver" 8 | } 9 | 10 | enabled_log { 11 | category = "kube-audit" 12 | } 13 | 14 | enabled_log { 15 | category = "kube-audit-admin" 16 | } 17 | 18 | enabled_log { 19 | category = "kube-controller-manager" 20 | } 21 | 22 | enabled_log { 23 | category = "kube-scheduler" 24 | } 25 | 26 | enabled_log { 27 | category = "cluster-autoscaler" 28 | } 29 | 30 | enabled_log { 31 | category = "guard" 32 | } 33 | 34 | metric { 35 | category = "AllMetrics" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /infrastructure/aks-nodepools.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_kubernetes_cluster_node_pool" "default_app_node_pool" { 2 | lifecycle { 3 | ignore_changes = [ 4 | node_count 5 | ] 6 | } 7 | 8 | name = "apps" 9 | kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id 10 | vnet_subnet_id = data.azurerm_subnet.k8s_nodes_subnet.id 11 | vm_size = "Standard_D4ads_v5" 12 | auto_scaling_enabled = true 13 | mode = "User" 14 | os_sku = "AzureLinux" 15 | os_disk_size_gb = 127 16 | node_count = 3 17 | min_count = 3 18 | max_count = 6 19 | kubelet_disk_type = "Temporary" 20 | max_pods = 250 21 | upgrade_settings { 22 | max_surge = "25%" 23 | } 24 | } 25 | 26 | # resource "azurerm_kubernetes_cluster_node_pool" "eshop_app_node_pool" { 27 | # lifecycle { 28 | # ignore_changes = [ 29 | # node_count 30 | # ] 31 | # } 32 | 33 | # name = "eshop" 34 | # kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id 35 | # vnet_subnet_id = data.azurerm_subnet.k8s_nodes_subnet.id 36 | # vm_size = "Standard_B4ms" 37 | # enable_auto_scaling = true 38 | # mode = "User" 39 | # os_sku = "AzureLinux" 40 | # os_disk_size_gb = 100 41 | # node_count = 3 42 | # min_count = 3 43 | # max_count = 6 44 | # kubelet_disk_type = "Temporary" 45 | # max_pods = 250 46 | 47 | # upgrade_settings { 48 | # max_surge = "25%" 49 | # } 50 | 51 | # node_taints = ["reservedFor=eShopOnDapr:NoSchedule"] 52 | # } 53 | 54 | # resource "azurerm_kubernetes_cluster_node_pool" "traduire_app_node_pool" { 55 | # lifecycle { 56 | # ignore_changes = [ 57 | # node_count 58 | # ] 59 | # } 60 | # name = "traduire" 61 | # kubernetes_cluster_id = azurerm_kubernetes_cluster.k8s.id 62 | # vnet_subnet_id = data.azurerm_subnet.k8s_nodes_subnet.id 63 | # vm_size = "Standard_B4ms" 64 | # enable_auto_scaling = true 65 | # mode = "User" 66 | # os_sku = "AzureLinux" 67 | # os_disk_size_gb = 100 68 | # node_count = 3 69 | # min_count = 3 70 | # max_count = 6 71 | # kubelet_disk_type = "Temporary" 72 | # max_pods = 250 73 | 74 | # upgrade_settings { 75 | # max_surge = "25%" 76 | # } 77 | 78 | # node_taints = [ "app=traduire:NoSchedule" ] 79 | # } 80 | -------------------------------------------------------------------------------- /infrastructure/aks-post-setup-config.tf: -------------------------------------------------------------------------------- 1 | resource "azapi_update_resource" "this" { 2 | depends_on = [ 3 | azurerm_kubernetes_cluster.k8s, 4 | azurerm_kubernetes_cluster_node_pool.eshop_app_node_pool, 5 | //azurerm_kubernetes_cluster_node_pool.default_app_node_pool, 6 | //azurerm_kubernetes_cluster_node_pool.traduire_app_node_pool 7 | ] 8 | 9 | type = "Microsoft.ContainerService/managedClusters@2023-01-02-preview" 10 | resource_id = azurerm_kubernetes_cluster.k8s.id 11 | 12 | body = jsonencode({ 13 | properties = { 14 | networkProfile = { 15 | advancedNetworking = { 16 | observability = { 17 | enabled = true 18 | } 19 | } 20 | } 21 | } 22 | }) 23 | } 24 | -------------------------------------------------------------------------------- /infrastructure/cluster.tfvars: -------------------------------------------------------------------------------- 1 | agent_count = 3 2 | location = "southcentralus" 3 | k8s_vnet_resource_group_name = "Apps02_Network_RG" 4 | k8s_vnet = "DevSub02-Vnet-001" 5 | k8s_nodes_subnet = "kubernetes-nodes" 6 | k8s_apiserver_subnet = "kubernetes-apiserver" 7 | dns_service_ip = "100.64.0.10" 8 | service_cidr = "100.64.0.0/16" 9 | pod_cidr = "100.99.0.0/16" 10 | core_subscription = "ccfc5dda-43af-4b5e-8cc2-1dda18f2382e" 11 | dns_resource_group_name = "Core_DNS_RG" 12 | acr_resource_group = "Core_ContainerRepo_RG" 13 | acr_name = "bjdcsa" 14 | github_actions_identity_name = "gha-identity" 15 | github_actions_identity_resource_group = "Core_DevOps_RG" 16 | certificate_name = "wildcard-certificate" 17 | vm_size = "Standard_D2a_v4" 18 | -------------------------------------------------------------------------------- /infrastructure/dapr.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_kubernetes_cluster_extension" "dapr" { 2 | depends_on = [ 3 | azurerm_kubernetes_cluster_extension.flux, 4 | ] 5 | name = "dapr" 6 | cluster_id = azurerm_kubernetes_cluster.k8s.id 7 | extension_type = "microsoft.dapr" 8 | release_namespace = "dapr-system" 9 | } 10 | -------------------------------------------------------------------------------- /infrastructure/flux.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_kubernetes_cluster_extension" "flux" { 2 | depends_on = [ 3 | azurerm_kubernetes_cluster_node_pool.default_app_node_pool, 4 | //azurerm_kubernetes_cluster_node_pool.istio_node_pool 5 | ] 6 | name = "flux" 7 | cluster_id = azurerm_kubernetes_cluster.k8s.id 8 | extension_type = "microsoft.flux" 9 | } 10 | 11 | resource "azurerm_kubernetes_flux_configuration" "flux_config" { 12 | depends_on = [ 13 | azurerm_kubernetes_cluster_extension.flux 14 | ] 15 | 16 | name = "aks-flux-extension" 17 | cluster_id = azurerm_kubernetes_cluster.k8s.id 18 | namespace = "flux-system" 19 | scope = "cluster" 20 | 21 | git_repository { 22 | url = local.flux_repository 23 | reference_type = "branch" 24 | reference_value = local.flux_branch 25 | timeout_in_seconds = 600 26 | sync_interval_in_seconds = 30 27 | } 28 | 29 | kustomizations { 30 | name = "istio-crd" 31 | path = local.crd_path 32 | timeout_in_seconds = 600 33 | sync_interval_in_seconds = 120 34 | retry_interval_in_seconds = 300 35 | garbage_collection_enabled = true 36 | depends_on = [] 37 | } 38 | 39 | kustomizations { 40 | name = "istio-cfg" 41 | path = local.istio_cfg_path 42 | timeout_in_seconds = 600 43 | sync_interval_in_seconds = 120 44 | retry_interval_in_seconds = 300 45 | garbage_collection_enabled = true 46 | depends_on = [ 47 | "istio-crd" 48 | ] 49 | } 50 | 51 | kustomizations { 52 | name = "istio-gw" 53 | path = local.istio_gw_path 54 | timeout_in_seconds = 600 55 | sync_interval_in_seconds = 120 56 | retry_interval_in_seconds = 300 57 | garbage_collection_enabled = true 58 | depends_on = [ 59 | "istio-cfg" 60 | ] 61 | } 62 | 63 | kustomizations { 64 | name = "apps" 65 | path = local.app_path 66 | 67 | timeout_in_seconds = 600 68 | sync_interval_in_seconds = 120 69 | retry_interval_in_seconds = 300 70 | garbage_collection_enabled = true 71 | depends_on = [ 72 | "istio-cfg" 73 | ] 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /infrastructure/github-actions.tf: -------------------------------------------------------------------------------- 1 | data "azurerm_user_assigned_identity" "github_actions" { 2 | name = var.github_actions_identity_name 3 | resource_group_name = var.github_actions_identity_resource_group 4 | provider = azurerm.core 5 | } 6 | 7 | resource "azurerm_role_assignment" "github_actions" { 8 | scope = azurerm_kubernetes_cluster.k8s.id 9 | role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin" 10 | principal_id = data.azurerm_user_assigned_identity.github_actions.principal_id 11 | skip_service_principal_aad_check = true 12 | } 13 | 14 | resource "azurerm_role_assignment" "github_actions_aks_read" { 15 | scope = azurerm_kubernetes_cluster.k8s.id 16 | role_definition_name = "Azure Kubernetes Service Cluster User Role" 17 | principal_id = data.azurerm_user_assigned_identity.github_actions.principal_id 18 | skip_service_principal_aad_check = true 19 | } 20 | -------------------------------------------------------------------------------- /infrastructure/identities.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_user_assigned_identity" "aks_identity" { 2 | name = "${var.cluster_name}-cluster-identity" 3 | resource_group_name = azurerm_resource_group.k8s.name 4 | location = azurerm_resource_group.k8s.location 5 | } 6 | 7 | resource "azurerm_user_assigned_identity" "aks_kubelet_identity" { 8 | name = "${var.cluster_name}-kubelet-identity" 9 | resource_group_name = azurerm_resource_group.k8s.name 10 | location = azurerm_resource_group.k8s.location 11 | } 12 | 13 | resource "azurerm_user_assigned_identity" "aks_service_mesh_ingress_identity" { 14 | name = local.ingress_identity 15 | resource_group_name = azurerm_resource_group.k8s.name 16 | location = azurerm_resource_group.k8s.location 17 | } 18 | 19 | resource "azurerm_federated_identity_credential" "aks_service_mesh_ingress_identity" { 20 | name = local.ingress_identity 21 | resource_group_name = azurerm_resource_group.k8s.name 22 | audience = ["api://AzureADTokenExchange"] 23 | issuer = azurerm_kubernetes_cluster.k8s.oidc_issuer_url 24 | parent_id = azurerm_user_assigned_identity.aks_service_mesh_ingress_identity.id 25 | subject = "system:serviceaccount:${var.ingress_namespace}:${local.ingress_identity}" 26 | } 27 | 28 | resource "azurerm_user_assigned_identity" "aks_zipkin_identity" { 29 | name = local.otel_identity 30 | resource_group_name = azurerm_resource_group.k8s.name 31 | location = azurerm_resource_group.k8s.location 32 | } 33 | 34 | resource "azurerm_federated_identity_credential" "aks_zipkin_identity" { 35 | name = local.otel_identity 36 | resource_group_name = azurerm_resource_group.k8s.name 37 | audience = ["api://AzureADTokenExchange"] 38 | issuer = azurerm_kubernetes_cluster.k8s.oidc_issuer_url 39 | parent_id = azurerm_user_assigned_identity.aks_zipkin_identity.id 40 | subject = "system:serviceaccount:${var.zipkin_namespace}:${local.otel_identity}" 41 | } 42 | -------------------------------------------------------------------------------- /infrastructure/keyvault.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_key_vault" "k8s" { 2 | name = "${var.cluster_name}-kv" 3 | resource_group_name = azurerm_resource_group.k8s.name 4 | location = azurerm_resource_group.k8s.location 5 | tenant_id = data.azurerm_client_config.current.tenant_id 6 | soft_delete_retention_days = 7 7 | purge_protection_enabled = false 8 | enable_rbac_authorization = true 9 | 10 | sku_name = "standard" 11 | 12 | network_acls { 13 | bypass = "AzureServices" 14 | default_action = "Deny" 15 | } 16 | } 17 | 18 | resource "azurerm_private_endpoint" "key_vault" { 19 | name = "${var.cluster_name}-kv-endpoint" 20 | resource_group_name = azurerm_resource_group.k8s.name 21 | location = azurerm_resource_group.k8s.location 22 | subnet_id = data.azurerm_subnet.private_endpoint_subnet.id 23 | 24 | private_service_connection { 25 | name = "kv-${var.cluster_name}-kv-endpoint" 26 | private_connection_resource_id = azurerm_key_vault.k8s.id 27 | subresource_names = ["vault"] 28 | is_manual_connection = false 29 | } 30 | 31 | private_dns_zone_group { 32 | name = data.azurerm_private_dns_zone.privatelink_vaultcore_azure_net.name 33 | private_dns_zone_ids = [data.azurerm_private_dns_zone.privatelink_vaultcore_azure_net.id] 34 | } 35 | } 36 | 37 | resource "azurerm_key_vault_certificate" "k8s" { 38 | name = var.certificate_name 39 | key_vault_id = azurerm_key_vault.k8s.id 40 | 41 | depends_on = [ 42 | azurerm_private_endpoint.key_vault, 43 | azurerm_kubernetes_cluster.k8s 44 | ] 45 | 46 | certificate { 47 | contents = var.certificate_base64_encoded 48 | password = var.certificate_password 49 | } 50 | 51 | certificate_policy { 52 | issuer_parameters { 53 | name = "Self" 54 | } 55 | 56 | key_properties { 57 | exportable = true 58 | key_size = 2048 59 | key_type = "RSA" 60 | reuse_key = false 61 | } 62 | 63 | secret_properties { 64 | content_type = "application/x-pkcs12" 65 | } 66 | } 67 | } 68 | 69 | resource "azurerm_key_vault_secret" "otel" { 70 | name = "otel-collector-config" 71 | value = replace(templatefile("otel-config.tpl", { app_insight_key = "${azurerm_application_insights.k8s.instrumentation_key}" }), "/\n/", "\n") 72 | key_vault_id = azurerm_key_vault.k8s.id 73 | 74 | depends_on = [ 75 | azurerm_private_endpoint.key_vault, 76 | azurerm_kubernetes_cluster.k8s 77 | ] 78 | } 79 | 80 | -------------------------------------------------------------------------------- /infrastructure/logging.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_log_analytics_workspace" "k8s" { 2 | name = "${var.cluster_name}-logs" 3 | location = azurerm_resource_group.k8s.location 4 | resource_group_name = azurerm_resource_group.k8s.name 5 | sku = "PerGB2018" 6 | daily_quota_gb = 5 7 | local_authentication_disabled = true 8 | } 9 | 10 | resource "azurerm_application_insights" "k8s" { 11 | name = "${var.cluster_name}-appinsights" 12 | location = azurerm_resource_group.k8s.location 13 | resource_group_name = azurerm_resource_group.k8s.name 14 | workspace_id = azurerm_log_analytics_workspace.k8s.id 15 | application_type = "web" 16 | local_authentication_disabled = false 17 | } 18 | 19 | resource "azurerm_monitor_data_collection_rule" "log_analytics" { 20 | name = "${var.cluster_name}-law-datacollection-rules" 21 | location = azurerm_resource_group.k8s.location 22 | resource_group_name = azurerm_resource_group.k8s.name 23 | depends_on = [ 24 | azurerm_log_analytics_workspace.k8s 25 | ] 26 | 27 | destinations { 28 | log_analytics { 29 | workspace_resource_id = azurerm_log_analytics_workspace.k8s.id 30 | name = "ciworkspace" 31 | } 32 | } 33 | 34 | data_flow { 35 | streams = ["Microsoft-ContainerInsights-Group-Default"] 36 | destinations = ["ciworkspace"] 37 | } 38 | 39 | data_sources { 40 | extension { 41 | streams = ["Microsoft-ContainerInsights-Group-Default"] 42 | extension_name = "ContainerInsights" 43 | name = "ContainerInsightsExtension" 44 | } 45 | } 46 | } 47 | 48 | resource "azapi_resource" "log_analytics_datacollection_rule_associations" { 49 | type = "Microsoft.Insights/dataCollectionRuleAssociations@2021-09-01-preview" 50 | name = "${var.cluster_name}-law-datacollection-rules-association" 51 | parent_id = azurerm_kubernetes_cluster.k8s.id 52 | body = jsonencode({ 53 | properties = { 54 | dataCollectionRuleId = azurerm_monitor_data_collection_rule.log_analytics.id 55 | } 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /infrastructure/main.tf: -------------------------------------------------------------------------------- 1 | data "azurerm_client_config" "current" {} 2 | data "azurerm_subscription" "current" {} 3 | 4 | resource "azurerm_resource_group" "k8s" { 5 | name = var.resource_group_name 6 | location = var.location 7 | tags = { 8 | Application = "Private AKS Kubernetes Demo" 9 | Components = "aks; key vault; istio" 10 | DeployedOn = timestamp() 11 | } 12 | } 13 | 14 | locals { 15 | ingress_identity = "${var.service_mesh_type}-ingress-sa-identity" 16 | otel_identity = "otel-sa-identity" 17 | flux_repository = "https://github.com/briandenicola/private-kubernetes-cluster-setup" 18 | flux_branch = "cluster/default_cluster_name" 19 | app_path = "./cluster-manifests" 20 | crd_path = "./cluster-manifests/common/customresourcedefinitions" 21 | istio_cfg_path = "./cluster-manifests/common/istio/configuration" 22 | istio_gw_path = "./cluster-manifests/common/istio/gateway" 23 | istio_version = [ "asm-1-24" ] 24 | } 25 | -------------------------------------------------------------------------------- /infrastructure/network.tf: -------------------------------------------------------------------------------- 1 | data "azurerm_virtual_network" "vnet" { 2 | name = var.k8s_vnet 3 | resource_group_name = var.k8s_vnet_resource_group_name 4 | } 5 | 6 | data "azurerm_subnet" "k8s_nodes_subnet" { 7 | name = var.k8s_nodes_subnet 8 | virtual_network_name = var.k8s_vnet 9 | resource_group_name = var.k8s_vnet_resource_group_name 10 | } 11 | 12 | data "azurerm_subnet" "k8s_apiserver_subnet" { 13 | name = var.k8s_apiserver_subnet 14 | virtual_network_name = var.k8s_vnet 15 | resource_group_name = var.k8s_vnet_resource_group_name 16 | } 17 | 18 | data "azurerm_subnet" "management_subnet" { 19 | name = "Servers" 20 | virtual_network_name = var.k8s_vnet 21 | resource_group_name = var.k8s_vnet_resource_group_name 22 | } 23 | 24 | data "azurerm_subnet" "private_endpoint_subnet" { 25 | name = "private-endpoints" 26 | virtual_network_name = var.k8s_vnet 27 | resource_group_name = var.k8s_vnet_resource_group_name 28 | } 29 | 30 | data "azurerm_private_dns_zone" "aks_private_zone" { 31 | name = "privatelink.${var.location}.azmk8s.io" 32 | resource_group_name = var.dns_resource_group_name 33 | provider = azurerm.core 34 | } 35 | 36 | data "azurerm_private_dns_zone" "privatelink_vaultcore_azure_net" { 37 | name = "privatelink.vaultcore.azure.net" 38 | resource_group_name = var.dns_resource_group_name 39 | provider = azurerm.core 40 | } 41 | -------------------------------------------------------------------------------- /infrastructure/otel-config.tpl: -------------------------------------------------------------------------------- 1 | receivers: 2 | zipkin: 3 | endpoint: 0.0.0.0:9411 4 | extensions: 5 | health_check: 6 | pprof: 7 | endpoint: :1888 8 | zpages: 9 | endpoint: :55679 10 | exporters: 11 | logging: 12 | loglevel: debug 13 | azuremonitor: 14 | endpoint: "https://dc.services.visualstudio.com/v2/track" 15 | instrumentation_key: ${app_insight_key} 16 | maxbatchsize: 100 17 | maxbatchinterval: 10s 18 | service: 19 | extensions: [pprof, zpages, health_check] 20 | pipelines: 21 | traces: 22 | receivers: [zipkin] 23 | exporters: [azuremonitor,logging] 24 | -------------------------------------------------------------------------------- /infrastructure/prereqs/azuredeploy.central.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "azure_firewall_name": { 6 | "value": "bjdproxy-central" 7 | }, 8 | "default_nsg_name": { 9 | "value": "default-nsg" 10 | }, 11 | "public_ip_address_name": { 12 | "value": "bjdproxy-central-pip" 13 | }, 14 | "routetable_name": { 15 | "value": "bjdproxy-central-routetable" 16 | }, 17 | "firewallpolicy_name": { 18 | "value": "bjdproxy-central-policy" 19 | }, 20 | "virtual_network_name": { 21 | "value": "DevSub02-Vnet-001" 22 | }, 23 | "virtual_network_cidr" :{ 24 | "value": "10.5" 25 | }, 26 | "firewall_next_hop": { 27 | "value": "10.5.254.4" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /infrastructure/prereqs/azuredeploy.northcentral.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "azure_firewall_name": { 6 | "value": "bjdproxy-northcentral" 7 | }, 8 | "default_nsg_name": { 9 | "value": "default-nsg" 10 | }, 11 | "public_ip_address_name": { 12 | "value": "bjdproxy-northcentral-pip" 13 | }, 14 | "routetable_name": { 15 | "value": "bjdproxy-northcentral-routetable" 16 | }, 17 | "firewallpolicy_name": { 18 | "value": "bjdproxy-northcentral-policy" 19 | }, 20 | "virtual_network_name": { 21 | "value": "DevSub02-Vnet-002" 22 | }, 23 | "virtual_network_cidr" :{ 24 | "value": "10.17" 25 | }, 26 | "firewall_next_hop": { 27 | "value": "10.17.254.4" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /infrastructure/prereqs/azuredeploy.nsg.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "internet_nsg_name": { 6 | "type": "String", 7 | "defaultValue": "internet-nsg" 8 | }, 9 | "location": { 10 | "type": "string", 11 | "defaultValue": "[resourceGroup().location]" 12 | } 13 | }, 14 | "variables": {}, 15 | "resources": [ 16 | { 17 | "type": "Microsoft.Network/networkSecurityGroups", 18 | "apiVersion": "2020-11-01", 19 | "name": "[parameters('internet_nsg_name')]", 20 | "location": "[parameters('location')]", 21 | "properties": { 22 | "securityRules": [ 23 | { 24 | "name": "HTTP", 25 | "properties": { 26 | "protocol": "TCP", 27 | "sourcePortRange": "*", 28 | "destinationPortRange": "80", 29 | "sourceAddressPrefix": "*", 30 | "destinationAddressPrefix": "*", 31 | "access": "Allow", 32 | "priority": 100, 33 | "direction": "Inbound", 34 | "sourcePortRanges": [], 35 | "destinationPortRanges": [], 36 | "sourceAddressPrefixes": [], 37 | "destinationAddressPrefixes": [] 38 | } 39 | }, 40 | { 41 | "name": "HTTPS", 42 | "properties": { 43 | "protocol": "TCP", 44 | "sourcePortRange": "*", 45 | "destinationPortRange": "443", 46 | "sourceAddressPrefix": "*", 47 | "destinationAddressPrefix": "*", 48 | "access": "Allow", 49 | "priority": 110, 50 | "direction": "Inbound", 51 | "sourcePortRanges": [], 52 | "destinationPortRanges": [], 53 | "sourceAddressPrefixes": [], 54 | "destinationAddressPrefixes": [] 55 | } 56 | }, 57 | { 58 | "name": "AllowGatewayManager", 59 | "properties": { 60 | "protocol": "*", 61 | "sourcePortRange": "*", 62 | "destinationPortRange": "443", 63 | "sourceAddressPrefix": "GatewayManager", 64 | "destinationAddressPrefix": "*", 65 | "access": "Allow", 66 | "priority": 120, 67 | "direction": "Inbound", 68 | "sourcePortRanges": [], 69 | "destinationPortRanges": [], 70 | "sourceAddressPrefixes": [], 71 | "destinationAddressPrefixes": [] 72 | } 73 | }, 74 | { 75 | "name": "AllowSshRdpOutbound", 76 | "properties": { 77 | "protocol": "*", 78 | "sourcePortRange": "*", 79 | "sourceAddressPrefix": "*", 80 | "destinationAddressPrefix": "VirtualNetwork", 81 | "access": "Allow", 82 | "priority": 130, 83 | "direction": "Outbound", 84 | "sourcePortRanges": [], 85 | "destinationPortRanges": [ 86 | "22", 87 | "3389" 88 | ], 89 | "sourceAddressPrefixes": [], 90 | "destinationAddressPrefixes": [] 91 | } 92 | }, 93 | { 94 | "name": "AllowAzureCloudOutbound", 95 | "properties": { 96 | "protocol": "*", 97 | "sourcePortRange": "*", 98 | "destinationPortRange": "443", 99 | "sourceAddressPrefix": "*", 100 | "destinationAddressPrefix": "AzureCloud", 101 | "access": "Allow", 102 | "priority": 140, 103 | "direction": "Outbound", 104 | "sourcePortRanges": [], 105 | "destinationPortRanges": [], 106 | "sourceAddressPrefixes": [], 107 | "destinationAddressPrefixes": [] 108 | } 109 | } 110 | ] 111 | } 112 | } 113 | ] 114 | } -------------------------------------------------------------------------------- /infrastructure/prereqs/azuredeploy.southcentral.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "azure_firewall_name": { 6 | "value": "bjdproxy-southcentral" 7 | }, 8 | "default_nsg_name": { 9 | "value": "default-nsg" 10 | }, 11 | "public_ip_address_name": { 12 | "value": "bjdproxy-southcentral-pip" 13 | }, 14 | "routetable_name": { 15 | "value": "bjdproxy-southcentral-routetable" 16 | }, 17 | "firewallpolicy_name": { 18 | "value": "bjdproxy-southcentral-policy" 19 | }, 20 | "virtual_network_name": { 21 | "value": "DevSub02-Vnet-001" 22 | }, 23 | "virtual_network_cidr" :{ 24 | "value": "10.7" 25 | }, 26 | "firewall_next_hop": { 27 | "value": "10.7.254.4" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /infrastructure/prereqs/azuredeploy.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "azure_firewall_name": { 6 | "type": "String" 7 | }, 8 | "default_nsg_name": { 9 | "type": "String" 10 | }, 11 | "public_ip_address_name": { 12 | "type": "String" 13 | }, 14 | "routetable_name": { 15 | "type": "String" 16 | }, 17 | "firewallpolicy_name": { 18 | "type": "String" 19 | }, 20 | "virtual_network_name": { 21 | "type": "String" 22 | }, 23 | "virtual_network_cidr": { 24 | "type": "string" 25 | }, 26 | "firewall_next_hop": { 27 | "type": "String" 28 | }, 29 | "location": { 30 | "defaultValue": "[resourceGroup().location]", 31 | "type": "string" 32 | } 33 | }, 34 | "variables": { 35 | 36 | }, 37 | "resources": [ 38 | { 39 | "type": "Microsoft.Network/firewallPolicies", 40 | "apiVersion": "2020-11-01", 41 | "name": "[parameters('firewallpolicy_name')]", 42 | "location": "[parameters('location')]", 43 | "properties": { 44 | "sku": { 45 | "tier": "Standard" 46 | }, 47 | "threatIntelMode": "Alert", 48 | "dnsSettings": { 49 | "servers": [], 50 | "enableProxy": true 51 | } 52 | } 53 | }, 54 | { 55 | "type": "Microsoft.Network/firewallPolicies/ruleCollectionGroups", 56 | "apiVersion": "2020-11-01", 57 | "name": "[concat(parameters('firewallpolicy_name'), '/default_rules_collection')]", 58 | "location": "[parameters('location')]", 59 | "dependsOn": [ 60 | "[resourceId('Microsoft.Network/firewallPolicies', parameters('firewallpolicy_name'))]" 61 | ], 62 | "properties": { 63 | "priority": 200, 64 | "ruleCollections": [ 65 | { 66 | "ruleCollectionType": "FirewallPolicyFilterRuleCollection", 67 | "action": { 68 | "type": "Allow" 69 | }, 70 | "rules": [ 71 | { 72 | "ruleType": "ApplicationRule", 73 | "name": "azure-management", 74 | "protocols": [ 75 | { 76 | "protocolType": "Https", 77 | "port": 443 78 | } 79 | ], 80 | "fqdnTags": [], 81 | "webCategories": [], 82 | "targetFqdns": [ 83 | "management.microsoft.com" 84 | ], 85 | "targetUrls": [], 86 | "terminateTLS": false, 87 | "sourceAddresses": [ 88 | "*" 89 | ], 90 | "destinationAddresses": [], 91 | "sourceIpGroups": [] 92 | }, 93 | { 94 | "ruleType": "ApplicationRule", 95 | "name": "docker", 96 | "protocols": [ 97 | { 98 | "protocolType": "Https", 99 | "port": 443 100 | } 101 | ], 102 | "fqdnTags": [], 103 | "webCategories": [], 104 | "targetFqdns": [ 105 | "production.cloudflare.docker.com", 106 | "*.docker.io", 107 | "download.docker.com" 108 | ], 109 | "targetUrls": [], 110 | "terminateTLS": false, 111 | "sourceAddresses": [ 112 | "*" 113 | ], 114 | "destinationAddresses": [], 115 | "sourceIpGroups": [] 116 | }, 117 | { 118 | "ruleType": "ApplicationRule", 119 | "name": "monitoring-ods", 120 | "protocols": [ 121 | { 122 | "protocolType": "Https", 123 | "port": 443 124 | } 125 | ], 126 | "fqdnTags": [], 127 | "webCategories": [], 128 | "targetFqdns": [ 129 | "*.ods.opinsights.azure.com" 130 | ], 131 | "targetUrls": [], 132 | "terminateTLS": false, 133 | "sourceAddresses": [ 134 | "*" 135 | ], 136 | "destinationAddresses": [], 137 | "sourceIpGroups": [] 138 | }, 139 | { 140 | "ruleType": "ApplicationRule", 141 | "name": "mcr.microsoft.com", 142 | "protocols": [ 143 | { 144 | "protocolType": "Https", 145 | "port": 443 146 | } 147 | ], 148 | "fqdnTags": [], 149 | "webCategories": [], 150 | "targetFqdns": [ 151 | "mcr.microsoft.com" 152 | ], 153 | "targetUrls": [], 154 | "terminateTLS": false, 155 | "sourceAddresses": [ 156 | "*" 157 | ], 158 | "destinationAddresses": [], 159 | "sourceIpGroups": [] 160 | }, 161 | { 162 | "ruleType": "ApplicationRule", 163 | "name": "policy-store", 164 | "protocols": [ 165 | { 166 | "protocolType": "Https", 167 | "port": 443 168 | } 169 | ], 170 | "fqdnTags": [], 171 | "webCategories": [], 172 | "targetFqdns": [ 173 | "store.policy.core.windows.net" 174 | ], 175 | "targetUrls": [], 176 | "terminateTLS": false, 177 | "sourceAddresses": [ 178 | "*" 179 | ], 180 | "destinationAddresses": [], 181 | "sourceIpGroups": [] 182 | }, 183 | { 184 | "ruleType": "ApplicationRule", 185 | "name": "aad-login", 186 | "protocols": [ 187 | { 188 | "protocolType": "Https", 189 | "port": 443 190 | } 191 | ], 192 | "fqdnTags": [], 193 | "webCategories": [], 194 | "targetFqdns": [ 195 | "login.microsoftonline.com" 196 | ], 197 | "targetUrls": [], 198 | "terminateTLS": false, 199 | "sourceAddresses": [ 200 | "*" 201 | ], 202 | "destinationAddresses": [], 203 | "sourceIpGroups": [] 204 | }, 205 | { 206 | "ruleType": "ApplicationRule", 207 | "name": "policy-data", 208 | "protocols": [ 209 | { 210 | "protocolType": "Https", 211 | "port": 443 212 | } 213 | ], 214 | "fqdnTags": [], 215 | "webCategories": [], 216 | "targetFqdns": [ 217 | "data.policy.core.windows.net" 218 | ], 219 | "targetUrls": [], 220 | "terminateTLS": false, 221 | "sourceAddresses": [ 222 | "*" 223 | ], 224 | "destinationAddresses": [], 225 | "sourceIpGroups": [] 226 | }, 227 | { 228 | "ruleType": "ApplicationRule", 229 | "name": "ubuntu-changelogs", 230 | "protocols": [ 231 | { 232 | "protocolType": "Http", 233 | "port": 80 234 | }, 235 | { 236 | "protocolType": "Https", 237 | "port": 443 238 | } 239 | ], 240 | "fqdnTags": [], 241 | "webCategories": [], 242 | "targetFqdns": [ 243 | "changelogs.ubuntu.com" 244 | ], 245 | "targetUrls": [], 246 | "terminateTLS": false, 247 | "sourceAddresses": [ 248 | "*" 249 | ], 250 | "destinationAddresses": [], 251 | "sourceIpGroups": [] 252 | }, 253 | { 254 | "ruleType": "ApplicationRule", 255 | "name": "ubuntu-security", 256 | "protocols": [ 257 | { 258 | "protocolType": "Http", 259 | "port": 80 260 | }, 261 | { 262 | "protocolType": "Https", 263 | "port": 443 264 | } 265 | ], 266 | "fqdnTags": [], 267 | "webCategories": [], 268 | "targetFqdns": [ 269 | "security.ubuntu.com" 270 | ], 271 | "targetUrls": [], 272 | "terminateTLS": false, 273 | "sourceAddresses": [ 274 | "*" 275 | ], 276 | "destinationAddresses": [], 277 | "sourceIpGroups": [] 278 | }, 279 | { 280 | "ruleType": "ApplicationRule", 281 | "name": "monitoring-uri", 282 | "protocols": [ 283 | { 284 | "protocolType": "Https", 285 | "port": 443 286 | } 287 | ], 288 | "fqdnTags": [], 289 | "webCategories": [], 290 | "targetFqdns": [ 291 | "*.monitoring.azure.com", 292 | "*.monitor.azure.com" 293 | ], 294 | "targetUrls": [], 295 | "terminateTLS": false, 296 | "sourceAddresses": [ 297 | "*" 298 | ], 299 | "destinationAddresses": [], 300 | "sourceIpGroups": [] 301 | }, 302 | { 303 | "ruleType": "ApplicationRule", 304 | "name": "aks-packages", 305 | "protocols": [ 306 | { 307 | "protocolType": "Https", 308 | "port": 443 309 | } 310 | ], 311 | "fqdnTags": [], 312 | "webCategories": [], 313 | "targetFqdns": [ 314 | "packages.microsoft.com" 315 | ], 316 | "targetUrls": [], 317 | "terminateTLS": false, 318 | "sourceAddresses": [ 319 | "*" 320 | ], 321 | "destinationAddresses": [], 322 | "sourceIpGroups": [] 323 | }, 324 | { 325 | "ruleType": "ApplicationRule", 326 | "name": "monitoring-oms", 327 | "protocols": [ 328 | { 329 | "protocolType": "Https", 330 | "port": 443 331 | } 332 | ], 333 | "fqdnTags": [], 334 | "webCategories": [], 335 | "targetFqdns": [ 336 | "*.oms.opinsights.azure.com" 337 | ], 338 | "targetUrls": [], 339 | "terminateTLS": false, 340 | "sourceAddresses": [ 341 | "*" 342 | ], 343 | "destinationAddresses": [], 344 | "sourceIpGroups": [] 345 | }, 346 | { 347 | "ruleType": "ApplicationRule", 348 | "name": "mcr-data", 349 | "protocols": [ 350 | { 351 | "protocolType": "Https", 352 | "port": 443 353 | } 354 | ], 355 | "fqdnTags": [], 356 | "webCategories": [], 357 | "targetFqdns": [ 358 | "*.data.mcr.microsoft.com" 359 | ], 360 | "targetUrls": [], 361 | "terminateTLS": false, 362 | "sourceAddresses": [ 363 | "*" 364 | ], 365 | "destinationAddresses": [], 366 | "sourceIpGroups": [] 367 | }, 368 | { 369 | "ruleType": "ApplicationRule", 370 | "name": "monitoring-telemetry", 371 | "protocols": [ 372 | { 373 | "protocolType": "Https", 374 | "port": 443 375 | } 376 | ], 377 | "fqdnTags": [], 378 | "webCategories": [], 379 | "targetFqdns": [ 380 | "dc.services.visualstudio.com" 381 | ], 382 | "targetUrls": [], 383 | "terminateTLS": false, 384 | "sourceAddresses": [ 385 | "*" 386 | ], 387 | "destinationAddresses": [], 388 | "sourceIpGroups": [] 389 | }, 390 | { 391 | "ruleType": "ApplicationRule", 392 | "name": "ubuntu-archive", 393 | "protocols": [ 394 | { 395 | "protocolType": "Http", 396 | "port": 80 397 | }, 398 | { 399 | "protocolType": "Https", 400 | "port": 443 401 | } 402 | ], 403 | "fqdnTags": [], 404 | "webCategories": [], 405 | "targetFqdns": [ 406 | "azure.archive.ubuntu.com" 407 | ], 408 | "targetUrls": [], 409 | "terminateTLS": false, 410 | "sourceAddresses": [ 411 | "*" 412 | ], 413 | "destinationAddresses": [], 414 | "sourceIpGroups": [] 415 | }, 416 | { 417 | "ruleType": "ApplicationRule", 418 | "name": "github", 419 | "protocols": [ 420 | { 421 | "protocolType": "Https", 422 | "port": 443 423 | } 424 | ], 425 | "fqdnTags": [], 426 | "webCategories": [], 427 | "targetFqdns": [ 428 | "github.com", 429 | "raw.githubusercontent.com" 430 | ], 431 | "targetUrls": [], 432 | "terminateTLS": false, 433 | "sourceAddresses": [ 434 | "*" 435 | ], 436 | "destinationAddresses": [], 437 | "sourceIpGroups": [] 438 | }, 439 | { 440 | "ruleType": "ApplicationRule", 441 | "name": "mcr-mirror", 442 | "protocols": [ 443 | { 444 | "protocolType": "Https", 445 | "port": 443 446 | } 447 | ], 448 | "fqdnTags": [], 449 | "webCategories": [], 450 | "targetFqdns": [ 451 | "acs-mirror.azureedge.net" 452 | ], 453 | "targetUrls": [], 454 | "terminateTLS": false, 455 | "sourceAddresses": [ 456 | "*" 457 | ], 458 | "destinationAddresses": [], 459 | "sourceIpGroups": [] 460 | }, 461 | { 462 | "ruleType": "ApplicationRule", 463 | "name": "container_repo", 464 | "protocols": [ 465 | { 466 | "protocolType": "Https", 467 | "port": 443 468 | } 469 | ], 470 | "fqdnTags": [], 471 | "webCategories": [], 472 | "targetFqdns": [ 473 | "bjdcsa.azurecr.io", 474 | "bjdcsa.centralus.data.azurecr.io", 475 | "bjdcsa.southcentralus.data.azurecr.io" 476 | ], 477 | "targetUrls": [], 478 | "terminateTLS": false, 479 | "sourceAddresses": [ 480 | "*" 481 | ], 482 | "destinationAddresses": [], 483 | "sourceIpGroups": [] 484 | }, 485 | { 486 | "ruleType": "ApplicationRule", 487 | "name": "github-container-repo", 488 | "protocols": [ 489 | { 490 | "protocolType": "Https", 491 | "port": 443 492 | } 493 | ], 494 | "fqdnTags": [], 495 | "webCategories": [], 496 | "targetFqdns": [ 497 | "ghcr.io", 498 | "pkg-containers.githubusercontent.com", 499 | "objects.githubusercontent.com" 500 | ], 501 | "targetUrls": [], 502 | "terminateTLS": false, 503 | "sourceAddresses": [ 504 | "*" 505 | ], 506 | "destinationAddresses": [], 507 | "sourceIpGroups": [] 508 | }, 509 | { 510 | "ruleType": "ApplicationRule", 511 | "name": "keyvault", 512 | "protocols": [ 513 | { 514 | "protocolType": "Https", 515 | "port": 443 516 | } 517 | ], 518 | "fqdnTags": [], 519 | "webCategories": [], 520 | "targetFqdns": [ 521 | "chat-e0ce7b6-kv.vault.azure.net" 522 | ], 523 | "targetUrls": [], 524 | "terminateTLS": false, 525 | "sourceAddresses": [ 526 | "*" 527 | ], 528 | "destinationAddresses": [], 529 | "sourceIpGroups": [] 530 | }, 531 | { 532 | "ruleType": "ApplicationRule", 533 | "name": "cognitive-services", 534 | "protocols": [ 535 | { 536 | "protocolType": "Https", 537 | "port": 443 538 | } 539 | ], 540 | "fqdnTags": [], 541 | "webCategories": [], 542 | "targetFqdns": [ 543 | "centralus.api.cognitive.microsoft.com", 544 | "southcentralus.api.cognitive.microsoft.com" 545 | ], 546 | "targetUrls": [], 547 | "terminateTLS": false, 548 | "sourceAddresses": [ 549 | "*" 550 | ], 551 | "destinationAddresses": [], 552 | "sourceIpGroups": [] 553 | }, 554 | { 555 | "ruleType": "ApplicationRule", 556 | "name": "b2c", 557 | "protocols": [ 558 | { 559 | "protocolType": "Https", 560 | "port": 443 561 | } 562 | ], 563 | "fqdnTags": [], 564 | "webCategories": [], 565 | "targetFqdns": [ 566 | "mmztkupi.b2clogin.com" 567 | ], 568 | "targetUrls": [], 569 | "terminateTLS": false, 570 | "sourceAddresses": [ 571 | "*" 572 | ], 573 | "destinationAddresses": [], 574 | "sourceIpGroups": [] 575 | }, 576 | { 577 | "ruleType": "ApplicationRule", 578 | "name": "osm", 579 | "protocols": [ 580 | { 581 | "protocolType": "Https", 582 | "port": 443 583 | } 584 | ], 585 | "fqdnTags": [], 586 | "webCategories": [], 587 | "targetFqdns": [ 588 | "openservicemesh.github.io" 589 | ], 590 | "targetUrls": [], 591 | "terminateTLS": false, 592 | "sourceAddresses": [ 593 | "*" 594 | ], 595 | "destinationAddresses": [], 596 | "sourceIpGroups": [] 597 | }, 598 | { 599 | "ruleType": "ApplicationRule", 600 | "name": "contour", 601 | "protocols": [ 602 | { 603 | "protocolType": "Https", 604 | "port": 443 605 | } 606 | ], 607 | "fqdnTags": [], 608 | "webCategories": [], 609 | "targetFqdns": [ 610 | "charts.bitnami.com" 611 | ], 612 | "targetUrls": [], 613 | "terminateTLS": false, 614 | "sourceAddresses": [ 615 | "*" 616 | ], 617 | "destinationAddresses": [], 618 | "sourceIpGroups": [] 619 | }, 620 | { 621 | "ruleType": "ApplicationRule", 622 | "name": "dapr", 623 | "protocols": [ 624 | { 625 | "protocolType": "Https", 626 | "port": 443 627 | } 628 | ], 629 | "fqdnTags": [], 630 | "webCategories": [], 631 | "targetFqdns": [ 632 | "dapr.github.io", 633 | "linuxgeneva-microsoft.azurecr.io", 634 | "*.blob.core.windows.net" 635 | ], 636 | "targetUrls": [], 637 | "terminateTLS": false, 638 | "sourceAddresses": [ 639 | "*" 640 | ], 641 | "destinationAddresses": [], 642 | "sourceIpGroups": [] 643 | }, 644 | { 645 | "ruleType": "ApplicationRule", 646 | "name": "kured", 647 | "protocols": [ 648 | { 649 | "protocolType": "Https", 650 | "port": 443 651 | } 652 | ], 653 | "fqdnTags": [], 654 | "webCategories": [], 655 | "targetFqdns": [ 656 | "weaveworks.github.io", 657 | "kubereboot.github.io" 658 | ], 659 | "targetUrls": [], 660 | "terminateTLS": false, 661 | "sourceAddresses": [ 662 | "*" 663 | ], 664 | "destinationAddresses": [], 665 | "sourceIpGroups": [] 666 | }, 667 | { 668 | "ruleType": "ApplicationRule", 669 | "name": "keda", 670 | "protocols": [ 671 | { 672 | "protocolType": "Https", 673 | "port": 443 674 | } 675 | ], 676 | "fqdnTags": [], 677 | "webCategories": [], 678 | "targetFqdns": [ 679 | "kedacore.github.io" 680 | ], 681 | "targetUrls": [], 682 | "terminateTLS": false, 683 | "sourceAddresses": [ 684 | "*" 685 | ], 686 | "destinationAddresses": [], 687 | "sourceIpGroups": [] 688 | }, 689 | { 690 | "ruleType": "ApplicationRule", 691 | "name": "traduire", 692 | "protocols": [ 693 | { 694 | "protocolType": "Https", 695 | "port": 443 696 | } 697 | ], 698 | "fqdnTags": [], 699 | "webCategories": [], 700 | "targetFqdns": [ 701 | "trad8b43xz-pubsub01.webpubsub.azure.com", 702 | "mp3trad8b43xz.blob.core.windows.net", 703 | "spsvcprodussc.blob.core.windows.net" 704 | ], 705 | "targetUrls": [], 706 | "terminateTLS": false, 707 | "sourceAddresses": [ 708 | "*" 709 | ], 710 | "destinationAddresses": [], 711 | "sourceIpGroups": [] 712 | }, 713 | { 714 | "ruleType": "ApplicationRule", 715 | "name": "Redhat", 716 | "protocols": [ 717 | { 718 | "protocolType": "Https", 719 | "port": 44 720 | }, 721 | { 722 | "protocolType": "Http", 723 | "port": 80 724 | } 725 | ], 726 | "fqdnTags": [], 727 | "webCategories": [], 728 | "targetFqdns": [ 729 | "cdn-ubi.redhat.com", 730 | "mirrorlist.centos.org", 731 | "centos.mirror.shastacoe.net" 732 | ], 733 | "targetUrls": [], 734 | "terminateTLS": false, 735 | "sourceAddresses": [ 736 | "*" 737 | ], 738 | "destinationAddresses": [], 739 | "sourceIpGroups": [] 740 | }, 741 | { 742 | "ruleType": "ApplicationRule", 743 | "name": "google-container-repo", 744 | "protocols": [ 745 | { 746 | "protocolType": "Https", 747 | "port": 443 748 | } 749 | ], 750 | "fqdnTags": [], 751 | "webCategories": [], 752 | "targetFqdns": [ 753 | "gcr.io", 754 | "storage.googleapis.com" 755 | ], 756 | "targetUrls": [], 757 | "terminateTLS": false, 758 | "sourceAddresses": [ 759 | "*" 760 | ], 761 | "destinationAddresses": [], 762 | "sourceIpGroups": [] 763 | }, 764 | { 765 | "ruleType": "ApplicationRule", 766 | "name": "workload-identity", 767 | "protocols": [ 768 | { 769 | "protocolType": "Https", 770 | "port": 443 771 | } 772 | ], 773 | "fqdnTags": [], 774 | "webCategories": [], 775 | "targetFqdns": [ 776 | "azure.github.io" 777 | ], 778 | "targetUrls": [], 779 | "terminateTLS": false, 780 | "sourceAddresses": [ 781 | "*" 782 | ], 783 | "destinationAddresses": [], 784 | "sourceIpGroups": [] 785 | }, 786 | { 787 | "ruleType": "ApplicationRule", 788 | "name": "Kubecost", 789 | "protocols": [ 790 | { 791 | "protocolType": "Https", 792 | "port": 443 793 | } 794 | ], 795 | "fqdnTags": [], 796 | "webCategories": [], 797 | "targetFqdns": [ 798 | "kubecost.github.io", 799 | "quay.io", 800 | "k8s.gcr.io" 801 | ], 802 | "targetUrls": [], 803 | "terminateTLS": false, 804 | "sourceAddresses": [ 805 | "*" 806 | ], 807 | "destinationAddresses": [], 808 | "sourceIpGroups": [] 809 | }, 810 | { 811 | "ruleType": "ApplicationRule", 812 | "name": "cert-manager", 813 | "protocols": [ 814 | { 815 | "protocolType": "Https", 816 | "port": 443 817 | } 818 | ], 819 | "fqdnTags": [], 820 | "webCategories": [], 821 | "targetFqdns": [ 822 | "*.quay.io", 823 | "charts.jetstack.io" 824 | ], 825 | "targetUrls": [], 826 | "terminateTLS": false, 827 | "sourceAddresses": [ 828 | "*" 829 | ], 830 | "destinationAddresses": [], 831 | "sourceIpGroups": [] 832 | }, 833 | { 834 | "ruleType": "ApplicationRule", 835 | "name": "AKS-Cluster-extensions", 836 | "protocols": [ 837 | { 838 | "protocolType": "Https", 839 | "port": 443 840 | } 841 | ], 842 | "fqdnTags": [], 843 | "webCategories": [], 844 | "targetFqdns": [ 845 | "*.dp.kubernetesconfiguration.azure.com", 846 | "arcmktplaceprod.azurecr.io", 847 | "*.ingestion.msftcloudes.com", 848 | "*.microsoftmetrics.com", 849 | "marketplaceapi.microsoft.com" 850 | ], 851 | "targetUrls": [], 852 | "terminateTLS": false, 853 | "sourceAddresses": [ 854 | "*" 855 | ], 856 | "destinationAddresses": [], 857 | "sourceIpGroups": [] 858 | } 859 | ], 860 | "name": "app_rule_collection", 861 | "priority": 500 862 | }, 863 | { 864 | "ruleCollectionType": "FirewallPolicyFilterRuleCollection", 865 | "action": { 866 | "type": "Allow" 867 | }, 868 | "rules": [ 869 | { 870 | "ruleType": "NetworkRule", 871 | "name": "time", 872 | "ipProtocols": [ 873 | "UDP" 874 | ], 875 | "sourceAddresses": [ 876 | "*" 877 | ], 878 | "sourceIpGroups": [], 879 | "destinationAddresses": [], 880 | "destinationIpGroups": [], 881 | "destinationFqdns": [ 882 | "ntp.ubuntu.com" 883 | ], 884 | "destinationPorts": [ 885 | "123" 886 | ] 887 | }, 888 | { 889 | "ruleType": "NetworkRule", 890 | "name": "monitor", 891 | "ipProtocols": [ 892 | "TCP" 893 | ], 894 | "sourceAddresses": [ 895 | "*" 896 | ], 897 | "sourceIpGroups": [], 898 | "destinationAddresses": [ 899 | "AzureMonitor" 900 | ], 901 | "destinationIpGroups": [], 902 | "destinationFqdns": [], 903 | "destinationPorts": [ 904 | "443" 905 | ] 906 | }, 907 | { 908 | "ruleType": "NetworkRule", 909 | "name": "github", 910 | "ipProtocols": [ 911 | "TCP" 912 | ], 913 | "sourceAddresses": [ 914 | "*" 915 | ], 916 | "sourceIpGroups": [], 917 | "destinationAddresses": [], 918 | "destinationIpGroups": [], 919 | "destinationFqdns": [ 920 | "github.com" 921 | ], 922 | "destinationPorts": [ 923 | "22" 924 | ] 925 | }, 926 | { 927 | "ruleType": "NetworkRule", 928 | "name": "github-raw", 929 | "ipProtocols": [ 930 | "TCP" 931 | ], 932 | "sourceAddresses": [ 933 | "*" 934 | ], 935 | "sourceIpGroups": [], 936 | "destinationAddresses": [], 937 | "destinationIpGroups": [], 938 | "destinationFqdns": [ 939 | "raw.githubusercontent.com" 940 | ], 941 | "destinationPorts": [ 942 | "22" 943 | ] 944 | } 945 | ], 946 | "name": "network_rule_collection", 947 | "priority": 400 948 | } 949 | ] 950 | } 951 | }, 952 | { 953 | "type": "Microsoft.Network/networkSecurityGroups", 954 | "apiVersion": "2020-11-01", 955 | "name": "[parameters('default_nsg_name')]", 956 | "location": "[parameters('location')]", 957 | "properties": { 958 | "securityRules": [] 959 | } 960 | }, 961 | { 962 | "type": "Microsoft.Network/publicIPAddresses", 963 | "apiVersion": "2020-11-01", 964 | "name": "[parameters('public_ip_address_name')]", 965 | "location": "[parameters('location')]", 966 | "sku": { 967 | "name": "Standard", 968 | "tier": "Regional" 969 | }, 970 | "properties": { 971 | "publicIPAddressVersion": "IPv4", 972 | "publicIPAllocationMethod": "Static", 973 | "idleTimeoutInMinutes": 4, 974 | "ipTags": [] 975 | } 976 | }, 977 | { 978 | "type": "Microsoft.Network/routeTables", 979 | "apiVersion": "2020-11-01", 980 | "name": "[parameters('routetable_name')]", 981 | "location": "[parameters('location')]", 982 | "tags": { 983 | }, 984 | "properties": { 985 | "disableBgpRoutePropagation": true, 986 | "routes": [ 987 | { 988 | "name": "DefaultRoute", 989 | "properties": { 990 | "addressPrefix": "0.0.0.0/0", 991 | "nextHopType": "VirtualAppliance", 992 | "nextHopIpAddress": "[parameters('firewall_next_hop')]", 993 | "hasBgpOverride": false 994 | } 995 | }, 996 | { 997 | "name": "HomeNetwork", 998 | "properties": { 999 | "addressPrefix": "172.16.201.0/24", 1000 | "nextHopType": "VirtualNetworkGateway", 1001 | "hasBgpOverride": false 1002 | } 1003 | } 1004 | ] 1005 | } 1006 | }, 1007 | { 1008 | "type": "Microsoft.Network/azureFirewalls", 1009 | "apiVersion": "2020-11-01", 1010 | "name": "[parameters('azure_firewall_name')]", 1011 | "location": "[parameters('location')]", 1012 | "dependsOn": [ 1013 | "[resourceId('Microsoft.Network/publicIPAddresses', parameters('public_ip_address_name'))]", 1014 | "[resourceId('Microsoft.Network/firewallPolicies', parameters('firewallpolicy_name'))]" 1015 | ], 1016 | "properties": { 1017 | "sku": { 1018 | "name": "AZFW_VNet", 1019 | "tier": "Standard" 1020 | }, 1021 | "threatIntelMode": "Alert", 1022 | "additionalProperties": {}, 1023 | "ipConfigurations": [ 1024 | { 1025 | "name": "[concat(parameters('public_ip_address_name'), '-pip')]", 1026 | "properties": { 1027 | "publicIPAddress": { 1028 | "id": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('public_ip_address_name'))]" 1029 | }, 1030 | "subnet": { 1031 | "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('virtual_network_name'), 'AzureFirewallSubnet')]" 1032 | } 1033 | } 1034 | } 1035 | ], 1036 | "networkRuleCollections": [], 1037 | "applicationRuleCollections": [], 1038 | "natRuleCollections": [], 1039 | "firewallPolicy": { 1040 | "id": "[resourceId('Microsoft.Network/firewallPolicies', parameters('firewallpolicy_name'))]" 1041 | } 1042 | } 1043 | }, 1044 | { 1045 | "type": "Microsoft.Network/virtualNetworks", 1046 | "apiVersion": "2020-11-01", 1047 | "name": "[parameters('virtual_network_name')]", 1048 | "location": "[parameters('location')]", 1049 | "dependsOn": [ 1050 | "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('default_nsg_name'))]", 1051 | "[resourceId('Microsoft.Network/routeTables', parameters('routetable_name'))]" 1052 | ], 1053 | "properties": { 1054 | "addressSpace": { 1055 | "addressPrefixes": [ 1056 | "[concat(parameters('virtual_network_cidr'),'.0.0/16')]" 1057 | ] 1058 | }, 1059 | "dhcpOptions": { 1060 | "dnsServers": [ 1061 | "10.1.7.4" 1062 | ] 1063 | }, 1064 | "subnets": [ 1065 | { 1066 | "name": "kubernetes-nodes", 1067 | "properties": { 1068 | "addressPrefix": "[concat(parameters('virtual_network_cidr'),'.3.0/24')]", 1069 | "networkSecurityGroup": { 1070 | "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('default_nsg_name'))]" 1071 | }, 1072 | "routeTable": { 1073 | "id": "[resourceId('Microsoft.Network/routeTables', parameters('routetable_name'))]" 1074 | }, 1075 | "serviceEndpoints": [ 1076 | { 1077 | "service": "Microsoft.KeyVault", 1078 | "locations": [ 1079 | "*" 1080 | ] 1081 | } 1082 | ], 1083 | "delegations": [], 1084 | "privateEndpointNetworkPolicies": "Disabled", 1085 | "privateLinkServiceNetworkPolicies": "Enabled" 1086 | } 1087 | }, 1088 | { 1089 | "name": "kubernetes-apiserver", 1090 | "properties": { 1091 | "addressPrefix": "[concat(parameters('virtual_network_cidr'),'.0.0/28')]", 1092 | "networkSecurityGroup": { 1093 | "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('default_nsg_name'))]" 1094 | }, 1095 | "routeTable": { 1096 | "id": "[resourceId('Microsoft.Network/routeTables', parameters('routetable_name'))]" 1097 | }, 1098 | "serviceEndpoints": [ 1099 | { 1100 | "service": "Microsoft.KeyVault", 1101 | "locations": [ 1102 | "*" 1103 | ] 1104 | } 1105 | ], 1106 | "delegations": [ 1107 | { 1108 | "name": "Microsoft.ContainerService.managedClusters", 1109 | "id": "[concat(resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('virtual_network_name'), 'kubernetes-apiserver'), '/delegations/Microsoft.ContainerService.managedClusters')]", 1110 | "properties": { 1111 | "serviceName": "Microsoft.ContainerService/managedClusters" 1112 | }, 1113 | "type": "Microsoft.Network/virtualNetworks/subnets/delegations" 1114 | } 1115 | ], 1116 | "privateEndpointNetworkPolicies": "Disabled", 1117 | "privateLinkServiceNetworkPolicies": "Enabled" 1118 | } 1119 | }, 1120 | { 1121 | "name": "AzureFirewallSubnet", 1122 | "properties": { 1123 | "addressPrefix": "[concat(parameters('virtual_network_cidr'),'.254.0/24')]", 1124 | "serviceEndpoints": [ 1125 | { 1126 | "service": "Microsoft.KeyVault", 1127 | "locations": [ 1128 | "*" 1129 | ] 1130 | } 1131 | ], 1132 | "delegations": [], 1133 | "privateEndpointNetworkPolicies": "Enabled", 1134 | "privateLinkServiceNetworkPolicies": "Enabled" 1135 | } 1136 | }, 1137 | { 1138 | "name": "private-endpoints", 1139 | "properties": { 1140 | "addressPrefix": "[concat(parameters('virtual_network_cidr'),'.2.0/24')]", 1141 | "networkSecurityGroup": { 1142 | "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('default_nsg_name'))]" 1143 | }, 1144 | "serviceEndpoints": [ 1145 | { 1146 | "service": "Microsoft.KeyVault", 1147 | "locations": [ 1148 | "*" 1149 | ] 1150 | } 1151 | ], 1152 | "delegations": [], 1153 | "privateEndpointNetworkPolicies": "Disabled", 1154 | "privateLinkServiceNetworkPolicies": "Enabled" 1155 | } 1156 | }, 1157 | { 1158 | "name": "Servers", 1159 | "properties": { 1160 | "addressPrefix": "[concat(parameters('virtual_network_cidr'),'.1.0/24')]", 1161 | "networkSecurityGroup": { 1162 | "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('default_nsg_name'))]" 1163 | }, 1164 | "serviceEndpoints": [ 1165 | { 1166 | "service": "Microsoft.KeyVault", 1167 | "locations": [ 1168 | "*" 1169 | ] 1170 | } 1171 | ], 1172 | "delegations": [], 1173 | "privateEndpointNetworkPolicies": "Enabled", 1174 | "privateLinkServiceNetworkPolicies": "Enabled" 1175 | } 1176 | } 1177 | ], 1178 | "virtualNetworkPeerings": [ 1179 | ], 1180 | "enableDdosProtection": false 1181 | } 1182 | } 1183 | ] 1184 | } -------------------------------------------------------------------------------- /infrastructure/prometheus-recording-rules.tf: -------------------------------------------------------------------------------- 1 | #Data Collection Prometheus Node Rule Group 2 | resource "azapi_resource" "prometheus_node_recording_rule_group" { 3 | depends_on = [ 4 | azurerm_monitor_data_collection_rule.prometheus 5 | ] 6 | 7 | type = "Microsoft.AlertsManagement/prometheusRuleGroups@2021-07-22-preview" 8 | name = "${var.cluster_name}-NodeRecordingRuleGroup" 9 | parent_id = azurerm_resource_group.k8s.id 10 | location = azurerm_resource_group.k8s.location 11 | 12 | body = jsonencode({ 13 | properties = { 14 | scopes = [ local.am_workspace_id ] 15 | enabled = true, 16 | clusterName = var.cluster_name 17 | interval = "PT1M" 18 | rules = [ 19 | { 20 | record = "instance:node_num_cpu:sum", 21 | expression = "count without (cpu, mode) ( node_cpu_seconds_total{job=\"node\",mode=\"idle\"})" 22 | }, 23 | { 24 | record = "instance:node_cpu_utilisation:rate5m", 25 | expression = "1 - avg without (cpu) ( sum without (mode) (rate(node_cpu_seconds_total{job=\"node\", mode=~\"idle|iowait|steal\"}[5m])))" 26 | }, 27 | { 28 | record = "instance:node_load1_per_cpu:ratio", 29 | expression = "( node_load1{job=\"node\"}/ instance:node_num_cpu:sum{job=\"node\"})" 30 | }, 31 | { 32 | record = "instance:node_memory_utilisation:ratio", 33 | expression = "1 - ( ( node_memory_MemAvailable_bytes{job=\"node\"} or ( node_memory_Buffers_bytes{job=\"node\"} + node_memory_Cached_bytes{job=\"node\"} + node_memory_MemFree_bytes{job=\"node\"} + node_memory_Slab_bytes{job=\"node\"} ) )/ node_memory_MemTotal_bytes{job=\"node\"})" 34 | }, 35 | { 36 | record = "instance:node_vmstat_pgmajfault:rate5m", 37 | expression = "rate(node_vmstat_pgmajfault{job=\"node\"}[5m])" 38 | }, 39 | { 40 | record = "instance_device:node_disk_io_time_seconds:rate5m", 41 | expression = "rate(node_disk_io_time_seconds_total{job=\"node\", device!=\"\"}[5m])" 42 | }, 43 | { 44 | record = "instance_device:node_disk_io_time_weighted_seconds:rate5m", 45 | expression = "rate(node_disk_io_time_weighted_seconds_total{job=\"node\", device!=\"\"}[5m])" 46 | }, 47 | { 48 | record = "instance:node_network_receive_bytes_excluding_lo:rate5m", 49 | expression = "sum without (device) ( rate(node_network_receive_bytes_total{job=\"node\", device!=\"lo\"}[5m]))" 50 | }, 51 | { 52 | record = "instance:node_network_transmit_bytes_excluding_lo:rate5m", 53 | expression = "sum without (device) ( rate(node_network_transmit_bytes_total{job=\"node\", device!=\"lo\"}[5m]))" 54 | }, 55 | { 56 | record = "instance:node_network_receive_drop_excluding_lo:rate5m", 57 | expression = "sum without (device) ( rate(node_network_receive_drop_total{job=\"node\", device!=\"lo\"}[5m]))" 58 | }, 59 | { 60 | record = "instance:node_network_transmit_drop_excluding_lo:rate5m", 61 | expression = "sum without (device) ( rate(node_network_transmit_drop_total{job=\"node\", device!=\"lo\"}[5m]))" 62 | } 63 | ] 64 | } 65 | }) 66 | } 67 | 68 | #Data Collection Prometheus Kubernetes Rule Group 69 | resource "azapi_resource" "prometheus_kubernetes_rule_groups" { 70 | depends_on = [ 71 | azurerm_monitor_data_collection_rule.prometheus 72 | ] 73 | 74 | type = "Microsoft.AlertsManagement/prometheusRuleGroups@2021-07-22-preview" 75 | name = "${var.cluster_name}-KubernetesRecordingRuleGroup" 76 | parent_id = azurerm_resource_group.k8s.id 77 | location = azurerm_resource_group.k8s.location 78 | 79 | body = jsonencode({ 80 | properties = { 81 | scopes = [ local.am_workspace_id ] 82 | enabled = true, 83 | clusterName = var.cluster_name 84 | interval = "PT1M" 85 | rules = [ 86 | { 87 | record = "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate", 88 | expression = "sum by (cluster, namespace, pod, container) ( irate(container_cpu_usage_seconds_total{job=\"cadvisor\", image!=\"\"}[5m])) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"}))" 89 | }, 90 | { 91 | record = "node_namespace_pod_container:container_memory_working_set_bytes", 92 | expression = "container_memory_working_set_bytes{job=\"cadvisor\", image!=\"\"}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=\"\"}))" 93 | }, 94 | { 95 | record = "node_namespace_pod_container:container_memory_rss", 96 | expression = "container_memory_rss{job=\"cadvisor\", image!=\"\"}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=\"\"}))" 97 | }, 98 | { 99 | record = "node_namespace_pod_container:container_memory_cache", 100 | expression = "container_memory_cache{job=\"cadvisor\", image!=\"\"}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=\"\"}))" 101 | }, 102 | { 103 | record = "node_namespace_pod_container:container_memory_swap", 104 | expression = "container_memory_swap{job=\"cadvisor\", image!=\"\"}* on (namespace, pod) group_left(node) topk by(namespace, pod) (1, max by(namespace, pod, node) (kube_pod_info{node!=\"\"}))" 105 | }, 106 | { 107 | record = "cluster:namespace:pod_memory:active:kube_pod_container_resource_requests", 108 | expression = "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ( (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1))" 109 | }, 110 | { 111 | record = "namespace_memory:kube_pod_container_resource_requests:sum", 112 | expression = "sum by (namespace, cluster) ( sum by (namespace, pod, cluster) ( max by (namespace, pod, container, cluster) ( kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( kube_pod_status_phase{phase=~\"Pending|Running\"} == 1 ) ))" 113 | }, 114 | { 115 | record = "cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests", 116 | expression = "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ( (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1))" 117 | }, 118 | { 119 | record = "namespace_cpu:kube_pod_container_resource_requests:sum", 120 | expression = "sum by (namespace, cluster) ( sum by (namespace, pod, cluster) ( max by (namespace, pod, container, cluster) ( kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( kube_pod_status_phase{phase=~\"Pending|Running\"} == 1 ) ))" 121 | }, 122 | { 123 | record = "cluster:namespace:pod_memory:active:kube_pod_container_resource_limits", 124 | expression = "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ( (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1))" 125 | }, 126 | { 127 | record = "namespace_memory:kube_pod_container_resource_limits:sum", 128 | expression = "sum by (namespace, cluster) ( sum by (namespace, pod, cluster) ( max by (namespace, pod, container, cluster) ( kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( kube_pod_status_phase{phase=~\"Pending|Running\"} == 1 ) ))" 129 | }, 130 | { 131 | record = "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits", 132 | expression = "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)group_left() max by (namespace, pod, cluster) ( (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1) )" 133 | }, 134 | { 135 | record = "namespace_cpu:kube_pod_container_resource_limits:sum", 136 | expression = "sum by (namespace, cluster) ( sum by (namespace, pod, cluster) ( max by (namespace, pod, container, cluster) ( kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( kube_pod_status_phase{phase=~\"Pending|Running\"} == 1 ) ))" 137 | }, 138 | { 139 | record = "namespace_workload_pod:kube_pod_owner:relabel", 140 | expression = "max by (cluster, namespace, workload, pod) ( label_replace( label_replace( kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"ReplicaSet\"}, \"replicaset\", \"$1\", \"owner_name\", \"(.*)\" ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( 1, max by (replicaset, namespace, owner_name) ( kube_replicaset_owner{job=\"kube-state-metrics\"} ) ), \"workload\", \"$1\", \"owner_name\", \"(.*)\" ))", 141 | labels = { 142 | workload_type = "deployment" 143 | } 144 | }, 145 | { 146 | record = "namespace_workload_pod:kube_pod_owner:relabel", 147 | expression = "max by (cluster, namespace, workload, pod) ( label_replace( kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"DaemonSet\"}, \"workload\", \"$1\", \"owner_name\", \"(.*)\" ))", 148 | labels = { 149 | workload_type = "daemonset" 150 | } 151 | }, 152 | { 153 | record = "namespace_workload_pod:kube_pod_owner:relabel", 154 | expression = "max by (cluster, namespace, workload, pod) ( label_replace( kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"StatefulSet\"}, \"workload\", \"$1\", \"owner_name\", \"(.*)\" ))", 155 | labels = { 156 | workload_type = "statefulset" 157 | } 158 | }, 159 | { 160 | record = "namespace_workload_pod:kube_pod_owner:relabel", 161 | expression = "max by (cluster, namespace, workload, pod) ( label_replace( kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"Job\"}, \"workload\", \"$1\", \"owner_name\", \"(.*)\" ))", 162 | labels = { 163 | workload_type = "job" 164 | } 165 | }, 166 | { 167 | record = ":node_memory_MemAvailable_bytes:sum", 168 | expression = "sum( node_memory_MemAvailable_bytes{job=\"node\"} or ( node_memory_Buffers_bytes{job=\"node\"} + node_memory_Cached_bytes{job=\"node\"} + node_memory_MemFree_bytes{job=\"node\"} + node_memory_Slab_bytes{job=\"node\"} )) by (cluster)" 169 | }, 170 | { 171 | record = "cluster:node_cpu:ratio_rate5m", 172 | expression = "sum(rate(node_cpu_seconds_total{job=\"node\",mode!=\"idle\",mode!=\"iowait\",mode!=\"steal\"}[5m])) by (cluster) /count(sum(node_cpu_seconds_total{job=\"node\"}) by (cluster, instance, cpu)) by (cluster)" 173 | } 174 | ] 175 | } 176 | }) 177 | } -------------------------------------------------------------------------------- /infrastructure/prometheus.tf: -------------------------------------------------------------------------------- 1 | resource "azapi_resource" "azure_monitor_workspace" { 2 | type = "microsoft.monitor/accounts@2021-06-03-preview" 3 | name = "${var.cluster_name}-workspace" 4 | parent_id = azurerm_resource_group.k8s.id 5 | location = "southcentralus" #azurerm_resource_group.k8s.location 6 | 7 | body = jsonencode({ 8 | }) 9 | } 10 | 11 | locals { 12 | am_workspace_id = "${data.azurerm_subscription.current.id}/resourcegroups/${azurerm_resource_group.k8s.name}/providers/microsoft.monitor/accounts/${var.cluster_name}-workspace" 13 | } 14 | 15 | resource "azurerm_monitor_data_collection_endpoint" "prometheus" { 16 | name = "${var.cluster_name}-prometheus-datacollection-ep" 17 | resource_group_name = azurerm_resource_group.k8s.name 18 | location = azurerm_resource_group.k8s.location 19 | kind = "Linux" 20 | public_network_access_enabled = true 21 | } 22 | 23 | resource "azurerm_monitor_data_collection_rule" "prometheus" { 24 | name = "${var.cluster_name}-prometheus-datacollection-rules" 25 | resource_group_name = azurerm_resource_group.k8s.name 26 | location = azurerm_resource_group.k8s.location 27 | depends_on = [ 28 | azapi_resource.azure_monitor_workspace, 29 | azurerm_monitor_data_collection_endpoint.prometheus 30 | ] 31 | kind = "Linux" 32 | data_collection_endpoint_id = azurerm_monitor_data_collection_endpoint.prometheus.id 33 | 34 | destinations { 35 | monitor_account { 36 | monitor_account_id = local.am_workspace_id 37 | name = "MonitoringAccount1" 38 | } 39 | } 40 | 41 | data_flow { 42 | destinations = ["MonitoringAccount1"] 43 | streams = ["Microsoft-PrometheusMetrics"] 44 | } 45 | 46 | data_sources { 47 | prometheus_forwarder { 48 | name = "PrometheusDataSource" 49 | streams = ["Microsoft-PrometheusMetrics"] 50 | } 51 | } 52 | } 53 | 54 | resource "azapi_resource" "monitor_datacollection_rule_associations" { 55 | type = "Microsoft.Insights/dataCollectionRuleAssociations@2021-09-01-preview" 56 | name = "${var.cluster_name}-prometheus-datacollection-rules-association" 57 | parent_id = azurerm_kubernetes_cluster.k8s.id 58 | body = jsonencode({ 59 | properties = { 60 | dataCollectionRuleId = azurerm_monitor_data_collection_rule.prometheus.id 61 | } 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /infrastructure/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | azurerm = { 4 | source = "hashicorp/azurerm" 5 | version = "~> 4" 6 | } 7 | azapi = { 8 | source = "Azure/azapi" 9 | version = "~> 2" 10 | } 11 | } 12 | 13 | backend "azurerm" { 14 | resource_group_name = "Core_Storage_RG" 15 | storage_account_name = "bjdterraform003" 16 | container_name = "plans" 17 | } 18 | } 19 | 20 | provider "azapi" { 21 | } 22 | 23 | provider "azurerm" { 24 | features {} 25 | } 26 | 27 | provider "azurerm" { 28 | alias = "core" 29 | features {} 30 | 31 | subscription_id = var.core_subscription 32 | } -------------------------------------------------------------------------------- /infrastructure/roles.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_role_assignment" "aks_role_assignemnt_dns" { 2 | scope = data.azurerm_private_dns_zone.aks_private_zone.id 3 | role_definition_name = "Private DNS Zone Contributor" 4 | principal_id = azurerm_user_assigned_identity.aks_identity.principal_id 5 | skip_service_principal_aad_check = true 6 | } 7 | 8 | resource "azurerm_role_assignment" "aks_role_assignemnt_nework" { 9 | scope = data.azurerm_virtual_network.vnet.id 10 | role_definition_name = "Network Contributor" 11 | principal_id = azurerm_user_assigned_identity.aks_identity.principal_id 12 | skip_service_principal_aad_check = true 13 | } 14 | 15 | resource "azurerm_role_assignment" "aks_role_assignemnt_msi" { 16 | scope = azurerm_user_assigned_identity.aks_kubelet_identity.id 17 | role_definition_name = "Managed Identity Operator" 18 | principal_id = azurerm_user_assigned_identity.aks_identity.principal_id 19 | skip_service_principal_aad_check = true 20 | } 21 | 22 | resource "azurerm_role_assignment" "acr_pullrole_nodepool" { 23 | scope = data.azurerm_container_registry.acr_repo.id 24 | role_definition_name = "AcrPull" 25 | principal_id = azurerm_user_assigned_identity.aks_kubelet_identity.principal_id 26 | provider = azurerm.core 27 | skip_service_principal_aad_check = true 28 | } 29 | 30 | resource "azurerm_role_assignment" "github_deployer_access" { 31 | scope = azurerm_key_vault.k8s.id 32 | role_definition_name = "Key Vault Administrator" 33 | principal_id = data.azurerm_user_assigned_identity.github_actions.principal_id 34 | provider = azurerm.core 35 | skip_service_principal_aad_check = true 36 | } 37 | 38 | resource "azurerm_role_assignment" "istio_ingress_secret_access" { 39 | scope = azurerm_key_vault.k8s.id 40 | role_definition_name = "Key Vault Secrets User" 41 | principal_id = azurerm_user_assigned_identity.aks_service_mesh_ingress_identity.principal_id 42 | skip_service_principal_aad_check = true 43 | } 44 | 45 | resource "azurerm_role_assignment" "zipkin_secret_access" { 46 | scope = azurerm_key_vault.k8s.id 47 | role_definition_name = "Key Vault Secrets User" 48 | principal_id = azurerm_user_assigned_identity.aks_zipkin_identity.principal_id 49 | skip_service_principal_aad_check = true 50 | } -------------------------------------------------------------------------------- /infrastructure/variables.tf: -------------------------------------------------------------------------------- 1 | variable "location" { 2 | description = "The Azure Region to deploy AKS" 3 | default = "southcentralus" 4 | } 5 | 6 | variable "k8s_vnet_resource_group_name" { 7 | description = "The Resource Group name that contains the Vnet for AKS" 8 | } 9 | 10 | variable "k8s_nodes_subnet" { 11 | description = "The subnet name where AKS nodes will be deployed to" 12 | } 13 | 14 | variable "k8s_apiserver_subnet" { 15 | description = "The subnet name where AKS master server will be projected into" 16 | } 17 | 18 | variable "k8s_vnet" { 19 | description = "The Vnet name where AKS will be deployed to" 20 | } 21 | 22 | variable "dns_service_ip" { 23 | description = "The IP address for the DNS serviced hosted inside AKS cluster" 24 | } 25 | 26 | variable "service_cidr" { 27 | description = "The IP range for internal services in AKS. Should not overlap any other IP space " 28 | } 29 | 30 | variable "pod_cidr" { 31 | description = "The IP range for internal pod in AKS. Should not overlap any other IP space " 32 | } 33 | 34 | variable "cluster_name" { 35 | description = "The cluster name" 36 | } 37 | 38 | variable "resource_group_name" { 39 | description = "The Azure Resource Group to deploy AKS" 40 | } 41 | 42 | variable "agent_count" { 43 | description = "The number of nodes in the cluster" 44 | default = "2" 45 | } 46 | 47 | variable "vm_size" { 48 | description = "The VM node size" 49 | default = "Standard_DS3_v2" 50 | } 51 | 52 | variable "core_subscription" { 53 | default = "2deb88fe-eca8-499a-adb9-6e0ea8b6c1d2" 54 | description = "The subscription where Azure Container Repo lives" 55 | } 56 | 57 | variable "acr_resource_group" { 58 | default = "Core_Infra_ContainerRepo_RG" 59 | description = "The Resource Grop where Azure Container Repo lives" 60 | } 61 | 62 | variable "acr_name" { 63 | default = "bjdcsa" 64 | description = "The Azure Container Repo name" 65 | } 66 | 67 | variable "dns_resource_group_name" { 68 | description = "The Resource Group name that contains Private DNS Zones" 69 | } 70 | 71 | variable "github_actions_identity_name" { 72 | description = "The name of the Github Task runner Managed Identity" 73 | } 74 | 75 | variable "github_actions_identity_resource_group" { 76 | description = "The Resource Group name that Github Taskrunner Identity" 77 | } 78 | 79 | variable "service_mesh_type" { 80 | default = "istio" 81 | description = "The type of Service Mesh to install onto the cluster" 82 | } 83 | 84 | variable "certificate_name" { 85 | description = "The name of the certificate to use for TLS" 86 | } 87 | 88 | variable "certificate_base64_encoded" { 89 | description = "The name of the certificate to use for TLS" 90 | } 91 | 92 | variable "certificate_password" { 93 | description = "The password for the certificate" 94 | } 95 | 96 | variable "ingress_namespace" { 97 | description = "The namespace to deploy Istio gateway components to" 98 | default = "aks-istio-ingress" 99 | } 100 | 101 | variable "zipkin_namespace" { 102 | description = "The namespace to deploy Istio gateway components to" 103 | default = "otel-system" 104 | } -------------------------------------------------------------------------------- /scripts/aks-application-bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while (( "$#" )); do 4 | case "$1" in 5 | -c|--cluster-name) 6 | CLUSTER_NAME+=($2) 7 | shift 2 8 | ;; 9 | -n|--namespace) 10 | NAMESPACE=$2 11 | shift 2 12 | ;; 13 | -g|--group-name) 14 | GROUP_NAME=$2 15 | shift 2 16 | ;; 17 | -i|--identity-name) 18 | IDENTITY_NAME=$2 19 | shift 2 20 | ;; 21 | -h|--help) 22 | echo "Usage: ./workload-identity.sh --cluster-name --namespace --group-name --identity-name --identity-resource-group 23 | Overview: This script will bootstrap a namespace for develoepers in an AKS cluster 24 | --cluster-name(c) - The AKS cluster where this identity will be used 25 | --namespace(n) - The Kuberentes namespace where for this development team 26 | --group-name(g) - The Azure AD Group Name to onboard 27 | --identity-name(i) - The User Assigned Managed Identity to onboard to this namespace 28 | " 29 | exit 0 30 | ;; 31 | --) 32 | shift 33 | break 34 | ;; 35 | -*|--*=) 36 | echo "Error: Unsupported flag $1" >&2 37 | exit 1 38 | ;; 39 | esac 40 | done 41 | 42 | DEV_ROLE="Azure Kubernetes Service RBAC Writer" 43 | 44 | CLUSTER_DETAILS=`az aks list --query "[?name=='${CLUSTER_NAME}']"` 45 | CLUSTER_RG=`echo ${CLUSTER_DETAILS} | jq -r ".[].resourceGroup"` 46 | 47 | az aks get-credentials -g ${CLUSTER_RG} -n ${CLUSTER_NAME} --overwrite-existing 48 | kubelogin convert-kubeconfig -l msi 49 | 50 | #Create Namespace 51 | kubectl create ns ${NAMESPACE} || true 52 | kubectl label ns ${NAMESPACE} istio-injection=enabled 53 | kubectl -n ${NAMESPACE} create quota ns-limits --hard=requests.cpu=2,limits.cpu=4,requests.memory=10Gi,limits.memory=20Gi,count/gateways.networking.istio.io=0 54 | 55 | #Create Servcie Account in Namespace 56 | IDENTITY_DETAILS=`az identity list --query "[?name=='${IDENTITY_NAME}']"` 57 | IDENTITY_ID=`echo ${IDENTITY_DETAILS} | jq -r ".[].clientId"` 58 | IDENTITY_TENANT_ID=`echo ${IDENTITY_DETAILS} | jq -r ".[].tenantId"` 59 | kubectl -n ${NAMESPACE} create serviceaccount ${IDENTITY_NAME} 60 | kubectl -n ${NAMESPACE} annotate serviceaccount ${IDENTITY_NAME} azure.workload.identity/client-id=${IDENTITY_ID} 61 | kubectl -n ${NAMESPACE} annotate serviceaccount ${IDENTITY_NAME} azure.workload.identity/tenant-id=${IDENTITY_TENANT_ID} 62 | kubectl -n ${NAMESPACE} label serviceaccount ${IDENTITY_NAME} azure.workload.identity/use: "true" 63 | 64 | #Assign RBAC permissions to namespace 65 | CLUSTER_ID=`echo ${CLUSTER_DETAILS} | jq -r ".[].id"` 66 | GROUP_ID=`az ad group show --group "${GROUP_NAME}" --query "objectId" --output tsv` 67 | az role assignment create --assignee ${GROUP_ID} --role "${DEV_ROLE}" --scope ${CLUSTER_ID}/namespaces/${NAMESPACE} 68 | 69 | cat <&2 36 | exit 1 37 | ;; 38 | esac 39 | done 40 | 41 | if [[ -z "${CLIENT_ID}" ]]; then 42 | CLIENT_ID=${ARM_CLIENT_ID} 43 | fi 44 | 45 | if [[ -z "${SUBSCRIPTION_ID}" ]]; then 46 | SUBSCRIPTION_ID=${ARM_SUBSCRIPTION_ID} 47 | fi 48 | 49 | az login --identity -u ${CLIENT_ID} 50 | az account set -s ${SUBSCRIPTION_ID} 51 | 52 | az aks get-credentials -g ${CLUSTER_RG} -n ${CLUSTER_NAME} --overwrite-existing --format azure 53 | kubelogin convert-kubeconfig -l azurecli 54 | 55 | az aks rotate-certs -g ${CLUSTER_RG} -n ${CLUSTER_NAME} -------------------------------------------------------------------------------- /scripts/aks-preview-features.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | az extension add --name aks-preview 4 | az extension update --name aks-preview 5 | 6 | #Features Have Gone GA and no longer require preview flag 7 | #az feature register --namespace Microsoft.ContainerService --name HTTPProxyConfigPreview 8 | #az feature register --namespace Microsoft.ContainerService --name MultiAgentpoolPreview 9 | #az feature register --namespace Microsoft.ContainerService --name EnablePodIdentityPreview 10 | #az feature register --namespace Microsoft.ContainerService --name RunCommandPreview 11 | #az feature register --namespace Microsoft.ContainerService --name AKS-OpenServiceMesh 12 | #az feature register --namespace Microsoft.ContainerService --name AKS-AzureKeyVaultSecretsProvider 13 | #az feature register --namespace Microsoft.ContainerService --name EnableOIDCIssuerPreview 14 | #az feature register --namespace Microsoft.ContainerService --name FleetResourcePreview 15 | #az feature register --namespace Microsoft.ContainerService --name AzureServiceMeshPreview 16 | #az feature register --namespace Microsoft.ContainerService --name EnableWorkloadIdentityPreview 17 | 18 | features=( 19 | "DisableLocalAccountsPreview" 20 | "AKS-ExtensionManager" 21 | "AKS-AzureDefender" 22 | "AzureOverlayPreview" 23 | "AKS-PrometheusAddonPreview" 24 | "EnableImageCleanerPreview" 25 | "AKS-KedaPreview" 26 | "EnableAPIServerVnetIntegrationPreview" 27 | "EnableAzureDiskCSIDriverV2" 28 | "AKS-Dapr" 29 | "EnableMultipleStandardLoadBalancers" 30 | "AKSNodelessPreview" 31 | "NodeOsUpgradeChannelPreview" 32 | "CiliumDataplanePreview" 33 | "TrustedAccessPreview" 34 | "KubeletDisk" 35 | "KataVMIsolationPreview" 36 | "KataCcIsolationPreview" 37 | "EnableBYOKOnEphemeralOSDiskPreview" 38 | "NetworkObservabilityPreview" 39 | "EnableCloudControllerManager" 40 | "EnableImageIntegrityPreview" 41 | "AKS-AzurePolicyExternalData" 42 | "KubeProxyConfigurationPreview" 43 | "NodeAutoProvisioningPreview" 44 | "NRGLockdownPreview" 45 | "IstioNativeSidecarModePreview" 46 | "SafeguardsPreview" 47 | "DisableSSHPreview" 48 | "AutomaticSKUPreview" 49 | "AdvancedNetworkingPreview" 50 | "AzureLinuxV3Preview" 51 | ) 52 | 53 | for feature in ${features[*]} 54 | do 55 | az feature register --namespace Microsoft.ContainerService --name $feature 56 | done 57 | 58 | watch -n 10 -g az feature list --namespace Microsoft.ContainerService -o table --query \"[?properties.state == \'Registering\']\" 59 | 60 | az provider register --namespace Microsoft.Kubernetes 61 | az provider register --namespace Microsoft.ContainerService 62 | az provider register --namespace Microsoft.KubernetesConfiguration 63 | 64 | az extension add --name k8s-extension 65 | az extension add --name fleet 66 | -------------------------------------------------------------------------------- /scripts/aks-repave.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while (( "$#" )); do 4 | case "$1" in 5 | -i) 6 | CLIENT_ID=$2 7 | shift 2 8 | ;; 9 | -s) 10 | SUBSCRIPTION_ID=$2 11 | shift 2 12 | ;; 13 | -n) 14 | CLUSTER_NAME=$2 15 | shift 2 16 | ;; 17 | -g) 18 | CLUSTER_RG=$2 19 | shift 2 20 | ;; 21 | -v) 22 | KEY_VAULT_NAME=$2 23 | shift 2 24 | ;; 25 | --secret-name) 26 | KEY_VAULT_SECRET_NAME=$2 27 | shift 2 28 | ;; 29 | -h|--help) 30 | echo "Usage: ./aks-repave.sh -i {MSI_GUID} -s {SUBSCRIPTION_GUID} -n {AKS_NAME} -g {AKS_RG} -v {KeyVault_NAME} --secret-name {SSH_SECRET_NAME} 31 | -i: The Managed Identity to access the Azure Resources 32 | -s: The Azure Subscription for the resources 33 | -n: The AKS name 34 | -g: The Resource Group for AKS and Key Vault 35 | -v: The Key Vault to store the SSH Private Key 36 | --secret-name: The name of the SSH secret in Key Vault 37 | " 38 | exit 0 39 | ;; 40 | --) 41 | shift 42 | break 43 | ;; 44 | -*|--*=) 45 | echo "Error: Unsupported flag $1" >&2 46 | exit 1 47 | ;; 48 | esac 49 | done 50 | 51 | if [[ -z "${CLIENT_ID}" ]]; then 52 | CLIENT_ID=${ARM_CLIENT_ID} 53 | fi 54 | 55 | if [[ -z "${SUBSCRIPTION_ID}" ]]; then 56 | SUBSCRIPTION_ID=${ARM_SUBSCRIPTION_ID} 57 | fi 58 | 59 | az login --identity -u ${CLIENT_ID} 60 | az account set -s ${SUBSCRIPTION_ID} 61 | 62 | az aks get-credentials -g ${CLUSTER_RG} -n ${CLUSTER_NAME} --overwrite-existing --format azure 63 | kubelogin convert-kubeconfig -l azurecli 64 | 65 | ID=${RANDOM} 66 | PRIVATE_KEY_FILE=/tmp/id_rsa_${ID} 67 | PUBLIC_KEY_FILE=/tmp/id_rsa_${ID}.pub 68 | ssh-keygen -q -b 4096 -f ${PRIVATE_KEY_FILE} -t rsa 69 | 70 | az keyvault secret set --name ${KEY_VAULT_SECRET_NAME} --vault-name ${KEY_VAULT_NAME} --file ${PRIVATE_KEY_FILE} --encoding ascii 71 | az aks update -g ${CLUSTER_RG} -n ${CLUSTER_NAME} --ssh-key-value ${PUBLIC_KEY_FILE} 72 | 73 | rm -rf ${PUBLIC_KEY_FILE} 74 | rm -rf ${PRIVATE_KEY_FILE} --------------------------------------------------------------------------------