├── .gitignore ├── codeql3000.yml ├── Dockerfile ├── CODE_OF_CONDUCT.md ├── SHIR ├── health-check.ps1 ├── setup.ps1 └── build.ps1 ├── LICENSE ├── SECURITY.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | SHIR/IntegrationRuntime*.msi 2 | -------------------------------------------------------------------------------- /codeql3000.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | Codeql.Enabled: true 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/windows/servercore:ltsc2019 2 | ARG INSTALL_JDK=false 3 | ARG INSTALL_CERT=false 4 | 5 | # Download the latest self-hosted integration runtime installer into the SHIR folder 6 | COPY SHIR C:/SHIR/ 7 | 8 | RUN ["powershell", "C:/SHIR/build.ps1"] 9 | 10 | ENTRYPOINT ["powershell", "C:/SHIR/setup.ps1"] 11 | 12 | ENV SHIR_WINDOWS_CONTAINER_ENV True 13 | 14 | HEALTHCHECK --start-period=120s CMD ["powershell", "C:/SHIR/health-check.ps1"] 15 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /SHIR/health-check.ps1: -------------------------------------------------------------------------------- 1 | $DmgcmdPath = "C:\Program Files\Microsoft Integration Runtime\5.0\Shared\dmgcmd.exe" 2 | 3 | function Check-Node-Connection() { 4 | $outputFile = "C:\SHIR\status-check-$([guid]::NewGuid().ToString()).txt" 5 | Start-Process $DmgcmdPath -Wait -ArgumentList "-cgc" -RedirectStandardOutput $outputFile 6 | $ConnectionResult = Get-Content $outputFile 7 | Remove-Item -Force $outputFile 8 | 9 | if ($ConnectionResult -like "Connected") { 10 | return $TRUE 11 | } 12 | else { 13 | exit 1 14 | } 15 | } 16 | 17 | if (Check-Node-Connection) { 18 | exit 0 19 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Azure Data Factory Integration Runtime in Windows Container Sample 2 | ======= 3 | This repo contains the sample for running the Azure Data Factory Integration Runtime in Windows Container 4 | 5 | Support SHIR version: 5.0 or later 6 | 7 | For more information about Azure Data Factory, see [https://docs.microsoft.com/en-us/azure/data-factory/concepts-integration-runtime](https://docs.microsoft.com/en-us/azure/data-factory/concepts-integration-runtime) 8 | 9 | # QuickStart 10 | 1. Prepare [Windows for containers](https://learn.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=dockerce) 11 | 2. Build the Windows container image in the project folder 12 | ```bash 13 | > docker build . -t [--build-arg="INSTALL_JDK=true"] [--build-arg="INSTALL_CERT=true"] 14 | ``` 15 | ### __Arguments list__ 16 | |Name|Necessity|Default|Description| 17 | |---|---|---|---| 18 | | `INSTALL_JDK` | Optional | `false` | The flag to install Microsoft's JDK 21 LTS. | 19 | | `INSTALL_CERT` | Optional | `false` | The flag to install Root or Intermittent certificate files and must match regex format [root.*.cer] or [intermittent.*.cer] to work. | 20 | 3. Run the container with specific arguments by passing environment variables 21 | ```bash 22 | > docker run -d -e AUTH_KEY= \ 23 | [-e NODE_NAME=] \ 24 | [-e ENABLE_HA={true|false}] \ 25 | [-e HA_PORT=] \ 26 | [-e ENABLE_AE={true|false}] \ 27 | [-e AE_TIME=] \ 28 | [-e TZ=] \ 29 | 30 | ``` 31 | ### __Arguments list__ 32 | |Name|Necessity|Default|Description| 33 | |---|---|---|---| 34 | | `AUTH_KEY` | Required | | The authentication key for the self-hosted integration runtime. | 35 | | `NODE_NAME` | Optional | `hostname` | The specified name of the node. | 36 | | `ENABLE_HA` | Optional | `false` | The flag to enable high availability and scalability.
It supports up to 4 nodes registered to the same IR when `HA` is enabled, otherwise only 1 is allowed. If set to true and in a kubernetes cluster - consider setting [spec.template.spec.dnsConfig.searches](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#hostname-and-name-resolution) to `service`.`namespace`.`svc.cluster.local` to allow intra-pod communications | 37 | | `HA_PORT` | Optional | `8060` | The port to set up a high availability cluster. | 38 | | `ENABLE_AE` | Optional | `false` | The flag to enable offline nodes auto-expiration.
If enabled, the node will be marked as expired when it has been offline for timeout duration defined by `AE_TIME`. | 39 | | `AE_TIME` | Optional | `600` | The expiration timeout duration for offline nodes in seconds.
Should be no less than 600 (10 minutes). | 40 | | `TZ` | Optional | `UTC` | Valid values can be found as Id from the command `Get-TimeZone -List` | 41 | 42 | # Contributing 43 | 44 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 45 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 46 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 47 | 48 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 49 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 50 | provided by the bot. You will only need to do this once across all repos using our CLA. 51 | 52 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 53 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 54 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 55 | -------------------------------------------------------------------------------- /SHIR/setup.ps1: -------------------------------------------------------------------------------- 1 | $DmgcmdPath = "C:\Program Files\Microsoft Integration Runtime\5.0\Shared\dmgcmd.exe" 2 | 3 | function Write-Log($Message) { 4 | function TS { Get-Date -Format 'MM/dd/yyyy HH:mm:ss' } 5 | Write-Host "[$(TS)] $Message" 6 | } 7 | 8 | function Check-Is-Registered() { 9 | $result = Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\DataTransfer\DataManagementGateway\ConfigurationManager' -Name HaveRun -ErrorAction SilentlyContinue 10 | if (($result -ne $null) -and ($result.HaveRun -eq 'Mdw')) { 11 | return $TRUE 12 | } 13 | return $FALSE 14 | } 15 | 16 | function Check-Main-Process() { 17 | $ProcessResult = Get-WmiObject Win32_Process -Filter "name = 'diahost.exe'" 18 | 19 | if ($ProcessResult) { 20 | return $TRUE 21 | } 22 | 23 | Write-Log "diahost.exe is not running" 24 | return $FALSE 25 | } 26 | 27 | 28 | function RegisterNewNode { 29 | Param( 30 | $AUTH_KEY, 31 | $NODE_NAME, 32 | $ENABLE_HA, 33 | $HA_PORT, 34 | $ENABLE_AE, 35 | $AE_TIME 36 | ) 37 | 38 | Write-Log "Start registering a new SHIR node" 39 | Write-Log "Registering SHIR node with the node key: $($AUTH_KEY)" 40 | 41 | if (!$NODE_NAME) { 42 | $NODE_NAME = $Env:COMPUTERNAME 43 | } 44 | Write-Log "Registering SHIR node with the node name: $($NODE_NAME)" 45 | 46 | if ($ENABLE_HA -eq "true") { 47 | Write-Log "Enable High Availability" 48 | $PORT = $HA_PORT 49 | if (!$HA_PORT) { 50 | $PORT = "8060" 51 | } 52 | Write-Log "Remote Access Port: $($PORT)" 53 | Start-Process $DmgcmdPath -Wait -ArgumentList "-EnableRemoteAccessInContainer", "$($PORT)" -RedirectStandardOutput "C:\SHIR\register-out.txt" -RedirectStandardError "C:\SHIR\register-error.txt" 54 | Start-Sleep -Seconds 15 55 | } 56 | 57 | if ($ENABLE_AE -eq "true") { 58 | Write-Log "Enable Offline Nodes Auto-Expiration" 59 | if (!$AE_TIME) { 60 | $AE_TIME = 600 61 | } 62 | 63 | Write-Log "Node Expiration Time In Seconds: $($AE_TIME)" 64 | Start-Process $DmgcmdPath -Wait -ArgumentList "-RegisterNewNode", "$($AUTH_KEY)", "$($NODE_NAME)", "$($AE_TIME)" -RedirectStandardOutput "C:\SHIR\register-out.txt" -RedirectStandardError "C:\SHIR\register-error.txt" 65 | Start-Sleep -Seconds 15 66 | } else { 67 | Start-Process $DmgcmdPath -Wait -ArgumentList "-RegisterNewNode", "$($AUTH_KEY)", "$($NODE_NAME)" -RedirectStandardOutput "C:\SHIR\register-out.txt" -RedirectStandardError "C:\SHIR\register-error.txt" 68 | } 69 | 70 | $StdOutResult = Get-Content "C:\SHIR\register-out.txt" 71 | $StdErrResult = Get-Content "C:\SHIR\register-error.txt" 72 | 73 | if ($StdOutResult) 74 | { 75 | Write-Log "Registration output:" 76 | $StdOutResult | ForEach-Object { Write-Log $_ } 77 | } 78 | 79 | if ($StdErrResult) 80 | { 81 | Write-Log "Registration errors:" 82 | $StdErrResult | ForEach-Object { Write-Log $_ } 83 | } 84 | } 85 | 86 | 87 | # Set timezone if set from input 88 | If (Test-Path Env.TZ) { 89 | try { 90 | Set-TimeZone -Id $Env:TZ 91 | } catch { 92 | Write-Log "Unable to set the $Env:TZ timezone" 93 | } 94 | } 95 | 96 | # Register SHIR with key from Env Variable: AUTH_KEY 97 | if (Check-Is-Registered) { 98 | Write-Log "Restart the existing node" 99 | 100 | if ((Test-Path Env:ENABLE_HA) -and ($Env:ENABLE_HA -eq "true")) { 101 | Write-Log "Enable High Availability" 102 | $PORT = $Env:HA_PORT 103 | if (!$Env:HA_PORT) { 104 | $PORT = "8060" 105 | } 106 | Write-Log "Remote Access Port: $($PORT)" 107 | Start-Process $DmgcmdPath -Wait -ArgumentList "-EnableRemoteAccessInContainer", "$($PORT)" 108 | Start-Sleep -Seconds 15 109 | } 110 | 111 | Start-Process $DmgcmdPath -Wait -ArgumentList "-Start" 112 | } elseif (Test-Path Env:AUTH_KEY) { 113 | Start-Process $DmgcmdPath -Wait -ArgumentList "-Start" 114 | 115 | RegisterNewNode $Env:AUTH_KEY $Env:NODE_NAME $Env:ENABLE_HA $Env:HA_PORT $Env:ENABLE_AE $Env:AE_TIME 116 | } else { 117 | Write-Log "Invalid AUTH_KEY Value" 118 | exit 1 119 | } 120 | 121 | Write-Log "Waiting 60 seconds for connecting" 122 | Start-Sleep -Seconds 60 123 | 124 | try { 125 | $COUNT = 0 126 | $IS_REGISTERED = $FALSE 127 | while ($TRUE) { 128 | if(!$IS_REGISTERED) { 129 | if (Check-Is-Registered) { 130 | $IS_REGISTERED = $TRUE 131 | Write-Log "Self-hosted Integration Runtime is connected to the cloud service" 132 | } 133 | } 134 | 135 | if (Check-Main-Process) { 136 | $COUNT = 0 137 | } else { 138 | $COUNT += 1 139 | if ($COUNT -gt 5) { 140 | throw "Diahost.exe is not running" 141 | } 142 | } 143 | 144 | Start-Sleep -Seconds 60 145 | } 146 | } 147 | finally { 148 | Write-Log "Stop the node connection" 149 | Start-Process $DmgcmdPath -Wait -ArgumentList "-Stop" 150 | Write-Log "Stop the node connection successfully" 151 | exit 0 152 | } 153 | 154 | exit 1 -------------------------------------------------------------------------------- /SHIR/build.ps1: -------------------------------------------------------------------------------- 1 | $DmgcmdPath = "C:\Program Files\Microsoft Integration Runtime\5.0\Shared\dmgcmd.exe" 2 | 3 | function Write-Log($Message) { 4 | function TS { Get-Date -Format 'MM/dd/yyyy hh:mm:ss' } 5 | Write-Host "[$(TS)] $Message" 6 | } 7 | 8 | function Install-SHIR() { 9 | Write-Log "Install the Self-hosted Integration Runtime in the Windows container" 10 | 11 | $MsiFiles = (Get-ChildItem -Path C:\SHIR | Where-Object { $_.Name -match [regex] "IntegrationRuntime.*.msi" }) 12 | if ($MsiFiles) { 13 | $MsiFileName = $MsiFiles[0].Name 14 | Write-Log "Using SHIR MSI file: $MsiFileName" 15 | } 16 | else { 17 | Write-Log "Downloading latest version of SHIR MSI file" 18 | $MsiFileName = 'IntegrationRuntime.latest.msi' 19 | 20 | # Temporarily disable progress updates to speed up the download process. (See https://stackoverflow.com/questions/69942663/invoke-webrequest-progress-becomes-irresponsive-paused-while-downloading-the-fil) 21 | $ProgressPreference = 'SilentlyContinue' 22 | Invoke-WebRequest -Uri 'https://go.microsoft.com/fwlink/?linkid=839822&clcid=0x409' -OutFile "C:\SHIR\$MsiFileName" 23 | $ProgressPreference = 'Continue' 24 | } 25 | 26 | Write-Log "Installing SHIR" 27 | Start-Process msiexec.exe -Wait -ArgumentList "/i C:\SHIR\$MsiFileName /qn" 28 | if (!$?) { 29 | Write-Log "SHIR MSI Install Failed" 30 | } 31 | 32 | Write-Log "SHIR MSI Install Successfully" 33 | Write-Log "Will remove C:\SHIR\$MsiFileName" 34 | Remove-Item "C:\SHIR\$MsiFileName" 35 | Write-Log "Removed C:\SHIR\$MsiFileName" 36 | } 37 | 38 | function Install-MSFT-JDK() { 39 | Write-Log "Install the Microsoft OpenJDK in the Windows container" 40 | Write-Log "Downloading Microsoft OpenJDK 21 LTS msi" 41 | $JDKMsiFileName = 'microsoft-jdk-21-windows-x64.msi' 42 | 43 | # Temporarily disable progress updates to speed up the download process. (See https://stackoverflow.com/questions/69942663/invoke-webrequest-progress-becomes-irresponsive-paused-while-downloading-the-fil) 44 | $ProgressPreference = 'SilentlyContinue' 45 | Invoke-WebRequest -Uri "https://aka.ms/download-jdk/$JDKMsiFileName" -OutFile "C:\SHIR\$JDKMsiFileName" 46 | $ProgressPreference = 'Continue' 47 | 48 | Write-Log "Installing Microsoft OpenJDK" 49 | # Arguments pulled from https://learn.microsoft.com/en-us/java/openjdk/install#install-via-msi 50 | Start-Process msiexec.exe -Wait -ArgumentList "/i C:\SHIR\$JDKMsiFileName ADDLOCAL=FeatureMain,FeatureEnvironment,FeatureJarFileRunWith,FeatureJavaHome INSTALLDIR=`"c:\Program Files\Microsoft\`" /quiet" 51 | if (!$?) { 52 | Write-Log "Microsoft OpenJDK MSI Install Failed" 53 | } 54 | Write-Log "Microsoft OpenJDK MSI Install Successfully" 55 | Write-Log "Will remove C:\SHIR\$JDKMsiFileName" 56 | Remove-Item "C:\SHIR\$JDKMsiFileName" 57 | Write-Log "Removed C:\SHIR\$JDKMsiFileName" 58 | } 59 | 60 | function SetupEnv() { 61 | Write-Log "Begin to Setup the SHIR Environment" 62 | Start-Process $DmgcmdPath -Wait -ArgumentList "-Stop -StopUpgradeService -TurnOffAutoUpdate" 63 | Write-Log "SHIR Environment Setup Successfully" 64 | } 65 | 66 | function Add-Monitor-User($theUser) { 67 | try { 68 | Add-LocalGroupMember -Group "Performance Monitor Users" -Member $theUser 69 | } catch { 70 | Write-Log "The user $theUser was already in the Performance Monitor Users group" 71 | } 72 | try { 73 | Add-LocalGroupMember -Group "Performance Log Users" -Member $theUser 74 | } catch { 75 | Write-Log "The user $theUser was already in the Performance Log Users group" 76 | } 77 | Write-Log "The user $theUser is now in groups Performance Monitor Users and Performance Log Users" 78 | } 79 | 80 | Function Install-Certificate { 81 | $rootcert = (Get-ChildItem -Path C:\SHIR | Where-Object { $_.Name -match [regex] "root.*.cer" }) 82 | $intermittent = (Get-ChildItem -Path C:\SHIR | Where-Object { $_.Name -match [regex] "intermittent.*.cer" }) 83 | if($rootcert) { 84 | Write-Log "Installing Root Certificate" 85 | Import-Certificate -FilePath "C:\SHIR\$rootcert" -CertStoreLocation Cert:\LocalMachine\Root 86 | Write-Log "Will remove C:\SHIR\$rootcert" 87 | Remove-Item "C:\SHIR\$rootcert" 88 | 89 | } 90 | if($intermittent) { 91 | Write-Log "Installing Intermittent Certificate" 92 | Import-Certificate -FilePath "C:\SHIR\$intermittent" -CertStoreLocation Cert:\LocalMachine\CA 93 | Write-Log "Will remove C:\SHIR\$intermittent" 94 | Remove-Item "C:\SHIR\$intermittent" 95 | } 96 | } 97 | 98 | Install-SHIR 99 | 100 | # ####################################################### 101 | # Add user to the monitoring groups 102 | # the current user was fetched from a running pod using the below code 103 | # $currentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name 104 | Add-Monitor-User "User Manager\ContainerAdministrator" # This is the user that a user will enter as when logging into the container 105 | Add-Monitor-User "NT SERVICE\DIAHostService" # This is the user that runs the SHIR backend 106 | 107 | 108 | if ([bool]::Parse($env:INSTALL_JDK)) { 109 | Install-MSFT-JDK 110 | } 111 | 112 | if ([bool]::Parse($env:INSTALL_CERT)) { 113 | Install-Certificate 114 | } 115 | 116 | exit 0 117 | --------------------------------------------------------------------------------