├── .deployment ├── .gitattributes ├── .gitignore ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── azure-function ├── .gitignore ├── .vscode │ ├── extensions.json │ ├── launch.json │ ├── settings.json │ └── tasks.json ├── ChangeFeedParser.cs ├── ProcessDebeziumPayload.cs ├── azure-deploy.sh ├── azure-function.csproj ├── host.json └── local.settings.json.template ├── debezium ├── azure │ ├── 00-deploy-debezium.sh │ └── 01-register-connector.sh ├── on-prem │ ├── .env.template │ ├── delete-connector.ps1 │ ├── delete-connector.sh │ ├── docker-compose.yaml │ ├── list-connectors.ps1 │ ├── list-connectors.sh │ ├── register-connector.ps1 │ ├── register-connector.sh │ ├── start-debezium.ps1 │ ├── start-debezium.sh │ ├── stop-debezium.ps1 │ └── stop-debezium.sh ├── sqlserver-connector-config.json.template └── utils │ └── clean-eventhubs.sh ├── documentation ├── SQL-Server-Connector-Configuration-Value.md └── sql-server-change-stream.gif └── sql ├── README.md ├── azure-sql ├── 00-setup-database-user.sql ├── 01-enable-cdc.sql ├── 02-create-new-sales-order.sql ├── 03-modify-warehouse-stock.sql └── 04-disable-cdc.sql └── sql-server ├── 00-setup-database-user.sql ├── 01-enable-cdc.sql ├── 02-create-new-sales-order.sql ├── 03-modify-warehouse-stock.sql └── 04-disable-cdc.sql /.deployment: -------------------------------------------------------------------------------- 1 | [config] 2 | project = azure-function -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015/2017 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # Visual Studio 2017 auto generated files 33 | Generated\ Files/ 34 | 35 | # MSTest test Results 36 | [Tt]est[Rr]esult*/ 37 | [Bb]uild[Ll]og.* 38 | 39 | # NUNIT 40 | *.VisualState.xml 41 | TestResult.xml 42 | 43 | # Build Results of an ATL Project 44 | [Dd]ebugPS/ 45 | [Rr]eleasePS/ 46 | dlldata.c 47 | 48 | # Benchmark Results 49 | BenchmarkDotNet.Artifacts/ 50 | 51 | # .NET Core 52 | project.lock.json 53 | project.fragment.lock.json 54 | artifacts/ 55 | **/Properties/launchSettings.json 56 | 57 | # StyleCop 58 | StyleCopReport.xml 59 | 60 | # Files built by Visual Studio 61 | *_i.c 62 | *_p.c 63 | *_i.h 64 | *.ilk 65 | *.meta 66 | *.obj 67 | *.iobj 68 | *.pch 69 | *.pdb 70 | *.ipdb 71 | *.pgc 72 | *.pgd 73 | *.rsp 74 | *.sbr 75 | *.tlb 76 | *.tli 77 | *.tlh 78 | *.tmp 79 | *.tmp_proj 80 | *.log 81 | *.vspscc 82 | *.vssscc 83 | .builds 84 | *.pidb 85 | *.svclog 86 | *.scc 87 | 88 | # Chutzpah Test files 89 | _Chutzpah* 90 | 91 | # Visual C++ cache files 92 | ipch/ 93 | *.aps 94 | *.ncb 95 | *.opendb 96 | *.opensdf 97 | *.sdf 98 | *.cachefile 99 | *.VC.db 100 | *.VC.VC.opendb 101 | 102 | # Visual Studio profiler 103 | *.psess 104 | *.vsp 105 | *.vspx 106 | *.sap 107 | 108 | # Visual Studio Trace Files 109 | *.e2e 110 | 111 | # TFS 2012 Local Workspace 112 | $tf/ 113 | 114 | # Guidance Automation Toolkit 115 | *.gpState 116 | 117 | # ReSharper is a .NET coding add-in 118 | _ReSharper*/ 119 | *.[Rr]e[Ss]harper 120 | *.DotSettings.user 121 | 122 | # JustCode is a .NET coding add-in 123 | .JustCode 124 | 125 | # TeamCity is a build add-in 126 | _TeamCity* 127 | 128 | # DotCover is a Code Coverage Tool 129 | *.dotCover 130 | 131 | # AxoCover is a Code Coverage Tool 132 | .axoCover/* 133 | !.axoCover/settings.json 134 | 135 | # Visual Studio code coverage results 136 | *.coverage 137 | *.coveragexml 138 | 139 | # NCrunch 140 | _NCrunch_* 141 | .*crunch*.local.xml 142 | nCrunchTemp_* 143 | 144 | # MightyMoose 145 | *.mm.* 146 | AutoTest.Net/ 147 | 148 | # Web workbench (sass) 149 | .sass-cache/ 150 | 151 | # Installshield output folder 152 | [Ee]xpress/ 153 | 154 | # DocProject is a documentation generator add-in 155 | DocProject/buildhelp/ 156 | DocProject/Help/*.HxT 157 | DocProject/Help/*.HxC 158 | DocProject/Help/*.hhc 159 | DocProject/Help/*.hhk 160 | DocProject/Help/*.hhp 161 | DocProject/Help/Html2 162 | DocProject/Help/html 163 | 164 | # Click-Once directory 165 | publish/ 166 | 167 | # Publish Web Output 168 | *.[Pp]ublish.xml 169 | *.azurePubxml 170 | # Note: Comment the next line if you want to checkin your web deploy settings, 171 | # but database connection strings (with potential passwords) will be unencrypted 172 | *.pubxml 173 | *.publishproj 174 | 175 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 176 | # checkin your Azure Web App publish settings, but sensitive information contained 177 | # in these scripts will be unencrypted 178 | PublishScripts/ 179 | 180 | # NuGet Packages 181 | *.nupkg 182 | # The packages folder can be ignored because of Package Restore 183 | **/[Pp]ackages/* 184 | # except build/, which is used as an MSBuild target. 185 | !**/[Pp]ackages/build/ 186 | # Uncomment if necessary however generally it will be regenerated when needed 187 | #!**/[Pp]ackages/repositories.config 188 | # NuGet v3's project.json files produces more ignorable files 189 | *.nuget.props 190 | *.nuget.targets 191 | 192 | # Microsoft Azure Build Output 193 | csx/ 194 | *.build.csdef 195 | 196 | # Microsoft Azure Emulator 197 | ecf/ 198 | rcf/ 199 | 200 | # Windows Store app package directories and files 201 | AppPackages/ 202 | BundleArtifacts/ 203 | Package.StoreAssociation.xml 204 | _pkginfo.txt 205 | *.appx 206 | 207 | # Visual Studio cache files 208 | # files ending in .cache can be ignored 209 | *.[Cc]ache 210 | # but keep track of directories ending in .cache 211 | !*.[Cc]ache/ 212 | 213 | # Others 214 | ClientBin/ 215 | ~$* 216 | *~ 217 | *.dbmdl 218 | *.dbproj.schemaview 219 | *.jfm 220 | *.pfx 221 | *.publishsettings 222 | orleans.codegen.cs 223 | 224 | # Including strong name files can present a security risk 225 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 226 | #*.snk 227 | 228 | # Since there are multiple workflows, uncomment next line to ignore bower_components 229 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 230 | #bower_components/ 231 | 232 | # RIA/Silverlight projects 233 | Generated_Code/ 234 | 235 | # Backup & report files from converting an old project file 236 | # to a newer Visual Studio version. Backup files are not needed, 237 | # because we have git ;-) 238 | _UpgradeReport_Files/ 239 | Backup*/ 240 | UpgradeLog*.XML 241 | UpgradeLog*.htm 242 | ServiceFabricBackup/ 243 | *.rptproj.bak 244 | 245 | # SQL Server files 246 | *.mdf 247 | *.ldf 248 | *.ndf 249 | 250 | # Business Intelligence projects 251 | *.rdl.data 252 | *.bim.layout 253 | *.bim_*.settings 254 | *.rptproj.rsuser 255 | 256 | # Microsoft Fakes 257 | FakesAssemblies/ 258 | 259 | # GhostDoc plugin setting file 260 | *.GhostDoc.xml 261 | 262 | # Node.js Tools for Visual Studio 263 | .ntvs_analysis.dat 264 | node_modules/ 265 | 266 | # Visual Studio 6 build log 267 | *.plg 268 | 269 | # Visual Studio 6 workspace options file 270 | *.opt 271 | 272 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 273 | *.vbw 274 | 275 | # Visual Studio LightSwitch build output 276 | **/*.HTMLClient/GeneratedArtifacts 277 | **/*.DesktopClient/GeneratedArtifacts 278 | **/*.DesktopClient/ModelManifest.xml 279 | **/*.Server/GeneratedArtifacts 280 | **/*.Server/ModelManifest.xml 281 | _Pvt_Extensions 282 | 283 | # Paket dependency manager 284 | .paket/paket.exe 285 | paket-files/ 286 | 287 | # FAKE - F# Make 288 | .fake/ 289 | 290 | # JetBrains Rider 291 | .idea/ 292 | *.sln.iml 293 | 294 | # CodeRush 295 | .cr/ 296 | 297 | # Python Tools for Visual Studio (PTVS) 298 | __pycache__/ 299 | *.pyc 300 | 301 | # Cake - Uncomment if you are using it 302 | # tools/** 303 | # !tools/packages.config 304 | 305 | # Tabs Studio 306 | *.tss 307 | 308 | # Telerik's JustMock configuration file 309 | *.jmconfig 310 | 311 | # BizTalk build output 312 | *.btp.cs 313 | *.btm.cs 314 | *.odx.cs 315 | *.xsd.cs 316 | 317 | # OpenCover UI analysis results 318 | OpenCover/ 319 | 320 | # Azure Stream Analytics local run output 321 | ASALocalRun/ 322 | 323 | # MSBuild Binary and Structured Log 324 | *.binlog 325 | 326 | # NVidia Nsight GPU debugger configuration file 327 | *.nvuser 328 | 329 | # MFractors (Xamarin productivity tool) working folder 330 | .mfractor/ 331 | 332 | # Additional files 333 | **.env 334 | __blobstorage__/ 335 | __azurite_db_* 336 | 337 | # SQL Server Debezium connector config (generated) 338 | sqlserver-connector-config.json -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to .NET Functions", 6 | "type": "coreclr", 7 | "request": "attach", 8 | "processId": "${command:azureFunctions.pickProcess}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "azure-function/bin/Release/net6.0/publish", 3 | "azureFunctions.projectLanguage": "C#", 4 | "azureFunctions.projectRuntime": "~2", 5 | "debug.internalConsoleOptions": "neverOpen", 6 | "azureFunctions.preDeployTask": "publish" 7 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "clean", 6 | "command": "dotnet clean", 7 | "type": "shell", 8 | "problemMatcher": "$msCompile", 9 | "options": { 10 | "cwd": "${workspaceFolder}/azure-function" 11 | } 12 | }, 13 | { 14 | "label": "build", 15 | "command": "dotnet build", 16 | "type": "shell", 17 | "dependsOn": "clean", 18 | "group": { 19 | "kind": "build", 20 | "isDefault": true 21 | }, 22 | "problemMatcher": "$msCompile", 23 | "options": { 24 | "cwd": "${workspaceFolder}/azure-function" 25 | } 26 | }, 27 | { 28 | "label": "clean release", 29 | "command": "dotnet clean --configuration Release", 30 | "type": "shell", 31 | "problemMatcher": "$msCompile", 32 | "options": { 33 | "cwd": "${workspaceFolder}/azure-function" 34 | } 35 | }, 36 | { 37 | "label": "publish", 38 | "command": "dotnet publish --configuration Release", 39 | "type": "shell", 40 | "dependsOn": "clean release", 41 | "problemMatcher": "$msCompile", 42 | "options": { 43 | "cwd": "${workspaceFolder}/azure-function" 44 | } 45 | }, 46 | { 47 | "type": "func", 48 | "dependsOn": "build", 49 | "options": { 50 | "cwd": "${workspaceFolder}/azure-function/bin/Debug/net6.0" 51 | }, 52 | "command": "host start", 53 | "isBackground": true, 54 | "problemMatcher": "$func-watch" 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Azure SQL Database Change Stream with Debezium Sample Changelog 2 | 3 | 4 | ## 1.0.0 (2021-08-11) 5 | 6 | - Added support for Azure SQL 7 | - Repo moved into Azure Samples organization -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Azure SQL Database Change Stream with Debezium Sample 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 6 | 7 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase master -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Davide Mauri 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | languages: 4 | - tsql 5 | - sql 6 | - csharp 7 | products: 8 | - azure-sql-database 9 | - sql-server 10 | - azure-sql-managed-instance 11 | - azure-sqlserver-vm 12 | - azure 13 | - dotnet 14 | - azure-functions 15 | - azure-event-hubs 16 | description: "Create a Change Data Feed from Azure SQL or SQL Server using the Open Source tool Debezium" 17 | --- 18 | 19 | # Azure SQL / SQL Server Change Stream with Debezium 20 | 21 | 28 | 29 | ![License](https://img.shields.io/badge/license-MIT-green.svg) 30 | 31 | Azure SQL Database and SQL Server Change Stream sample using [Debezium](https://debezium.io/). A change feed or change stream allow applications to access real-time data changes, using standard technologies and well-known API, to create modern applications using the full power of database like SQL Server. 32 | 33 | Debezium make use of [Change Data Capture](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-2017), so it can be used with SQL Server on-premises and the whole [Azure SQL](https://azure.microsoft.com/en-us/products/azure-sql) family (https://azure.microsoft.com/en-us/products/azure-sql) (Azure SQL MI, Azure SQL DB, SQL Server on VM). 34 | 35 | With Debezium and Azure SQL / SQL Server you can not only create more modern and reactive applications that handle data changes in near real time with a minium impact on the database, but you can also use it to implement your Hybrid IT strategy, still using On-Prem SQL Server but relying on Azure for all your computing needs, taking advantage of PaaS offerings like EventHubs and Azure Functions. This sample will show how to do that. 36 | 37 | ![SQL Server Change Stream](./documentation/sql-server-change-stream.gif) 38 | 39 | For those who want more details, Debezium uses Apache Kafka, which is not natively available on Azure as a PaaS offer. But luckily EventHubs offer almost 100% support, so we can use it instead of Kafka and make maintenance and scalability easier. 40 | 41 | [Event Hubs for Kafka](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-for-kafka-ecosystem-overview) 42 | 43 | ## Step by Step Guide 44 | 45 | This step by step guide uses Wide World Importers sample database from here: 46 | 47 | https://github.com/Microsoft/sql-server-samples/releases/tag/wide-world-importers-v1.0 48 | 49 | Make sure you download the OLTP database if you want to follow this guide without having to change a thing. To restore the database on Azure SQL, you can use the scripts provided here: [Restore Database in Azure SQL](https://github.com/yorek/azure-sql-db-samples/tree/master/samples/01-restore-database). 50 | 51 | ### Create Debezium User 52 | 53 | Debezium needs to query the database so a dedicated user is used throughout the sample. User has `db_owner` access to make the script simpler. In real world you may want to tighten security a bit more. 54 | 55 | To create the login and user run the script `00-setup-database-user.sql` from the `sql` folder (pertinent to the service or product you are using, Azure SQL or SQL Server / Azure SQL MI) on the database where you have restored Wide World Importers database. 56 | 57 | ### Enable Change Data Capture 58 | 59 | Debezium uses Change Data Capture to capture all the changes done to selected tables. 60 | 61 | In this samples only two tables are monitored: 62 | 63 | - Sales.Orders 64 | - Warehouse.StockItems 65 | 66 | The script `01-enable-cdc.sql` enable Change Data Capture on the aforementioned tables. 67 | 68 | ### Create an Azure Event Hubs 69 | 70 | All data gathered by Change Data Capture will be send to Event Hubs, so create an Azure Event Hubs in your Azure Subscription. Using the [Azure Cloud Shell](https://shell.azure.com/) Bash: 71 | 72 | ```bash 73 | # Set vars 74 | RESOURCE_GROUP=debezium 75 | LOCATION=eastus 76 | EVENTHUB_NAMESPACE=debezium 77 | EVENTHUB_SCHEMA_HISTORY=schemahistory 78 | 79 | # create group 80 | az group create \ 81 | --name $RESOURCE_GROUP \ 82 | --location $LOCATION 83 | 84 | # create eventhub namespace with kafka enabled 85 | az eventhubs namespace create \ 86 | --name $EVENTHUB_NAMESPACE \ 87 | --resource-group $RESOURCE_GROUP \ 88 | --location $LOCATION \ 89 | --enable-kafka 90 | 91 | # create eventhub for schema history 92 | az eventhubs eventhub create \ 93 | --resource-group $RESOURCE_GROUP \ 94 | --namespace $EVENTHUB_NAMESPACE \ 95 | --name $EVENTHUB_SCHEMA_HISTORY \ 96 | --partition-count 1 \ 97 | --cleanup-policy Delete \ 98 | --retention-time-in-hours 168 99 | ``` 100 | 101 | Later in the configuration process you'll need the EventHubs connection string, so grab it and store it somewhere: 102 | 103 | ```bash 104 | RESOURCE_GROUP=debezium 105 | EVENTHUB_NAMESPACE=debezium 106 | 107 | az eventhubs namespace authorization-rule keys list \ 108 | --resource-group $RESOURCE_GROUP \ 109 | --namespace-name $EVENTHUB_NAMESPACE \ 110 | --name RootManageSharedAccessKey \ 111 | --query "primaryConnectionString" \ 112 | --output tsv 113 | ``` 114 | 115 | ### Run Debezium 116 | 117 | #### Pre-Requisites 118 | 119 | In order to run Debezium you have to install and configure Apache Kafka, Apache Zookeper and Kafka Connect. If you already know how to do that, or your already have a testing or development environment, well, perfect. Go and install Debezium SQL Server Connector: [Installing a Debezium Connector](https://debezium.io/docs/install/stable/#installing_a_debezium_connector). 120 | 121 | If prefer a more lean and quick easy to start using Debezium, you can just use the [Debezium Docker Image](https://github.com/debezium/docker-images), that provides anything you need to run a test instance of Debezium. Just make sure you have [Docker](https://docs.docker.com/install/) and [Docker Compose](https://docs.docker.com/compose) installed. In the `debezium/on-prem` folder you can find all the scripts needed to run Debezium using Docker. 122 | 123 | #### Configure Environment 124 | 125 | Docker Compose will use `.env` to get the environment variables values used in the `.yaml` configuration file. The provided `.env.template` file look like the following: 126 | 127 | ```bash 128 | DEBEZIUM_VERSION=2.7 129 | EVENTHUB_NAMESPACE= 130 | EVENTHUB_CONNECTION_STRING= 131 | ``` 132 | 133 | Copy it and create a new `.env` file. Leave the version set to `2.7`. Change the `EVENTHUB_NAMESPACE` to the EventHubs name you created before. Also set `EVENTHUB_CONNECTION_STRING` to hold the EventHubs connection string you got before. Make sure not to use any additional quotes or double quotes. 134 | 135 | #### The .yaml file 136 | 137 | If you are just interested in testing Debezium you can safely skip this section and move to the next one to start Debezium. If you want to understand how to make Debezium work with Event Hubs, read on. 138 | 139 | Debezium needs Apache Kafka to run, NOT Azure Event Hubs. Luckily for us, Azure Event Hubs exposes a Kafka-Compatible endpoint, so we can still enjoy Kafka with all the comfort of a PaaS offering. There are a few tweeks needed in order to make Debezium working with Azure Event Hubs. 140 | 141 | First of all EventHubs requires authentication. This part is taken care from the configuration settings that looks like the following: 142 | 143 | ```yaml 144 | - *_SECURITY_PROTOCOL=SASL_SSL 145 | - *_SASL_MECHANISM=PLAIN 146 | - *_SASL_JAAS_CONFIG=[...] 147 | ``` 148 | 149 | Documentation on EventHubs Kafka Authentication and Kafka Connect is available here: 150 | 151 | [Integrate Apache Kafka Connect support on Azure Event Hubs](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-kafka-connect-tutorial) 152 | 153 | Since we're running a Docker Image, we cannot really change the configuration file, but Debezium allows pass-through configurations: 154 | 155 | [Debezium Connect-Base](https://github.com/debezium/docker-images/tree/master/connect-base) 156 | 157 | There is additional caveat to keep in mind. EventHubs security uses the string `$ConnectionString` as username. In order to avoid to have Docker Compose to treat it as a variable instead, a double dollar sign `$$` needs to be used: 158 | 159 | [Docker Compose Config File Variable Substitution](https://docs.docker.com/compose/compose-file/#variable-substitution) 160 | 161 | Two other options useful for running Debezium on Azure Event Hubs are the following: 162 | 163 | ```yaml 164 | - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE=false 165 | - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE=true 166 | ``` 167 | 168 | They control if the schema is sent with the data or not. Since the Azure Event Hubs only support values, as opposed to Apache Kafka, which everything is actually a key-value pair, the schema generation for the key section can be safely turned off. While this can also be done for the value part, it is not recommended as some data type are serialized in a Kafka-specific way and you need to know their "sematic" type in order to recreate the correct value. 169 | 170 | [Debezium SQL Server Connector Data Types](https://debezium.io/docs/connectors/sqlserver/#data-types) 171 | 172 | Here's a sample of a schema for a "create" (INSERT) event: 173 | 174 | [Debezium SQL Server Connector Create Event Sample](https://debezium.io/docs/connectors/sqlserver/#create-events) 175 | 176 | #### Start Debezium 177 | 178 | Debezium can now be started. If you're using the Docker Images you can just do this by running `debezium/on-prem/start-debezium.ps1` (or the `.sh` file if you're on Linux/WSL) 179 | 180 | Once the startup has finished, you'll see something like 181 | 182 | ```text 183 | [Worker clientId=connect-1, groupId=1] Finished starting connectors and tasks [org.apache.kafka.connect.runtime.distributed.DistributedHerder] 184 | ``` 185 | 186 | you will see four topics (or EventHub to use the Azure EventHubs nomenclature): 187 | 188 | ```bash 189 | 190 | RESOURCE_GROUP=debezium 191 | EVENTHUB_NAMESPACE=debezium 192 | 193 | az eventhubs eventhub list \ 194 | --resource-group $RESOURCE_GROUP \ 195 | --namespace-name $EVENTHUB_NAMESPACE \ 196 | --output table 197 | ``` 198 | 199 | and the result will show: 200 | 201 | - debezium_configs 202 | - debezium_offsets 203 | - debezium_statuses 204 | - schemahistory 205 | 206 | to explore Azure Event Hubs is strongly suggest to download and use [Service Bus Explorer](https://github.com/paolosalvatori/ServiceBusExplorer) 207 | 208 | #### Register SQL Server Connector 209 | 210 | Now that Debezium is running, the SQL Server Connector (which is used both for connecting to Azure SQL or SQL Server) can be registered. Before doing that, make sure to specify the correct connection for your SQL Server instance in a file named `sqlserver-connector-config.json`. You can create one using the template file [sqlserver-connector-config.json.template](debezium/sqlserver-connector-config.json.template) file. 211 | 212 | Make sure to change the following properties to match your SQL Server configuration: 213 | 214 | ```json 215 | "database.hostname": ".database.windows.net", 216 | "database.names": "", 217 | "database.user" : "", 218 | "database.password" : "", 219 | ``` 220 | 221 | You would need to replace ``, ``, ``, and `` with proper values. 222 | 223 | Also, make sure to change the following properties to match your Azure Event Hub configuration: 224 | 225 | ```json 226 | "schema.history.internal.kafka.bootstrap.servers": ".servicebus.windows.net:9093", 227 | "schema.history.internal.kafka.topic": "", 228 | "schema.history.internal.consumer.sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"\";", 229 | "schema.history.internal.producer.sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"\";", 230 | ``` 231 | 232 | You would need to replace ``, ``, and `` with proper values. 233 | 234 | All the other values used are explained in detail here: 235 | 236 | [SQL Server Connector Configuration Values](./documentation/SQL-Server-Connector-Configuration-Value.md) 237 | 238 | Once the configuration file is set, just register that using `debezium/on-prem/register-connector.ps1`. 239 | 240 | Depending on how big your tables are, it make take a while (more on this later). Once you see the following message: 241 | 242 | ```text 243 | Snapshot step 8 - Finalizing [io.debezium.relational.HistorizedRelationalSnapshotChangeEventSource] 244 | ``` 245 | 246 | and no other errors or exception before that, you'll know that the SQL Server Connector is correctly running. 247 | 248 | ### Make sample changes 249 | 250 | Now that Debezium is running and fully configured, you can generate a new Sales Order and insert, update and delete some data in the Stock table. You can use the following scripts: 251 | 252 | ```bash 253 | ./sql/.../02-create-new-sales-order.sql 254 | ./sql/.../03-modify-warehouse-stock.sql 255 | ``` 256 | 257 | After running the script you can use Service Bus Explorer or VS Code Event Hub Explorer to consume the stream of changes sent to Azure Event Hubs. You'll notice a new topic named `wwi`. That's where we instructed Debezium to send all the changes detected to the monitored tables. 258 | 259 | ### Consume Change Stream using an Azure Functions 260 | 261 | One way to quickly react to the Change Stream data coming from Debezium is to use Azure Functions. A sample is available in folder `azure-function`. The easiest way to run the sample is to open it from VS Code or via [Azure Function Core Tools](https://docs.microsoft.com/azure/azure-functions/functions-run-local), via `func start`. It will automatically recognize it as an Azure Function and download everything needed to run it. 262 | 263 | Make sure you have a `local.setting.json` that looks like the provided template. Copy the Azure Event Hubs connection string you got at the beginning into the `Debezium` configuration option. 264 | 265 | Start the function. As soon as the Azure Function runtime is running, the code will start to process the changes already available in EventHubs and you'll see something like this: 266 | 267 | ```text 268 | Event from Change Feed received: 269 | - Object: Sales.Orders 270 | - Operation: Insert 271 | - Captured At: 2019-08-04T22:35:59.0100000Z 272 | > OrderID = 73625 273 | > CustomerID = 941 274 | > SalespersonPersonID = 3 275 | > PickedByPersonID = 276 | > ContactPersonID = 3141 277 | > BackorderOrderID = 278 | > OrderDate = 8/4/2019 12:00:00 AM 279 | > ExpectedDeliveryDate = 8/5/2019 12:00:00 AM 280 | > CustomerPurchaseOrderNumber = 4923 281 | > IsUndersupplyBackordered = False 282 | > Comments = Auto-generated 283 | > DeliveryInstructions = Unit 17, 1466 Deilami Road 284 | > InternalComments = 285 | > PickingCompletedWhen = 286 | > LastEditedBy = 3 287 | > LastEditedWhen = 8/4/2019 10:35:58 PM 288 | Executed 'ProcessDebeziumPayload' (Succeeded, Id=ee9d1080-64ff-4039-83af-69c4b12fa85f) 289 | ``` 290 | 291 | ### Done 292 | 293 | Congratulations, you now have a working Change Stream from SQL Server. This opens up a whole new set of possibilities! Have fun! 294 | 295 | ## Notes 296 | 297 | ### Running Debezium on Azure 298 | 299 | If you're using Debezium with Azure SQL MI or Azure SQL DB, you may want to run Debezium on Azure. Sample script to run the Debezium container on Azure Container Instances are available in the `debezium/azure` folder. 300 | 301 | ### Connector Configuration 302 | 303 | More details on SQL Server and Event Hubs specific configuration here: 304 | 305 | [SQL Server Connector Configuration Values](./documentation/SQL-Server-Connector-Configuration-Value.md) 306 | 307 | ## Troubleshooting 308 | 309 | ### Local Azure Function not working 310 | 311 | If, after starting the Azure Function via `func start` you get the following error: 312 | 313 | ``` 314 | The listener for function 'ProcessDebeziumPayload' was unable to start. Microsoft.WindowsAzure.Storage: No connection could be made because the target machine actively refused it. System.Net.Http: No connection could be made because the target machine actively refused it. System.Private.CoreLib: No connection could be made because the target machine actively refused it. 315 | ``` 316 | 317 | you need to make sure that [Azure Storage Emulator](https://docs.microsoft.com/azure/storage/common/storage-use-azurite) is installed and started. 318 | -------------------------------------------------------------------------------- /azure-function/.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /azure-function/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-dotnettools.csharp" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /azure-function/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to .NET Functions", 6 | "type": "coreclr", 7 | "request": "attach", 8 | "processId": "${command:azureFunctions.pickProcess}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /azure-function/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "bin/Release/net6.0/publish", 3 | "azureFunctions.projectLanguage": "C#", 4 | "azureFunctions.projectRuntime": "~3", 5 | "debug.internalConsoleOptions": "neverOpen", 6 | "azureFunctions.preDeployTask": "publish (functions)" 7 | } -------------------------------------------------------------------------------- /azure-function/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "clean (functions)", 6 | "command": "dotnet", 7 | "args": [ 8 | "clean", 9 | "/property:GenerateFullPaths=true", 10 | "/consoleloggerparameters:NoSummary" 11 | ], 12 | "type": "process", 13 | "problemMatcher": "$msCompile" 14 | }, 15 | { 16 | "label": "build (functions)", 17 | "command": "dotnet", 18 | "args": [ 19 | "build", 20 | "/property:GenerateFullPaths=true", 21 | "/consoleloggerparameters:NoSummary" 22 | ], 23 | "type": "process", 24 | "dependsOn": "clean (functions)", 25 | "group": { 26 | "kind": "build", 27 | "isDefault": true 28 | }, 29 | "problemMatcher": "$msCompile" 30 | }, 31 | { 32 | "label": "clean release (functions)", 33 | "command": "dotnet", 34 | "args": [ 35 | "clean", 36 | "--configuration", 37 | "Release", 38 | "/property:GenerateFullPaths=true", 39 | "/consoleloggerparameters:NoSummary" 40 | ], 41 | "type": "process", 42 | "problemMatcher": "$msCompile" 43 | }, 44 | { 45 | "label": "publish (functions)", 46 | "command": "dotnet", 47 | "args": [ 48 | "publish", 49 | "--configuration", 50 | "Release", 51 | "/property:GenerateFullPaths=true", 52 | "/consoleloggerparameters:NoSummary" 53 | ], 54 | "type": "process", 55 | "dependsOn": "clean release (functions)", 56 | "problemMatcher": "$msCompile" 57 | }, 58 | { 59 | "type": "func", 60 | "dependsOn": "build (functions)", 61 | "options": { 62 | "cwd": "${workspaceFolder}/bin/Debug/net6.0" 63 | }, 64 | "command": "host start", 65 | "isBackground": true, 66 | "problemMatcher": "$func-dotnet-watch" 67 | } 68 | ] 69 | } -------------------------------------------------------------------------------- /azure-function/ChangeFeedParser.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using Microsoft.Extensions.Logging; 8 | using Newtonsoft.Json.Linq; 9 | 10 | namespace Azure.SQLDB.ChangeDataCapture.Debezium.Sample 11 | { 12 | public class Utils 13 | { 14 | public static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); 15 | } 16 | 17 | public enum Operation 18 | { 19 | Insert, 20 | Update, 21 | Delete, 22 | Read 23 | } 24 | 25 | public class TableInfo 26 | { 27 | public readonly string Schema; 28 | public readonly string Table; 29 | public readonly DateTime ChangedAt; 30 | 31 | public TableInfo(string schema, string table, DateTime timeStamp) 32 | { 33 | this.Schema = schema; 34 | this.Table = table; 35 | this.ChangedAt = timeStamp; 36 | } 37 | } 38 | 39 | public class Field 40 | { 41 | public readonly string Name; 42 | public readonly object Value; 43 | 44 | public Field(string name, object value) 45 | { 46 | this.Name = name; 47 | this.Value = value; 48 | } 49 | } 50 | 51 | public class Fields : IEnumerable 52 | { 53 | private JObject _body; 54 | private readonly string SectionName; 55 | private JObject _schema => (JObject)(_body["schema"]); 56 | private JObject _section => (JObject)(_body["payload"][SectionName]); 57 | private Dictionary _fields = new Dictionary(); 58 | 59 | public object this[string fieldName] 60 | { 61 | get 62 | { 63 | return GetValue(fieldName); 64 | } 65 | } 66 | 67 | public IEnumerator GetEnumerator() 68 | { 69 | foreach (var f in _fields) 70 | { 71 | yield return new Field(f.Key, this[f.Key]); 72 | } 73 | } 74 | 75 | IEnumerator IEnumerable.GetEnumerator() 76 | { 77 | return GetEnumerator(); 78 | } 79 | 80 | public Fields(JObject body, string sectionName) 81 | { 82 | _body = body; 83 | SectionName = sectionName; 84 | 85 | var fields = (JArray)(_schema["fields"][0]["fields"]); 86 | 87 | foreach (var f in fields.ToArray()) 88 | { 89 | string name = f["field"].ToString(); 90 | string type = f["type"].ToString(); 91 | string debeziumType = f["name"]?.ToString(); 92 | _fields.Add(name, debeziumType ?? ""); 93 | } 94 | } 95 | 96 | public object GetValue(string fieldName) 97 | { 98 | var property = (_section).Property(fieldName); 99 | 100 | string result = property.Value.ToString(); 101 | string debeziumType = _fields[property.Name]; 102 | 103 | if (string.IsNullOrEmpty(result)) 104 | return result; 105 | 106 | if (string.IsNullOrEmpty(_fields[property.Name])) // not a debezium data type 107 | return result; 108 | 109 | switch (_fields[property.Name]) 110 | { 111 | case "io.debezium.time.Date": 112 | var daysFromEoch = Int32.Parse(result); 113 | return Utils.Epoch.AddDays(daysFromEoch).Date; 114 | 115 | case "io.debezium.time.Time": 116 | var millisecondFromMidnight = Int32.Parse(result); 117 | return Utils.Epoch.AddMilliseconds(millisecondFromMidnight).TimeOfDay; 118 | 119 | case "io.debezium.time.MicroTime": 120 | var elapsedMicroSeconds = Int64.Parse(result); 121 | return Utils.Epoch.AddTicks(elapsedMicroSeconds * 10).TimeOfDay; 122 | 123 | case "io.debezium.time.NanoTime": 124 | var elapsedNanoSeconds = Int64.Parse(result); 125 | return Utils.Epoch.AddTicks(elapsedNanoSeconds / 100).TimeOfDay; 126 | 127 | case "io.debezium.time.Timestamp": 128 | var elapsedMilliseconds = Int64.Parse(result); 129 | return Utils.Epoch.AddMilliseconds(elapsedMilliseconds); 130 | 131 | case "io.debezium.time.MicroTimestamp": 132 | var elapsedMicroSeconds2 = Int64.Parse(result); 133 | return Utils.Epoch.AddMilliseconds(elapsedMicroSeconds2 * 10); 134 | 135 | case "io.debezium.time.NanoTimestamp": 136 | var elapsedNanoSeconds2 = Int64.Parse(result); 137 | return Utils.Epoch.AddTicks(elapsedNanoSeconds2 / 100); 138 | 139 | default: 140 | throw new ApplicationException($"'{debeziumType}' is unknown"); 141 | } 142 | } 143 | } 144 | 145 | public class ChangeFeedParser 146 | { 147 | private JObject _body; 148 | private JObject _payload => (JObject)(_body["payload"]); 149 | private JObject _source => (JObject)(_payload["source"]); 150 | public TableInfo TableInfo { get; private set; } 151 | public readonly Fields After; 152 | public readonly Fields Before; 153 | public readonly Operation Operation; 154 | 155 | public ChangeFeedParser(JObject body) 156 | { 157 | _body = body; 158 | 159 | TableInfo = new TableInfo( 160 | schema: _source["schema"].ToString(), 161 | table: _source["table"].ToString(), 162 | timeStamp: Utils.Epoch.AddMilliseconds(Int64.Parse(_source["ts_ms"].ToString())) 163 | ); 164 | 165 | Before = new Fields(body, "before"); 166 | After = new Fields(body, "after"); 167 | 168 | switch (_payload["op"]?.ToString()) 169 | { 170 | case "c": 171 | Operation = Operation.Insert; 172 | break; 173 | 174 | case "u": 175 | Operation = Operation.Update; 176 | break; 177 | 178 | case "d": 179 | Operation = Operation.Delete; 180 | break; 181 | 182 | default: 183 | throw new ApplicationException("Field 'op' contains an unknown value or doesn't exists."); 184 | } 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /azure-function/ProcessDebeziumPayload.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using Microsoft.Azure.EventHubs; 7 | using Microsoft.Azure.WebJobs; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json.Linq; 10 | 11 | namespace Azure.SQLDB.ChangeDataCapture.Debezium.Sample 12 | { 13 | public static class ProcessDebeziumPayload 14 | { 15 | [FunctionName("ProcessDebeziumPayload")] 16 | public static async Task Run([EventHubTrigger("wwi", Connection = "Debezium")] EventData[] events, ILogger log) 17 | { 18 | var exceptions = new List(); 19 | 20 | foreach (EventData eventData in events) 21 | { 22 | try 23 | { 24 | if (eventData.Body.Array.Length > 0) 25 | { 26 | string messageBody = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count); 27 | 28 | //log.LogInformation(messageBody); 29 | 30 | var body = JObject.Parse(messageBody); 31 | var parser = new ChangeFeedParser(body); 32 | 33 | log.LogInformation("Event from Change Feed received:"); 34 | log.LogInformation("- Object: " + parser.TableInfo.Schema + "." + parser.TableInfo.Table); 35 | log.LogInformation("- Operation: " + parser.Operation.ToString()); 36 | log.LogInformation("- Captured At: " + parser.TableInfo.ChangedAt.ToString("O")); 37 | 38 | Fields fields; 39 | if (parser.Operation == Operation.Insert || parser.Operation == Operation.Update) 40 | fields = parser.After; 41 | else 42 | fields = parser.Before; 43 | 44 | foreach(var f in fields) 45 | { 46 | log.LogInformation($"> {f.Name} = {f.Value}"); 47 | } 48 | } 49 | await Task.Yield(); 50 | } 51 | catch (Exception e) 52 | { 53 | // We need to keep processing the rest of the batch - capture this exception and continue. 54 | // Also, consider capturing details of the message that failed processing so it can be processed again later. 55 | exceptions.Add(e); 56 | } 57 | } 58 | 59 | // Once processing of the batch is complete, if any messages in the batch failed processing throw an exception so that there is a record of the failure. 60 | if (exceptions.Count > 1) 61 | throw new AggregateException(exceptions); 62 | 63 | if (exceptions.Count == 1) 64 | throw exceptions.Single(); 65 | } 66 | } 67 | } -------------------------------------------------------------------------------- /azure-function/azure-deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | # Requirements: 6 | # - AZ CLI: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest 7 | # - jq: https://stedolan.github.io/jq/download/ 8 | 9 | # Make sure these values are correct for your environment 10 | resourceGroup="dm-debezium" 11 | appName="dm-debezium" 12 | storageName="dmdebezium" 13 | location="WestUS2" 14 | 15 | # Change this if you are using your own github repository 16 | gitSource="https://github.com/Azure-Samples/azure-sql-db-change-stream-debezium" 17 | 18 | # Check that local.settings.json exists 19 | settingsFile="./local.settings.json" 20 | if ! [ -f $settingsFile ]; then 21 | echo "$settingsFile does not exists. Please create it." 22 | exit 23 | fi 24 | 25 | echo "Creating Resource Group..."; 26 | az group create \ 27 | -n $resourceGroup \ 28 | -l $location 29 | 30 | echo "Creating Application Insight..." 31 | az resource create \ 32 | -g $resourceGroup \ 33 | -n $appName-ai \ 34 | --resource-type "Microsoft.Insights/components" \ 35 | --properties '{"Application_Type":"web"}' 36 | 37 | echo "Reading Application Insight Key..." 38 | aikey=`az resource show -g $resourceGroup -n $appName-ai --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv` 39 | 40 | echo "Creating Storage Account..."; 41 | az storage account create \ 42 | -g $resourceGroup \ 43 | -l $location \ 44 | -n $storageName \ 45 | --sku Standard_LRS 46 | 47 | echo "Creating Function App..."; 48 | az functionapp create \ 49 | -g $resourceGroup \ 50 | -n $appName \ 51 | --storage-account $storageName \ 52 | --app-insights-key $aikey \ 53 | --consumption-plan-location $location \ 54 | --deployment-source-url $gitSource \ 55 | --deployment-source-branch master \ 56 | --functions-version 3 \ 57 | --os-type Windows 58 | 59 | echo "Configuring Settings..."; 60 | settings=(Debezium) 61 | for i in "${settings[@]}" 62 | do 63 | v=`cat $settingsFile | jq .Values.$i -r` 64 | c="az functionapp config appsettings set -g $resourceGroup -n $appName --settings $i='$v'" 65 | #echo $c 66 | eval $c 67 | done 68 | -------------------------------------------------------------------------------- /azure-function/azure-function.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | net6.0 4 | v3 5 | Azure.SQLDB.ChangeDataCapture.Debezium 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | PreserveNewest 14 | 15 | 16 | PreserveNewest 17 | Never 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /azure-function/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /azure-function/local.settings.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "IsEncrypted": false, 3 | "Values": { 4 | "AzureWebJobsStorage": "UseDevelopmentStorage=true", 5 | "FUNCTIONS_WORKER_RUNTIME": "dotnet", 6 | "Debezium":"" 7 | } 8 | } -------------------------------------------------------------------------------- /debezium/azure/00-deploy-debezium.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Strict mode, fail on any error 4 | set -euo pipefail 5 | 6 | DEBEZIUM_VERSION=2.7 7 | RESOURCE_GROUP="debezium" 8 | EVENTHUB_NAMESPACE="debezium" 9 | EVENTHUB_SCHEMA_HISTORY="schemahistory" 10 | CONTAINER_NAME="debezium" 11 | LOCATION="WestUS2" 12 | 13 | echo "deploying resource group" 14 | az group create \ 15 | --name $RESOURCE_GROUP \ 16 | --location $LOCATION 17 | 18 | echo "deploying eventhubs namespace" 19 | az eventhubs namespace create \ 20 | --resource-group $RESOURCE_GROUP \ 21 | --location $LOCATION \ 22 | --name $EVENTHUB_NAMESPACE \ 23 | --enable-kafka=true 24 | 25 | echo "deploying schema history event hub" 26 | az eventhubs eventhub create \ 27 | --resource-group $RESOURCE_GROUP \ 28 | --namespace-name $EVENTHUB_NAMESPACE \ 29 | --name $EVENTHUB_SCHEMA_HISTORY \ 30 | --partition-count 1 \ 31 | --cleanup-policy Delete \ 32 | --retention-time-in-hours 168 \ 33 | --output none 34 | 35 | echo "gathering eventhubs connection string" 36 | EVENTHUB_CONNECTION_STRING=`az eventhubs namespace authorization-rule keys list --resource-group $RESOURCE_GROUP --name RootManageSharedAccessKey --namespace-name $EVENTHUB_NAMESPACE --output tsv --query 'primaryConnectionString'` 37 | 38 | echo "deploying debezium container" 39 | az container create \ 40 | --resource-group $RESOURCE_GROUP \ 41 | --location $LOCATION \ 42 | --name $CONTAINER_NAME \ 43 | --image debezium/connect:${DEBEZIUM_VERSION} \ 44 | --ports 8083 \ 45 | --ip-address Public \ 46 | --os-type Linux \ 47 | --cpu 2 \ 48 | --memory 4 \ 49 | --environment-variables \ 50 | BOOTSTRAP_SERVERS=${EVENTHUB_NAMESPACE}.servicebus.windows.net:9093 \ 51 | GROUP_ID=1 \ 52 | CONFIG_STORAGE_TOPIC=debezium_configs \ 53 | OFFSET_STORAGE_TOPIC=debezium_offsets \ 54 | STATUS_STORAGE_TOPIC=debezium_statuses \ 55 | CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE=false \ 56 | CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE=true \ 57 | CONNECT_REQUEST_TIMEOUT_MS=60000 \ 58 | CONNECT_SECURITY_PROTOCOL=SASL_SSL \ 59 | CONNECT_SASL_MECHANISM=PLAIN \ 60 | CONNECT_SASL_JAAS_CONFIG="org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\$ConnectionString\" password=\"${EVENTHUB_CONNECTION_STRING}\";" \ 61 | CONNECT_PRODUCER_SECURITY_PROTOCOL=SASL_SSL \ 62 | CONNECT_PRODUCER_SASL_MECHANISM=PLAIN \ 63 | CONNECT_PRODUCER_SASL_JAAS_CONFIG="org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\$ConnectionString\" password=\"${EVENTHUB_CONNECTION_STRING}\";" \ 64 | CONNECT_CONSUMER_SECURITY_PROTOCOL=SASL_SSL \ 65 | CONNECT_CONSUMER_SASL_MECHANISM=PLAIN \ 66 | CONNECT_CONSUMER_SASL_JAAS_CONFIG="org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\$ConnectionString\" password=\"${EVENTHUB_CONNECTION_STRING}\";" 67 | 68 | echo "eventhub connection string" 69 | echo $EVENTHUB_CONNECTION_STRING -------------------------------------------------------------------------------- /debezium/azure/01-register-connector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Strict mode, fail on any error 4 | set -euo pipefail 5 | 6 | RESOURCE_GROUP="debezium" 7 | CONTAINER_NAME="debezium" 8 | 9 | echo "finding debezium ip" 10 | DEBEZIUM_IP=`az container show --resource-group $RESOURCE_GROUP --name $CONTAINER_NAME --output tsv --query "ipAddress.ip"` 11 | 12 | echo "registering connector" 13 | curl -i -X POST \ 14 | -H "Accept:application/json" -H "Content-Type:application/json" \ 15 | http://${DEBEZIUM_IP}:8083/connectors/ \ 16 | -d @../sqlserver-connector-config.json 17 | -------------------------------------------------------------------------------- /debezium/on-prem/.env.template: -------------------------------------------------------------------------------- 1 | DEBEZIUM_VERSION=2.7 2 | EVENTHUB_NAMESPACE= 3 | EVENTHUB_CONNECTION_STRING= -------------------------------------------------------------------------------- /debezium/on-prem/delete-connector.ps1: -------------------------------------------------------------------------------- 1 | Invoke-RestMethod http://localhost:8083/connectors/wwi -Method DELETE -------------------------------------------------------------------------------- /debezium/on-prem/delete-connector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -X DELETE http://localhost:8083/connectors/wwi -w "\n" 4 | -------------------------------------------------------------------------------- /debezium/on-prem/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | connect: 4 | image: debezium/connect:${DEBEZIUM_VERSION} 5 | ports: 6 | - 8083:8083 7 | environment: 8 | - BOOTSTRAP_SERVERS=${EVENTHUB_NAMESPACE}.servicebus.windows.net:9093 9 | - GROUP_ID=1 10 | - CONFIG_STORAGE_TOPIC=debezium_configs 11 | - OFFSET_STORAGE_TOPIC=debezium_offsets 12 | - STATUS_STORAGE_TOPIC=debezium_statuses 13 | - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE=false 14 | - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE=true 15 | - CONNECT_REQUEST_TIMEOUT_MS=60000 16 | - CONNECT_SECURITY_PROTOCOL=SASL_SSL 17 | - CONNECT_SASL_MECHANISM=PLAIN 18 | - CONNECT_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="$$ConnectionString" password="${EVENTHUB_CONNECTION_STRING}"; 19 | - CONNECT_PRODUCER_SECURITY_PROTOCOL=SASL_SSL 20 | - CONNECT_PRODUCER_SASL_MECHANISM=PLAIN 21 | - CONNECT_PRODUCER_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="$$ConnectionString" password="${EVENTHUB_CONNECTION_STRING}"; 22 | - CONNECT_CONSUMER_SECURITY_PROTOCOL=SASL_SSL 23 | - CONNECT_CONSUMER_SASL_MECHANISM=PLAIN 24 | - CONNECT_CONSUMER_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="$$ConnectionString" password="${EVENTHUB_CONNECTION_STRING}"; 25 | -------------------------------------------------------------------------------- /debezium/on-prem/list-connectors.ps1: -------------------------------------------------------------------------------- 1 | Invoke-RestMethod http://localhost:8083/connectors 2 | -------------------------------------------------------------------------------- /debezium/on-prem/list-connectors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -i -X GET http://localhost:8083/connectors -w "\n" 4 | -------------------------------------------------------------------------------- /debezium/on-prem/register-connector.ps1: -------------------------------------------------------------------------------- 1 | $JSON = Get-Content '..\sqlserver-connector-config.json' | Out-String 2 | Invoke-RestMethod http://localhost:8083/connectors/ -Method POST -Body $JSON -ContentType "application/json" 3 | -------------------------------------------------------------------------------- /debezium/on-prem/register-connector.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -i -X POST \ 4 | -H "Accept:application/json" \ 5 | -H "Content-Type:application/json" \ 6 | http://localhost:8083/connectors/ \ 7 | -d @../sqlserver-connector-config.json \ 8 | -w "\n" 9 | -------------------------------------------------------------------------------- /debezium/on-prem/start-debezium.ps1: -------------------------------------------------------------------------------- 1 | docker compose up -------------------------------------------------------------------------------- /debezium/on-prem/start-debezium.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | docker compose up -------------------------------------------------------------------------------- /debezium/on-prem/stop-debezium.ps1: -------------------------------------------------------------------------------- 1 | docker compose down -------------------------------------------------------------------------------- /debezium/on-prem/stop-debezium.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | docker compose down -------------------------------------------------------------------------------- /debezium/sqlserver-connector-config.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wwi", 3 | "config": { 4 | "snapshot.mode": "schema_only", 5 | "connector.class": "io.debezium.connector.sqlserver.SqlServerConnector", 6 | "database.hostname": ".database.windows.net", 7 | "database.port": "1433", 8 | "database.user": "", 9 | "database.password": "", 10 | "database.names": "", 11 | "driver.encrypt": "false", 12 | "driver.trustServerCertificate": "true", 13 | "schema.history.internal.kafka.bootstrap.servers": ".servicebus.windows.net:9093", 14 | "schema.history.internal.kafka.topic": "", 15 | "schema.history.internal.consumer.security.protocol": "SASL_SSL", 16 | "schema.history.internal.consumer.sasl.mechanism": "PLAIN", 17 | "schema.history.internal.consumer.sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"\";", 18 | "schema.history.internal.producer.security.protocol": "SASL_SSL", 19 | "schema.history.internal.producer.sasl.mechanism": "PLAIN", 20 | "schema.history.internal.producer.sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"\";", 21 | "table.include.list": "Sales.Orders,Warehouse.StockItems", 22 | "tombstones.on.delete": "false", 23 | "topic.prefix": "SQLAzure", 24 | "transforms": "Reroute", 25 | "transforms.Reroute.type": "io.debezium.transforms.ByLogicalTableRouter", 26 | "transforms.Reroute.topic.regex": "(.*)", 27 | "transforms.Reroute.topic.replacement": "wwi" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /debezium/utils/clean-eventhubs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Strict mode, fail on any error 4 | set -euo pipefail 5 | 6 | RESOURCE_GROUP="debezium" 7 | EVENTHUB_NAMESPACE="debezium" 8 | 9 | echo "deleting debezium-created eventhubs" 10 | ehs=("debezium_configs" "debezium_offsets" "debezium_statuses" "dbzschemahistory") 11 | for e in "${ehs[@]}"; do 12 | echo "deleting $e..." 13 | az eventhubs eventhub delete \ 14 | --resource-group $RESOURCE_GROUP \ 15 | --namespace-name $EVENTHUB_NAMESPACE \ 16 | --name $e 17 | done 18 | 19 | echo "done" 20 | -------------------------------------------------------------------------------- /documentation/SQL-Server-Connector-Configuration-Value.md: -------------------------------------------------------------------------------- 1 | # Debezium SQL Server Connector Configuration Notes 2 | 3 | In order to properly work with Azure EventHubs and somehow big databases, the Debezium SQL Server Connector needs to be configured with some specific options. Here's the details: 4 | 5 | ## Snapshot Mode 6 | 7 | ```json 8 | "snapshot.mode": "schema_only", 9 | ``` 10 | 11 | As also explained in the README in more detail, this option is needed to make sure data already present in the table, when Debezium is firstly activated, is not sent into the Change Stream. Debezium default behavior is to read all the existing data, send it to the Change Stream, and the start to monitor the table for changes: 12 | 13 | [Debezium Snapshots](https://debezium.io/documentation/reference/1.6/connectors/sqlserver.html#sqlserver-snapshots) 14 | 15 | For this sample we don't need to have all existing data into the Change Stream, as we only need the changes from now on. So the option is set to `initial_schema_only`. 16 | 17 | ## Decimal Handling 18 | 19 | ``` json 20 | "decimal.handling.mode": "string", 21 | ``` 22 | 23 | By default decimal values are sent as Base64 encoded byte arrays. It is much easier to just have it as a string instead, especially if you plan to use .NET to handle the Change Stream. 24 | 25 | ## Whitelist Tables 26 | 27 | ```json 28 | "table.include.list": "Sales.Orders,Warehouse.StockItems", 29 | ``` 30 | 31 | If you don't specify which tables should be monitored by Debezium, it will try to analyze *all* tables, not only those who have Change Data Capture enabled. To avoid errors and excessive locking, manually specify the tables you want Debezium to monitor. 32 | 33 | ## Topic Routing 34 | 35 | ```json 36 | "transforms": "Reroute", 37 | "transforms.Reroute.type": "io.debezium.transforms.ByLogicalTableRouter", 38 | "transforms.Reroute.topic.regex": "(.*)", 39 | "transforms.Reroute.topic.replacement": "wwi", 40 | ``` 41 | 42 | This configuration section is really useful especially with EventHubs. By default Debezium will create one Topic for each table. With Apache Kafka this is pretty standard, as you can use wildcards to data coming from different topics. EventHubs doesn't have that option, so having all the event routed to one well-known topic is more convenient. The configuration shown above will take data from *any* table (`transforms.Reroute.topic.regex`) and will send it to just one topic (`transforms.Reroute.topic.replacement`). As you can guess from the support of Regular Expression, you can do quite complex re-routings. More info here: 43 | 44 | [Topic Routing](https://debezium.io/documentation/reference/1.6/configuration/topic-routing.html) 45 | 46 | ## Tombstones 47 | 48 | ```json 49 | "tombstones.on.delete": false, 50 | ``` 51 | 52 | Apache Kafka uses tombstones, completely empty messages, to know when it can delete data during log compaction. (More info on this process here: [Apache Kafka Log Compaction](http://cloudurable.com/blog/kafka-architecture-log-compaction/index.html)). Azure Event Hubs doesn't support Log Compaction at the moment, so we don't need tombstones. 53 | 54 | ## Database History 55 | 56 | ```json 57 | "database.history":"io.debezium.relational.history.MemoryDatabaseHistory" 58 | ``` 59 | 60 | This is a trick configuration that *should not* be used, but it is unfortunately needed due to limited compatibility support that Azure Event Hubs offers for Kafka API. GitHub Issue here for those interested in the details: 61 | 62 | [Error "The broker does not support DESCRIBE_CONFIGS](https://github.com/Azure/azure-event-hubs-for-kafka/issues/61) 63 | 64 | In brief what this is what happens: Debezium will try to create a new topic to store all DDL changes done to the database, in order to keep schema history. Unfortunately it won'y be able to do so, due to the aforementioned issus. The specified configuration tells Debezium to store the schema changes in memory, without using a Kafka Topic. This is tricky since if you do *a lot* of schema changes than it may happen that the machine running Debezium could go out of memory. But schema changes are usually not that common, so you should be pretty safe, until Azure Event Hubs is 100% Kafka compatible. 65 | -------------------------------------------------------------------------------- /documentation/sql-server-change-stream.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-sql-db-change-stream-debezium/1e3abbdaab5ae2790f1fed899fa5a3273d9ed3f4/documentation/sql-server-change-stream.gif -------------------------------------------------------------------------------- /sql/README.md: -------------------------------------------------------------------------------- 1 | # SQL Scripts 2 | 3 | ## azure-sql 4 | 5 | Script to be used if you want to test this sample using Azure SQL database 6 | 7 | ## sql-server 8 | 9 | Script to be used if you want to test this sample using Azure SQL MI, SQL Server on-prem or SQL Server running in a VM. -------------------------------------------------------------------------------- /sql/azure-sql/00-setup-database-user.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-sql-db-change-stream-debezium/1e3abbdaab5ae2790f1fed899fa5a3273d9ed3f4/sql/azure-sql/00-setup-database-user.sql -------------------------------------------------------------------------------- /sql/azure-sql/01-enable-cdc.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Make sure you are connected to the database you want to use in this sample. 3 | For example the database "WideWorldImporters" 4 | */ 5 | 6 | -- Enable CDC on database 7 | EXEC sys.sp_cdc_enable_db 8 | GO 9 | 10 | -- Enable CDC on selected tables 11 | EXEC sys.sp_cdc_enable_table N'Sales', N'Orders', @role_name=null, @supports_net_changes=0 12 | EXEC sys.sp_cdc_enable_table N'Warehouse', N'StockItems', @role_name=null, @supports_net_changes=0 13 | GO 14 | 15 | -- Verify the CDC has been enabled for the selected tables 16 | EXEC sys.sp_cdc_help_change_data_capture 17 | GO -------------------------------------------------------------------------------- /sql/azure-sql/02-create-new-sales-order.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Make sure you are connected to the database you want to use in this sample. 3 | For example the database "WideWorldImporters" 4 | */ 5 | 6 | -- Issue a new random order 7 | 8 | DECLARE @personId INT 9 | SELECT TOP(1) @personId = PersonID FROM [Application].People WHERE IsEmployee <> 0 ORDER BY NEWID(); 10 | 11 | DECLARE @orders AS [Website].[OrderList] 12 | INSERT INTO @orders 13 | SELECT TOP(1) 1 AS OrderReference, c.CustomerID, c.PrimaryContactPersonID AS ContactPersonID, CAST(DATEADD(day, 1, SYSDATETIME()) AS date) AS ExpectedDeliveryDate, CAST(FLOOR(RAND() * 10000) + 1 AS nvarchar(20)) AS CustomerPurchaseOrderNumber, CAST(0 AS bit) AS IsUndersupplyBackordered, N'Auto-generated' AS Comments, c.DeliveryAddressLine1 + N', ' + c.DeliveryAddressLine2 AS DeliveryInstructions FROM Sales.Customers AS c ORDER BY NEWID(); 14 | 15 | DECLARE @r INT = CAST(RAND() * 100 AS INT) 16 | DECLARE @orderlinelist AS [Website].[OrderLineList] 17 | INSERT INTO @orderlinelist 18 | SELECT TOP(7) 1 AS OrderReference, si.StockItemID, si.StockItemName AS [Description], FLOOR(RAND() * 10) + 1 AS Quantity FROM Warehouse.StockItems AS si WHERE IsChillerStock = 0 ORDER BY NEWID() 19 | INSERT INTO @orderlinelist 20 | SELECT TOP(1) 1 AS OrderReference, si.StockItemID, si.StockItemName AS [Description], FLOOR(RAND() * 10) + 1 AS Quantity FROM Warehouse.StockItems AS si WHERE IsChillerStock <> 0 AND @r < 4 ORDER BY NEWID() 21 | 22 | EXEC Website.InsertCustomerOrders @orders, @orderlinelist, @personId, @personId 23 | GO 24 | 25 | -- View inserted order header 26 | SELECT TOP (1) * FROM Sales.Orders ORDER BY OrderID DESC -------------------------------------------------------------------------------- /sql/azure-sql/03-modify-warehouse-stock.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Make sure you are connected to the database you want to use in this sample. 3 | For example the database "WideWorldImporters" 4 | */ 5 | 6 | -- Update some stock items 7 | WITH cte AS 8 | ( 9 | SELECT TOP (1) * FROM [Warehouse].[StockItems] ORDER BY NEWID() DESC 10 | ) 11 | UPDATE 12 | cte 13 | SET 14 | InternalComments = 'Just some random thoughts...', 15 | Barcode = '123ABC123' 16 | GO 17 | 18 | WAITFOR DELAY '00:00:05' 19 | GO 20 | 21 | WITH cte AS 22 | ( 23 | SELECT TOP (1) * FROM [Warehouse].[StockItems] ORDER BY NEWID() DESC 24 | ) 25 | UPDATE 26 | cte 27 | SET 28 | MarketingComments = 'Some important marketing message.', 29 | Barcode = '456QAZ567' 30 | GO 31 | 32 | WAITFOR DELAY '00:00:05' 33 | GO 34 | 35 | -- Insert a new dummy stock item 36 | 37 | INSERT INTO [Warehouse].[StockItems] 38 | ([StockItemID], [StockItemName], [SupplierID], [ColorID], [UnitPackageID], [OuterPackageID], [Brand], [Size], [LeadTimeDays], [QuantityPerOuter], [IsChillerStock], [Barcode], [TaxRate], [UnitPrice], [RecommendedRetailPrice], [TypicalWeightPerUnit], [MarketingComments], [InternalComments], [Photo], [CustomFields], [LastEditedBy]) 39 | VALUES 40 | (999, 'Dummy Stock', 1, NULL, 1, 6, NULL, NULL, 1, 1, 0, NULL, 0.0, 0.0, 0.0, 0.0, NULL, NULL, NULL, '{}', 1) 41 | GO 42 | 43 | SELECT * FROM [Warehouse].[StockItems] WHERE StockItemID = 999 44 | GO 45 | 46 | WAITFOR DELAY '00:00:05' 47 | GO 48 | 49 | -- Delete the inserted dummy item 50 | DELETE FROM [Warehouse].[StockItems] WHERE StockItemID = 999 51 | GO 52 | -------------------------------------------------------------------------------- /sql/azure-sql/04-disable-cdc.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Make sure you are connected to the database you want to use in this sample. 3 | For example the database "WideWorldImporters" 4 | */ 5 | 6 | -- Disable CDC 7 | exec sys.sp_cdc_disable_db 8 | go 9 | -------------------------------------------------------------------------------- /sql/sql-server/00-setup-database-user.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-sql-db-change-stream-debezium/1e3abbdaab5ae2790f1fed899fa5a3273d9ed3f4/sql/sql-server/00-setup-database-user.sql -------------------------------------------------------------------------------- /sql/sql-server/01-enable-cdc.sql: -------------------------------------------------------------------------------- 1 | USE [WideWorldImporters] 2 | GO 3 | 4 | /* 5 | Enable CDC on database (make sure SQL Server Agent is running!) 6 | */ 7 | EXEC sys.sp_cdc_enable_db 8 | GO 9 | 10 | /* 11 | Enable CDC on selected tables 12 | */ 13 | EXEC sys.sp_cdc_enable_table N'Sales', N'Orders', @role_name=null, @supports_net_changes=0 14 | EXEC sys.sp_cdc_enable_table N'Warehouse', N'StockItems', @role_name=null, @supports_net_changes=0 15 | GO 16 | 17 | /* 18 | Verify the CDC has been enabled for the selected tables 19 | */ 20 | EXEC sys.sp_cdc_help_change_data_capture 21 | GO -------------------------------------------------------------------------------- /sql/sql-server/02-create-new-sales-order.sql: -------------------------------------------------------------------------------- 1 | USE WideWorldImporters 2 | GO 3 | 4 | -- Issue a new random order 5 | 6 | DECLARE @personId INT 7 | SELECT TOP(1) @personId = PersonID FROM [Application].People WHERE IsEmployee <> 0 ORDER BY NEWID(); 8 | 9 | DECLARE @orders AS [Website].[OrderList] 10 | INSERT INTO @orders 11 | SELECT TOP(1) 1 AS OrderReference, c.CustomerID, c.PrimaryContactPersonID AS ContactPersonID, CAST(DATEADD(day, 1, SYSDATETIME()) AS date) AS ExpectedDeliveryDate, CAST(FLOOR(RAND() * 10000) + 1 AS nvarchar(20)) AS CustomerPurchaseOrderNumber, CAST(0 AS bit) AS IsUndersupplyBackordered, N'Auto-generated' AS Comments, c.DeliveryAddressLine1 + N', ' + c.DeliveryAddressLine2 AS DeliveryInstructions FROM Sales.Customers AS c ORDER BY NEWID(); 12 | 13 | DECLARE @r INT = CAST(RAND() * 100 AS INT) 14 | DECLARE @orderlinelist AS [Website].[OrderLineList] 15 | INSERT INTO @orderlinelist 16 | SELECT TOP(7) 1 AS OrderReference, si.StockItemID, si.StockItemName AS [Description], FLOOR(RAND() * 10) + 1 AS Quantity FROM Warehouse.StockItems AS si WHERE IsChillerStock = 0 ORDER BY NEWID() 17 | INSERT INTO @orderlinelist 18 | SELECT TOP(1) 1 AS OrderReference, si.StockItemID, si.StockItemName AS [Description], FLOOR(RAND() * 10) + 1 AS Quantity FROM Warehouse.StockItems AS si WHERE IsChillerStock <> 0 AND @r < 4 ORDER BY NEWID() 19 | 20 | EXEC Website.InsertCustomerOrders @orders, @orderlinelist, @personId, @personId 21 | GO 22 | 23 | -- View inserted order header 24 | SELECT TOP (1) * FROM Sales.Orders ORDER BY OrderID DESC -------------------------------------------------------------------------------- /sql/sql-server/03-modify-warehouse-stock.sql: -------------------------------------------------------------------------------- 1 | USE WideWorldImporters 2 | GO 3 | 4 | /* 5 | Update some stock items 6 | */ 7 | WITH cte AS 8 | ( 9 | SELECT TOP (1) * FROM [Warehouse].[StockItems] ORDER BY NEWID() DESC 10 | ) 11 | UPDATE 12 | cte 13 | SET 14 | InternalComments = 'Just some random thoughts...', 15 | Barcode = '123ABC123' 16 | GO 17 | 18 | WAITFOR DELAY '00:00:05' 19 | GO 20 | 21 | WITH cte AS 22 | ( 23 | SELECT TOP (1) * FROM [Warehouse].[StockItems] ORDER BY NEWID() DESC 24 | ) 25 | UPDATE 26 | cte 27 | SET 28 | MarketingComments = 'Some important marketing message.', 29 | Barcode = '456QAZ567' 30 | GO 31 | 32 | WAITFOR DELAY '00:00:05' 33 | GO 34 | 35 | /* 36 | Insert a new dummy stock item 37 | */ 38 | 39 | INSERT INTO [Warehouse].[StockItems] 40 | ([StockItemID], [StockItemName], [SupplierID], [ColorID], [UnitPackageID], [OuterPackageID], [Brand], [Size], [LeadTimeDays], [QuantityPerOuter], [IsChillerStock], [Barcode], [TaxRate], [UnitPrice], [RecommendedRetailPrice], [TypicalWeightPerUnit], [MarketingComments], [InternalComments], [Photo], [CustomFields], [LastEditedBy]) 41 | VALUES 42 | (999, 'Dummy Stock', 1, NULL, 1, 6, NULL, NULL, 1, 1, 0, NULL, 0.0, 0.0, 0.0, 0.0, NULL, NULL, NULL, '{}', 1) 43 | GO 44 | 45 | SELECT * FROM [Warehouse].[StockItems] WHERE StockItemID = 999 46 | GO 47 | 48 | WAITFOR DELAY '00:00:05' 49 | GO 50 | 51 | /* 52 | Delete the inserted dummy item 53 | */ 54 | DELETE FROM [Warehouse].[StockItems] WHERE StockItemID = 999 55 | GO 56 | -------------------------------------------------------------------------------- /sql/sql-server/04-disable-cdc.sql: -------------------------------------------------------------------------------- 1 | use WideWorldImporters 2 | go 3 | 4 | exec sys.sp_cdc_disable_db 5 | go 6 | --------------------------------------------------------------------------------