├── .gitattributes ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE.md ├── README.md ├── images └── materialized-view-architecture.png ├── materialized-view-processor ├── .gitignore ├── Entities.cs ├── Function.cs ├── ViewProcessor.cs ├── host.json └── materialized-view-processor.csproj ├── scripts └── deploy.sh └── sensor-data-producer ├── .dockerignore ├── .vscode ├── launch.json └── tasks.json ├── App.config.template ├── Dockerfile ├── Program.cs ├── build.bat └── sensor-data-producer.csproj /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text=auto eol=lf 2 | *.cs text=auto eol=lf 3 | *.md text=auto eol=lf 4 | *.json text=auto eol=lf 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015/2017 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # Visual Studio 2017 auto generated files 33 | Generated\ Files/ 34 | 35 | # MSTest test Results 36 | [Tt]est[Rr]esult*/ 37 | [Bb]uild[Ll]og.* 38 | 39 | # NUNIT 40 | *.VisualState.xml 41 | TestResult.xml 42 | 43 | # Build Results of an ATL Project 44 | [Dd]ebugPS/ 45 | [Rr]eleasePS/ 46 | dlldata.c 47 | 48 | # Benchmark Results 49 | BenchmarkDotNet.Artifacts/ 50 | 51 | # .NET Core 52 | project.lock.json 53 | project.fragment.lock.json 54 | artifacts/ 55 | **/Properties/launchSettings.json 56 | 57 | # StyleCop 58 | StyleCopReport.xml 59 | 60 | # Files built by Visual Studio 61 | *_i.c 62 | *_p.c 63 | *_i.h 64 | *.ilk 65 | *.meta 66 | *.obj 67 | *.iobj 68 | *.pch 69 | *.pdb 70 | *.ipdb 71 | *.pgc 72 | *.pgd 73 | *.rsp 74 | *.sbr 75 | *.tlb 76 | *.tli 77 | *.tlh 78 | *.tmp 79 | *.tmp_proj 80 | *.log 81 | *.vspscc 82 | *.vssscc 83 | .builds 84 | *.pidb 85 | *.svclog 86 | *.scc 87 | 88 | # Chutzpah Test files 89 | _Chutzpah* 90 | 91 | # Visual C++ cache files 92 | ipch/ 93 | *.aps 94 | *.ncb 95 | *.opendb 96 | *.opensdf 97 | *.sdf 98 | *.cachefile 99 | *.VC.db 100 | *.VC.VC.opendb 101 | 102 | # Visual Studio profiler 103 | *.psess 104 | *.vsp 105 | *.vspx 106 | *.sap 107 | 108 | # Visual Studio Trace Files 109 | *.e2e 110 | 111 | # TFS 2012 Local Workspace 112 | $tf/ 113 | 114 | # Guidance Automation Toolkit 115 | *.gpState 116 | 117 | # ReSharper is a .NET coding add-in 118 | _ReSharper*/ 119 | *.[Rr]e[Ss]harper 120 | *.DotSettings.user 121 | 122 | # JustCode is a .NET coding add-in 123 | .JustCode 124 | 125 | # TeamCity is a build add-in 126 | _TeamCity* 127 | 128 | # DotCover is a Code Coverage Tool 129 | *.dotCover 130 | 131 | # AxoCover is a Code Coverage Tool 132 | .axoCover/* 133 | !.axoCover/settings.json 134 | 135 | # Visual Studio code coverage results 136 | *.coverage 137 | *.coveragexml 138 | 139 | # NCrunch 140 | _NCrunch_* 141 | .*crunch*.local.xml 142 | nCrunchTemp_* 143 | 144 | # MightyMoose 145 | *.mm.* 146 | AutoTest.Net/ 147 | 148 | # Web workbench (sass) 149 | .sass-cache/ 150 | 151 | # Installshield output folder 152 | [Ee]xpress/ 153 | 154 | # DocProject is a documentation generator add-in 155 | DocProject/buildhelp/ 156 | DocProject/Help/*.HxT 157 | DocProject/Help/*.HxC 158 | DocProject/Help/*.hhc 159 | DocProject/Help/*.hhk 160 | DocProject/Help/*.hhp 161 | DocProject/Help/Html2 162 | DocProject/Help/html 163 | 164 | # Click-Once directory 165 | publish/ 166 | 167 | # Publish Web Output 168 | *.[Pp]ublish.xml 169 | *.azurePubxml 170 | # Note: Comment the next line if you want to checkin your web deploy settings, 171 | # but database connection strings (with potential passwords) will be unencrypted 172 | *.pubxml 173 | *.publishproj 174 | 175 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 176 | # checkin your Azure Web App publish settings, but sensitive information contained 177 | # in these scripts will be unencrypted 178 | PublishScripts/ 179 | 180 | # NuGet Packages 181 | *.nupkg 182 | # The packages folder can be ignored because of Package Restore 183 | **/[Pp]ackages/* 184 | # except build/, which is used as an MSBuild target. 185 | !**/[Pp]ackages/build/ 186 | # Uncomment if necessary however generally it will be regenerated when needed 187 | #!**/[Pp]ackages/repositories.config 188 | # NuGet v3's project.json files produces more ignorable files 189 | *.nuget.props 190 | *.nuget.targets 191 | 192 | # Microsoft Azure Build Output 193 | csx/ 194 | *.build.csdef 195 | 196 | # Microsoft Azure Emulator 197 | ecf/ 198 | rcf/ 199 | 200 | # Windows Store app package directories and files 201 | AppPackages/ 202 | BundleArtifacts/ 203 | Package.StoreAssociation.xml 204 | _pkginfo.txt 205 | *.appx 206 | 207 | # Visual Studio cache files 208 | # files ending in .cache can be ignored 209 | *.[Cc]ache 210 | # but keep track of directories ending in .cache 211 | !*.[Cc]ache/ 212 | 213 | # Others 214 | ClientBin/ 215 | ~$* 216 | *~ 217 | *.dbmdl 218 | *.dbproj.schemaview 219 | *.jfm 220 | *.pfx 221 | *.publishsettings 222 | orleans.codegen.cs 223 | 224 | # Including strong name files can present a security risk 225 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 226 | #*.snk 227 | 228 | # Since there are multiple workflows, uncomment next line to ignore bower_components 229 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 230 | #bower_components/ 231 | 232 | # RIA/Silverlight projects 233 | Generated_Code/ 234 | 235 | # Backup & report files from converting an old project file 236 | # to a newer Visual Studio version. Backup files are not needed, 237 | # because we have git ;-) 238 | _UpgradeReport_Files/ 239 | Backup*/ 240 | UpgradeLog*.XML 241 | UpgradeLog*.htm 242 | ServiceFabricBackup/ 243 | *.rptproj.bak 244 | 245 | # SQL Server files 246 | *.mdf 247 | *.ldf 248 | *.ndf 249 | 250 | # Business Intelligence projects 251 | *.rdl.data 252 | *.bim.layout 253 | *.bim_*.settings 254 | *.rptproj.rsuser 255 | 256 | # Microsoft Fakes 257 | FakesAssemblies/ 258 | 259 | # GhostDoc plugin setting file 260 | *.GhostDoc.xml 261 | 262 | # Node.js Tools for Visual Studio 263 | .ntvs_analysis.dat 264 | node_modules/ 265 | 266 | # Visual Studio 6 build log 267 | *.plg 268 | 269 | # Visual Studio 6 workspace options file 270 | *.opt 271 | 272 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 273 | *.vbw 274 | 275 | # Visual Studio LightSwitch build output 276 | **/*.HTMLClient/GeneratedArtifacts 277 | **/*.DesktopClient/GeneratedArtifacts 278 | **/*.DesktopClient/ModelManifest.xml 279 | **/*.Server/GeneratedArtifacts 280 | **/*.Server/ModelManifest.xml 281 | _Pvt_Extensions 282 | 283 | # Paket dependency manager 284 | .paket/paket.exe 285 | paket-files/ 286 | 287 | # FAKE - F# Make 288 | .fake/ 289 | 290 | # JetBrains Rider 291 | .idea/ 292 | *.sln.iml 293 | 294 | # CodeRush 295 | .cr/ 296 | 297 | # Python Tools for Visual Studio (PTVS) 298 | __pycache__/ 299 | *.pyc 300 | 301 | # Cake - Uncomment if you are using it 302 | # tools/** 303 | # !tools/packages.config 304 | 305 | # Tabs Studio 306 | *.tss 307 | 308 | # Telerik's JustMock configuration file 309 | *.jmconfig 310 | 311 | # BizTalk build output 312 | *.btp.cs 313 | *.btm.cs 314 | *.odx.cs 315 | *.xsd.cs 316 | 317 | # OpenCover UI analysis results 318 | OpenCover/ 319 | 320 | # Azure Stream Analytics local run output 321 | ASALocalRun/ 322 | 323 | # MSBuild Binary and Structured Log 324 | *.binlog 325 | 326 | # NVidia Nsight GPU debugger configuration file 327 | *.nvuser 328 | 329 | # MFractors (Xamarin productivity tool) working folder 330 | .mfractor/ 331 | 332 | # Additional Items 333 | .vscode/ 334 | *.zip 335 | *.pptx 336 | mvp2/ 337 | logs/ -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to [project-title] 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 6 | 7 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase master -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! 77 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | languages: 4 | - csharp 5 | products: 6 | - azure 7 | description: "Shows how materialized view can be kept updated in near-real time using a serverless approach with Azure Functions, Cosmos DB and Cosmos DB Change Feed." 8 | urlFragment: real-time-view-cosomos-azure-functions 9 | --- 10 | 11 | # Near-Real Time Updated Materialized View With Cosmos DB and Azure Functions 12 | 13 | This sample shows how materialized view can be kept updated in near-real time using a completely serverless approach with 14 | 15 | - Azure Function 16 | - Cosmos DB 17 | - Cosmos DB Change Feed 18 | 19 | The high-level architecture is the following one: 20 | 21 | ![High Level Materialized View Architecture](./images/materialized-view-architecture.png) 22 | 23 | Device simulator writes JSON data to Cosmos DB into `raw` collection. Such data is exposed by Cosmos DB Change Feed and consumed by an Azure Function (via Change Feed Processor), that get the JSON document and uses it to create or updated the related materialized views, stored in the `view` collection. 24 | 25 | - [Change feed in Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db/change-feed) 26 | - [Change feed processor in Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db/change-feed-processor) 27 | 28 | A more detailed discussion on the architecture and solution can be found here: 29 | 30 | [Real-Time Materialized Views with Cosmos DB](https://medium.com/@mauridb/real-time-materialized-views-with-cosmos-db-90ecea84f650) 31 | 32 | The sample simulates one or more IoT Devices whose generated data needs to be sent, received and processed in near-real time. In this context, "processed" means: 33 | 34 | - Provide, for each device, the sum of the sent `value` data and also the last sent value 35 | - Provide one view with all devices and the last data sent by each one 36 | 37 | ## Sample data 38 | 39 | The simulated IoT devices will send this sample data: 40 | 41 | { 42 | "deviceId": "036", 43 | "value": 164.91290226807487, 44 | "timestamp": "2019-03-22T19:46:20.8633068Z" 45 | } 46 | 47 | ## Processed data 48 | 49 | The resulting processed data for each device will look like the following document: 50 | 51 | { 52 | "id": "030", 53 | "aggregationSum": 3519.8782286699293, 54 | "lastValue": 155.41897977488998, 55 | "type": "device", 56 | "deviceId": "030", 57 | "lastUpdate": "2019-03-22T19:50:17Z" 58 | } 59 | 60 | The document contain aggregated data for the device specified in `id`, as long as the aggregated value in `aggregationSum` and the last sent value in the `lastValue` field. 61 | 62 | The `global` materialized view is where the last value *for each device* is stored: 63 | 64 | { 65 | "id": "global", 66 | "deviceId": "global", 67 | "type": "global", 68 | "deviceSummary": { 69 | "035": 104.3423159533843, 70 | "016": 129.1018793494915, 71 | ... 72 | "023": 177.62450146378228, 73 | "033": 178.97744880941576 74 | }, 75 | } 76 | 77 | Values are updated in near-real time by using the Change Feed feature provided by Cosmos DB. The sample is using processing data coming from the Change Feed every second, but it can easily changed to a much lower value if you need more "real time" updates. 78 | 79 | - [Trigger Azure Functions from Azure Cosmos DB](https://docs.microsoft.com/en-us/azure/cosmos-db/change-feed-functions) 80 | - [Azure Function Cosmos DB Triggers](https://docs.microsoft.com/en-us/azure/azure-functions/functions-bindings-cosmosdb-v2#trigger---c-attributes) 81 | 82 | ## Implementation Notes 83 | 84 | Data is partitioned by `deviceId` in both the `raw` and `view` collection to make sure order is preserved within data sent by a single device. Collections do not use indexing as only point-lookups are done and since in this sample Cosmos DB is used a key-value store, switching off indexing allows to spare RUs. 85 | 86 | ## Prerequisites 87 | 88 | If you don't have an Azure subscription, create a [free account](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio) before you begin. 89 | 90 | In addition: 91 | 92 | - [Visual Studio 2017](https://visualstudio.microsoft.com/downloads/) or [Visual Studio Code](https://code.visualstudio.com/) 93 | - [.NET Core SDK](https://dotnet.microsoft.com/download) 94 | - [Git](https://www.git-scm.com/downloads) 95 | - [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) 96 | - [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/) 97 | 98 | ## Getting Started 99 | 100 | Make sure you have WSL (Windows System For Linux) installed and have AZ CLI version > 2.0.50. Before running any script also make sure you are authenticated on AZ CLI using 101 | 102 | ```bash 103 | az login 104 | ``` 105 | 106 | and have selected the Azure Subscription you want to use for the tests: 107 | 108 | ```bash 109 | az account list --output table 110 | az account set --subscription "" 111 | ``` 112 | 113 | ## Clone the sample project 114 | 115 | Clone the repository and open the code-samples directory from your command line tool. 116 | 117 | ```bash 118 | git clone https://github.com/Azure-Samples/cosmosdb-materialized-views 119 | cd cosmosdb-materialized-views 120 | ``` 121 | 122 | ## Create Azure Resources 123 | 124 | To create and configure the Azure Resources needed for the project, you just have to run the `deploy.sh` script in the `script` folder. 125 | 126 | Script has been tested on Mac OSX and Ubuntu Bash. 127 | 128 | ./script/deploy 129 | 130 | The following resources will be created: 131 | 132 | - Resource Group 133 | - Azure Storage 134 | - Azure Function (using Consumption Plan) 135 | - Application Insight 136 | - Cosmos DB with 3 Collections (`raw` and `view` with 1000 RU each, `leases` with 400 RU) 137 | 138 | By default all deployed resources will have the `mvsample` prefix and the location will be `eastus`. If you don't have any specific naming requirements, by default the generated `ROOT_NAME` will be *uniquified* by postfixing random numbers to make sure you don't have any name collision with someone else trying the sample at the same time. 139 | 140 | If needed you can change the following defauly settings 141 | 142 | export ROOT_NAME='mvsample${UNIQUIFIER}' 143 | export LOCATION='eastus' 144 | 145 | in `./script/deploy` to make sure they work for you. 146 | 147 | ## Run the Producer application 148 | 149 | The producer application will generate sample sensor data as described before. The application takes the device ids to generate as parameter: 150 | 151 | cd sensor-data-producer 152 | dotnet run 1 153 | 154 | The above sample will generate random data for device id 001. If you want to generate more data just specify how many sensor you need to be simulated: 155 | 156 | dotnet run 10 157 | 158 | will generate data for 10 sensors, from 001 to 010. If you want to generate more workload and you're planning to distribute the work on different clients (using Azure Container Instances or Azure Kubernetes Service for example), you'll find useful the ability to specify which device id range you want to be generated by each running application. For example 159 | 160 | dotnet run 15-25 161 | 162 | will generate data with Device Ids starting from 015 up to 025. 163 | 164 | ### Docker Image 165 | 166 | If you want to build and run the simulator using Docker, just run `build.bat` to build the docker image. The run it using 167 | 168 | docker run -it iot-simulator 1-10 169 | 170 | to simulate device from 001 to 010, for example. 171 | 172 | ## Check results 173 | 174 | Once the producer is started you can see the result by using Azure Portal or Azure Storage Explorer to look for document create int the `view` collection of the created Cosmos DB database. 175 | 176 | You can also take a look at the Application Insight Live Metric Streams to see in real time function processing incoming data from the Change Feed 177 | 178 | ## Additional Reference 179 | 180 | - [Azure Cosmos DB + Functions Cookbook — Connection modes](https://medium.com/microsoftazure/azure-cosmos-db-functions-cookbook-connection-modes-ecf405a750d9) 181 | -------------------------------------------------------------------------------- /images/materialized-view-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/cosmosdb-materialized-views/ef81bbf6c0012b75cb95c5e9ad773a7b973405c8/images/materialized-view-architecture.png -------------------------------------------------------------------------------- /materialized-view-processor/.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /materialized-view-processor/Entities.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Newtonsoft.Json; 3 | using Newtonsoft.Json.Linq; 4 | using System.Threading.Tasks; 5 | using System.Linq; 6 | using System.Collections.Generic; 7 | using Microsoft.Azure.Documents; 8 | using Microsoft.Azure.WebJobs; 9 | using Microsoft.Azure.WebJobs.Host; 10 | using Microsoft.Extensions.Logging; 11 | using Microsoft.Azure.Documents.Client; 12 | using System.Net; 13 | 14 | namespace Azure.Samples.Entities 15 | { 16 | public class Device 17 | { 18 | public string DeviceId; 19 | public double Value; 20 | public string TimeStamp; 21 | 22 | public static Device FromDocument(Document document) 23 | { 24 | var result = new Device() 25 | { 26 | DeviceId = document.GetPropertyValue("deviceId"), 27 | Value = document.GetPropertyValue("value"), 28 | TimeStamp = document.GetPropertyValue("timestamp").ToString("yyyy-MM-ddTHH:mm:ssK") 29 | }; 30 | 31 | return result; 32 | } 33 | } 34 | 35 | public class DeviceMaterializedView 36 | { 37 | [JsonProperty("id")] 38 | public string Name; 39 | 40 | [JsonProperty("aggregationSum")] 41 | public double AggregationSum; 42 | 43 | [JsonProperty("lastValue")] 44 | public double LastValue; 45 | 46 | [JsonProperty("type")] 47 | public string Type; 48 | 49 | [JsonProperty("deviceId")] 50 | public string DeviceId; 51 | 52 | [JsonProperty("lastUpdate")] 53 | public string TimeStamp; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /materialized-view-processor/Function.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Newtonsoft.Json; 3 | using Newtonsoft.Json.Linq; 4 | using System.Threading.Tasks; 5 | using System.Linq; 6 | using System.Collections.Generic; 7 | using Microsoft.Azure.Documents; 8 | using Microsoft.Azure.WebJobs; 9 | using Microsoft.Azure.WebJobs.Host; 10 | using Microsoft.Extensions.Logging; 11 | using Microsoft.Azure.Documents.Client; 12 | using System.Net; 13 | using Azure.Samples.Entities; 14 | using Azure.Samples.Processor; 15 | 16 | namespace Azure.Samples 17 | { 18 | public static class Function 19 | { 20 | [FunctionName("MaterializedViewProcessor")] 21 | public static async Task Run( 22 | [CosmosDBTrigger( 23 | databaseName: "%DatabaseName%", 24 | collectionName: "%RawCollectionName%", 25 | ConnectionStringSetting = "ConnectionString", 26 | LeaseCollectionName = "leases", 27 | FeedPollDelay=1000 28 | )]IReadOnlyList input, 29 | [CosmosDB( 30 | databaseName: "%DatabaseName%", 31 | collectionName: "%ViewCollectionName%", 32 | ConnectionStringSetting = "ConnectionString" 33 | )]DocumentClient client, 34 | ILogger log 35 | ) 36 | { 37 | if (input != null && input.Count > 0) 38 | { 39 | var p = new ViewProcessor(client, log); 40 | 41 | log.LogInformation($"Processing {input.Count} events"); 42 | 43 | foreach(var d in input) 44 | { 45 | var device = Device.FromDocument(d); 46 | 47 | var tasks = new List(); 48 | 49 | tasks.Add(p.UpdateDeviceMaterializedView(device)); 50 | tasks.Add(p.UpdateGlobalMaterializedView(device)); 51 | 52 | await Task.WhenAll(tasks); 53 | } 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /materialized-view-processor/ViewProcessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Newtonsoft.Json; 3 | using Newtonsoft.Json.Linq; 4 | using System.Threading.Tasks; 5 | using System.Linq; 6 | using System.Collections.Generic; 7 | using Microsoft.Azure.Documents; 8 | using Microsoft.Azure.WebJobs; 9 | using Microsoft.Azure.WebJobs.Host; 10 | using Microsoft.Extensions.Logging; 11 | using Microsoft.Azure.Documents.Client; 12 | using System.Net; 13 | using Azure.Samples.Entities; 14 | 15 | namespace Azure.Samples.Processor 16 | { 17 | public class ViewProcessor 18 | { 19 | private DocumentClient _client; 20 | private Uri _collectionUri; 21 | private ILogger _log; 22 | 23 | private string _databaseName = Environment.GetEnvironmentVariable("DatabaseName"); 24 | private string _collectionName = Environment.GetEnvironmentVariable("ViewCollectionName"); 25 | 26 | 27 | public ViewProcessor(DocumentClient client, ILogger log) 28 | { 29 | _log = log; 30 | _client = client; 31 | _collectionUri = UriFactory.CreateDocumentCollectionUri(_databaseName, _collectionName); 32 | } 33 | 34 | public async Task UpdateGlobalMaterializedView(Device device) 35 | { 36 | _log.LogInformation("Updating global materialized view"); 37 | 38 | Document viewAll = null; 39 | var optionsAll = new RequestOptions() { PartitionKey = new PartitionKey("global") }; 40 | 41 | int attempts = 0; 42 | 43 | while (attempts < 10) 44 | { 45 | try 46 | { 47 | var uriAll = UriFactory.CreateDocumentUri(_databaseName, _collectionName, "global"); 48 | 49 | _log.LogInformation($"Materialized view: {uriAll.ToString()}"); 50 | 51 | viewAll = await _client.ReadDocumentAsync(uriAll, optionsAll); 52 | } 53 | catch (DocumentClientException ex) 54 | { 55 | if (ex.StatusCode != HttpStatusCode.NotFound) 56 | throw ex; 57 | } 58 | 59 | if (viewAll == null) 60 | { 61 | viewAll = new Document(); 62 | viewAll.SetPropertyValue("id", "global"); 63 | viewAll.SetPropertyValue("deviceId", "global"); 64 | viewAll.SetPropertyValue("type", "global"); 65 | viewAll.SetPropertyValue("id", "global"); 66 | viewAll.SetPropertyValue("deviceSummary", new JObject()); 67 | } 68 | 69 | var ds = viewAll.GetPropertyValue("deviceSummary"); 70 | ds[device.DeviceId] = device.Value; 71 | viewAll.SetPropertyValue("deviceSummary", ds); 72 | viewAll.SetPropertyValue("deviceLastUpdate", device.TimeStamp); 73 | viewAll.SetPropertyValue("lastUpdate", DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssK")); 74 | 75 | AccessCondition acAll = new AccessCondition() { 76 | Type = AccessConditionType.IfMatch, 77 | Condition = viewAll.ETag 78 | }; 79 | optionsAll.AccessCondition = acAll; 80 | 81 | try 82 | { 83 | await UpsertDocument(viewAll, optionsAll); 84 | return; 85 | } 86 | catch (DocumentClientException de) 87 | { 88 | if (de.StatusCode == HttpStatusCode.PreconditionFailed) 89 | { 90 | attempts += 1; 91 | _log.LogWarning($"Optimistic concurrency pre-condition check failed. Trying again ({attempts}/10)"); 92 | } 93 | else 94 | { 95 | throw; 96 | } 97 | } 98 | } 99 | 100 | throw new ApplicationException("Could not insert document after retring 10 times, due to concurrency violations"); 101 | } 102 | 103 | public async Task UpdateDeviceMaterializedView(Device device) 104 | { 105 | var optionsSingle = new RequestOptions() { PartitionKey = new PartitionKey(device.DeviceId) }; 106 | 107 | DeviceMaterializedView viewSingle = null; 108 | 109 | try 110 | { 111 | var uriSingle = UriFactory.CreateDocumentUri(_databaseName, _collectionName, device.DeviceId); 112 | 113 | _log.LogInformation($"Materialized view: {uriSingle.ToString()}"); 114 | 115 | viewSingle = await _client.ReadDocumentAsync(uriSingle, optionsSingle); 116 | } 117 | catch (DocumentClientException ex) 118 | { 119 | if (ex.StatusCode != HttpStatusCode.NotFound) 120 | throw ex; 121 | } 122 | 123 | //log.LogInformation("Document: " + viewSingle.ToString()); 124 | 125 | if (viewSingle == null) 126 | { 127 | _log.LogInformation("Creating new materialized view"); 128 | viewSingle = new DeviceMaterializedView() 129 | { 130 | Name = device.DeviceId, 131 | Type = "device", 132 | DeviceId = device.DeviceId, 133 | AggregationSum = device.Value, 134 | LastValue = device.Value, 135 | TimeStamp = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssK") 136 | }; 137 | } 138 | else 139 | { 140 | _log.LogInformation("Updating materialized view"); 141 | viewSingle.AggregationSum += device.Value; 142 | viewSingle.LastValue = device.Value; 143 | viewSingle.TimeStamp = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssK"); 144 | } 145 | 146 | await UpsertDocument(viewSingle, optionsSingle); 147 | } 148 | 149 | private async Task> UpsertDocument(object document, RequestOptions options) 150 | { 151 | int attempts = 0; 152 | 153 | while (attempts < 3) 154 | { 155 | try 156 | { 157 | var result = await _client.UpsertDocumentAsync(_collectionUri, document, options); 158 | _log.LogInformation($"{options.PartitionKey} RU Used: {result.RequestCharge:0.0}"); 159 | return result; 160 | } 161 | catch (DocumentClientException de) 162 | { 163 | if (de.StatusCode == HttpStatusCode.TooManyRequests) 164 | { 165 | _log.LogWarning($"Waiting for {de.RetryAfter} msec..."); 166 | await Task.Delay(de.RetryAfter); 167 | attempts += 1; 168 | } 169 | else 170 | { 171 | throw; 172 | } 173 | } 174 | } 175 | 176 | throw new ApplicationException("Could not insert document after being throttled 3 times"); 177 | } 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /materialized-view-processor/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "cosmosDB": { 4 | "connectionMode": "Direct", 5 | "protocol": "Tcp" 6 | } 7 | } -------------------------------------------------------------------------------- /materialized-view-processor/materialized-view-processor.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | netcoreapp2.1 4 | v2 5 | Azure.Samples 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | PreserveNewest 14 | 15 | 16 | PreserveNewest 17 | Never 18 | 19 | 20 | -------------------------------------------------------------------------------- /scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # generate common uniquifier 4 | export UNIQUIFIER=`openssl rand 128 -base64 | tr -cd '[:digit:]' | cut -c 1-7` 5 | 6 | export ROOT_NAME="mvsample${UNIQUIFIER}" 7 | export LOCATION='eastus' 8 | 9 | export RESOURCE_GROUP=$ROOT_NAME 10 | export STORAGE_ACCOUNT=$ROOT_NAME 11 | export COSMOSDB_SERVER_NAME=$ROOT_NAME 12 | export COSMOSDB_DATABASE_NAME=$ROOT_NAME 13 | export COSMOSDB_COLLECTION_NAME_RAW='raw' 14 | export COSMOSDB_COLLECTION_NAME_MV='view' 15 | export COSMOSDB_RU=1000 16 | export PLAN_NAME=$ROOT_NAME 17 | export FUNCTIONAPP_NAME=$ROOT_NAME 18 | 19 | echo "starting deployment: $ROOT_NAME" 20 | 21 | PP=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) 22 | 23 | mkdir $PP/logs &>/dev/null 24 | 25 | set -euo pipefail 26 | 27 | echo "checking prerequisites" 28 | 29 | HAS_AZ=`command -v az` 30 | if [ -z HAS_AZ ]; then 31 | echo "AZ CLI not found" 32 | echo "please install it as described here:" 33 | echo "https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-apt?view=azure-cli-latest" 34 | exit 1 35 | fi 36 | 37 | HAS_ZIP=`command -v zip` 38 | if [ -z HAS_ZIP ]; then 39 | echo "zip not found" 40 | echo "please install it as it is needed by the script" 41 | exit 1 42 | fi 43 | 44 | HAS_DOTNET=`command -v dotnet` 45 | if [ -z HAS_DOTNET ]; then 46 | echo "dotnet not found" 47 | echo "please install .NET Core it as it is needed by the script" 48 | echo "https://dotnet.microsoft.com/download" 49 | exit 1 50 | fi 51 | 52 | echo 'creating resource group' 53 | az group create -n $RESOURCE_GROUP -l $LOCATION -o json \ 54 | 1> $PP/logs/010-group-create.log 55 | 56 | echo 'creating storage account' 57 | az storage account create -n $STORAGE_ACCOUNT -g $RESOURCE_GROUP --sku Standard_LRS \ 58 | 1> $PP/logs/020-storage-account.log 59 | 60 | echo 'creating cosmosdb account' 61 | SERVER_EXISTS=`az cosmosdb check-name-exists -n $COSMOSDB_SERVER_NAME -o tsv` 62 | if [ $SERVER_EXISTS == "false" ]; then 63 | az cosmosdb create -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME \ 64 | -o json \ 65 | 1> $PP/logs/030-cosmosdb-create.log 66 | fi 67 | 68 | echo 'creating cosmosdb database' 69 | DB_EXISTS=`az cosmosdb database exists -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME --db-name $COSMOSDB_DATABASE_NAME -o tsv` 70 | if [ $DB_EXISTS == "false" ]; then 71 | az cosmosdb database create -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME \ 72 | --db-name $COSMOSDB_DATABASE_NAME \ 73 | -o json \ 74 | 1> $PP/logs/040-cosmosdb-database-create.log 75 | fi 76 | 77 | echo 'creating cosmosdb raw collection' 78 | COLLECTION_EXISTS=`az cosmosdb collection exists -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME --db-name $COSMOSDB_DATABASE_NAME --collection-name $COSMOSDB_COLLECTION_NAME_RAW -o tsv` 79 | if [ $COLLECTION_EXISTS == "false" ]; then 80 | az cosmosdb collection create -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME -d $COSMOSDB_DATABASE_NAME \ 81 | --collection-name $COSMOSDB_COLLECTION_NAME_RAW \ 82 | --partition-key-path "/deviceId" \ 83 | --indexing-policy '{ "indexingMode": "none", "automatic": false }' \ 84 | --throughput $COSMOSDB_RU \ 85 | -o json \ 86 | 1> $PP/logs/050-cosmosdb-collection-create-raw.log 87 | fi 88 | 89 | echo 'creating cosmosdb view collection' 90 | COLLECTION_EXISTS=`az cosmosdb collection exists -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME --db-name $COSMOSDB_DATABASE_NAME --collection-name $COSMOSDB_COLLECTION_NAME_MV -o tsv` 91 | if [ $COLLECTION_EXISTS == "false" ]; then 92 | az cosmosdb collection create -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME -d $COSMOSDB_DATABASE_NAME \ 93 | --collection-name $COSMOSDB_COLLECTION_NAME_MV \ 94 | --partition-key-path "/deviceId" \ 95 | --indexing-policy '{ "indexingMode": "none", "automatic": false }' \ 96 | --throughput $COSMOSDB_RU \ 97 | -o json \ 98 | 1> $PP/logs/060-cosmosdb-collection-create-mv.log 99 | fi 100 | 101 | echo 'creating cosmosdb leases collection' 102 | COLLECTION_EXISTS=`az cosmosdb collection exists -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME --db-name $COSMOSDB_DATABASE_NAME --collection-name leases -o tsv` 103 | if [ $COLLECTION_EXISTS == "false" ]; then 104 | az cosmosdb collection create -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME -d $COSMOSDB_DATABASE_NAME \ 105 | --collection-name leases \ 106 | --throughput 400 \ 107 | -o json \ 108 | 1> $PP/logs/070-cosmosdb-collection-create-leases.log 109 | fi 110 | 111 | echo 'creating appinsights' 112 | az resource create --resource-group $RESOURCE_GROUP --resource-type "Microsoft.Insights/components" \ 113 | --name $ROOT_NAME --location $LOCATION --properties '{"ApplicationId":"$ROOT_NAME","Application_Type":"other","Flow_Type":"Redfield"}' \ 114 | -o json \ 115 | 1> $PP/logs/080-appinsights.log 116 | 117 | echo 'getting appinsights instrumentation key' 118 | APPINSIGHTS_INSTRUMENTATIONKEY=`az resource show -g $RESOURCE_GROUP -n $ROOT_NAME --resource-type "Microsoft.Insights/components" --query properties.InstrumentationKey -o tsv` 119 | 120 | echo 'creating function app' 121 | az functionapp create -g $RESOURCE_GROUP -n $FUNCTIONAPP_NAME \ 122 | --consumption-plan-location $LOCATION \ 123 | --app-insights-key $APPINSIGHTS_INSTRUMENTATIONKEY \ 124 | --storage-account $STORAGE_ACCOUNT \ 125 | -o json \ 126 | 1> $PP/logs/090-functionapp.log 127 | 128 | echo 'adding app settings for connection strings' 129 | 130 | echo ". DatabaseName" 131 | az functionapp config appsettings set --name $FUNCTIONAPP_NAME \ 132 | --resource-group $RESOURCE_GROUP \ 133 | --settings DatabaseName=$COSMOSDB_DATABASE_NAME \ 134 | -o json \ 135 | 1>> $PP/logs/090-functionapp.log 136 | 137 | echo ". RawCollectionName" 138 | az functionapp config appsettings set --name $FUNCTIONAPP_NAME \ 139 | --resource-group $RESOURCE_GROUP \ 140 | --settings RawCollectionName=$COSMOSDB_COLLECTION_NAME_RAW \ 141 | -o json \ 142 | 1>> $PP/logs/090-functionapp.log 143 | 144 | echo ". ViewCollectionName" 145 | az functionapp config appsettings set --name $FUNCTIONAPP_NAME \ 146 | --resource-group $RESOURCE_GROUP \ 147 | --settings ViewCollectionName=$COSMOSDB_COLLECTION_NAME_MV \ 148 | -o json \ 149 | 1>> $PP/logs/090-functionapp.log 150 | 151 | echo ". ConnectionString" 152 | COSMOSDB_CONNECTIONSTRING=`az cosmosdb list-connection-strings -g $RESOURCE_GROUP --name $COSMOSDB_SERVER_NAME --query 'connectionStrings[0].connectionString' -o tsv` 153 | az functionapp config appsettings set --name $FUNCTIONAPP_NAME \ 154 | --resource-group $RESOURCE_GROUP \ 155 | --settings ConnectionString=$COSMOSDB_CONNECTIONSTRING \ 156 | -o json \ 157 | 1>> $PP/logs/090-functionapp.log 158 | 159 | echo 'building function app' 160 | FUNCTION_SRC_PATH=$PP/../materialized-view-processor 161 | CURDIR=$PWD 162 | cd $FUNCTION_SRC_PATH 163 | dotnet publish . --configuration Release 164 | 165 | echo 'creating zip file' 166 | ZIPFOLDER="./bin/Release/netcoreapp2.1/publish/" 167 | rm -f publish.zip 168 | cd $ZIPFOLDER 169 | zip -r $PP/publish.zip . 170 | cd $CURDIR 171 | 172 | echo 'deploying function' 173 | az functionapp deployment source config-zip \ 174 | --resource-group $RESOURCE_GROUP \ 175 | --name $FUNCTIONAPP_NAME \ 176 | --src $PP/publish.zip \ 177 | 1> $PP/logs/100-functionapp-deploy.log 178 | 179 | echo 'removing local zip file' 180 | rm -f $PP/publish.zip 181 | 182 | echo 'creating App.Config' 183 | 184 | COSMOSDB_URI=`az cosmosdb list -g $RESOURCE_GROUP --query '[0].documentEndpoint' -o tsv` 185 | COSMOSDB_KEY=`az cosmosdb list-keys -g $RESOURCE_GROUP -n $COSMOSDB_SERVER_NAME --query 'primaryMasterKey' -o tsv` 186 | APP=$PP/../sensor-data-producer 187 | 188 | sed "s|{URI}|${COSMOSDB_URI}|g" $APP/App.config.template > $APP/App.config 189 | 190 | sed -i.bak "s|{KEY}|${COSMOSDB_KEY}|g" $APP/App.config 191 | 192 | sed -i.bak "s|{DB}|${COSMOSDB_DATABASE_NAME}|g" $APP/App.config 193 | 194 | sed -i.bak "s|{RAW}|${COSMOSDB_COLLECTION_NAME_RAW}|g" $APP/App.config 195 | 196 | rm -f $APP/App.config.bak 197 | 198 | echo 'deployment done' 199 | -------------------------------------------------------------------------------- /sensor-data-producer/.dockerignore: -------------------------------------------------------------------------------- 1 | .dockerignore 2 | .env 3 | .git 4 | .gitignore 5 | .vs 6 | .vscode 7 | docker-compose.yml 8 | docker-compose.*.yml 9 | */bin 10 | */obj 11 | -------------------------------------------------------------------------------- /sensor-data-producer/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to find out which attributes exist for C# debugging 3 | // Use hover for the description of the existing attributes 4 | // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (console)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | // If you have changed target frameworks, make sure to update the program path. 13 | "program": "${workspaceFolder}/bin/Debug/netcoreapp2.1/sensor-data-producer.dll", 14 | "args": ["1-10"], 15 | "cwd": "${workspaceFolder}", 16 | // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window 17 | "console": "internalConsole", 18 | "stopAtEntry": false, 19 | "internalConsoleOptions": "openOnSessionStart", 20 | "justMyCode": false 21 | }, 22 | { 23 | "name": ".NET Core Attach", 24 | "type": "coreclr", 25 | "request": "attach", 26 | "processId": "${command:pickProcess}" 27 | } 28 | ,] 29 | } -------------------------------------------------------------------------------- /sensor-data-producer/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "build", 6 | "command": "dotnet", 7 | "type": "process", 8 | "args": [ 9 | "build", 10 | "${workspaceFolder}/sensor-data-producer.csproj" 11 | ], 12 | "problemMatcher": "$msCompile" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /sensor-data-producer/App.config.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /sensor-data-producer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM microsoft/dotnet:2.1-runtime AS base 2 | WORKDIR /app 3 | 4 | FROM microsoft/dotnet:2.1-sdk AS publish 5 | WORKDIR /src 6 | COPY . . 7 | RUN dotnet restore 8 | RUN dotnet publish -c Release -o /app 9 | 10 | FROM base AS final 11 | WORKDIR /app 12 | COPY --from=publish ./app . 13 | 14 | ENTRYPOINT ["dotnet", "sensor-data-producer.dll"] 15 | -------------------------------------------------------------------------------- /sensor-data-producer/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using System.Net; 5 | using Microsoft.Azure.Documents; 6 | using Microsoft.Azure.Documents.Client; 7 | using Newtonsoft.Json; 8 | using System.Threading; 9 | using System.IO; 10 | using System.Collections.Generic; 11 | using System.Configuration; 12 | 13 | namespace SensorDataProducer 14 | { 15 | public class SensorData 16 | { 17 | [JsonProperty("deviceId")] 18 | public string Id { get; set; } 19 | 20 | [JsonProperty("value")] 21 | public double Value { get; set; } 22 | 23 | [JsonProperty("timestamp")] 24 | public string TimeStamp { get; set; } 25 | 26 | public override string ToString() 27 | { 28 | return string.Format($"{Id}: {TimeStamp} - {Value}"); 29 | } 30 | } 31 | 32 | public class CosmosDBInfo { 33 | public string EndpointUri; 34 | public string Key; 35 | public string Database; 36 | public string Collection; 37 | } 38 | 39 | class Program 40 | { 41 | static async Task Main(string[] args) 42 | { 43 | string sensorId = string.Empty; 44 | 45 | var cosmosDBInfo = new CosmosDBInfo() 46 | { 47 | EndpointUri = ConfigurationManager.AppSettings?["CosmosDB:EndpointURI"], 48 | Key = ConfigurationManager.AppSettings?["CosmosDB:Key"], 49 | Database = ConfigurationManager.AppSettings?["CosmosDB:Database"], 50 | Collection = ConfigurationManager.AppSettings?["CosmosDB:Collection:Raw"] 51 | }; 52 | 53 | 54 | if (args.Count() == 1) 55 | { 56 | sensorId = args[0]; 57 | } 58 | 59 | if (string.IsNullOrEmpty(sensorId)) 60 | { 61 | Console.WriteLine("Please specify SensorId range. Eg: sensor-data-producer 1-10"); 62 | return; 63 | } 64 | 65 | int s = 0; 66 | int e = 0; 67 | 68 | if (sensorId.Contains("-")) 69 | { 70 | var split = sensorId.Split('-'); 71 | if (split.Count() != 2) { 72 | Console.WriteLine("Range must be in the form N-M, where N and M are positive integers. Eg; 1-10"); 73 | return; 74 | } 75 | 76 | Int32.TryParse(split[0], out s); 77 | Int32.TryParse(split[1], out e); 78 | } else 79 | { 80 | s = 1; 81 | Int32.TryParse(sensorId, out e); 82 | } 83 | 84 | if (s == 0 || e == 0) 85 | { 86 | Console.WriteLine("Provided SensorId must be an integer number or a range of positive integers in the form N-M. Eg: 1-10"); 87 | return; 88 | } 89 | 90 | var tasks = new List(); 91 | var cts = new CancellationTokenSource(); 92 | 93 | var simulator = new Simulator(cosmosDBInfo, cts.Token); 94 | 95 | foreach (int i in Enumerable.Range(s, e)) 96 | { 97 | tasks.Add(new Task(async () => await simulator.Run(i), TaskCreationOptions.LongRunning)); 98 | } 99 | 100 | tasks.ForEach(t => t.Start()); 101 | 102 | Console.WriteLine("Press any key to terminate simulator"); 103 | Console.ReadKey(true); 104 | 105 | cts.Cancel(); 106 | Console.WriteLine("Cancel requested..."); 107 | 108 | await Task.WhenAll(tasks.ToArray()); 109 | 110 | Console.WriteLine("Done."); 111 | } 112 | } 113 | 114 | class Simulator { 115 | 116 | private CancellationToken _token; 117 | private DocumentClient _client; 118 | private CosmosDBInfo _cosmosDB; 119 | 120 | public Simulator(CosmosDBInfo cosmosDB, CancellationToken token) 121 | { 122 | _token = token; 123 | _cosmosDB = cosmosDB; 124 | _client = new DocumentClient( 125 | new Uri(_cosmosDB.EndpointUri), 126 | _cosmosDB.Key, 127 | new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp } 128 | ); 129 | } 130 | 131 | public async Task Run(int sensorId) 132 | { 133 | var database = await _client.CreateDatabaseIfNotExistsAsync(new Database { Id = _cosmosDB.Database }); 134 | 135 | var collection = await _client.CreateDocumentCollectionIfNotExistsAsync( 136 | UriFactory.CreateDatabaseUri(_cosmosDB.Database), 137 | new DocumentCollection { Id = _cosmosDB.Collection } 138 | ); 139 | 140 | var collectionUri = UriFactory.CreateDocumentCollectionUri(_cosmosDB.Database, _cosmosDB.Collection); 141 | 142 | Random random = new Random(); 143 | 144 | while (!_token.IsCancellationRequested) 145 | { 146 | var sensorData = new SensorData() 147 | { 148 | Id = sensorId.ToString().PadLeft(3, '0'), 149 | Value = 100 + random.NextDouble() * 100, 150 | TimeStamp = DateTime.UtcNow.ToString("o") 151 | }; 152 | 153 | Console.WriteLine(sensorData); 154 | 155 | bool documentCreated = false; 156 | int tryCount = 0; 157 | 158 | while(!documentCreated && tryCount<3) 159 | { 160 | try 161 | { 162 | await _client.CreateDocumentAsync(collectionUri, sensorData); 163 | documentCreated = true; 164 | } 165 | catch (DocumentClientException de) 166 | { 167 | if (de.StatusCode == HttpStatusCode.TooManyRequests) 168 | { 169 | Console.WriteLine($"{sensorData.Id}: Waiting for ${de.RetryAfter} msec..."); 170 | documentCreated = false; 171 | await Task.Delay(de.RetryAfter); 172 | tryCount =+ 1; 173 | } 174 | else 175 | { 176 | throw; 177 | } 178 | } 179 | } 180 | 181 | if (documentCreated == false) 182 | { 183 | throw new ApplicationException("Cannot create document after trying 3 times"); 184 | } 185 | 186 | await Task.Delay(random.Next(500) + 750); 187 | } 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /sensor-data-producer/build.bat: -------------------------------------------------------------------------------- 1 | docker build . -t iot-simulator -------------------------------------------------------------------------------- /sensor-data-producer/sensor-data-producer.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp2.1 6 | SensorDataProducer 7 | latest 8 | 9 | 10 | 11 | TRACE;DEBUG 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | --------------------------------------------------------------------------------