├── .gitignore ├── .vscode ├── .ropeproject │ ├── config.py │ └── objectdb ├── launch.json └── settings.json ├── README.md ├── _config.yml ├── config ├── deployment.amd64.json └── deployment.arm32v7.json ├── deployment.template.json ├── docs ├── Architecture.jpg ├── Architecture.vsdx ├── Azure IoT Edge, Python Azure Functions and SignalR.pdf ├── Azure IoT Edge, Python Azure Functions and SignalR.pptx ├── Creating an image recognition solution with Azure IoT Edge and Azure Cognitive Services.pptx ├── README.1.md ├── README.old.md ├── SystemArchitecture.vsdx ├── azure-iotedge-monitoring.png ├── azure-portal-iotedge-device-details.png ├── build-architecture.pdf ├── build-architecture.vsdx ├── congratulations.jpg ├── custom-vision-domain.png ├── deploy-to-device.png ├── export-as-docker.png ├── export-choose-your-platform.png ├── exportmodel.png ├── index-1.html ├── iot-edge-arch.png ├── iot-edge-in-action.jpg ├── iotedge-architecture.png ├── iotedge-list.png ├── raspberry-pi-3a-image-classifier.png ├── raspberry-pi-3a-image-classifier.xcf ├── raspberry-pi-image-classifier.jpg ├── select-processor-architecture.jpg ├── solution-build-push-docker.png ├── speech-key.png ├── speech-service.png └── visual-studio-code-open-project.png ├── modules ├── CameraCaptureOpenCV │ ├── Dockerfile.amd64 │ ├── Dockerfile.amd64.debug │ ├── Dockerfile.arm32v7 │ ├── app │ │ ├── CameraCapture.py │ │ ├── VideoStream.py │ │ ├── azure_text_speech.py │ │ ├── azure_text_translate.py │ │ ├── iotedge_camera.py │ │ ├── speech_map_australian.json │ │ ├── speech_map_chinese.json │ │ ├── speech_map_korean.json │ │ └── text2speech.py │ ├── build │ │ ├── amd64-requirements.txt │ │ └── arm32v7-requirements.txt │ └── module.json └── ImageClassifierService │ ├── Dockerfile.amd64 │ ├── Dockerfile.amd64.debug │ ├── Dockerfile.arm32v7 │ ├── app │ ├── iotedge_model.py │ ├── labels.txt │ ├── model.pb │ └── predict.py │ ├── build │ ├── amd64-requirements.txt │ └── arm32v7-requirements.txt │ └── module.json ├── set-camera-sh └── version.py /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015/2017 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # Visual Studio 2017 auto generated files 33 | Generated\ Files/ 34 | 35 | # MSTest test Results 36 | [Tt]est[Rr]esult*/ 37 | [Bb]uild[Ll]og.* 38 | 39 | # NUNIT 40 | *.VisualState.xml 41 | TestResult.xml 42 | 43 | # Build Results of an ATL Project 44 | [Dd]ebugPS/ 45 | [Rr]eleasePS/ 46 | dlldata.c 47 | 48 | # Benchmark Results 49 | BenchmarkDotNet.Artifacts/ 50 | 51 | # .NET Core 52 | project.lock.json 53 | project.fragment.lock.json 54 | artifacts/ 55 | **/Properties/launchSettings.json 56 | 57 | # StyleCop 58 | StyleCopReport.xml 59 | 60 | # Files built by Visual Studio 61 | *_i.c 62 | *_p.c 63 | *_i.h 64 | *.ilk 65 | *.meta 66 | *.obj 67 | *.iobj 68 | *.pch 69 | *.pdb 70 | *.ipdb 71 | *.pgc 72 | *.pgd 73 | *.rsp 74 | *.sbr 75 | *.tlb 76 | *.tli 77 | *.tlh 78 | *.tmp 79 | *.tmp_proj 80 | *.log 81 | *.vspscc 82 | *.vssscc 83 | .builds 84 | *.pidb 85 | *.svclog 86 | *.scc 87 | 88 | # Chutzpah Test files 89 | _Chutzpah* 90 | 91 | # Visual C++ cache files 92 | ipch/ 93 | *.aps 94 | *.ncb 95 | *.opendb 96 | *.opensdf 97 | *.sdf 98 | *.cachefile 99 | *.VC.db 100 | *.VC.VC.opendb 101 | 102 | # Visual Studio profiler 103 | *.psess 104 | *.vsp 105 | *.vspx 106 | *.sap 107 | 108 | # Visual Studio Trace Files 109 | *.e2e 110 | 111 | # TFS 2012 Local Workspace 112 | $tf/ 113 | 114 | # Guidance Automation Toolkit 115 | *.gpState 116 | 117 | # ReSharper is a .NET coding add-in 118 | _ReSharper*/ 119 | *.[Rr]e[Ss]harper 120 | *.DotSettings.user 121 | 122 | # JustCode is a .NET coding add-in 123 | .JustCode 124 | 125 | # TeamCity is a build add-in 126 | _TeamCity* 127 | 128 | # DotCover is a Code Coverage Tool 129 | *.dotCover 130 | 131 | # AxoCover is a Code Coverage Tool 132 | .axoCover/* 133 | !.axoCover/settings.json 134 | 135 | # Visual Studio code coverage results 136 | *.coverage 137 | *.coveragexml 138 | 139 | # NCrunch 140 | _NCrunch_* 141 | .*crunch*.local.xml 142 | nCrunchTemp_* 143 | 144 | # MightyMoose 145 | *.mm.* 146 | AutoTest.Net/ 147 | 148 | # Web workbench (sass) 149 | .sass-cache/ 150 | 151 | # Installshield output folder 152 | [Ee]xpress/ 153 | 154 | # DocProject is a documentation generator add-in 155 | DocProject/buildhelp/ 156 | DocProject/Help/*.HxT 157 | DocProject/Help/*.HxC 158 | DocProject/Help/*.hhc 159 | DocProject/Help/*.hhk 160 | DocProject/Help/*.hhp 161 | DocProject/Help/Html2 162 | DocProject/Help/html 163 | 164 | # Click-Once directory 165 | publish/ 166 | 167 | # Publish Web Output 168 | *.[Pp]ublish.xml 169 | *.azurePubxml 170 | # Note: Comment the next line if you want to checkin your web deploy settings, 171 | # but database connection strings (with potential passwords) will be unencrypted 172 | *.pubxml 173 | *.publishproj 174 | 175 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 176 | # checkin your Azure Web App publish settings, but sensitive information contained 177 | # in these scripts will be unencrypted 178 | PublishScripts/ 179 | 180 | # NuGet Packages 181 | *.nupkg 182 | # The packages folder can be ignored because of Package Restore 183 | **/[Pp]ackages/* 184 | # except build/, which is used as an MSBuild target. 185 | !**/[Pp]ackages/build/ 186 | # Uncomment if necessary however generally it will be regenerated when needed 187 | #!**/[Pp]ackages/repositories.config 188 | # NuGet v3's project.json files produces more ignorable files 189 | *.nuget.props 190 | *.nuget.targets 191 | 192 | # Microsoft Azure Build Output 193 | csx/ 194 | *.build.csdef 195 | 196 | # Microsoft Azure Emulator 197 | ecf/ 198 | rcf/ 199 | 200 | # Windows Store app package directories and files 201 | AppPackages/ 202 | BundleArtifacts/ 203 | Package.StoreAssociation.xml 204 | _pkginfo.txt 205 | *.appx 206 | 207 | # Visual Studio cache files 208 | # files ending in .cache can be ignored 209 | *.[Cc]ache 210 | # but keep track of directories ending in .cache 211 | !*.[Cc]ache/ 212 | 213 | # Others 214 | ClientBin/ 215 | ~$* 216 | *~ 217 | *.dbmdl 218 | *.dbproj.schemaview 219 | *.jfm 220 | *.pfx 221 | *.publishsettings 222 | orleans.codegen.cs 223 | 224 | # Including strong name files can present a security risk 225 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 226 | #*.snk 227 | 228 | # Since there are multiple workflows, uncomment next line to ignore bower_components 229 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 230 | #bower_components/ 231 | 232 | # RIA/Silverlight projects 233 | Generated_Code/ 234 | 235 | # Backup & report files from converting an old project file 236 | # to a newer Visual Studio version. Backup files are not needed, 237 | # because we have git ;-) 238 | _UpgradeReport_Files/ 239 | Backup*/ 240 | UpgradeLog*.XML 241 | UpgradeLog*.htm 242 | ServiceFabricBackup/ 243 | *.rptproj.bak 244 | 245 | # SQL Server files 246 | *.mdf 247 | *.ldf 248 | *.ndf 249 | 250 | # Business Intelligence projects 251 | *.rdl.data 252 | *.bim.layout 253 | *.bim_*.settings 254 | *.rptproj.rsuser 255 | 256 | # Microsoft Fakes 257 | FakesAssemblies/ 258 | 259 | # GhostDoc plugin setting file 260 | *.GhostDoc.xml 261 | 262 | # Node.js Tools for Visual Studio 263 | .ntvs_analysis.dat 264 | node_modules/ 265 | 266 | # Visual Studio 6 build log 267 | *.plg 268 | 269 | # Visual Studio 6 workspace options file 270 | *.opt 271 | 272 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 273 | *.vbw 274 | 275 | # Visual Studio LightSwitch build output 276 | **/*.HTMLClient/GeneratedArtifacts 277 | **/*.DesktopClient/GeneratedArtifacts 278 | **/*.DesktopClient/ModelManifest.xml 279 | **/*.Server/GeneratedArtifacts 280 | **/*.Server/ModelManifest.xml 281 | _Pvt_Extensions 282 | 283 | # Paket dependency manager 284 | .paket/paket.exe 285 | paket-files/ 286 | 287 | # FAKE - F# Make 288 | .fake/ 289 | 290 | # JetBrains Rider 291 | .idea/ 292 | *.sln.iml 293 | 294 | # CodeRush 295 | .cr/ 296 | 297 | # Python Tools for Visual Studio (PTVS) 298 | __pycache__/ 299 | *.pyc 300 | 301 | # Cake - Uncomment if you are using it 302 | # tools/** 303 | # !tools/packages.config 304 | 305 | # Tabs Studio 306 | *.tss 307 | 308 | # Telerik's JustMock configuration file 309 | *.jmconfig 310 | 311 | # BizTalk build output 312 | *.btp.cs 313 | *.btm.cs 314 | *.odx.cs 315 | *.xsd.cs 316 | 317 | # OpenCover UI analysis results 318 | OpenCover/ 319 | 320 | # Azure Stream Analytics local run output 321 | ASALocalRun/ 322 | 323 | # MSBuild Binary and Structured Log 324 | *.binlog 325 | 326 | # NVidia Nsight GPU debugger configuration file 327 | *.nvuser 328 | 329 | # MFractors (Xamarin productivity tool) working folder 330 | .mfractor/ 331 | -------------------------------------------------------------------------------- /.vscode/.ropeproject/config.py: -------------------------------------------------------------------------------- 1 | # The default ``config.py`` 2 | # flake8: noqa 3 | 4 | 5 | def set_prefs(prefs): 6 | """This function is called before opening the project""" 7 | 8 | # Specify which files and folders to ignore in the project. 9 | # Changes to ignored resources are not added to the history and 10 | # VCSs. Also they are not returned in `Project.get_files()`. 11 | # Note that ``?`` and ``*`` match all characters but slashes. 12 | # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc' 13 | # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc' 14 | # '.svn': matches 'pkg/.svn' and all of its children 15 | # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o' 16 | # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o' 17 | prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject', 18 | '.hg', '.svn', '_svn', '.git', '.tox'] 19 | 20 | # Specifies which files should be considered python files. It is 21 | # useful when you have scripts inside your project. Only files 22 | # ending with ``.py`` are considered to be python files by 23 | # default. 24 | # prefs['python_files'] = ['*.py'] 25 | 26 | # Custom source folders: By default rope searches the project 27 | # for finding source folders (folders that should be searched 28 | # for finding modules). You can add paths to that list. Note 29 | # that rope guesses project source folders correctly most of the 30 | # time; use this if you have any problems. 31 | # The folders should be relative to project root and use '/' for 32 | # separating folders regardless of the platform rope is running on. 33 | # 'src/my_source_folder' for instance. 34 | # prefs.add('source_folders', 'src') 35 | 36 | # You can extend python path for looking up modules 37 | # prefs.add('python_path', '~/python/') 38 | 39 | # Should rope save object information or not. 40 | prefs['save_objectdb'] = True 41 | prefs['compress_objectdb'] = False 42 | 43 | # If `True`, rope analyzes each module when it is being saved. 44 | prefs['automatic_soa'] = True 45 | # The depth of calls to follow in static object analysis 46 | prefs['soa_followed_calls'] = 0 47 | 48 | # If `False` when running modules or unit tests "dynamic object 49 | # analysis" is turned off. This makes them much faster. 50 | prefs['perform_doa'] = True 51 | 52 | # Rope can check the validity of its object DB when running. 53 | prefs['validate_objectdb'] = True 54 | 55 | # How many undos to hold? 56 | prefs['max_history_items'] = 32 57 | 58 | # Shows whether to save history across sessions. 59 | prefs['save_history'] = True 60 | prefs['compress_history'] = False 61 | 62 | # Set the number spaces used for indenting. According to 63 | # :PEP:`8`, it is best to use 4 spaces. Since most of rope's 64 | # unit-tests use 4 spaces it is more reliable, too. 65 | prefs['indent_size'] = 4 66 | 67 | # Builtin and c-extension modules that are allowed to be imported 68 | # and inspected by rope. 69 | prefs['extension_modules'] = [] 70 | 71 | # Add all standard c-extensions to extension_modules list. 72 | prefs['import_dynload_stdmods'] = True 73 | 74 | # If `True` modules with syntax errors are considered to be empty. 75 | # The default value is `False`; When `False` syntax errors raise 76 | # `rope.base.exceptions.ModuleSyntaxError` exception. 77 | prefs['ignore_syntax_errors'] = False 78 | 79 | # If `True`, rope ignores unresolvable imports. Otherwise, they 80 | # appear in the importing namespace. 81 | prefs['ignore_bad_imports'] = False 82 | 83 | # If `True`, rope will insert new module imports as 84 | # `from import ` by default. 85 | prefs['prefer_module_from_imports'] = False 86 | 87 | # If `True`, rope will transform a comma list of imports into 88 | # multiple separate import statements when organizing 89 | # imports. 90 | prefs['split_imports'] = False 91 | 92 | # If `True`, rope will remove all top-level import statements and 93 | # reinsert them at the top of the module when making changes. 94 | prefs['pull_imports_to_top'] = True 95 | 96 | # If `True`, rope will sort imports alphabetically by module name instead 97 | # of alphabetically by import statement, with from imports after normal 98 | # imports. 99 | prefs['sort_imports_alphabetically'] = False 100 | 101 | # Location of implementation of 102 | # rope.base.oi.type_hinting.interfaces.ITypeHintingFactory In general 103 | # case, you don't have to change this value, unless you're an rope expert. 104 | # Change this value to inject you own implementations of interfaces 105 | # listed in module rope.base.oi.type_hinting.providers.interfaces 106 | # For example, you can add you own providers for Django Models, or disable 107 | # the search type-hinting in a class hierarchy, etc. 108 | prefs['type_hinting_factory'] = ( 109 | 'rope.base.oi.type_hinting.factory.default_type_hinting_factory') 110 | 111 | 112 | def project_opened(project): 113 | """This function is called after opening the project""" 114 | # Do whatever you like here! 115 | -------------------------------------------------------------------------------- /.vscode/.ropeproject/objectdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/.vscode/.ropeproject/objectdb -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Terminal (integrated)", 9 | "type": "python", 10 | "request": "launch", 11 | "program": "${file}", 12 | "console": "integratedTerminal" 13 | }, 14 | { 15 | "name": "Python: CameraCaptureOpenCV Remote Attach", 16 | "type": "python", 17 | "request": "attach", 18 | "port": 5678, 19 | "host": "192.168.1.198", 20 | "pathMappings": [ 21 | { 22 | "localRoot": "${workspaceFolder}/modules/CameraCaptureOpenCV/app", 23 | "remoteRoot": "/app/." 24 | } 25 | ] 26 | }, 27 | { 28 | "name": "Python: Image Classifier Remote Attach", 29 | "type": "python", 30 | "request": "attach", 31 | "port": 5679, 32 | "host": "localhost", 33 | "pathMappings": [ 34 | { 35 | "localRoot": "${workspaceFolder}/modules/ImageClassifierService/app", 36 | "remoteRoot": "/app/." 37 | } 38 | ] 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.pythonPath": "C:\\Users\\dglover\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", 3 | "python.linting.pylintEnabled": true, 4 | "azure-iot-edge.defaultPlatform": { 5 | "platform": "arm32v7", 6 | "alias": null 7 | } 8 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | |Author|[Dave Glover](https://developer.microsoft.com/advocates/dave-glover?WT.mc_id=iot-0000-dglover), Microsoft Cloud Developer Advocate | 4 | |----|---| 5 | |Solution| [Creating an image recognition solution with Azure IoT Edge and Azure Cognitive Services](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services)| 6 | |Documentation|[README](https://gloveboxes.github.io/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/) | 7 | |Platform| [Azure IoT Edge](https://docs.microsoft.com/azure/iot-edge/?WT.mc_id=iot-0000-dglover)| 8 | |Documentation | [Azure IoT Edge](https://docs.microsoft.com/azure/iot-edge/?WT.mc_id=iot-0000-dglover), [Azure Custom Vision](https://docs.microsoft.com/azure/cognitive-services/custom-vision-service/getting-started-build-a-classifier/?WT.mc_id=iot-0000-dglover), [Azure Speech Services](https://docs.microsoft.com/azure/cognitive-services/speech-service/overview/?WT.mc_id=iot-0000-dglover), [Azure Functions on Edge](https://docs.microsoft.com/azure/iot-edge/tutorial-deploy-function/?WT.mc_id=iot-0000-dglover), [Azure Stream Analytics](https://docs.microsoft.com/azure/iot-edge/tutorial-deploy-stream-analytics/?WT.mc_id=iot-0000-dglover), [Azure Machine Learning Services](https://docs.microsoft.com/azure/iot-edge/tutorial-deploy-machine-learning/?WT.mc_id=iot-0000-dglover) | 9 | |Video Training|[Enable edge intelligence with Azure IoT Edge](https://channel9.msdn.com/events/Connect/2017/T253?WT.mc_id=iot-0000-dglover)| 10 | |Programming Language| Python| 11 | |Date|As at April 2019| 12 | 13 | ![raspberry pi 3a azure iot edge](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/raspberry-pi-3a-image-classifier.png) 14 | 15 | **Raspberry Pi 3A+ running Azure IoT Edge Image Classifier** 16 | 17 | 18 | * [Parts Required](#PartsRequired) 19 | * [Quick Installation Guide for Raspberry Pi](#QuickInstallationGuideforRaspberryPi) 20 | * [Solution Overview](#SolutionOverview) 21 | * [Azure IoT Edge in Action](#AzureIoTEdgeinAction) 22 | * [Solution Architectural Considerations](#SolutionArchitecturalConsiderations) 23 | * [Creating the Fruit Classification Model](#CreatingtheFruitClassificationModel) 24 | * [Exporting an Azure Custom Vision Model](#ExportinganAzureCustomVisionModel) 25 | * [Azure Speech Services](#AzureSpeechServices) 26 | * [Understanding the Project Structure](#UnderstandingtheProjectStructure) 27 | * [Building the Solution](#BuildingtheSolution) 28 | * [Deploying the Solution](#DeployingtheSolution) 29 | * [Monitoring the Solution on the IoT Edge Device](#MonitoringtheSolutionontheIoTEdgeDevice) 30 | * [Monitoring the Solution from the Azure IoT Edge Blade](#MonitoringtheSolutionfromtheAzureIoTEdgeBlade) 31 | 32 | 36 | 37 | 38 | # Image Classification with Azure IoT Edge 39 | 40 | There are lots of applications for image recognition but what I had in mind when developing this application was a solution for vision impaired people scanning fruit and vegetables at a self-service checkout. 41 | 42 | ## Parts Required 43 | 44 | 1. Raspberry Pi 3B or better, USB Camera, and a Speaker. 45 | 46 | Note, the solution will run on a Raspberry Pi 3A+, it has enough processing power, but the device is limited to 512MB RAM. I would recommend a Raspberry Pi 3B+ as it has 1GB of RAM and is faster than the older 3B model. Azure IoT Edge requires an ARM32v7 or better processor. It will not run on the ARM32v6 processor found in the Raspberry Pi Zero. 47 | 2. Alternatively, you can run the solution on desktop Linux - such as Ubuntu 18.04. This solution requires USB camera pass through into a Docker container as well as Azure IoT Edge support. So for now, that is Linux. 48 | 49 | 50 | ## Quick Installation Guide for Raspberry Pi 51 | 52 | If you do not want to download and build the solution you can use the prebuilt Azure IoT Edge configuration from my [GitHub](https://github.com/gloveboxes?tab=repositories) repository and use the associated Docker images. 53 | 54 | 55 | 1. Set up [Raspbian Stretch Lite](https://learn.pimoroni.com/tutorial/sandyj/setting-up-a-headless-pi) on Raspberry Pi. Be sure to configure the correct [Country Code](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements) in your **wpa_supplicant.conf** file. 56 | 2. If you don't already have an Azure account then sign up for a [free Azure account](https://azure.microsoft.com/free/?WT.mc_id=iot-0000-dglover). If you are a student then sign up for an [Azure for Students](https://azure.microsoft.com/free/students/?WT.mc_id=iot-0000-dglover) account, no credit card required. 57 | 3. [Follow these instructions](https://docs.microsoft.com/azure/iot-edge/how-to-register-device-portal/?WT.mc_id=iot-0000-dglover) to create an Azure IoT Hub, and an Azure IoT Edge device. 58 | 1. [Install Azure IoT Edge runtime on Raspberry Pi](https://docs.microsoft.com/azure/iot-edge/how-to-install-iot-edge-linux-arm/?WT.mc_id=iot-0000-dglover) 59 | 1. Download the deployment configuration file that describes the Azure IoT Edge Modules and Routes for this solution. Open the [deployment.arm32v7.json](https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/master/config/deployment.arm32v7.json) link and save the deployment.arm32v7.json in a known location on your computer. 60 | 1. Install the [Azure CLI](https://docs.microsoft.com/cli/azure/install-azure-cli/?WT.mc_id=iot-0000-dglover) and the [IoT extension for Azure CLI](https://github.com/Azure/azure-iot-cli-extension) command line tools. For more information, see [Deploy Azure IoT Edge modules with Azure CLI](https://docs.microsoft.com/azure/iot-edge/how-to-deploy-modules-cli/?WT.mc_id=iot-0000-dglover) 61 | 1. Open a command line console/terminal and change directory to the location where you saved the deployment.arm32v7.json file. 62 | 1. Finally, from the command line run the following command, be sure to substitute [device id] and the [hub name] values. 63 | 64 | ```bash 65 | az iot edge set-modules --device-id [device id] --hub-name [hub name] --content deployment.arm32v7.json 66 | ``` 67 | 68 | 1. The modules will now start to deploy to your Raspberry Pi, the Raspberry Pi green activity LED will flicker until the deployment completes. Approximately 1.5 GB of Dockers modules will be downloaded and decompressed on the Raspberry Pi. This is a one off operation. 69 | 70 | 71 | ## Solution Overview 72 | 73 | The system identifies the item scanned against a pre-trained machine learning model, tells the person what they have just scanned, then sends a record of the transaction to a central inventory system. 74 | 75 | The solution runs on [Azure IoT Edge](#2-what-is-azure-iot-edge) and consists of a number of services. 76 | 77 | 1. The **Camera Capture Module** handles scanning items using a camera. It then calls the Image Classification module to identify the item, a call is then made to the "Text to Speech" module to convert item label to speech, and the name of the item scanned is played on the attached speaker. 78 | 79 | 2. The **Image Classification Module** runs a Tensorflow machine learning model that has been trained with images of fruit. It handles classifying the scanned items. 80 | 81 | 3. The **Text to Speech Module** converts the name of the item scanned from text to speech using Azure Speech Services. 82 | 83 | 4. A USB Camera is used to capture images of items to be bought. 84 | 85 | 5. A Speaker for text to speech playback. 86 | 87 | 6. **Azure IoT Hub** (Free tier) is used for managing, deploying, and reporting Azure IoT Edge devices running the solution. 88 | 89 | 7. **Azure Speech Services** (free tier) is used to generate very natural speech telling the shopper what they have just scanned. 90 | 91 | 8. **Azure Custom Vision service** was used to build the fruit model used for image classification. 92 | 93 | ![IoT Edge Solution Architecture](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/Architecture.jpg) 94 | 95 | # What is Azure IoT Edge 96 | 97 | The solution is built on [Azure IoT Edge](https://docs.microsoft.com/azure/iot-edge/?WT.mc_id=iot-0000-dglover) which is part of the Azure IoT Hub service and is used to define, secure and deploy a solution to an edge device. It also provides cloud-based central monitoring and reporting of the edge device. 98 | 99 | The main components for an IoT Edge solution are:- 100 | 101 | 1. The [IoT Edge Runtime](https://docs.microsoft.com/azure/iot-edge/iot-edge-runtime/?WT.mc_id=iot-0000-dglover) which is installed on the local edge device and consists of two main components. The **IoT Edge "hub"**, responsible for communications, and the **IoT Edge "agent"**, responsible for running and monitoring modules on the edge device. 102 | 103 | 2. [Modules](https://docs.microsoft.com/azure/iot-edge/iot-edge-modules/?WT.mc_id=iot-0000-dglover). Modules are the unit of deployment. Modules are docker images pulled from a registry such as the [Azure Container Registry](https://azure.microsoft.com/services/container-registry/?WT.mc_id=iot-0000-dglover), or [Docker Hub](https://hub.docker.com/). Modules can be custom developed, built as [Azure Functions](https://docs.microsoft.com/azure/iot-edge/tutorial-deploy-function/?WT.mc_id=iot-0000-dglover), or as exported services from [Azure Custom Vision](https://docs.microsoft.com/azure/iot-edge/tutorial-deploy-stream-analytics/?WT.mc_id=iot-0000-dglover), [Azure Machine Learning](https://docs.microsoft.com/azure/iot-edge/tutorial-deploy-machine-learning/?WT.mc_id=iot-0000-dglover), or [Azure Stream Analytics](https://docs.microsoft.com/azure/iot-edge/tutorial-deploy-stream-analytics/?WT.mc_id=iot-0000-dglover). 104 | 105 | 3. Routes. Routes define message paths between modules and with Azure IoT Hub. 106 | 107 | 4. Properties. You can set "desired" properties for a module from Azure IoT Hub. For example, you might want to set a threshold property for a temperature alert. 108 | 109 | 5. Create Options. Create Options tell the Docker runtime what options to start the module with. For example, you may wish to open ports for REST APIs or debugging ports, define paths to devices such as a USB Camera, set environment variables, or enable privilege mode for certain hardware operations. For more information see the [Docker API](https://docs.docker.com/engine/api/latest/) documentation. 110 | 111 | 6. [Deployment Manifest](https://docs.microsoft.com/azure/iot-edge/module-composition/?WT.mc_id=iot-0000-dglover). The Deployment Manifest pulls everything together and tells the Azure IoT Edge runtime what modules to deploy, from where, plus what message routes to set up, and what create options to start each module with. 112 | 113 | ## Azure IoT Edge in Action 114 | 115 | ![iot edge in action](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/iot-edge-in-action.jpg) 116 | 117 | ## Solution Architectural Considerations 118 | 119 | So, with that overview of Azure IoT Edge here were my considerations and constraints for the solution. 120 | 121 | 1. The solution should scale from a Raspberry Pi (running Raspbian Linux) on ARM32v7, to my desktop development environment, to an industrial capable IoT Edge device such as those found in the [Certified IoT Edge Catalog](https://catalog.azureiotsolutions.com/). 122 | 123 | 2. The solution needs camera input, I used a USB Webcam for image capture as it was supported across all target devices. 124 | 125 | 3. The camera capture module needed Docker USB device pass-through (not supported by Docker on Windows) so that plus targeting Raspberry Pi meant that I need to target Azure IoT Edge on Linux. 126 | 127 | 4. I wanted my developer experience to mirror the devices I was targeting plus I needed Docker support for the USB webcam, so I developed the solution on my Ubuntu 18.04 developer desktop. See my [Ubuntu for Azure Developers](https://gloveboxes.github.io/Ubuntu-for-Azure-Developers/) guide. 128 | 129 | As a workaround, if your development device is locked to Windows then use Ubuntu in Virtual Box which allows USB device pass-through which you can then pass-through to Docker in the Virtual Machine. A bit convoluted but it does work. 130 | 131 | ![raspberry pi image classifier](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/raspberry-pi-image-classifier.jpg) 132 | 133 | # Azure Services 134 | 135 | ## Creating the Fruit Classification Model 136 | 137 | The [Azure Custom Vision](https://docs.microsoft.com/azure/cognitive-services/custom-vision-service/?WT.mc_id=iot-0000-dglover) service is a simple way to create an image classification machine learning model without having to be a data science or machine learning expert. You simply upload multiple collections of labelled images. For example, you could upload a collection of banana images and label them as 'banana'. 138 | 139 | To create your own classification model read [How to build a classifier with Custom Vision](https://docs.microsoft.com/azure/cognitive-services/custom-vision-service/getting-started-build-a-classifier/?WT.mc_id=iot-0000-dglover) for more information. It is important to have a good variety of labelled images so be sure to read [How to improve your classifier](https://docs.microsoft.com/azure/cognitive-services/custom-vision-service/getting-started-improving-your-classifier/?WT.mc_id=iot-0000-dglover). 140 | 141 | ## Exporting an Azure Custom Vision Model 142 | 143 | This "Image Classification" module includes a simple fruit classification model that was exported from Azure Custom Vision. For more information read how to [Export your model for use with mobile devices](https://docs.microsoft.com/azure/cognitive-services/custom-vision-service/export-your-model/?WT.mc_id=iot-0000-dglover). It is important to select one of the "**compact**" domains from the project settings page otherwise you will not be able to export the model. 144 | 145 | Follow these steps to export your Custom Vision project model. 146 | 147 | 1. From the **Performance** tab of your Custom Vision project click **Export**. 148 | 149 | ![export model](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/exportmodel.png) 150 | 151 | 2. Select Dockerfile from the list of available options 152 | 153 | ![export-as-docker.png](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/export-as-docker.png) 154 | 155 | 3. Then select the Linux version of the Dockerfile. 156 | 157 | ![choose docker](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/export-choose-your-platform.png) 158 | 159 | 4. Download the docker file and unzip and you have a ready-made Docker solution with a Python Flask REST API. This was how I created the Azure IoT Edge Image Classification module in this solution. Too easy:) 160 | 161 | ## Azure Speech Services 162 | 163 | [Azure Speech Services](https://docs.microsoft.com/azure/cognitive-services/speech-service/overview/?WT.mc_id=iot-0000-dglover) supports both "speech to text" and "text to speech". For this solution, I'm using the text to speech (F0) free tier which is limited to 5 million characters per month. You will need to add the Speech service using the Azure Portal and "Grab your key" from the service. 164 | 165 | ![azure speech service](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/speech-service.png) 166 | 167 | Open the deployment.template.json file and update the BingKey with the key you copied from the Azure Speech service. 168 | 169 | ![speech key](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/speech-key.png) 170 | 171 | # How to install, build and deploy the solution 172 | 173 | 1. Clone this GitHub repository. 174 | 175 | ```bash 176 | git clone https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services.git 177 | ``` 178 | 179 | 2. Install the Azure IoT Edge runtime on your Linux desktop or device (eg Raspberry Pi). 180 | 181 | Follow the instructions to [Deploy your first IoT Edge module to a Linux x64 device](https://docs.microsoft.com/azure/iot-edge/quickstart-linux/?WT.mc_id=iot-0000-dglover). 182 | 183 | 3. Install the following software development tools. 184 | 185 | 1. [Visual Studio Code](https://code.visualstudio.com/?WT.mc_id=iot-0000-dglover) 186 | 2. Plus, the following Visual Studio Code Extensions 187 | - [Azure IoT Edge](https://marketplace.visualstudio.com/items?itemName=vsciot-vscode.azure-iot-edge&WT.mc_id=iot-0000-dglover) 188 | - [JSON Tools](https://marketplace.visualstudio.com/items?itemName=eriklynd.json-tools&WT.mc_id=iot-0000-dglover) useful for changing the "Create Options" for a module. 189 | 3. [Docker Community Edition](https://docs.docker.com/install/) on your development machine 190 | 191 | 4. With Visual Studio Code, open the IoT Edge solution you cloned from GitHub to your developer desktop. 192 | 193 | ## Understanding the Project Structure 194 | 195 | The following describes the highlighted sections of the project. 196 | 197 | 1. There are two modules: CameraCaptureOpenCV and ImageClassifierService. 198 | 199 | 2. The module.json file defines the Docker build process, the module version, and your docker registry. Updating the version number, pushing the updated module to an image registry, and updating the deployment manifest for an edge device triggers the Azure IoT Edge runtime to pull down the new module to the edge device. 200 | 201 | 3. The deployment.template.json file is used by the build process. It defines what modules to build, what message routes to set up, and what version of the IoT Edge runtime to run. 202 | 203 | 4. The deployment.json file is generated from the deployment.template.json and is the [Deployment Manifest](https://docs.microsoft.com/azure/iot-edge/module-composition/?WT.mc_id=iot-0000-dglover) 204 | 205 | 5. The version.py in the project root folder is a helper app you can run on your development machine that updates the version number of each module. Useful as a change in the version number is what triggers Azure IoT Edge runtime to pull the updated module and it is easy to forget to change the module version numbers:) 206 | 207 | ![visual studio code project structure](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/visual-studio-code-open-project.png) 208 | 209 | ## Building the Solution 210 | 211 | You need to ensure the image you plan to build matches the target processor architecture specified in the deployment.template.json file. 212 | 213 | 1. Specify your Docker repository in the module.json file for each module. If you are using a supported Linux Azure IoT Edge Distribution, such as Ubuntu 18.04 as your development machine and you have Azure IoT Edge installed locally then I strongly recommend setting up a local Docker Registry. It will significantly speed up your development, deployment and test cycle. 214 | 215 | To set up a local Docker Registry for prototyping and testing purposes. 216 | 217 | ```bash 218 | docker run -d -p 5000:5000 --restart always --name registry registry:2 219 | ``` 220 | 3. If pushing the image to a local Docker repository the specify localhost:5000. 221 | ```json 222 | "repository": "localhost:5000/camera-capture-opencv" 223 | ``` 224 | 4. Confirm processor architecture you plan to build for. 225 | From the Visual Studio Code bottom bar click the currently selected processor architecture, then from the popup select the desired processor architecture. 226 | ![](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/select-processor-architecture.jpg) 227 | 228 | 6. Next, Build and Push the solution to Docker by right mouse clicking the deployment.template.json file and select "**Build and Push IoT Edge Solution**". The first build will be slow as Docker needs to pull the base layers to your local machine. If you are cross compiling to arm32v7 then the first build will be very slow as OpenCV and Python requirements need to be compiled. On a fast Intel i7-8750H processor cross compiling this solution will take approximately 40 minutes. 229 | 230 | ![docker build and push](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/solution-build-push-docker.png) 231 | 232 | ## Deploying the Solution 233 | 234 | When the Docker Build and Push process has completed select the Azure IoT Hub device you want to deploy the solution to. Right mouse click the deployment.json file found in the config folder and select the target device from the drop-down list. 235 | 236 | ![deploy to device](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/deploy-to-device.png) 237 | 238 | ## Monitoring the Solution on the IoT Edge Device 239 | 240 | Once the solution has been deployed you can monitor it on the IoT Edge device itself using the ```iotedge list``` command. 241 | 242 | ```bash 243 | iotedge list 244 | ``` 245 | 246 | ![watch iotedge list](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/iotedge-list.png) 247 | 248 | ## Monitoring the Solution from the Azure IoT Edge Blade 249 | 250 | You can monitor the state of the Azure IoT Edge module from the Azure IoT Hub blade on the [Azure Portal](https://portal.azure.com/?WT.mc_id=iot-0000-dglover). 251 | 252 | ![azure iot edge devices](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/azure-iotedge-monitoring.png) 253 | 254 | Click on the device from the Azure IoT Edge blade to view more details about the modules running on the device. 255 | 256 | ![azure iot edge device details](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/azure-portal-iotedge-device-details.png) 257 | 258 | # Done! 259 | 260 | When the solution is finally deployed to the IoT Edge device the system will start telling you what items it thinks have been scanned. 261 | 262 | Congratulations you have deployed your first Azure IoT Edge Solution! 263 | 264 | ![congratulations](https://github.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/raw/master/docs/congratulations.jpg) 265 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-dinky -------------------------------------------------------------------------------- /config/deployment.amd64.json: -------------------------------------------------------------------------------- 1 | { 2 | "modulesContent": { 3 | "$edgeAgent": { 4 | "properties.desired": { 5 | "schemaVersion": "1.0", 6 | "runtime": { 7 | "type": "docker", 8 | "settings": { 9 | "minDockerVersion": "v1.25", 10 | "loggingOptions": "", 11 | "registryCredentials": {} 12 | } 13 | }, 14 | "systemModules": { 15 | "edgeAgent": { 16 | "type": "docker", 17 | "settings": { 18 | "image": "mcr.microsoft.com/azureiotedge-agent:1.0.8", 19 | "createOptions": "{}" 20 | } 21 | }, 22 | "edgeHub": { 23 | "type": "docker", 24 | "status": "running", 25 | "restartPolicy": "always", 26 | "settings": { 27 | "image": "mcr.microsoft.com/azureiotedge-hub:1.0.8", 28 | "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" 29 | } 30 | } 31 | }, 32 | "modules": { 33 | "camera-capture": { 34 | "version": "1.0", 35 | "type": "docker", 36 | "status": "running", 37 | "restartPolicy": "always", 38 | "settings": { 39 | "image": "localhost:5000/camera-capture-opencv:1.1.126-amd64", 40 | "createOptions": "{\"Env\":[\"Video=0\",\"azureSpeechServicesKey=2f57f2d9f1074faaa0e9484e1f1c08c1\",\"AiEndpoint=http://image-classifier-service:80/image\"],\"HostConfig\":{\"PortBindings\":{\"5678/tcp\":[{\"HostPort\":\"5678\"}]},\"Devices\":[{\"PathOnHost\":\"/dev/video0\",\"PathInContainer\":\"/dev/video0\",\"CgroupPermissions\":\"mrw\"},{\"PathOnHost\":\"/dev/snd\",\"PathInContainer\":\"/dev/snd\",\"CgroupPermissions\":\"mrw\"}]}}" 41 | } 42 | }, 43 | "image-classifier-service": { 44 | "version": "1.0", 45 | "type": "docker", 46 | "status": "running", 47 | "restartPolicy": "always", 48 | "settings": { 49 | "image": "localhost:5000/image-classifier-service:1.1.111-amd64", 50 | "createOptions": "{\"HostConfig\":{\"Binds\":[\"/home/pi/images:/images\"],\"PortBindings\":{\"8000/tcp\":[{\"HostPort\":\"80\"}],\"5679/tcp\":[{\"HostPort\":\"5679\"}]}}}" 51 | } 52 | } 53 | } 54 | } 55 | }, 56 | "$edgeHub": { 57 | "properties.desired": { 58 | "schemaVersion": "1.0", 59 | "routes": { 60 | "camera-capture": "FROM /messages/modules/camera-capture/outputs/output1 INTO $upstream" 61 | }, 62 | "storeAndForwardConfiguration": { 63 | "timeToLiveSecs": 7200 64 | } 65 | } 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /config/deployment.arm32v7.json: -------------------------------------------------------------------------------- 1 | { 2 | "modulesContent": { 3 | "$edgeAgent": { 4 | "properties.desired": { 5 | "schemaVersion": "1.0", 6 | "runtime": { 7 | "type": "docker", 8 | "settings": { 9 | "minDockerVersion": "v1.25", 10 | "loggingOptions": "", 11 | "registryCredentials": {} 12 | } 13 | }, 14 | "systemModules": { 15 | "edgeAgent": { 16 | "type": "docker", 17 | "settings": { 18 | "image": "mcr.microsoft.com/azureiotedge-agent:1.0.8", 19 | "createOptions": "{}" 20 | } 21 | }, 22 | "edgeHub": { 23 | "type": "docker", 24 | "status": "running", 25 | "restartPolicy": "always", 26 | "settings": { 27 | "image": "mcr.microsoft.com/azureiotedge-hub:1.0.8", 28 | "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" 29 | } 30 | } 31 | }, 32 | "modules": { 33 | "camera-capture": { 34 | "version": "1.0", 35 | "type": "docker", 36 | "status": "running", 37 | "restartPolicy": "always", 38 | "settings": { 39 | "image": "glovebox/camera-capture-opencv:1.1.128-arm32v7", 40 | "createOptions": "{\"Env\":[\"Video=0\",\"azureSpeechServicesKey=2f57f2d9f1074faaa0e9484e1f1c08c1\",\"AiEndpoint=http://image-classifier-service:80/image\",\"SpeechMapFilename=speech_map_australian.json\"],\"HostConfig\":{\"PortBindings\":{\"5678/tcp\":[{\"HostPort\":\"5678\"}]},\"Devices\":[{\"PathOnHost\":\"/dev/video0\",\"PathInContainer\":\"/dev/video0\",\"CgroupPermissions\":\"mrw\"},{\"PathOnHost\":\"/dev/snd\",\"PathInContainer\":\"/dev/snd\",\"CgroupPermissions\":\"mrw\"}]}}" 41 | } 42 | }, 43 | "image-classifier-service": { 44 | "version": "1.0", 45 | "type": "docker", 46 | "status": "running", 47 | "restartPolicy": "always", 48 | "settings": { 49 | "image": "glovebox/image-classifier-service:1.1.111-arm32v7", 50 | "createOptions": "{\"HostConfig\":{\"Binds\":[\"/home/pi/images:/images\"],\"PortBindings\":{\"8000/tcp\":[{\"HostPort\":\"80\"}],\"5679/tcp\":[{\"HostPort\":\"5679\"}]}}}" 51 | } 52 | } 53 | } 54 | } 55 | }, 56 | "$edgeHub": { 57 | "properties.desired": { 58 | "schemaVersion": "1.0", 59 | "routes": { 60 | "camera-capture": "FROM /messages/modules/camera-capture/outputs/output1 INTO $upstream" 61 | }, 62 | "storeAndForwardConfiguration": { 63 | "timeToLiveSecs": 7200 64 | } 65 | } 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /deployment.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "1.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.0.8", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.0.8", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "modules": { 54 | "camera-capture": { 55 | "version": "1.0", 56 | "type": "docker", 57 | "status": "running", 58 | "restartPolicy": "always", 59 | "settings": { 60 | "image": "${MODULES.CameraCaptureOpenCV}", 61 | "createOptions": { 62 | "Env": [ 63 | "Video=0", 64 | "azureSpeechServicesKey=2f57f2d9f1074faaa0e9484e1f1c08c1", 65 | "AiEndpoint=http://image-classifier-service:80/image", 66 | "SpeechMapFilename=speech_map_australian.json" 67 | ], 68 | "HostConfig": { 69 | "PortBindings": { 70 | "5678/tcp": [ 71 | { 72 | "HostPort": "5678" 73 | } 74 | ] 75 | }, 76 | "Devices": [ 77 | { 78 | "PathOnHost": "/dev/video0", 79 | "PathInContainer": "/dev/video0", 80 | "CgroupPermissions": "mrw" 81 | }, 82 | { 83 | "PathOnHost": "/dev/snd", 84 | "PathInContainer": "/dev/snd", 85 | "CgroupPermissions": "mrw" 86 | } 87 | ] 88 | } 89 | } 90 | } 91 | }, 92 | "image-classifier-service": { 93 | "version": "1.0", 94 | "type": "docker", 95 | "status": "running", 96 | "restartPolicy": "always", 97 | "settings": { 98 | "image": "${MODULES.ImageClassifierService}", 99 | "createOptions": { 100 | "HostConfig": { 101 | "Binds": [ 102 | "/home/pi/images:/images" 103 | ], 104 | "PortBindings": { 105 | "8000/tcp": [ 106 | { 107 | "HostPort": "80" 108 | } 109 | ], 110 | "5679/tcp": [ 111 | { 112 | "HostPort": "5679" 113 | } 114 | ] 115 | } 116 | } 117 | } 118 | } 119 | } 120 | } 121 | } 122 | }, 123 | "$edgeHub": { 124 | "properties.desired": { 125 | "schemaVersion": "1.0", 126 | "routes": { 127 | "camera-capture": "FROM /messages/modules/camera-capture/outputs/output1 INTO $upstream" 128 | }, 129 | "storeAndForwardConfiguration": { 130 | "timeToLiveSecs": 7200 131 | } 132 | } 133 | } 134 | } 135 | } -------------------------------------------------------------------------------- /docs/Architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/Architecture.jpg -------------------------------------------------------------------------------- /docs/Architecture.vsdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/Architecture.vsdx -------------------------------------------------------------------------------- /docs/Azure IoT Edge, Python Azure Functions and SignalR.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/Azure IoT Edge, Python Azure Functions and SignalR.pdf -------------------------------------------------------------------------------- /docs/Azure IoT Edge, Python Azure Functions and SignalR.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/Azure IoT Edge, Python Azure Functions and SignalR.pptx -------------------------------------------------------------------------------- /docs/Creating an image recognition solution with Azure IoT Edge and Azure Cognitive Services.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/Creating an image recognition solution with Azure IoT Edge and Azure Cognitive Services.pptx -------------------------------------------------------------------------------- /docs/README.1.md: -------------------------------------------------------------------------------- 1 | # Developing IoT Edge Solutions on Linux 2 | 3 | Focus of this article are my experiences building an Azure IoT Edge solution with Python and targeting IoT Edge running on a Linux Host, both desktop x64 and arm32v7 (Raspberry Pi). 4 | 5 | This article should be read in conjunction with the [Azure IoT Edge documents and tutorials](https://docs.microsoft.com/en-us/azure/iot-edge/). 6 | 7 | ## Developer Productivity 8 | 9 | Building for IoT Edge is not too dissimilar from building a mobile app as there is a level of indirection as soon as you start developing, deploying and testing your code to a device. Everything becomes slower from deploying code across the network, to starting the code on a lower powered device, to remotely attaching a debugger. My recommendation is that you aim to build and test as much as you can on your developer machine before you start deploying and testing on an Iot Edge device. 10 | 11 | ### Solution architecture 12 | 13 | Think of your solution as a set of (micro) services with defined contracts that can communicate via messages or REST calls. In the world of IoT Edge these are called modules and are synonymous with Docker containers. You should code defensively across network boundaries and plan for failure using various [circuit breaker](https://en.wikipedia.org/wiki/Circuit_breaker_design_pattern)/backoff strategies because at some point the services you call will not be available for a number of reasons including but not limited to service updates. 14 | 15 | The following example is the approach I took for backing off requests to the Image Classification Service. 16 | 17 | ```python 18 | retry = 0 19 | while retry < maxRetry: 20 | try: 21 | response = requests.post(self.imageProcessingEndpoint, headers=headers, 22 | params=self.imageProcessingParams, data=frame) 23 | break 24 | except: 25 | retry = retry + 1 26 | print('Image Classification REST Endpoint - Retry attempt # ' + str(retry)) 27 | time.sleep(retry) 28 | 29 | if retry >= maxRetry: 30 | return [] 31 | ``` 32 | 33 | ### Naming Convention 34 | 35 | As a general rule use lower case names for modules. This is especially important if making REST calls from Python modules as the Python Requests 36 | package and the get call will lower case the endpoint name and the IoT Edge runtime will not resolve the request against modules that use mixed case names. 37 | 38 | 1. Environment variables - UPPERCASE 39 | 1. Modules Names in lowercase 40 | 41 | Modules you need to write and modules you don’t 42 | 43 | Microsoft 1st party 44 | 45 | - https://docs.microsoft.com/en-au/azure/stream-analytics/stream-analytics-edge 46 | - https://docs.microsoft.com/en-us/azure/iot-edge/how-to-store-data-blob 47 | - https://docs.microsoft.com/en-us/azure/iot-edge/how-to-ci-cd 48 | 49 | IoT Edge Extensions 50 | 51 | Use the Visual Studio IoT Edge Extension to scaffold your solution. This will create your project structure, the modules, dockerfiles and the deployment.json file which instructs IoT Edge what modules to load their version and location to pull from. 52 | 53 | There is nothing to stop you closing the overall solution and opening each individual module in Visual Studio Code. It can make the development simpler, and enable others to work on other modules more easily. 54 | 55 | Building for IoT edge is like building for mobile, the code, deploy, test, debug cycle is elongated as there is a degree of indirection. Everything takes longer as you typically need to get the code to the device, starting a remote debugging session takes longer, stepping through the code over a network connect is slower etc - you get the idea. 56 | 57 | Exceptions 58 | 59 | Ensure all REST calls to modules are wrapped in exception handlers with some sort of retry/back off mechanism as modules maybe be in indeterminate states - it could be starting after a system boot, it might be being updated and the IoT Edge runtime has torn down the existing module in readiness to start the new version of the module. 60 | 61 | Build as much as you can on your development PC 62 | 63 | So when building a solution develop as much as you can on your developer desktop and mock out physical sensors that you may not have access to on your desktop. The easiest way is to set an environment variable to toggle between mocked and real sensors. Make a note of all of the Python PIP packages you use as you'll need to added them to the docker pip requirements file when you come to build the docker image. 64 | 65 | Mock out REST APIs with Function Proxies 66 | 67 | See Dean's article on Function proxies 68 | 69 | Linux example 70 | 71 | Windows example 72 | 73 | Python example with .env file 74 | 75 | So you have a viable module to start testing with IoT Edge 76 | 77 | First you need to Dockerise your module. When you created a module with IoT Edge it also created dockerfile as a start point for your solution. 78 | You'll need to customize the dockerfile to ensure all the prerequisites libraries and Python packages for the module are loaded in to the image. 79 | 80 | If you are cross compiling for another processor architecture then allow a lot of time for building an image - the example … 81 | took close to an hour to build. 82 | 83 | Dockerise 84 | 85 | If possible then use the same Docker base image as it will reduce the number of docker layers that need to be distributed to a device. 86 | 87 | https://hub.docker.com/r/microsoft/azureiotedge-azure-stream-analytics/ 88 | 89 | ## IoT Edge Module Create Options 90 | 91 | Build local, then Dockerise then wrap up as an IoT Module 92 | 93 | 1. Run a Local Docker Registry 94 | 95 | - https://docs.docker.com/registry/deploying/ 96 | - docker run -d -p 5000:5000 --restart=always --name registry registry:2 97 | 98 | ## Disk Space Management 99 | 100 | Docker images accumulate fast and gobble up disk space. So run a regular job to clean up old containers and images. 101 | 102 | ## Debugging 103 | 104 | 1. Python 4 and multi Threaded 105 | 2. https://code.visualstudio.com/docs/python/debugging 106 | 3. [Flask Apps](https://code.visualstudio.com/docs/python/debugging#_flask-debugging) 107 | 4. Local into a container running in the context of Iot Edge 108 | 109 | ## Microsoft 1st Party Modules 110 | -------------------------------------------------------------------------------- /docs/README.old.md: -------------------------------------------------------------------------------- 1 | while (1) {clear; docker ps; sleep 5} 2 | 3 | import base64 4 | base64.b64encode(frame) 5 | 6 | --- 7 | services: iot-edge, custom-vision-service, iot-hub 8 | platforms: python 9 | author: ebertrams 10 | --- 11 | 12 | # Custom Vision + Azure IoT Edge on a Raspberry Pi 3 13 | 14 | This is a sample showing how to deploy a Custom Vision model to a Raspberry Pi 3 device running Azure IoT Edge. This solution is made of 3 modules: 15 | 16 | - **Camera capture** - this module captures the video stream from a USB camera, sends the frames for analysis to the custom vision module and shares the output of this analysis to the edgeHub. This module is written in python and uses [OpenCV](https://opencv.org/) to read the video feed. 17 | - **Custom vision** - it is a web service over HTTP running locally that takes in images and classifies them based on a custom model built via the [Custom Vision website](https://azure.microsoft.com/en-us/services/cognitive-services/custom-vision-service/). This module has been exported from the Custom Vision website and slightly modified to run on a ARM architecture. You can modify it by updating the model.pb and label.txt files to update the model. 18 | - **SenseHat display** - this module gets messages from the edgeHub and blinks the raspberry Pi's senseHat according to the tags specified in the inputs messages. This module is written in python and requires a [SenseHat](https://www.raspberrypi.org/products/sense-hat/) to work. 19 | 20 | ## Get started 21 | 1- Update the module.json files of the 3 modules above to point to your own Azure Container Registry 22 | 23 | 2- Build the full solution by running the `Build IoT Edge Solution` command from the [Azure IoT Edge extension in VS Code](https://marketplace.visualstudio.com/items?itemName=vsciot-vscode.azure-iot-edge). 24 | 25 | ## Prerequisites 26 | 27 | You can run this solution on either of the following hardware: 28 | 29 | - **Raspberry Pi 3**: Set up Azure IoT Edge on a Raspberry Pi 3 ([instructions](https://blog.jongallant.com/2017/11/azure-iot-edge-raspberrypi/)) with a [SenseHat](https://www.raspberrypi.org/products/sense-hat/) and use the arm32v7 module tags. 30 | 31 | - **Simulated Azure IoT Edge device** (such as a PC): Set up Azure IoT Edge ([instructions on Windows](https://docs.microsoft.com/en-us/azure/iot-edge/tutorial-simulate-device-windows), [instructions on Linux](https://docs.microsoft.com/en-us/azure/iot-edge/tutorial-simulate-device-linux)) and use the amd64 module tags. 32 | -------------------------------------------------------------------------------- /docs/SystemArchitecture.vsdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/SystemArchitecture.vsdx -------------------------------------------------------------------------------- /docs/azure-iotedge-monitoring.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/azure-iotedge-monitoring.png -------------------------------------------------------------------------------- /docs/azure-portal-iotedge-device-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/azure-portal-iotedge-device-details.png -------------------------------------------------------------------------------- /docs/build-architecture.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/build-architecture.pdf -------------------------------------------------------------------------------- /docs/build-architecture.vsdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/build-architecture.vsdx -------------------------------------------------------------------------------- /docs/congratulations.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/congratulations.jpg -------------------------------------------------------------------------------- /docs/custom-vision-domain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/custom-vision-domain.png -------------------------------------------------------------------------------- /docs/deploy-to-device.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/deploy-to-device.png -------------------------------------------------------------------------------- /docs/export-as-docker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/export-as-docker.png -------------------------------------------------------------------------------- /docs/export-choose-your-platform.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/export-choose-your-platform.png -------------------------------------------------------------------------------- /docs/exportmodel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/exportmodel.png -------------------------------------------------------------------------------- /docs/index-1.html: -------------------------------------------------------------------------------- 1 | Unknown 2 |

Developing on Linux

3 |

Gotchas

4 |
    5 |
  1. Naming Convention
    2. 6 |
7 |

Developer Productivity

8 |
    9 |
  1. Crawl, Walk, Run
    2. 10 |
  2. Module Versioning
    3. 11 |
12 |

IoT Edge Module Create Options

13 |

Build local, then Dockerise then wrap up as an IoT Module

14 |
    15 |
  1. Run a Local Docker Registry
      16 |
    • https://docs.docker.com/registry/deploying/
      • 17 |
    • docker run -d -p 5000:5000 --restart=always --name registry registry:2
      • 18 |
    19 |
    2. 20 |
21 |

Disk Space Management

22 |

Docker images accumulate fast and gobble up disk space. So run a regular job to clean up old containers and images.

23 |

Debugging

24 |
    25 |
  1. Python 4 and multi Threaded
    2. 26 |
  2. https://code.visualstudio.com/docs/python/debugging
    3. 27 |
  3. 28 |

    Flask Apps

    29 |
    4. 30 |
  4. 31 |

    Local into a container running in the context of Iot Edge

    32 |
    5. 33 |
34 |

Microsoft 1st Party Modules

35 | -------------------------------------------------------------------------------- /docs/iot-edge-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/iot-edge-arch.png -------------------------------------------------------------------------------- /docs/iot-edge-in-action.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/iot-edge-in-action.jpg -------------------------------------------------------------------------------- /docs/iotedge-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/iotedge-architecture.png -------------------------------------------------------------------------------- /docs/iotedge-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/iotedge-list.png -------------------------------------------------------------------------------- /docs/raspberry-pi-3a-image-classifier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/raspberry-pi-3a-image-classifier.png -------------------------------------------------------------------------------- /docs/raspberry-pi-3a-image-classifier.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/raspberry-pi-3a-image-classifier.xcf -------------------------------------------------------------------------------- /docs/raspberry-pi-image-classifier.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/raspberry-pi-image-classifier.jpg -------------------------------------------------------------------------------- /docs/select-processor-architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/select-processor-architecture.jpg -------------------------------------------------------------------------------- /docs/solution-build-push-docker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/solution-build-push-docker.png -------------------------------------------------------------------------------- /docs/speech-key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/speech-key.png -------------------------------------------------------------------------------- /docs/speech-service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/speech-service.png -------------------------------------------------------------------------------- /docs/visual-studio-code-open-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/docs/visual-studio-code-open-project.png -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/Dockerfile.amd64: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | 3 | WORKDIR /app 4 | 5 | RUN apt-get update && \ 6 | apt-get install -y --no-install-recommends libcurl4-openssl-dev \ 7 | python3-pip python3-dev python3-numpy python-opencv build-essential \ 8 | libgtk2.0-dev libboost-python-dev git portaudio19-dev && \ 9 | rm -rf /var/lib/apt/lists/* 10 | 11 | RUN pip3 install --upgrade setuptools && pip3 install --upgrade pip 12 | # RUN python -m pip install --upgrade pip setuptools wheel 13 | COPY /build/amd64-requirements.txt ./ 14 | 15 | RUN pip3 install -r amd64-requirements.txt 16 | 17 | RUN apt-get update && \ 18 | apt-get install -y libsdl1.2-dev libsdl-image1.2-dev libsdl-mixer1.2-dev && \ 19 | rm -rf /var/lib/apt/lists/* 20 | 21 | ADD /app/ . 22 | # ADD /build/ . 23 | 24 | ENV PYTHONUNBUFFERED=1 25 | 26 | EXPOSE 5678 27 | 28 | CMD [ "python3", "-u", "./iotedge_camera.py" ] -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/Dockerfile.amd64.debug: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | 3 | WORKDIR /app 4 | 5 | RUN apt-get update && \ 6 | apt-get install -y --no-install-recommends libcurl4-openssl-dev \ 7 | python3-pip python3-dev python3-numpy python-opencv build-essential \ 8 | libgtk2.0-dev libboost-python-dev git portaudio19-dev && \ 9 | rm -rf /var/lib/apt/lists/* 10 | 11 | RUN pip3 install --upgrade setuptools && pip3 install --upgrade pip 12 | # RUN python -m pip install --upgrade pip setuptools wheel 13 | COPY /build/amd64-requirements.txt ./ 14 | 15 | RUN pip3 install -r amd64-requirements.txt 16 | 17 | RUN pip3 install pyaudio wave 18 | 19 | ADD /app/ . 20 | # ADD /build/ . 21 | 22 | EXPOSE 5678 23 | 24 | ENV PYTHONUNBUFFERED=1 25 | 26 | CMD [ "python3", "-u", "./main.py" ] -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/Dockerfile.arm32v7: -------------------------------------------------------------------------------- 1 | FROM balenalib/raspberrypi3:stretch 2 | 3 | # Enable cross building of ARM on x64 hardware, Remove this and the cross-build-end if building on ARM hardware. 4 | RUN [ "cross-build-start" ] 5 | 6 | # Install dependencies 7 | RUN apt-get update && apt-get install -y --no-install-recommends \ 8 | libatlas3-base libsz2 libharfbuzz0b libtiff5 libjasper1 libilmbase12 \ 9 | libopenexr22 libilmbase12 libgstreamer1.0-0 libavcodec57 libavformat57 \ 10 | libavutil55 libswscale4 libqtgui4 libqt4-test libqtcore4 \ 11 | libboost-python-dev python3-pip git wget \ 12 | python3-numpy build-essential libhdf5-100 portaudio19-dev \ 13 | && rm -rf /var/lib/apt/lists/* \ 14 | && apt-get -y autoremove 15 | 16 | RUN export PIP_DEFAULT_TIMEOUT=100 17 | RUN pip3 install --upgrade setuptools && pip3 install --upgrade pip 18 | RUN pip3 install azure-iothub-device-client opencv-contrib-python requests ptvsd requests pyaudio wave 19 | 20 | # Raspberry Kernel 4.19 Sound Issues - Stretch and Buster 21 | # Kernel 4.19 Sound & latest updates... https://www.raspberrypi.org/forums/viewtopic.php?t=241814 22 | 23 | ENV PA_ALSA_PLUGHW=1 24 | 25 | WORKDIR /app 26 | 27 | COPY /app/*.py ./ 28 | 29 | # disable python buffering to console out (https://docs.python.org/2/using/cmdline.html#envvar-PYTHONUNBUFFERED) 30 | ENV PYTHONUNBUFFERED=1 31 | 32 | EXPOSE 5678 33 | 34 | RUN [ "cross-build-end" ] 35 | 36 | ENTRYPOINT [ "python3", "iotedge_camera.py" ] -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/CameraCapture.py: -------------------------------------------------------------------------------- 1 | # To make python 2 and python 3 compatible code 2 | # from __future__ import division 3 | # from __future__ import absolute_import 4 | 5 | # Imports 6 | import text2speech 7 | from VideoStream import VideoStream 8 | # import VideoStream 9 | import os.path 10 | import base64 11 | import time 12 | import json 13 | import requests 14 | import numpy 15 | import sys 16 | if sys.version_info[0] < 3: # e.g python version <3 17 | import cv2 18 | else: 19 | import cv2 20 | from cv2 import cv2 21 | 22 | 23 | maxRetry = 5 24 | lastTagSpoken = '' 25 | count = 0 26 | 27 | 28 | class CameraCapture(object): 29 | 30 | def __IsInt(self, string): 31 | try: 32 | int(string) 33 | return True 34 | except ValueError: 35 | return False 36 | 37 | def __localize_text(self, key): 38 | value = None 39 | if self.speech_map is not None: 40 | result = list( 41 | filter(lambda text: text['key'] == key, self.speech_map)) 42 | if len(result) > 0: 43 | value = result[0]['value'] 44 | return value 45 | 46 | def __init__( 47 | self, 48 | videoPath, 49 | azureSpeechServiceKey, 50 | predictThreshold, 51 | imageProcessingEndpoint, 52 | sendToHubCallback, 53 | speechMapFileName 54 | ): 55 | self.videoPath = videoPath 56 | 57 | self.predictThreshold = predictThreshold 58 | self.imageProcessingEndpoint = imageProcessingEndpoint 59 | self.imageProcessingParams = "" 60 | self.sendToHubCallback = sendToHubCallback 61 | 62 | 63 | if self.__IsInt(videoPath): 64 | # case of a usb camera (usually mounted at /dev/video* where * is an int) 65 | self.isWebcam = True 66 | 67 | self.vs = None 68 | 69 | self.speech_map = None 70 | self.speech_voice = 'en-AU-Catherine' 71 | 72 | self.speech_map_filename = speechMapFileName 73 | 74 | if speechMapFileName is not None and os.path.isfile(self.speech_map_filename): 75 | with open(self.speech_map_filename, encoding='utf-8') as f: 76 | json_data = json.load(f) 77 | self.speech_voice = json_data.get('voice') 78 | self.speech_map = json_data.get('map') 79 | 80 | self.tts = text2speech.TextToSpeech( 81 | azureSpeechServiceKey, enableMemCache=True, enableDiskCache=True, voice=self.speech_voice) 82 | 83 | text = self.__localize_text('Starting scanner') 84 | self.tts.play('Starting scanner' if text is None else text) 85 | 86 | 87 | def __buildSentence(self, tag): 88 | vowels = ('a', 'e', 'i', 'o', 'u', 'A', 'E', 'I', 'O', 'U') 89 | sentence = 'You scanned ' 90 | if tag.startswith(vowels): 91 | sentence = sentence + 'an ' 92 | else: 93 | sentence = sentence + 'a ' 94 | return sentence + tag 95 | 96 | def __sendFrameForProcessing(self, frame): 97 | global count, lastTagSpoken 98 | count = count + 1 99 | print("sending frame to model: " + str(count)) 100 | 101 | headers = {'Content-Type': 'application/octet-stream'} 102 | 103 | retry = 0 104 | while retry < maxRetry: 105 | try: 106 | response = requests.post(self.imageProcessingEndpoint, headers=headers, 107 | params=self.imageProcessingParams, data=frame) 108 | break 109 | except: 110 | retry = retry + 1 111 | print( 112 | 'Image Classification REST Endpoint - Retry attempt # ' + str(retry)) 113 | time.sleep(retry) 114 | 115 | if retry >= maxRetry: 116 | print("retry inference") 117 | return [] 118 | 119 | predictions = response.json()['predictions'] 120 | sortResponse = sorted( 121 | predictions, key=lambda k: k['probability'], reverse=True)[0] 122 | probability = sortResponse['probability'] 123 | 124 | print("label: {}, probability {}".format( 125 | sortResponse['tagName'], sortResponse['probability'])) 126 | 127 | if sortResponse['tagName'] == 'Hand': 128 | lastTagSpoken = sortResponse['tagName'] 129 | return [] 130 | 131 | if probability > self.predictThreshold and sortResponse['tagName'] != lastTagSpoken: 132 | lastTagSpoken = sortResponse['tagName'] 133 | print('text to speech ' + lastTagSpoken) 134 | 135 | text = self.__localize_text(lastTagSpoken) 136 | self.tts.play(self.__buildSentence(lastTagSpoken) if text is None else text) 137 | 138 | return json.dumps(predictions) 139 | else: 140 | return [] 141 | 142 | def __displayTimeDifferenceInMs(self, endTime, startTime): 143 | return str(int((endTime-startTime) * 1000)) + " ms" 144 | 145 | def __enter__(self): 146 | self.vs = VideoStream(int(self.videoPath)).start() 147 | # needed to load at least one frame into the VideoStream class 148 | time.sleep(1.0) 149 | 150 | return self 151 | 152 | def start(self): 153 | 154 | frameCounter = 0 155 | while True: 156 | frameCounter += 1 157 | frame = self.vs.read() 158 | 159 | if self.imageProcessingEndpoint != "": 160 | 161 | encodedFrame = cv2.imencode(".jpg", frame)[1].tostring() 162 | try: 163 | response = self.__sendFrameForProcessing(encodedFrame) 164 | # print(response) 165 | # forwarding outcome of external processing to the EdgeHub 166 | if response != "[]" and self.sendToHubCallback is not None: 167 | try: 168 | self.sendToHubCallback(response) 169 | except: 170 | print( 171 | 'Issue sending telemetry') 172 | except: 173 | print('connectivity issue') 174 | 175 | # slow things down a bit - 4 frame a second is fine for demo purposes and less battery drain and lower Raspberry Pi CPU Temperature 176 | time.sleep(0.25) 177 | 178 | def __exit__(self, exception_type, exception_value, traceback): 179 | pass 180 | -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/VideoStream.py: -------------------------------------------------------------------------------- 1 | # To make python 2 and python 3 compatible code 2 | from __future__ import absolute_import 3 | 4 | from threading import Thread 5 | import time 6 | import sys 7 | if sys.version_info[0] < 3: # e.g python version <3 8 | import cv2 9 | else: 10 | import cv2 11 | # from cv2 import cv2 12 | 13 | # import the Queue class from Python 3 14 | if sys.version_info >= (3, 0): 15 | from queue import Queue 16 | # otherwise, import the Queue class for Python 2.7 17 | else: 18 | from Queue import Queue 19 | 20 | # This class reads all the video frames in a separate thread and always has the keeps only the latest frame in its queue to be grabbed by another thread 21 | 22 | 23 | class VideoStream(object): 24 | def __init__(self, path, queueSize=3): 25 | print('opening camera') 26 | self.stream = cv2.VideoCapture(0) 27 | # self.stream.set(cv2.CAP_PROP_FRAME_WIDTH, 640) 28 | # self.stream.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) 29 | # self.stream.set(cv2.CAP_PROP_SETTINGS, 1 ) 30 | self.stopped = False 31 | self.Q = Queue(maxsize=queueSize) 32 | 33 | def start(self): 34 | # start a thread to read frames from the video stream 35 | t = Thread(target=self.update, args=()) 36 | t.daemon = True 37 | t.start() 38 | return self 39 | 40 | def update(self): 41 | previousFrame = None 42 | previousDiff = 0 43 | delta = 0 44 | skippedFrames = 0 45 | queuedFrames = 0 46 | 47 | try: 48 | while True: 49 | if self.stopped: 50 | return 51 | 52 | (grabbed, frame) = self.stream.read() 53 | 54 | # if the `grabbed` boolean is `False`, then we have 55 | # reached the end of the video file 56 | if not grabbed: 57 | self.stop() 58 | return 59 | 60 | if previousFrame is None: 61 | previousFrame = frame 62 | continue 63 | 64 | difference = cv2.subtract(frame, previousFrame) 65 | b, g, r = cv2.split(difference) 66 | diff = cv2.countNonZero(b) + cv2.countNonZero(g) + cv2.countNonZero(r) 67 | delta = abs(diff - previousDiff) 68 | 69 | if delta > 80000: 70 | # Clean the queue 71 | while not self.Q.empty(): 72 | self.Q.get() 73 | self.Q.put(frame) 74 | queuedFrames = queuedFrames + 1 75 | 76 | previousFrame = frame 77 | previousDiff = diff 78 | 79 | else: 80 | skippedFrames = skippedFrames + 1 81 | 82 | time.sleep(0.15) 83 | 84 | except Exception as e: 85 | print("got error: "+str(e)) 86 | 87 | def read(self): 88 | return self.Q.get(block=True) 89 | 90 | def more(self): 91 | return self.Q.qsize() > 0 92 | 93 | def stop(self): 94 | self.stopped = True 95 | 96 | def __exit__(self, exception_type, exception_value, traceback): 97 | self.stream.release() 98 | -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/azure_text_speech.py: -------------------------------------------------------------------------------- 1 | # https://docs.microsoft.com/en-us/azure/cognitive-services/speech-service/rest-text-to-speech#get-a-list-of-voices 2 | 3 | 4 | import os 5 | import requests 6 | import time 7 | from xml.etree import ElementTree 8 | 9 | TOKEN_URL = "https://southeastasia.api.cognitive.microsoft.com/sts/v1.0/issuetoken" 10 | BASE_URL = "https://southeastasia.tts.speech.microsoft.com/" 11 | TEXT_TO_SPEECH_PATH = "cognitiveservices/v1" 12 | VOICES_PATH = "cognitiveservices/voices/list" 13 | 14 | 15 | class AzureSpeechServices(object): 16 | # Short name for 'Microsoft Server Speech Text to Speech Voice (en-US, GuyNeural)' 17 | def __init__(self, subscription_key, voice='en-US-GuyNeural'): 18 | self.subscription_key = subscription_key 19 | self.short_voice_name = voice 20 | self.access_token = None 21 | self.access_token_ttl = 0 22 | 23 | def get_token(self): 24 | ''' 25 | The TTS endpoint requires an access token. This method exchanges your 26 | subscription key for an access token that is valid for ten minutes. 27 | 28 | If time is less than 10 minutes then use the cached token 29 | ''' 30 | 31 | try: 32 | if self.subscription_key is None: 33 | return 34 | 35 | if abs(time.time() - self.access_token_ttl) < 10 * 60: 36 | return 37 | 38 | fetch_token_url = TOKEN_URL 39 | headers = { 40 | 'Ocp-Apim-Subscription-Key': self.subscription_key 41 | } 42 | response = requests.post(fetch_token_url, headers=headers) 43 | self.access_token = str(response.text) 44 | self.access_token_ttl = time.time() 45 | except: 46 | self.access_token = None 47 | 48 | def get_voice_list(self): 49 | if self.subscription_key is None: 50 | return 51 | 52 | self.get_token() 53 | 54 | if self.access_token is None: 55 | return None 56 | 57 | constructed_url = BASE_URL + VOICES_PATH 58 | headers = { 59 | 'Authorization': 'Bearer ' + self.access_token 60 | } 61 | 62 | response = requests.get(constructed_url, headers=headers) 63 | if response.status_code == 200: 64 | return response.content 65 | return None 66 | 67 | def get_audio(self, text): 68 | if self.subscription_key is None: 69 | return 70 | 71 | self.get_token() 72 | 73 | if self.access_token is None: 74 | return None 75 | 76 | constructed_url = BASE_URL + TEXT_TO_SPEECH_PATH 77 | headers = { 78 | 'Authorization': 'Bearer ' + self.access_token, 79 | 'Content-Type': 'application/ssml+xml', 80 | 'X-Microsoft-OutputFormat': 'riff-16khz-16bit-mono-pcm', 81 | 'User-Agent': 'YOUR_RESOURCE_NAME' 82 | } 83 | xml_body = ElementTree.Element('speak', version='1.0') 84 | xml_body.set('{http://www.w3.org/XML/1998/namespace}lang', 'en-us') 85 | voice = ElementTree.SubElement(xml_body, 'voice') 86 | voice.set('{http://www.w3.org/XML/1998/namespace}lang', 'en-US') 87 | voice.set('name', self.short_voice_name) 88 | voice.text = text 89 | body = ElementTree.tostring(xml_body) 90 | 91 | response = requests.post(constructed_url, headers=headers, data=body) 92 | 93 | if response.status_code == 200: 94 | return response.content 95 | else: 96 | print("\nStatus code: " + str(response.status_code) + 97 | "\nSomething went wrong. Check your subscription key and headers.\n") 98 | return None 99 | -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/azure_text_translate.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | import uuid 4 | import json 5 | 6 | 7 | class AzureTranslationServices(): 8 | 9 | def __init__(self, azureTranslatorServiceKey, language): 10 | self.azureTranslatorServiceKey = azureTranslatorServiceKey 11 | self.language = language 12 | 13 | def translate(self, text): 14 | try: 15 | # If you want to set your subscription key as a string, uncomment the next line. 16 | #subscriptionKey = 'put_your_key_here' 17 | 18 | # If you encounter any issues with the base_url or path, make sure 19 | # that you are using the latest endpoint: https://docs.microsoft.com/azure/cognitive-services/translator/reference/v3-0-translate 20 | base_url = 'https://api.cognitive.microsofttranslator.com' 21 | path = '/translate?api-version=3.0' 22 | params = '&to={}'.format(self.language) 23 | constructed_url = base_url + path + params 24 | 25 | headers = { 26 | 'Ocp-Apim-Subscription-Key': self.azureTranslatorServiceKey, 27 | 'Content-type': 'application/json', 28 | 'X-ClientTraceId': str(uuid.uuid4()) 29 | } 30 | 31 | # You can pass more than one object in body. 32 | body = [{ 33 | 'text': text 34 | }] 35 | 36 | request = requests.post(constructed_url, headers=headers, json=body) 37 | response = request.json() 38 | if len(response) > 0: 39 | jsonData = response[0].get('translations') 40 | if len(jsonData) > 0: 41 | return jsonData[0]['text'] 42 | 43 | return None 44 | except: 45 | return None 46 | -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/iotedge_camera.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | # Licensed under the MIT license. See LICENSE file in the project root for 3 | # full license information. 4 | 5 | import os 6 | import random 7 | import sys 8 | import time 9 | import ptvsd 10 | 11 | # ptvsd.enable_attach(address=('0.0.0.0', 5678)) 12 | # ptvsd.wait_for_attach() 13 | 14 | 15 | from iothub_client import IoTHubModuleClient, IoTHubClientError, IoTHubTransportProvider 16 | from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError 17 | 18 | import CameraCapture 19 | from CameraCapture import CameraCapture 20 | 21 | 22 | # global counters 23 | SEND_CALLBACKS = 0 24 | 25 | 26 | def send_to_Hub_callback(strMessage): 27 | if strMessage == []: 28 | return 29 | message = IoTHubMessage(bytearray(strMessage, 'utf8')) 30 | prop_map = message.properties() 31 | prop_map.add("appid", "scanner") 32 | hubManager.send_event_to_output("output1", message, 0) 33 | print('sent from send_to_Hub_callback') 34 | 35 | # Callback received when the message that we're forwarding is processed. 36 | 37 | 38 | def send_confirmation_callback(message, result, user_context): 39 | global SEND_CALLBACKS 40 | SEND_CALLBACKS += 1 41 | 42 | 43 | class HubManager(object): 44 | 45 | def __init__( 46 | self, 47 | messageTimeout, 48 | protocol 49 | ): 50 | ''' 51 | Communicate with the Edge Hub 52 | 53 | :param str connectionString: Edge Hub connection string 54 | :param int messageTimeout: the maximum time in milliseconds until a message times out. The timeout period starts at IoTHubClient.send_event_async. By default, messages do not expire. 55 | :param IoTHubTransportProvider protocol: Choose HTTP, AMQP or MQTT as transport protocol. Currently only MQTT is supported. 56 | ''' 57 | 58 | self.messageTimeout = messageTimeout 59 | self.protocol = protocol 60 | 61 | self.client_protocol = self.protocol 62 | self.client = IoTHubModuleClient() 63 | self.client.create_from_environment(protocol) 64 | 65 | self.client.set_option("messageTimeout", self.messageTimeout) 66 | 67 | def send_event_to_output(self, outputQueueName, event, send_context): 68 | self.client.send_event_async( 69 | outputQueueName, event, send_confirmation_callback, send_context) 70 | 71 | 72 | def main( 73 | videoPath, 74 | bingSpeechKey, 75 | predictThreshold, 76 | imageProcessingEndpoint="", 77 | speechMapFileName = None 78 | ): 79 | ''' 80 | Capture a camera feed, send it to processing and forward outputs to EdgeHub 81 | 82 | :param str connectionString: Edge Hub connection string. Mandatory. 83 | :param int videoPath: camera device path such as /dev/video0 or a test video file such as /TestAssets/myvideo.avi. Mandatory. 84 | :param str imageProcessingEndpoint: service endpoint to send the frames to for processing. Example: "http://face-detect-service:8080". Leave empty when no external processing is needed (Default). Optional. 85 | 86 | ''' 87 | try: 88 | print("\nPython %s\n" % sys.version) 89 | print("Camera Capture Azure IoT Edge Module. Press Ctrl-C to exit.") 90 | try: 91 | global hubManager 92 | hubManager = HubManager(10000, IoTHubTransportProvider.MQTT) 93 | except IoTHubError as iothub_error: 94 | print("Unexpected error %s from IoTHub" % iothub_error) 95 | return 96 | with CameraCapture(videoPath, bingSpeechKey, predictThreshold, imageProcessingEndpoint, send_to_Hub_callback, speechMapFileName) as cameraCapture: 97 | cameraCapture.start() 98 | except KeyboardInterrupt: 99 | print("Camera capture module stopped") 100 | 101 | 102 | def __convertStringToBool(env): 103 | if env in ['True', 'TRUE', '1', 'y', 'YES', 'Y', 'Yes']: 104 | return True 105 | elif env in ['False', 'FALSE', '0', 'n', 'NO', 'N', 'No']: 106 | return False 107 | else: 108 | raise ValueError('Could not convert string to bool.') 109 | 110 | 111 | if __name__ == '__main__': 112 | try: 113 | VIDEO_PATH = os.getenv('Video', '0') 114 | PREDICT_THRESHOLD = os.getenv('Threshold', .75) 115 | IMAGE_PROCESSING_ENDPOINT = os.getenv('AiEndpoint') 116 | AZURE_SPEECH_SERVICES_KEY = os.getenv('azureSpeechServicesKey', None) 117 | SPEECH_MAP_FILENAME = os.getenv('SpeechMapFilename', None) 118 | 119 | print(os.getenv('IOTEDGE_AUTHSCHEME')) 120 | 121 | except ValueError as error: 122 | print(error) 123 | sys.exit(1) 124 | 125 | main(VIDEO_PATH, AZURE_SPEECH_SERVICES_KEY, 126 | PREDICT_THRESHOLD, IMAGE_PROCESSING_ENDPOINT, SPEECH_MAP_FILENAME) 127 | -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/speech_map_australian.json: -------------------------------------------------------------------------------- 1 | { 2 | "voice": "en-AU-Catherine", 3 | "map": [ 4 | ] 5 | } -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/speech_map_chinese.json: -------------------------------------------------------------------------------- 1 | { 2 | "voice": "zh-CN-XiaoxiaoNeural", 3 | "map": [ 4 | { 5 | "key": "Green Apple", 6 | "value": "你扫描了一个青苹果" 7 | }, 8 | { 9 | "key": "Red Apple", 10 | "value": "你扫描了一个红苹果" 11 | }, 12 | { 13 | "key": "Orange", 14 | "value": "你扫描了一个橘子" 15 | }, 16 | { 17 | "key": "Banana", 18 | "value": "你扫描了一根香蕉" 19 | }, 20 | { 21 | "key": "Starting scanner", 22 | "value": "启动扫描仪" 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/speech_map_korean.json: -------------------------------------------------------------------------------- 1 | { 2 | "voice": "ko-KR-HeamiRUS", 3 | "map": [ 4 | { 5 | "key": "Green Apple", 6 | "value": "녹색 사과 스캔" 7 | }, 8 | { 9 | "key": "Red Apple", 10 | "value": "빨간 사과 스캔" 11 | }, 12 | { 13 | "key": "Orange", 14 | "value": "오렌지 스캔" 15 | }, 16 | { 17 | "key": "Banana", 18 | "value": "바나나 스캔" 19 | }, 20 | { 21 | "key": "Starting scanner", 22 | "value": "스캐너를 시작합니다" 23 | } 24 | ] 25 | } -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/app/text2speech.py: -------------------------------------------------------------------------------- 1 | from azure_text_speech import AzureSpeechServices 2 | from azure_text_translate import AzureTranslationServices 3 | import time 4 | import hashlib 5 | from pathlib import Path 6 | import os 7 | from datetime import datetime 8 | import pyaudio 9 | import wave 10 | import io 11 | 12 | 13 | class TextToSpeech(): 14 | def __init__(self, azureSpeechServiceKey, voice='en-US-GuyNeural', azureTranslatorServiceKey=None, translateToLanguage=None, enableMemCache=False, enableDiskCache=False): 15 | self.text2Speech = AzureSpeechServices(azureSpeechServiceKey, voice) 16 | self.translateText = AzureTranslationServices( 17 | azureTranslatorServiceKey, translateToLanguage) 18 | 19 | self.azureTranslatorServiceKey = azureTranslatorServiceKey 20 | self.translateToLanguage = translateToLanguage 21 | self.voice = voice 22 | self.enableMemCache = enableMemCache 23 | self.enableDiskCache = enableDiskCache 24 | 25 | self.ttsAudio = {} 26 | 27 | self.startSoundTime = datetime.min 28 | self.soundLength = 0.0 29 | 30 | if not Path('.cache-audio').is_dir(): 31 | os.mkdir('.cache-audio') 32 | 33 | def _playAudio(self, audio): 34 | CHUNK = 1024 35 | 36 | f = io.BytesIO() 37 | f.write(audio) 38 | f.seek(0) 39 | wf = wave.Wave_read(f) 40 | 41 | p = pyaudio.PyAudio() 42 | 43 | stream = p.open(format=p.get_format_from_width(wf.getsampwidth()), 44 | channels=wf.getnchannels(), 45 | rate=wf.getframerate(), 46 | output=True) 47 | 48 | data = wf.readframes(CHUNK) 49 | 50 | while data != b'': 51 | stream.write(data) 52 | data = wf.readframes(CHUNK) 53 | 54 | stream.stop_stream() 55 | stream.close() 56 | p.terminate() 57 | 58 | def play(self, text): 59 | if text is None or text == '': 60 | return 61 | 62 | digestKey = hashlib.md5(text.encode()).hexdigest() 63 | 64 | audio = self.ttsAudio.get(digestKey) if self.enableMemCache else None 65 | 66 | if audio is None: 67 | cacheFileName = "{}-{}.wav".format( 68 | self.voice, digestKey) 69 | 70 | cacheFileName = os.path.join('.cache-audio', cacheFileName) 71 | 72 | if self.enableDiskCache and Path(cacheFileName).is_file(): 73 | with open(cacheFileName, 'rb') as audiofile: 74 | audio = audiofile.read() 75 | else: 76 | if self.azureTranslatorServiceKey is not None and self.translateToLanguage is not None: 77 | translatedText = self.translateText.translate(text) 78 | if translatedText is None: 79 | print( 80 | 'Text to Speech problem: Check internet connection or Translation key or language') 81 | return 82 | else: 83 | translatedText = text 84 | 85 | audio = self.text2Speech.get_audio(translatedText) 86 | if audio is None: 87 | print( 88 | 'Text to Speech problem: Check internet connection or Speech key') 89 | return 90 | 91 | if self.enableDiskCache and audio is not None: 92 | with open(cacheFileName, 'wb') as audiofile: 93 | audiofile.write(audio) 94 | if self.enableMemCache: 95 | self.ttsAudio[digestKey] = audio 96 | 97 | self._playAudio(audio) 98 | -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/build/amd64-requirements.txt: -------------------------------------------------------------------------------- 1 | azure-iothub-device-client 2 | requests 3 | ptvsd 4 | opencv-contrib-python 5 | pyaudio 6 | wave -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/build/arm32v7-requirements.txt: -------------------------------------------------------------------------------- 1 | wheel 2 | opencv-contrib-python 3 | # numpy 4 | azure-iothub-device-client==1.4.0 5 | requests 6 | ptvsd 7 | pyaudio 8 | wave -------------------------------------------------------------------------------- /modules/CameraCaptureOpenCV/module.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-version": "0.0.1", 3 | "description": "", 4 | "image": { 5 | "repository": "glovebox/camera-capture-opencv", 6 | "tag": { 7 | "version": "1.1.128", 8 | "platforms": { 9 | "amd64": "./Dockerfile.amd64", 10 | "amd64.debug": "./Dockerfile.amd64.debug", 11 | "arm32v7": "./Dockerfile.arm32v7" 12 | } 13 | }, 14 | "buildOptions": [] 15 | }, 16 | "language": "python" 17 | } -------------------------------------------------------------------------------- /modules/ImageClassifierService/Dockerfile.amd64: -------------------------------------------------------------------------------- 1 | FROM python:3.5 2 | 3 | RUN pip install --upgrade pip 4 | 5 | COPY /build/amd64-requirements.txt ./ 6 | 7 | RUN export PIP_DEFAULT_TIMEOUT=100 8 | RUN pip3 install --upgrade pip 9 | RUN pip3 install --upgrade setuptools 10 | RUN pip install -r amd64-requirements.txt 11 | 12 | # Expose the port 13 | EXPOSE 80 14 | EXPOSE 5679 15 | 16 | ADD app /app 17 | 18 | # Set the working directory 19 | WORKDIR /app 20 | 21 | # Run the flask server for the endpoints 22 | CMD python -u iotedge_model.py -------------------------------------------------------------------------------- /modules/ImageClassifierService/Dockerfile.amd64.debug: -------------------------------------------------------------------------------- 1 | FROM python:3.5 2 | 3 | ADD app /app 4 | 5 | RUN pip install --upgrade pip 6 | 7 | COPY /build/amd64-requirements.txt ./ 8 | RUN pip install -r amd64-requirements.txt 9 | 10 | # Expose the port 11 | EXPOSE 80 12 | EXPOSE 5679 13 | 14 | # Set the working directory 15 | WORKDIR /app 16 | 17 | # Run the flask server for the endpoints 18 | CMD python -u app.py -------------------------------------------------------------------------------- /modules/ImageClassifierService/Dockerfile.arm32v7: -------------------------------------------------------------------------------- 1 | FROM balenalib/raspberrypi3:stretch 2 | 3 | # Enable cross building of ARM on x64 hardware, Remove this and the cross-build-end if building on ARM hardware. 4 | RUN [ "cross-build-start" ] 5 | 6 | # Install dependencies 7 | RUN apt-get update && apt-get install -y --no-install-recommends \ 8 | python3 \ 9 | python3-pip \ 10 | build-essential \ 11 | python3-dev \ 12 | libopenjp2-7-dev \ 13 | libtiff5-dev \ 14 | zlib1g-dev \ 15 | libjpeg-dev \ 16 | libatlas-base-dev \ 17 | && rm -rf /var/lib/apt/lists/* \ 18 | && apt-get -y autoremove 19 | 20 | # Python dependencies 21 | 22 | # RUN pip3 install --upgrade setuptools && pip3 install --upgrade pip 23 | # RUN pip3 install pillow numpy flask tensorflow ptvsd 24 | 25 | # Python dependencies 26 | RUN export PIP_DEFAULT_TIMEOUT=100 27 | RUN pip3 install --upgrade pip 28 | RUN pip3 install --upgrade setuptools 29 | RUN pip3 install pillow 30 | RUN pip3 install numpy 31 | RUN pip3 install flask 32 | RUN pip3 install tensorflow 33 | 34 | # Add the application 35 | ADD app /app 36 | 37 | # Expose the port 38 | EXPOSE 80 39 | EXPOSE 5679 40 | 41 | # Set the working directory 42 | WORKDIR /app 43 | 44 | # End cross building of ARM on x64 hardware, Remove this and the cross-build-start if building on ARM hardware. 45 | RUN [ "cross-build-end" ] 46 | 47 | # Run the flask server for the endpoints 48 | CMD ["python3","iotedge_model.py"] -------------------------------------------------------------------------------- /modules/ImageClassifierService/app/iotedge_model.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import os 4 | import io 5 | from flask import Flask, request, jsonify 6 | from PIL import Image 7 | 8 | from predict import Predict 9 | 10 | # import ptvsd 11 | # ptvsd.enable_attach(address = ('0.0.0.0', 5679)) 12 | 13 | app = Flask(__name__) 14 | predict = Predict() 15 | 16 | # 4MB Max image size limit 17 | app.config['MAX_CONTENT_LENGTH'] = 4 * 1024 * 1024 18 | 19 | # Default route just shows simple text 20 | @app.route('/') 21 | def index(): 22 | return 'CustomVision.ai model host harness' 23 | 24 | # Like the CustomVision.ai Prediction service /image route handles either 25 | # - octet-stream image file 26 | # - a multipart/form-data with files in the imageData parameter 27 | @app.route('/image', methods=['POST']) 28 | @app.route('//image', methods=['POST']) 29 | @app.route('//image/nostore', methods=['POST']) 30 | @app.route('//classify/iterations//image', methods=['POST']) 31 | @app.route('//classify/iterations//image/nostore', methods=['POST']) 32 | @app.route('//detect/iterations//image', methods=['POST']) 33 | @app.route('//detect/iterations//image/nostore', methods=['POST']) 34 | def predict_image_handler(project=None, publishedName=None): 35 | try: 36 | imageData = None 37 | if ('imageData' in request.files): 38 | imageData = request.files['imageData'] 39 | elif ('imageData' in request.form): 40 | imageData = request.form['imageData'] 41 | else: 42 | imageData = io.BytesIO(request.get_data()) 43 | 44 | img = Image.open(imageData) 45 | results = predict.predict_image(img) 46 | return jsonify(results) 47 | except Exception as e: 48 | print('EXCEPTION:', str(e)) 49 | return 'Error processing image', 500 50 | 51 | 52 | # Like the CustomVision.ai Prediction service /url route handles url's 53 | # in the body of hte request of the form: 54 | # { 'Url': ''} 55 | @app.route('/url', methods=['POST']) 56 | @app.route('//url', methods=['POST']) 57 | @app.route('//url/nostore', methods=['POST']) 58 | @app.route('//classify/iterations//url', methods=['POST']) 59 | @app.route('//classify/iterations//url/nostore', methods=['POST']) 60 | @app.route('//detect/iterations//url', methods=['POST']) 61 | @app.route('//detect/iterations//url/nostore', methods=['POST']) 62 | def predict_url_handler(project=None, publishedName=None): 63 | try: 64 | image_url = json.loads(request.get_data().decode('utf-8'))['url'] 65 | results = predict.predict_url(image_url) 66 | return jsonify(results) 67 | except Exception as e: 68 | print('EXCEPTION:', str(e)) 69 | return 'Error processing image' 70 | 71 | if __name__ == '__main__': 72 | # Run the server 73 | app.run(host='0.0.0.0', port=80) 74 | 75 | -------------------------------------------------------------------------------- /modules/ImageClassifierService/app/labels.txt: -------------------------------------------------------------------------------- 1 | Avocado 2 | Banana 3 | Green Apple 4 | Hand 5 | Orange 6 | Red Apple -------------------------------------------------------------------------------- /modules/ImageClassifierService/app/model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gloveboxes/Creating-an-image-recognition-solution-with-Azure-IoT-Edge-and-Azure-Cognitive-Services/317b910822570ff7cf5b06f49d125a69617d7c54/modules/ImageClassifierService/app/model.pb -------------------------------------------------------------------------------- /modules/ImageClassifierService/app/predict.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | from datetime import datetime 3 | import time 4 | import tensorflow as tf 5 | from PIL import Image 6 | import numpy as np 7 | import sys 8 | 9 | 10 | class Predict(): 11 | 12 | def __init__(self): 13 | 14 | self.filename = 'model.pb' 15 | self.labels_filename = 'labels.txt' 16 | self.network_input_size = 0 17 | self.output_layer = 'loss:0' 18 | self.input_node = 'Placeholder:0' 19 | self.graph_def = tf.compat.v1.GraphDef() 20 | self.labels = [] 21 | self.graph = None 22 | 23 | self._initialize() 24 | 25 | def _initialize(self): 26 | print('Loading model...', end=''), 27 | with tf.io.gfile.GFile(self.filename, 'rb') as f: 28 | self.graph_def.ParseFromString(f.read()) 29 | 30 | tf.import_graph_def(self.graph_def, name='') 31 | self.graph = tf.compat.v1.get_default_graph() 32 | 33 | # Retrieving 'network_input_size' from shape of 'input_node' 34 | input_tensor_shape = self.graph.get_tensor_by_name( 35 | self.input_node).shape.as_list() 36 | 37 | assert len(input_tensor_shape) == 4 38 | assert input_tensor_shape[1] == input_tensor_shape[2] 39 | 40 | self.network_input_size = input_tensor_shape[1] 41 | 42 | with open(self.labels_filename, 'rt') as lf: 43 | self.labels = [l.strip() for l in lf.readlines()] 44 | 45 | def _log_msg(self, msg): 46 | print("{}: {}".format(time.time(), msg)) 47 | 48 | def _resize_to_256_square(self, image): 49 | w, h = image.size 50 | new_w = int(256 / h * w) 51 | image.thumbnail((new_w, 256), Image.ANTIALIAS) 52 | return image 53 | 54 | def _crop_center(self, image): 55 | w, h = image.size 56 | xpos = (w - self.network_input_size) / 2 57 | ypos = (h - self.network_input_size) / 2 58 | box = (xpos, ypos, xpos + self.network_input_size, 59 | ypos + self.network_input_size) 60 | return image.crop(box) 61 | 62 | def _resize_down_to_1600_max_dim(self, image): 63 | w, h = image.size 64 | if h < 1600 and w < 1600: 65 | return image 66 | 67 | new_size = (1600 * w // h, 1600) if (h > w) else (1600, 1600 * h // w) 68 | self._log_msg("resize: " + str(w) + "x" + str(h) + " to " + 69 | str(new_size[0]) + "x" + str(new_size[1])) 70 | if max(new_size) / max(image.size) >= 0.5: 71 | method = Image.BILINEAR 72 | else: 73 | method = Image.BICUBIC 74 | return image.resize(new_size, method) 75 | 76 | def _convert_to_nparray(self, image): 77 | # RGB -> BGR 78 | image = np.array(image) 79 | return image[:, :, (2, 1, 0)] 80 | 81 | def _update_orientation(self, image): 82 | exif_orientation_tag = 0x0112 83 | if hasattr(image, '_getexif'): 84 | exif = image._getexif() 85 | if exif != None and exif_orientation_tag in exif: 86 | orientation = exif.get(exif_orientation_tag, 1) 87 | self._log_msg('Image has EXIF Orientation: ' + 88 | str(orientation)) 89 | # orientation is 1 based, shift to zero based and flip/transpose based on 0-based values 90 | orientation -= 1 91 | if orientation >= 4: 92 | image = image.transpose(Image.TRANSPOSE) 93 | if orientation == 2 or orientation == 3 or orientation == 6 or orientation == 7: 94 | image = image.transpose(Image.FLIP_TOP_BOTTOM) 95 | if orientation == 1 or orientation == 2 or orientation == 5 or orientation == 6: 96 | image = image.transpose(Image.FLIP_LEFT_RIGHT) 97 | return image 98 | 99 | def predict_url(self, imageUrl): 100 | self._log_msg("Predicting from url: " + imageUrl) 101 | with urlopen(imageUrl) as testImage: 102 | image = Image.open(testImage) 103 | return self.predict_image(image) 104 | 105 | def predict_image(self, image): 106 | try: 107 | if image.mode != "RGB": 108 | self._log_msg("Converting to RGB") 109 | image = image.convert("RGB") 110 | 111 | # Update orientation based on EXIF tags 112 | image = self._update_orientation(image) 113 | 114 | image = self._resize_down_to_1600_max_dim(image) 115 | 116 | image = self._resize_to_256_square(image) 117 | 118 | image = self._crop_center(image) 119 | 120 | cropped_image = self._convert_to_nparray(image) 121 | 122 | with self.graph.as_default(): 123 | with tf.Session() as sess: 124 | prob_tensor = sess.graph.get_tensor_by_name( 125 | self.output_layer) 126 | predictions, = sess.run( 127 | prob_tensor, {self.input_node: [cropped_image]}) 128 | 129 | result = [] 130 | for p, label in zip(predictions, self.labels): 131 | truncated_probablity = np.float64(round(p, 8)) 132 | if truncated_probablity > 1e-8: 133 | result.append({ 134 | 'tagName': label, 135 | 'probability': truncated_probablity, 136 | 'tagId': '', 137 | 'boundingBox': None}) 138 | 139 | response = { 140 | 'id': '', 141 | 'project': '', 142 | 'iteration': '', 143 | 'created': datetime.utcnow().isoformat(), 144 | 'predictions': result 145 | } 146 | 147 | return response 148 | 149 | except Exception as e: 150 | self._log_msg(str(e)) 151 | return 'Error: Could not preprocess image for prediction. ' + str(e) 152 | -------------------------------------------------------------------------------- /modules/ImageClassifierService/build/amd64-requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow 2 | pillow 3 | numpy 4 | flask 5 | ptvsd -------------------------------------------------------------------------------- /modules/ImageClassifierService/build/arm32v7-requirements.txt: -------------------------------------------------------------------------------- 1 | pillow 2 | flask 3 | -------------------------------------------------------------------------------- /modules/ImageClassifierService/module.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-version": "0.0.1", 3 | "description": "", 4 | "image": { 5 | "repository": "glovebox/image-classifier-service", 6 | "tag": { 7 | "version": "1.1.111", 8 | "platforms": { 9 | "amd64": "./Dockerfile.amd64", 10 | "amd64.debug": "./Dockerfile.amd64.debug", 11 | "arm32v7": "./Dockerfile.arm32v7" 12 | } 13 | }, 14 | "buildOptions": [] 15 | }, 16 | "language": "python" 17 | } -------------------------------------------------------------------------------- /set-camera-sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # USB Camera Settings - most import is to turn off the auto focus 4 | # https://www.kurokesu.com/main/2016/01/16/manual-usb-camera-settings-in-linux/ 5 | # v4l2-ctl -d /dev/video0 --list-ctrls 6 | 7 | v4l2-ctl --set-ctrl=power_line_frequency=1 8 | v4l2-ctl --set-ctrl=focus_auto=0 9 | v4l2-ctl --set-ctrl=brightness=150 10 | v4l2-ctl --set-ctrl=contrast=7 11 | v4l2-ctl --set-ctrl=saturation=100 12 | v4l2-ctl --set-ctrl=focus_absolute=20 13 | v4l2-ctl --set-ctrl=zoom_absolute=20 14 | 15 | 16 | 17 | # Potential new settings 18 | 19 | # http://www.ideasonboard.org/uvc/faq/#faq6 20 | # https://stackoverflow.com/questions/25619309/how-do-i-enable-the-uvc-quirk-fix-bandwidth-quirk-in-linux-uvc-driverhttps://stackoverflow.com/questions/25619309/how-do-i-enable-the-uvc-quirk-fix-bandwidth-quirk-in-linux-uvc-driver 21 | # https://stackoverflow.com/questions/25619309/how-do-i-enable-the-uvc-quirk-fix-bandwidth-quirk-in-linux-uvc-driver 22 | # https://stackoverflow.com/questions/9781770/capturing-multiple-webcams-uvcvideo-with-opencv-on-linux/23881125#23881125 23 | 24 | # o make umläute's answer survive reboots, I created file /etc/modprobe.d/uvcvideo.conf with content 25 | 26 | #options uvcvideo quirks=0x80 27 | #To get the module to reload uvcvideo.conf, unload and load the module: 28 | 29 | #rmmod uvcvideo 30 | #modprobe uvcvideo 31 | 32 | 33 | sudo rmmod uvcvideo 34 | sudo modprobe uvcvideo nodrop=1 timeout=5000 quirks=0x80 35 | modinfo uvcvideo 36 | 37 | v4l2-ctl --set-ctrl=power_line_frequency=1 38 | v4l2-ctl --set-ctrl=focus_auto=0 39 | v4l2-ctl --set-ctrl=focus_absolute=20 -------------------------------------------------------------------------------- /version.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import random 4 | from pprint import pprint 5 | import os.path 6 | from os.path import isdir 7 | from os import listdir 8 | from packaging import version 9 | # from os.path import * 10 | modulesFileBase = "modules/" 11 | 12 | 13 | def updateModule(moduleName, versionX): 14 | filename = modulesFileBase + moduleName + "/module.json" 15 | if not os.path.exists(filename): 16 | return False 17 | 18 | with open(filename) as f: 19 | data = json.load(f) 20 | 21 | # todo increment micro release 22 | # major.minor.micro 23 | # https://www.python.org/dev/peps/pep-0440/ 24 | 25 | v = data["image"]["tag"]["version"] 26 | 27 | ver = tuple(map(int, (v.split(".")))) 28 | micro = ver[2:3][0] 29 | micro = micro + 1 30 | new_ver = f"{ver[0:1][0]}.{ver[1:2][0]}.{micro}" 31 | 32 | data["image"]["tag"]["version"] = new_ver 33 | 34 | with open(filename, 'w') as outfile: 35 | json.dump(data, outfile, indent=4) 36 | 37 | return True 38 | 39 | 40 | def updateVersion(moduleName): 41 | print(moduleName) 42 | version = "0.1." + str(random.randint(1, 10000)) 43 | 44 | if not updateModule(moduleName, version): 45 | return 46 | 47 | # updateDeployment(moduleName, version) 48 | 49 | 50 | dirs = listdir("modules") 51 | 52 | for d in dirs: 53 | updateVersion(d) 54 | --------------------------------------------------------------------------------