├── src ├── edge │ ├── .gitignore │ ├── modules │ │ ├── objectCounter │ │ │ ├── requirements.txt │ │ │ ├── Dockerfile.amd64 │ │ │ ├── Dockerfile.amd64.debug │ │ │ ├── module.json │ │ │ ├── .gitignore │ │ │ └── main.py │ │ ├── grpcExtension │ │ │ ├── app │ │ │ │ ├── exception_handler.py │ │ │ │ ├── batchImageProcessor.py │ │ │ │ ├── shared_memory.py │ │ │ │ ├── main.py │ │ │ │ ├── grpc-proto │ │ │ │ │ ├── extension_pb2_grpc.py │ │ │ │ │ ├── media_pb2.py │ │ │ │ │ └── extension_pb2.py │ │ │ │ └── inference_server.py │ │ │ ├── docker │ │ │ │ └── Dockerfile │ │ │ └── readme.md │ │ └── httpExtension │ │ │ ├── docker │ │ │ └── Dockerfile │ │ │ ├── app │ │ │ ├── imageProcessor.py │ │ │ └── main.py │ │ │ └── readme.md │ ├── deployment.template.json │ ├── deployment.yolov3.template.json │ ├── deployment.openvino.template.json │ ├── deployment.grpcyolov3icpu.template.json │ ├── deployment.grpctinyyolov3icpu.template.json │ ├── deployment.customvision.template.json │ ├── deployment.openvino.grpc.template.json │ ├── deployment.httpExtension.template.json │ ├── deployment.openvino.grpc.xpu.template.json │ ├── deployment.objectCounter.template.json │ ├── deployment.grpc.template.json │ ├── deployment.composite.template.json │ ├── deployment.spatialAnalysis.ase.template.json │ ├── deployment.spatialAnalysis.generic.template.json │ └── readme.md ├── .gitignore └── cloud-to-device-console-app │ ├── .gitignore │ ├── requirements.txt │ ├── main.py │ ├── operations.json │ └── readme.md ├── .github ├── CODE_OF_CONDUCT.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── CODE_OF_CONDUCT.md ├── LICENSE ├── SECURITY.md ├── README.md └── .gitignore /src/edge/.gitignore: -------------------------------------------------------------------------------- 1 | config/ -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | topologies/ 2 | -------------------------------------------------------------------------------- /src/cloud-to-device-console-app/.gitignore: -------------------------------------------------------------------------------- 1 | appsettings.json 2 | -------------------------------------------------------------------------------- /src/edge/modules/objectCounter/requirements.txt: -------------------------------------------------------------------------------- 1 | azure-iot-device~=2.1.2 -------------------------------------------------------------------------------- /src/cloud-to-device-console-app/requirements.txt: -------------------------------------------------------------------------------- 1 | ## No specific version. Install latest. ## 2 | certifi 3 | azure-iot-device 4 | azure-iot-hub -------------------------------------------------------------------------------- /src/edge/modules/objectCounter/Dockerfile.amd64: -------------------------------------------------------------------------------- 1 | FROM amd64/python:3.7-slim-buster 2 | 3 | WORKDIR /app 4 | 5 | COPY requirements.txt ./ 6 | RUN pip install -r requirements.txt 7 | 8 | COPY . . 9 | 10 | CMD [ "python3", "-u", "./main.py" ] -------------------------------------------------------------------------------- /src/edge/modules/objectCounter/Dockerfile.amd64.debug: -------------------------------------------------------------------------------- 1 | FROM amd64/python:3.7-slim-buster 2 | 3 | WORKDIR /app 4 | 5 | RUN pip install ptvsd==4.3.2 6 | COPY requirements.txt ./ 7 | RUN pip install -r requirements.txt 8 | 9 | COPY . . 10 | 11 | CMD [ "python3", "-u", "./main.py" ] -------------------------------------------------------------------------------- /src/edge/modules/objectCounter/module.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-version": "0.0.1", 3 | "description": "", 4 | "image": { 5 | "repository": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io/objectcounter", 6 | "tag": { 7 | "version": "0.0.1", 8 | "platforms": { 9 | "amd64": "./Dockerfile.amd64", 10 | "amd64.debug": "./Dockerfile.amd64.debug" 11 | } 12 | }, 13 | "buildOptions": [], 14 | "contextPath": "./" 15 | }, 16 | "language": "python" 17 | } -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/app/exception_handler.py: -------------------------------------------------------------------------------- 1 | import linecache 2 | import sys 3 | import logging 4 | 5 | def PrintGetExceptionDetails(): 6 | exType, exValue, exTraceback = sys.exc_info() 7 | 8 | tbFrame = exTraceback.tb_frame 9 | lineNo = exTraceback.tb_lineno 10 | fileName = tbFrame.f_code.co_filename 11 | 12 | linecache.checkcache(fileName) 13 | line = linecache.getline(fileName, lineNo, tbFrame.f_globals) 14 | 15 | exMessage = 'Exception:\n\tFile name: {0}\n\tLine number: {1}\n\tLine: {2}\n\tValue: {3}'.format(fileName, lineNo, line.strip(), exValue) 16 | 17 | logging.info(exMessage) 18 | 19 | return exType, exValue, exTraceback -------------------------------------------------------------------------------- /src/edge/modules/httpExtension/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | FROM ubuntu:18.04 4 | 5 | # Install python 6 | RUN apt-get update -y && \ 7 | apt-get install -y --no-install-recommends python3-pip python3-dev libglib2.0-0 libsm6 libxext6 libxrender-dev&& \ 8 | cd /usr/local/bin && \ 9 | ln -s /usr/bin/python3 python && \ 10 | pip3 install --upgrade pip 11 | 12 | # Install python packages 13 | RUN pip install numpy flask gunicorn requests opencv-python && \ 14 | apt-get install -y libgl1-mesa-dev 15 | 16 | # Copy the app file and the tags file 17 | RUN mkdir /app 18 | COPY ./app/*.py /app/ 19 | 20 | # Start http extension module 21 | WORKDIR /app 22 | ENTRYPOINT ["python3", "main.py"] 23 | -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | FROM ubuntu:18.04 as base 4 | 5 | # Install python 6 | RUN apt-get update && apt-get install -y --no-install-recommends \ 7 | python3-pip python3-dev libglib2.0-0 libsm6 libxext6 libxrender-dev\ 8 | && cd /usr/local/bin \ 9 | && ln -s /usr/bin/python3 python \ 10 | && pip3 install --upgrade pip \ 11 | && pip3 install numpy pillow opencv-python \ 12 | && pip3 install requests protobuf grpcio \ 13 | && apt-get install -y libgl1-mesa-dev && apt-get clean 14 | 15 | # Copy AVA extension specific files 16 | RUN mkdir /app 17 | COPY ./app/*.py /app/ 18 | COPY ./app/grpc-proto/*.py /app/ 19 | 20 | # Starts the AVA gRPC extension server 21 | FROM base as final 22 | WORKDIR /app/ 23 | CMD ["python3", "main.py"] 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | > Please provide us with the following information: 5 | > --------------------------------------------------------------- 6 | 7 | ### This issue is for a: (mark with an `x`) 8 | ``` 9 | - [ ] bug report -> please search issues before submitting 10 | - [ ] feature request 11 | - [ ] documentation issue or request 12 | - [ ] regression (a behavior that used to work and stopped in a new release) 13 | ``` 14 | 15 | ### Minimal steps to reproduce 16 | > 17 | 18 | ### Any log messages given by the failure 19 | > 20 | 21 | ### Expected/desired behavior 22 | > 23 | 24 | ### OS and Version? 25 | > Windows 7, 8 or 10. Linux (which distribution). macOS (Yosemite? El Capitan? Sierra?) 26 | 27 | ### Versions 28 | > 29 | 30 | ### Mention any other details that might be useful 31 | 32 | > --------------------------------------------------------------- 33 | > Thanks! We'll be in touch soon. 34 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | * ... 4 | 5 | ## Does this introduce a breaking change? 6 | 7 | ``` 8 | [ ] Yes 9 | [ ] No 10 | ``` 11 | 12 | ## Pull Request Type 13 | What kind of change does this Pull Request introduce? 14 | 15 | 16 | ``` 17 | [ ] Bugfix 18 | [ ] Feature 19 | [ ] Code style update (formatting, local variables) 20 | [ ] Refactoring (no functional changes, no api changes) 21 | [ ] Documentation content changes 22 | [ ] Other... Please describe: 23 | ``` 24 | 25 | ## How to Test 26 | * Get the code 27 | 28 | ``` 29 | git clone [repo-address] 30 | cd [repo-name] 31 | git checkout [branch-name] 32 | npm install 33 | ``` 34 | 35 | * Test the code 36 | 37 | ``` 38 | ``` 39 | 40 | ## What to Check 41 | Verify that the following are valid 42 | * ... 43 | 44 | ## Other Information 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /src/edge/modules/httpExtension/app/imageProcessor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | import logging 5 | import io 6 | import numpy as np 7 | import cv2 as cv 8 | 9 | class ImageProcessor(): 10 | def __init__(self): 11 | return 12 | 13 | def process_images(self, imgBytes): 14 | try: 15 | # Read image raw bytes 16 | imgBuf = io.BytesIO(imgBytes) 17 | imgBytes = np.frombuffer(imgBuf.getvalue(), dtype=np.uint8) 18 | 19 | # Convert to grayscale 20 | cvGrayImage = cv.imdecode(imgBytes, cv.COLOR_BGR2RGB) 21 | grayBytes = cvGrayImage.tobytes() 22 | 23 | # Calculate intensity 24 | totalColor = cvGrayImage.sum() 25 | avgColor = totalColor / len(grayBytes) 26 | 27 | colorIntensity = 'dark' if avgColor < 127 else 'light' 28 | 29 | logging.info('Color intensity: {}'.format(colorIntensity)) 30 | 31 | result = [{ 32 | "type" : "entity", 33 | "subtype" : "colorIntensity", 34 | "entity" : { 35 | "tag" : { 36 | "value" : colorIntensity, 37 | "confidence" : 1.0 38 | } 39 | } 40 | }] 41 | 42 | return result 43 | except: 44 | raise 45 | -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/app/batchImageProcessor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from PIL import Image, ImageDraw 3 | import io 4 | import numpy as np 5 | import cv2 as cv 6 | import inferencing_pb2 7 | import media_pb2 8 | import extension_pb2 9 | 10 | class BatchImageProcessor(): 11 | def __init__(self): 12 | return 13 | 14 | def process_images(self, mediaStreamMessage, rawBytes, size): 15 | # Read image raw bytes 16 | im = Image.frombytes('RGB', size, rawBytes.tobytes()) 17 | draw = ImageDraw.Draw(im) 18 | imgBuf = io.BytesIO() 19 | im.save(imgBuf, format='JPEG') 20 | imgBytes = np.frombuffer(imgBuf.getvalue(), dtype=np.uint8) 21 | 22 | # Convert to grayscale 23 | cvGrayImage = cv.imdecode(imgBytes, cv.COLOR_BGR2RGB) 24 | grayBytes = cvGrayImage.tobytes() 25 | 26 | # Calculate intensity 27 | totalColor = cvGrayImage.sum() 28 | avgColor = totalColor / len(grayBytes) 29 | 30 | colorIntensity = 'dark' if avgColor < 127 else 'light' 31 | 32 | logging.info('Color intensity: {}'.format(colorIntensity)) 33 | 34 | inference = mediaStreamMessage.media_sample.inferences.add() 35 | inference.subtype = 'colorIntensity' 36 | classification = inferencing_pb2.Classification( 37 | tag = inferencing_pb2.Tag( 38 | value = colorIntensity, 39 | confidence = 1.0 40 | ) 41 | ) 42 | inference.classification.CopyFrom(classification) 43 | 44 | return mediaStreamMessage 45 | -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/app/shared_memory.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | import mmap 3 | import os 4 | import logging 5 | from exception_handler import PrintGetExceptionDetails 6 | 7 | # *********************************************************************************** 8 | # Shared memory management 9 | # 10 | class SharedMemoryManager: 11 | def __init__(self, name, size): 12 | try: 13 | self._shmFilePath = '/dev/shm' 14 | self._shmFileName = name 15 | 16 | self._shmFileSize = size 17 | 18 | self._shmFileFullPath = os.path.join(self._shmFilePath, self._shmFileName) 19 | 20 | self._shmFile = open(self._shmFileFullPath, 'r+b') 21 | self._shm = mmap.mmap(self._shmFile.fileno(), self._shmFileSize) 22 | 23 | self._memSlots = dict() 24 | 25 | logging.info('Shared memory name: {0}'.format(self._shmFileFullPath)) 26 | except: 27 | PrintGetExceptionDetails() 28 | raise 29 | 30 | def ReadBytes(self, memorySlotOffset, memorySlotLength): 31 | try: 32 | # This is Non-Zero Copy operation 33 | # self._shm.seek(memorySlotOffset, os.SEEK_SET) 34 | # bytesRead = self._shm.read(memorySlotLength) 35 | # return bytesRead 36 | 37 | #Zero-copy version 38 | return memoryview(self._shm)[memorySlotOffset:memorySlotOffset+memorySlotLength] 39 | 40 | except: 41 | PrintGetExceptionDetails() 42 | raise 43 | 44 | 45 | 46 | 47 | def __del__(self): 48 | try: 49 | self._shmFile.close() 50 | except: 51 | PrintGetExceptionDetails() 52 | raise 53 | 54 | -------------------------------------------------------------------------------- /src/edge/modules/objectCounter/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /src/edge/modules/httpExtension/app/main.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | from datetime import datetime 5 | import io 6 | import json 7 | import os 8 | import logging 9 | import time 10 | from flask import Flask, Response, Request, abort, request 11 | import requests 12 | import argparse 13 | from imageProcessor import ImageProcessor 14 | 15 | def init_logging(): 16 | gunicorn_logger = logging.getLogger('gunicorn.error') 17 | if gunicorn_logger != None: 18 | app.logger.handlers = gunicorn_logger.handlers 19 | app.logger.setLevel(gunicorn_logger.level) 20 | 21 | # Get application arguments 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('-p', help='Port number to listen on.', type=int, default=8080) 24 | 25 | _arguments = parser.parse_args() 26 | 27 | # Default to port 8080 28 | httpServerPort = _arguments.p 29 | 30 | app = Flask(__name__) 31 | 32 | init_logging() 33 | 34 | processor = ImageProcessor() 35 | app.logger.info('Http extension listening on port: {}'.format(httpServerPort)) 36 | 37 | # /score routes to scoring function 38 | # This function returns a JSON object with inference result 39 | @app.route("/score", methods=['POST']) 40 | def score(): 41 | try: 42 | image_data = request.get_data() 43 | 44 | result = processor.process_images(image_data) 45 | 46 | if(result is not None): 47 | respBody = { 48 | "inferences" : result 49 | } 50 | 51 | respBody = json.dumps(respBody) 52 | return Response(respBody, status = 200, mimetype ='application/json') 53 | else: 54 | return Response(status = 400) 55 | except Exception as ex: 56 | logging.info('error: {}'.format(ex)) 57 | abort(Response(response='Could not decode image', status = 400)) 58 | 59 | if __name__ == '__main__': 60 | # Running the file directly 61 | app.run(host='0.0.0.0', port=httpServerPort) 62 | -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/app/main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | from exception_handler import PrintGetExceptionDetails 4 | from inference_server import InferenceServer 5 | import grpc 6 | import extension_pb2_grpc 7 | from concurrent import futures 8 | import argparse 9 | import os 10 | 11 | # Main thread 12 | def Main(): 13 | try: 14 | # Get application arguments 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('-p', metavar=('grpc_server_port'), help='Port number to serve gRPC server.', type=int, default=5001) 17 | parser.add_argument('-b', metavar=('batch_size'), help='Batch size.', type=int, default=1) 18 | 19 | _arguments = parser.parse_args() 20 | 21 | # Default to port 5001 22 | grpcServerPort = _arguments.p 23 | 24 | # Default batch size 1 25 | batchSize = _arguments.b 26 | 27 | # Get port from environment variable (overrides argument) 28 | envPort = os.getenv('port') 29 | 30 | # Get batch size from environment variable (overrides argument) 31 | envBatchSize = os.getenv('batchSize') 32 | 33 | if(envPort is not None): 34 | grpcServerPort = envPort 35 | 36 | if(envBatchSize is not None): 37 | batchSize = int(envBatchSize) 38 | 39 | logging.info('gRPC server port with: {0}'.format(grpcServerPort)) 40 | logging.info('Batch size set to: {0}'.format(batchSize)) 41 | 42 | # create gRPC server and start running 43 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=3)) 44 | extension_pb2_grpc.add_MediaGraphExtensionServicer_to_server(InferenceServer(batchSize), server) 45 | server.add_insecure_port(f'[::]:{grpcServerPort}') 46 | server.start() 47 | server.wait_for_termination() 48 | 49 | except: 50 | PrintGetExceptionDetails() 51 | exit(-1) 52 | 53 | if __name__ == "__main__": 54 | # Set logging parameters 55 | logging.basicConfig( 56 | level=logging.INFO, 57 | format='[%(asctime)-15s] [%(threadName)-12.12s] [%(levelname)s]: %(message)s', 58 | handlers=[ 59 | logging.StreamHandler(sys.stdout) # write in stdout 60 | ] 61 | ) 62 | 63 | # Call Main function 64 | Main() -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /src/cloud-to-device-console-app/main.py: -------------------------------------------------------------------------------- 1 | import json 2 | from os import path 3 | import pathlib 4 | import logging 5 | from builtins import input 6 | import ssl 7 | import urllib.request 8 | from azure.iot.hub import IoTHubRegistryManager 9 | from azure.iot.hub.models import CloudToDeviceMethod, CloudToDeviceMethodResult 10 | 11 | def read_url(url): 12 | url = url.replace(path.sep, '/') 13 | resp = urllib.request.urlopen(url, context=ssl._create_unverified_context()) 14 | return resp.read() 15 | 16 | class LivePipelineManager: 17 | 18 | def __init__(self): 19 | config_data = pathlib.Path('appsettings.json').read_text() 20 | config = json.loads(config_data) 21 | 22 | self.device_id = config['deviceId'] 23 | self.module_id = config['moduleId'] 24 | self.api_version = '1.0' 25 | 26 | self.registry_manager = IoTHubRegistryManager(config['IoThubConnectionString']) 27 | 28 | def invoke(self, method_name, payload): 29 | if method_name=='pipelineTopologySet': 30 | self.pipeline_topology_set(payload) 31 | return 32 | 33 | if method_name=='WaitForInput': 34 | print(payload['message']) 35 | input() 36 | return 37 | 38 | self.invoke_module_method(method_name, payload) 39 | 40 | def invoke_module_method(self, method_name, payload): 41 | # make sure '@apiVersion' has been set 42 | payload['@apiVersion'] = self.api_version 43 | 44 | module_method = CloudToDeviceMethod( 45 | method_name=method_name, 46 | payload=payload, 47 | response_timeout_in_seconds=30) 48 | 49 | print("\n----------------------- Request: %s --------------------------------------------------\n" % method_name) 50 | print(json.dumps(payload, indent=4)) 51 | 52 | resp = self.registry_manager.invoke_device_module_method(self.device_id, self.module_id, module_method) 53 | 54 | print("\n--------------- Response: %s - Status: %s ---------------\n" % (method_name, resp.status)) 55 | 56 | if resp.payload is not None: 57 | print(json.dumps(resp.payload, indent=4)) 58 | 59 | def pipeline_topology_set(self, op_parameters): 60 | if op_parameters is None: 61 | raise Exception('Operation parameters missing') 62 | 63 | if op_parameters.get('pipelineTopologyUrl') is not None: 64 | topology_json = read_url(op_parameters['pipelineTopologyUrl']) 65 | elif op_parameters.get('pipelineTopologyFile') is not None: 66 | topology_path = pathlib.Path(__file__).parent.joinpath(op_parameters['pipelineTopologyFile']) 67 | topology_json = topology_path.read_text() 68 | else: 69 | raise Exception('Neither pipelineTopologyUrl nor pipelineTopologyFile is specified') 70 | 71 | topology = json.loads(topology_json) 72 | 73 | self.invoke_module_method('pipelineTopologySet', topology) 74 | 75 | 76 | if __name__ == '__main__': 77 | manager = LivePipelineManager() 78 | 79 | operations_data_json = pathlib.Path('operations.json').read_text() 80 | operations_data = json.loads(operations_data_json) 81 | 82 | for operation in operations_data['operations']: 83 | manager.invoke(operation['opName'], operation['opParams']) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | languages: 4 | - python 5 | products: 6 | - azure 7 | - azure-video-analyzer 8 | description: "The samples in this repo show how to use the Azure Video Analyzer that enables you to capture, record, and analyze videos using AI." 9 | --- 10 | 11 | # Deprecated - Azure Video Analyzer samples 12 | 13 | We’re retiring the Azure Video Analyzer preview service, you're advised to transition your applications off of Video Analyzer by 01 December 2022. This repo is no longer being maintained. 14 | 15 | ## Contents 16 | 17 | | File/folder | Description | 18 | |----------------------|--------------------------------------------| 19 | | `src` | Sample source code. | 20 | | `.gitignore` | Defines what to ignore at commit time. | 21 | | `README.md` | This README file. | 22 | | `LICENSE` | The license for the sample. | 23 | | `SECURITY` | Guidelines for reporting security issues | 24 | | `CODE_OF_CONDUCT.md` | Open source code of conduct | 25 | 26 | The 'src' folder contains two sub-folders 27 | 28 | * **cloud-to-device-console-app** - This folder contains a Python app that enables you to invoke direct methods of Azure Video Analyzer module, with parameters defined by you in a JSON file (operations.json). 29 | * **edge** - This folder has a few IoT Edge deployment manifest templates, along with sample code for an IoT Edge module (under 'modules' folder) that can be used in conjunction with the Azure Video Analyzer module. 30 | 31 | ## Prerequisites 32 | 33 | 1. An active Azure subscription 34 | 2. Azure resources deployed in the Azure subscription 35 | 36 | a. Video Analyzer account 37 | 38 | b. Storage account 39 | 40 | c. Managed Identity 41 | 42 | d. IoT Hub 43 | 44 | e. Azure container registry 45 | 46 | 3. A Linux edge device with [IoT Edge runtime](https://docs.microsoft.com/en-us/azure/iot-edge/how-to-install-iot-edge-linux) 47 | 48 | 4. [Visual Studio Code](https://code.visualstudio.com/) on your development machine with following extensions 49 | 50 | a. [Azure IoT Tools](https://marketplace.visualstudio.com/items?itemName=vsciot-vscode.azure-iot-tools) 51 | 52 | b. [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python) 53 | 54 | 5. [Docker](https://docs.docker.com/engine/install/) on your development machine 55 | 56 | Set up Azure resources: 57 | 58 | [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/ava-click-to-deploy) 59 | 60 | 61 | ## Setup 62 | 63 | After cloning the repository, follow instructions outlined in **src/cloud-to-device-console-app/readme.md** to setup the console app. 64 | 65 | ## Running the sample 66 | 67 | Follow instructions outlined in **src/cloud-to-device-console-app/readme.md** to run the console app. 68 | 69 | ## Key concepts 70 | 71 | Read [Azure Video Analyzer concepts](https://docs.microsoft.com/azure/azure-video-analyzer/video-analyzer-docs/overview) 72 | 73 | ## Code of conduct 74 | 75 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 76 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 77 | -------------------------------------------------------------------------------- /src/cloud-to-device-console-app/operations.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiVersion": "1.1", 3 | "operations": [ 4 | { 5 | "opName": "pipelineTopologyList", 6 | "opParams": {} 7 | }, 8 | { 9 | "opName": "WaitForInput", 10 | "opParams": { 11 | "message": "Press Enter to continue" 12 | } 13 | }, 14 | { 15 | "opName": "livePipelineList", 16 | "opParams": {} 17 | }, 18 | { 19 | "opName": "WaitForInput", 20 | "opParams": { 21 | "message": "Press Enter to continue" 22 | } 23 | }, 24 | { 25 | "opName": "pipelineTopologySet", 26 | "opParams": { 27 | "pipelineTopologyUrl": "https://" 28 | } 29 | }, 30 | { 31 | "opName": "livePipelineSet", 32 | "opParams": { 33 | "name": "Sample-Pipeline-1", 34 | "properties": { 35 | "topologyName": "TopologyName", 36 | "description": "Sample pipeline description", 37 | "parameters": [ 38 | { 39 | "name": "rtspUrl", 40 | "value": "rtsp://rtspsim:554/media/camera-300s.mkv" 41 | }, 42 | { 43 | "name": "rtspUserName", 44 | "value": "testuser" 45 | }, 46 | { 47 | "name": "rtspPassword", 48 | "value": "testpassword" 49 | } 50 | ] 51 | } 52 | } 53 | }, 54 | { 55 | "opName": "livePipelineActivate", 56 | "opParams": { 57 | "name": "Sample-Pipeline-1" 58 | } 59 | }, 60 | { 61 | "opName": "livePipelineList", 62 | "opParams": {} 63 | }, 64 | { 65 | "opName": "WaitForInput", 66 | "opParams": { 67 | "message": "The livePipeline has been activated. Press Enter to continue and deactivate the livePipeline." 68 | } 69 | }, 70 | { 71 | "opName": "livePipelineDeactivate", 72 | "opParams": { 73 | "name": "Sample-Pipeline-1" 74 | } 75 | }, 76 | { 77 | "opName": "livePipelineDelete", 78 | "opParams": { 79 | "name": "Sample-Pipeline-1" 80 | } 81 | }, 82 | { 83 | "opName": "livePipelineList", 84 | "opParams": {} 85 | }, 86 | { 87 | "opName": "WaitForInput", 88 | "opParams": { 89 | "message": "Press Enter to continue" 90 | } 91 | }, 92 | { 93 | "opName": "pipelineTopologyDelete", 94 | "opParams": { 95 | "name": "TopologyName" 96 | } 97 | }, 98 | { 99 | "opName": "WaitForInput", 100 | "opParams": { 101 | "message": "Press Enter to continue" 102 | } 103 | }, 104 | { 105 | "opName": "pipelineTopologyList", 106 | "opParams": {} 107 | }, 108 | { 109 | "opName": "WaitForInput", 110 | "opParams": { 111 | "message": "Press Enter to continue" 112 | } 113 | } 114 | ] 115 | } 116 | 117 | -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/app/grpc-proto/extension_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | """Client and server classes corresponding to protobuf-defined services.""" 3 | import grpc 4 | 5 | import extension_pb2 as extension__pb2 6 | 7 | 8 | class MediaGraphExtensionStub(object): 9 | """ 10 | Media Graph Extension Service 11 | 12 | Media graph extension service definition allows graphs to be extended through a 13 | gRPC server implementation of a graph processor node. 14 | 15 | """ 16 | 17 | def __init__(self, channel): 18 | """Constructor. 19 | 20 | Args: 21 | channel: A grpc.Channel. 22 | """ 23 | self.ProcessMediaStream = channel.stream_stream( 24 | '/microsoft.azure.media.live_video_analytics.extensibility.grpc.v1.MediaGraphExtension/ProcessMediaStream', 25 | request_serializer=extension__pb2.MediaStreamMessage.SerializeToString, 26 | response_deserializer=extension__pb2.MediaStreamMessage.FromString, 27 | ) 28 | 29 | 30 | class MediaGraphExtensionServicer(object): 31 | """ 32 | Media Graph Extension Service 33 | 34 | Media graph extension service definition allows graphs to be extended through a 35 | gRPC server implementation of a graph processor node. 36 | 37 | """ 38 | 39 | def ProcessMediaStream(self, request_iterator, context): 40 | """ 41 | Continuously process a single media stream (audio/video) 42 | 43 | Note to Implementers: 44 | Client authentication can be achieved through an authentication token set on the "x-ms-authentication" 45 | request metadata key. The token format follows standard HTTP Basic auth scheme. Implementers of this 46 | service are responsible for validating this token. This token may be set through the REST API. 47 | 48 | """ 49 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 50 | context.set_details('Method not implemented!') 51 | raise NotImplementedError('Method not implemented!') 52 | 53 | 54 | def add_MediaGraphExtensionServicer_to_server(servicer, server): 55 | rpc_method_handlers = { 56 | 'ProcessMediaStream': grpc.stream_stream_rpc_method_handler( 57 | servicer.ProcessMediaStream, 58 | request_deserializer=extension__pb2.MediaStreamMessage.FromString, 59 | response_serializer=extension__pb2.MediaStreamMessage.SerializeToString, 60 | ), 61 | } 62 | generic_handler = grpc.method_handlers_generic_handler( 63 | 'microsoft.azure.media.live_video_analytics.extensibility.grpc.v1.MediaGraphExtension', rpc_method_handlers) 64 | server.add_generic_rpc_handlers((generic_handler,)) 65 | 66 | 67 | # This class is part of an EXPERIMENTAL API. 68 | class MediaGraphExtension(object): 69 | """ 70 | Media Graph Extension Service 71 | 72 | Media graph extension service definition allows graphs to be extended through a 73 | gRPC server implementation of a graph processor node. 74 | 75 | """ 76 | 77 | @staticmethod 78 | def ProcessMediaStream(request_iterator, 79 | target, 80 | options=(), 81 | channel_credentials=None, 82 | call_credentials=None, 83 | insecure=False, 84 | compression=None, 85 | wait_for_ready=None, 86 | timeout=None, 87 | metadata=None): 88 | return grpc.experimental.stream_stream(request_iterator, target, '/microsoft.azure.media.live_video_analytics.extensibility.grpc.v1.MediaGraphExtension/ProcessMediaStream', 89 | extension__pb2.MediaStreamMessage.SerializeToString, 90 | extension__pb2.MediaStreamMessage.FromString, 91 | options, channel_credentials, 92 | insecure, call_credentials, compression, wait_for_ready, timeout, metadata) 93 | -------------------------------------------------------------------------------- /src/edge/deployment.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "modules": { 54 | "avaedge": { 55 | "version": "1.0", 56 | "type": "docker", 57 | "status": "running", 58 | "restartPolicy": "always", 59 | "settings": { 60 | "image": "mcr.microsoft.com/media/video-analyzer:1", 61 | "createOptions": { 62 | "HostConfig": { 63 | "LogConfig": { 64 | "Type": "", 65 | "Config": { 66 | "max-size": "10m", 67 | "max-file": "10" 68 | } 69 | }, 70 | "Binds": [ 71 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 72 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 73 | ], 74 | "IpcMode": "host", 75 | "ShmSize": 1536870912 76 | } 77 | } 78 | } 79 | }, 80 | "rtspsim": { 81 | "version": "1.0", 82 | "type": "docker", 83 | "status": "running", 84 | "restartPolicy": "always", 85 | "settings": { 86 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 87 | "createOptions": { 88 | "HostConfig": { 89 | "LogConfig": { 90 | "Type": "", 91 | "Config": { 92 | "max-size": "10m", 93 | "max-file": "10" 94 | } 95 | }, 96 | "Binds": [ 97 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 98 | ] 99 | } 100 | } 101 | } 102 | } 103 | } 104 | } 105 | }, 106 | "$edgeHub": { 107 | "properties.desired": { 108 | "schemaVersion": "1.0", 109 | "routes": { 110 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 111 | }, 112 | "storeAndForwardConfiguration": { 113 | "timeToLiveSecs": 7200 114 | } 115 | } 116 | }, 117 | "avaedge": { 118 | "properties.desired": { 119 | "applicationDataDirectory": "/var/lib/videoanalyzer", 120 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 121 | "diagnosticsEventsOutputName": "diagnostics", 122 | "operationalEventsOutputName": "operational", 123 | "logLevel": "information", 124 | "logCategories": "application, events", 125 | "debugLogsDirectory": "/tmp/logs", 126 | "allowUnsecuredEndpoints": true, 127 | "telemetryOptOut": false 128 | } 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/edge/deployment.yolov3.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "modules": { 54 | "avaedge": { 55 | "version": "1.0", 56 | "type": "docker", 57 | "status": "running", 58 | "restartPolicy": "always", 59 | "settings": { 60 | "image": "mcr.microsoft.com/media/video-analyzer:1", 61 | "createOptions": { 62 | "HostConfig": { 63 | "LogConfig": { 64 | "Type": "", 65 | "Config": { 66 | "max-size": "10m", 67 | "max-file": "10" 68 | } 69 | }, 70 | "Binds": [ 71 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 72 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 73 | ] 74 | } 75 | } 76 | } 77 | }, 78 | "yolov3": { 79 | "version": "1.0", 80 | "type": "docker", 81 | "status": "running", 82 | "restartPolicy": "always", 83 | "settings": { 84 | "image": "mcr.microsoft.com/ava-utilities/avaextension:http-yolov3-onnx-v1.0", 85 | "createOptions": {} 86 | } 87 | }, 88 | "rtspsim": { 89 | "version": "1.0", 90 | "type": "docker", 91 | "status": "running", 92 | "restartPolicy": "always", 93 | "settings": { 94 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 95 | "createOptions": { 96 | "HostConfig": { 97 | "LogConfig": { 98 | "Type": "", 99 | "Config": { 100 | "max-size": "10m", 101 | "max-file": "10" 102 | } 103 | }, 104 | "Binds": [ 105 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 106 | ] 107 | } 108 | } 109 | } 110 | } 111 | } 112 | } 113 | }, 114 | "$edgeHub": { 115 | "properties.desired": { 116 | "schemaVersion": "1.0", 117 | "routes": { 118 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 119 | }, 120 | "storeAndForwardConfiguration": { 121 | "timeToLiveSecs": 7200 122 | } 123 | } 124 | }, 125 | "avaedge": { 126 | "properties.desired": { 127 | "applicationDataDirectory": "/var/lib/videoanalyzer", 128 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 129 | "diagnosticsEventsOutputName": "diagnostics", 130 | "operationalEventsOutputName": "operational", 131 | "logLevel": "information", 132 | "logCategories": "application, events", 133 | "debugLogsDirectory": "/tmp/logs", 134 | "allowUnsecuredEndpoints": true, 135 | "telemetryOptOut": false 136 | } 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/cloud-to-device-console-app/readme.md: -------------------------------------------------------------------------------- 1 | # AVA cloud to device sample console app 2 | 3 | This directory contains a dotnet core sample app that would enable you to invoke AVA on IoT Edge Direct Methods in a sequence and with parameters, defined by you in a JSON file (operations.json) 4 | 5 | ## Contents 6 | 7 | | File/folder | Description | 8 | |-------------------------|---------------------------------------------------------------| 9 | | `.gitignore` | Define what to ignore at commit time. | 10 | | `README.md` | This README file. | 11 | | `operations.json` | JSON file defining the sequence of operations to execute upon.| 12 | | `main.py` | The main program file | 13 | | `requirements.txt` | List of all dependent Python libraries | 14 | 15 | 16 | ## Setup 17 | 18 | Create a file named `appsettings.json` in this folder. Add the following text and provide values for all parameters. 19 | 20 | ```JSON 21 | { 22 | "IoThubConnectionString" : "", 23 | "deviceId" : "", 24 | "moduleId" : "" 25 | } 26 | ``` 27 | 28 | * **IoThubConnectionString** - Refers to the connection string of your IoT hub. This should have registry write and service connect access. 29 | * **deviceId** - Refers to your IoT Edge device ID (registered with your IoT hub) 30 | * **moduleId** - Refers to the module id of Azure Video Analyzer edge module (when deployed to the IoT Edge device) 31 | 32 | 33 | Create a file named `.env` in the `src/edge` folder and add the following text to it. Provide values for all variables. 34 | 35 | ```env 36 | AVA_PROVISIONING_TOKEN="" 37 | SUBSCRIPTION_ID="" 38 | RESOURCE_GROUP="" 39 | IOTHUB_CONNECTION_STRING="" 40 | VIDEO_INPUT_FOLDER_ON_DEVICE="" 41 | VIDEO_OUTPUT_FOLDER_ON_DEVICE="" 42 | APPDATA_FOLDER_ON_DEVICE="" 43 | CONTAINER_REGISTRY_USERNAME_myacr="" 44 | CONTAINER_REGISTRY_PASSWORD_myacr="" 45 | ``` 46 | 47 | Set up Azure resources: 48 | 49 | [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/ava-click-to-deploy) will generate the **appsettings.json** and **.env** file with values filled out. 50 | 51 | 52 | ## Running the sample from Visual Studio Code 53 | 54 | Detailed instructions for running the sample can be found in the tutorials for AVA on IoT Edge. Below is a summary of key steps. Make sure you have installed the required [prerequisites](./../../README.md#prerequisites). 55 | 56 | * Open your local clone of this git repository in Visual Studio Code, have the [Azure Iot Tools](https://marketplace.visualstudio.com/items?itemName=vsciot-vscode.azure-iot-tools) extension installed. 57 | * Right click on src/edge/deployment.template.json and select **“Generate Iot Edge deployment manifest”**. This will create an IoT Edge deployment manifest file in src/edge/config folder named deployment.amd64.json. 58 | * Right click on src/edge/config /deployment.amd64.json and select **"Create Deployment for single device"** and select the name of your edge device. This will trigger the deployment of the IoT Edge modules to your Edge device. You can view the status of the deployment in the Azure IoT Hub extension (expand 'Devices' and then 'Modules' under your IoT Edge device). 59 | * Right click on your edge device in Azure IoT Hub extension and select **"Start Monitoring Built-in Event Endpoint"**. 60 | * Install python dependencies from `requirements.txt`. This can be done by running `pip install -r src\cloud-to-device-console-app\requirements.txt`. 61 | * Select the "Cloud to Device - Console App" configuration in the run tab and start a debugging session (hit F5). You will start seeing some messages printed in the TERMINAL window. In the OUTPUT window, you will see messages that are being sent to the IoT Hub, by the AVAEdge module. 62 | 63 | ❗**Note:** *When running the debugger with the cloud-to-device-console project, the default launch.json creates a configuration with the parameter "console": "internalConsole". This does not work since internalConsole does not allow keyboard input. Changing the parameter to "console" : "integratedTerminal" fixes the problem.* 64 | 65 | ## Troubleshooting 66 | 67 | See the [Azure Video Analyzer Troubleshooting page](https://docs.microsoft.com/azure/azure-video-analyzer/video-analyzer-docs/troubleshoot.md). 68 | 69 | ## Next steps 70 | 71 | Experiment with different [pipeline topologies](https://docs.microsoft.com/azure/azure-video-analyzer/video-analyzer-docs/pipeline) by modifying `pipelineTopologyUrl` in operations.json. 72 | -------------------------------------------------------------------------------- /src/edge/deployment.openvino.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "modules": { 54 | "avaedge": { 55 | "version": "1.0", 56 | "type": "docker", 57 | "status": "running", 58 | "restartPolicy": "always", 59 | "settings": { 60 | "image": "mcr.microsoft.com/media/video-analyzer:1", 61 | "createOptions": { 62 | "HostConfig": { 63 | "LogConfig": { 64 | "Type": "", 65 | "Config": { 66 | "max-size": "10m", 67 | "max-file": "10" 68 | } 69 | }, 70 | "Binds": [ 71 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 72 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 73 | ] 74 | } 75 | } 76 | } 77 | }, 78 | "openvino": { 79 | "version": "1.0", 80 | "type": "docker", 81 | "status": "running", 82 | "restartPolicy": "always", 83 | "settings": { 84 | "image": "marketplace.azurecr.io/intel_corporation/open_vino:latest", 85 | "createOptions": { 86 | "ExposedPorts": { 87 | "4000/tcp": {} 88 | }, 89 | "Cmd": [ 90 | "/ams_wrapper/start_ams.py", 91 | "--ams_port=4000", 92 | "--ovms_port=9000" 93 | ] 94 | } 95 | } 96 | }, 97 | "rtspsim": { 98 | "version": "1.0", 99 | "type": "docker", 100 | "status": "running", 101 | "restartPolicy": "always", 102 | "settings": { 103 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 104 | "createOptions": { 105 | "HostConfig": { 106 | "LogConfig": { 107 | "Type": "", 108 | "Config": { 109 | "max-size": "10m", 110 | "max-file": "10" 111 | } 112 | }, 113 | "Binds": [ 114 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 115 | ] 116 | } 117 | } 118 | } 119 | } 120 | } 121 | } 122 | }, 123 | "$edgeHub": { 124 | "properties.desired": { 125 | "schemaVersion": "1.0", 126 | "routes": { 127 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 128 | }, 129 | "storeAndForwardConfiguration": { 130 | "timeToLiveSecs": 7200 131 | } 132 | } 133 | }, 134 | "avaedge": { 135 | "properties.desired": { 136 | "applicationDataDirectory": "/var/lib/videoanalyzer", 137 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 138 | "diagnosticsEventsOutputName": "diagnostics", 139 | "operationalEventsOutputName": "operational", 140 | "logLevel": "information", 141 | "logCategories": "application, events", 142 | "debugLogsDirectory": "/tmp/logs", 143 | "allowUnsecuredEndpoints": true, 144 | "telemetryOptOut": false 145 | } 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/edge/deployment.grpcyolov3icpu.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "modules": { 54 | "avaedge": { 55 | "version": "1.0", 56 | "type": "docker", 57 | "status": "running", 58 | "restartPolicy": "always", 59 | "settings": { 60 | "image": "mcr.microsoft.com/media/video-analyzer:1", 61 | "createOptions": { 62 | "HostConfig": { 63 | "LogConfig": { 64 | "Type": "", 65 | "Config": { 66 | "max-size": "10m", 67 | "max-file": "10" 68 | } 69 | }, 70 | "Binds": [ 71 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 72 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 73 | ], 74 | "IpcMode": "host", 75 | "ShmSize": 1536870912 76 | } 77 | } 78 | } 79 | }, 80 | "avaextension": { 81 | "version": "1.0", 82 | "type": "docker", 83 | "status": "running", 84 | "restartPolicy": "always", 85 | "settings": { 86 | "image": "mcr.microsoft.com/ava-utilities/avaextension:grpc-yolov3-onnx-v1.0", 87 | "createOptions": { 88 | "HostConfig": { 89 | "IpcMode": "host" 90 | } 91 | } 92 | } 93 | }, 94 | "rtspsim": { 95 | "version": "1.0", 96 | "type": "docker", 97 | "status": "running", 98 | "restartPolicy": "always", 99 | "settings": { 100 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 101 | "createOptions": { 102 | "HostConfig": { 103 | "LogConfig": { 104 | "Type": "", 105 | "Config": { 106 | "max-size": "10m", 107 | "max-file": "10" 108 | } 109 | }, 110 | "Binds": [ 111 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 112 | ] 113 | } 114 | } 115 | } 116 | } 117 | } 118 | } 119 | }, 120 | "$edgeHub": { 121 | "properties.desired": { 122 | "schemaVersion": "1.0", 123 | "routes": { 124 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 125 | }, 126 | "storeAndForwardConfiguration": { 127 | "timeToLiveSecs": 7200 128 | } 129 | } 130 | }, 131 | "avaedge": { 132 | "properties.desired": { 133 | "applicationDataDirectory": "/var/lib/videoanalyzer", 134 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 135 | "diagnosticsEventsOutputName": "diagnostics", 136 | "operationalEventsOutputName": "operational", 137 | "logLevel": "information", 138 | "logCategories": "application, events", 139 | "debugLogsDirectory": "/tmp/logs", 140 | "allowUnsecuredEndpoints": true, 141 | "telemetryOptOut": false 142 | } 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/edge/deployment.grpctinyyolov3icpu.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "modules": { 54 | "avaedge": { 55 | "version": "1.0", 56 | "type": "docker", 57 | "status": "running", 58 | "restartPolicy": "always", 59 | "settings": { 60 | "image": "mcr.microsoft.com/media/video-analyzer:1", 61 | "createOptions": { 62 | "HostConfig": { 63 | "LogConfig": { 64 | "Type": "", 65 | "Config": { 66 | "max-size": "10m", 67 | "max-file": "10" 68 | } 69 | }, 70 | "Binds": [ 71 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 72 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 73 | ], 74 | "IpcMode": "host", 75 | "ShmSize": 1536870912 76 | } 77 | } 78 | } 79 | }, 80 | "avaextension": { 81 | "version": "1.0", 82 | "type": "docker", 83 | "status": "running", 84 | "restartPolicy": "always", 85 | "settings": { 86 | "image": "mcr.microsoft.com/ava-utilities/avaextension:grpc-yolov3-tiny-onnx-v1.0", 87 | "createOptions": { 88 | "HostConfig": { 89 | "IpcMode": "host" 90 | } 91 | } 92 | } 93 | }, 94 | "rtspsim": { 95 | "version": "1.0", 96 | "type": "docker", 97 | "status": "running", 98 | "restartPolicy": "always", 99 | "settings": { 100 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 101 | "createOptions": { 102 | "HostConfig": { 103 | "LogConfig": { 104 | "Type": "", 105 | "Config": { 106 | "max-size": "10m", 107 | "max-file": "10" 108 | } 109 | }, 110 | "Binds": [ 111 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 112 | ] 113 | } 114 | } 115 | } 116 | } 117 | } 118 | } 119 | }, 120 | "$edgeHub": { 121 | "properties.desired": { 122 | "schemaVersion": "1.0", 123 | "routes": { 124 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 125 | }, 126 | "storeAndForwardConfiguration": { 127 | "timeToLiveSecs": 7200 128 | } 129 | } 130 | }, 131 | "avaedge": { 132 | "properties.desired": { 133 | "applicationDataDirectory": "/var/lib/videoanalyzer", 134 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 135 | "diagnosticsEventsOutputName": "diagnostics", 136 | "operationalEventsOutputName": "operational", 137 | "logLevel": "information", 138 | "logCategories": "application, events", 139 | "debugLogsDirectory": "/tmp/logs", 140 | "allowUnsecuredEndpoints": true, 141 | "telemetryOptOut": false 142 | } 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/edge/deployment.customvision.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": { 13 | "$CONTAINER_REGISTRY_USERNAME_myacr": { 14 | "username": "$CONTAINER_REGISTRY_USERNAME_myacr", 15 | "password": "$CONTAINER_REGISTRY_PASSWORD_myacr", 16 | "address": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io" 17 | } 18 | } 19 | } 20 | }, 21 | "systemModules": { 22 | "edgeAgent": { 23 | "type": "docker", 24 | "settings": { 25 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 26 | "createOptions": {} 27 | } 28 | }, 29 | "edgeHub": { 30 | "type": "docker", 31 | "status": "running", 32 | "restartPolicy": "always", 33 | "settings": { 34 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 35 | "createOptions": { 36 | "HostConfig": { 37 | "PortBindings": { 38 | "5671/tcp": [ 39 | { 40 | "HostPort": "5671" 41 | } 42 | ], 43 | "8883/tcp": [ 44 | { 45 | "HostPort": "8883" 46 | } 47 | ], 48 | "443/tcp": [ 49 | { 50 | "HostPort": "443" 51 | } 52 | ] 53 | } 54 | } 55 | } 56 | } 57 | } 58 | }, 59 | "modules": { 60 | "avaedge": { 61 | "version": "1.0", 62 | "type": "docker", 63 | "status": "running", 64 | "restartPolicy": "always", 65 | "settings": { 66 | "image": "mcr.microsoft.com/media/video-analyzer:1", 67 | "createOptions": { 68 | "HostConfig": { 69 | "LogConfig": { 70 | "Type": "", 71 | "Config": { 72 | "max-size": "10m", 73 | "max-file": "10" 74 | } 75 | }, 76 | "Binds": [ 77 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 78 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 79 | ] 80 | } 81 | } 82 | } 83 | }, 84 | "cv": { 85 | "version": "1.0", 86 | "type": "docker", 87 | "status": "running", 88 | "restartPolicy": "always", 89 | "settings": { 90 | "image": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io/cvtruck:latest", 91 | "createOptions": "{}" 92 | } 93 | }, 94 | "rtspsim": { 95 | "version": "1.0", 96 | "type": "docker", 97 | "status": "running", 98 | "restartPolicy": "always", 99 | "settings": { 100 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 101 | "createOptions": { 102 | "HostConfig": { 103 | "LogConfig": { 104 | "Type": "", 105 | "Config": { 106 | "max-size": "10m", 107 | "max-file": "10" 108 | } 109 | }, 110 | "Binds": [ 111 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 112 | ] 113 | } 114 | } 115 | } 116 | } 117 | } 118 | } 119 | }, 120 | "$edgeHub": { 121 | "properties.desired": { 122 | "schemaVersion": "1.0", 123 | "routes": { 124 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 125 | }, 126 | "storeAndForwardConfiguration": { 127 | "timeToLiveSecs": 7200 128 | } 129 | } 130 | }, 131 | "avaedge": { 132 | "properties.desired": { 133 | "applicationDataDirectory": "/var/lib/videoanalyzer", 134 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 135 | "diagnosticsEventsOutputName": "diagnostics", 136 | "operationalEventsOutputName": "operational", 137 | "logLevel": "information", 138 | "logCategories": "application, events", 139 | "debugLogsDirectory": "/tmp/logs", 140 | "allowUnsecuredEndpoints": true, 141 | "telemetryOptOut": false 142 | } 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/edge/deployment.openvino.grpc.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "" 12 | } 13 | }, 14 | "systemModules": { 15 | "edgeAgent": { 16 | "type": "docker", 17 | "settings": { 18 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 19 | "createOptions": "{}" 20 | } 21 | }, 22 | "edgeHub": { 23 | "type": "docker", 24 | "status": "running", 25 | "restartPolicy": "always", 26 | "settings": { 27 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 28 | "createOptions": "{ \"HostConfig\": { \"PortBindings\": { \"5671/tcp\": [ { \"HostPort\": \"5671\" } ], \"8883/tcp\": [ { \"HostPort\": \"8883\" } ], \"443/tcp\": [ { \"HostPort\": \"443\" } ] } } }" 29 | }, 30 | "env": { 31 | "UpstreamProtocol": { 32 | "value": "AMQPWS" 33 | } 34 | } 35 | } 36 | }, 37 | "modules": { 38 | "avaedge": { 39 | "version": "1.0", 40 | "type": "docker", 41 | "status": "running", 42 | "restartPolicy": "always", 43 | "settings": { 44 | "image": "mcr.microsoft.com/media/video-analyzer:1", 45 | "createOptions": { 46 | "HostConfig": { 47 | "LogConfig": { 48 | "Type": "", 49 | "Config": { 50 | "max-size": "10m", 51 | "max-file": "10" 52 | } 53 | }, 54 | "Binds": [ 55 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 56 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 57 | ], 58 | "IpcMode": "host", 59 | "ShmSize": 1536870912 60 | } 61 | } 62 | } 63 | }, 64 | "avaextension": { 65 | "version": "1.0", 66 | "type": "docker", 67 | "status": "running", 68 | "restartPolicy": "always", 69 | "settings": { 70 | "image": "intel/video-analytics-serving:0.5.0-dlstreamer-edge-ai-extension", 71 | "createOptions": { 72 | "ExposedPorts": { 73 | "80/tcp": {}, 74 | "5001/tcp": {} 75 | }, 76 | "HostConfig": { 77 | "Binds": ["/tmp:/tmp"], 78 | "PortBindings": { 79 | "80/tcp": [ 80 | { 81 | "HostPort": "8080" 82 | } 83 | ], 84 | "5001/tcp": [ 85 | { 86 | "HostPort": "5001" 87 | } 88 | ] 89 | }, 90 | "LogConfig": { 91 | "Type": "", 92 | "Config": { 93 | "max-size": "10m", 94 | "max-file": "10" 95 | } 96 | }, 97 | "IpcMode": "host" 98 | } 99 | } 100 | } 101 | }, 102 | "rtspsim": { 103 | "version": "1.0", 104 | "type": "docker", 105 | "status": "running", 106 | "restartPolicy": "always", 107 | "settings": { 108 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 109 | "createOptions": { 110 | "HostConfig": { 111 | "LogConfig": { 112 | "Type": "", 113 | "Config": { 114 | "max-size": "10m", 115 | "max-file": "10" 116 | } 117 | }, 118 | "Binds": [ 119 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 120 | ] 121 | } 122 | } 123 | } 124 | } 125 | } 126 | } 127 | }, 128 | "$edgeHub": { 129 | "properties.desired": { 130 | "schemaVersion": "1.0", 131 | "routes": { 132 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 133 | }, 134 | "storeAndForwardConfiguration": { 135 | "timeToLiveSecs": 7200 136 | } 137 | } 138 | }, 139 | "avaedge": { 140 | "properties.desired": { 141 | "applicationDataDirectory": "/var/lib/videoanalyzer", 142 | "provisioningToken": "$AVA_PROVISIONING_TOKEN", 143 | "diagnosticsEventsOutputName": "diagnostics", 144 | "operationalEventsOutputName": "operational", 145 | "logLevel": "information", 146 | "logCategories": "application, events", 147 | "debugLogsDirectory": "/tmp/logs", 148 | "allowUnsecuredEndpoints": true, 149 | "telemetryOptOut": false 150 | } 151 | } 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/edge/modules/objectCounter/main.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft. All rights reserved. 2 | # Licensed under the MIT license. See LICENSE file in the project root for 3 | # full license information. 4 | 5 | import time 6 | import sys 7 | import asyncio 8 | import json 9 | from six.moves import input 10 | from azure.iot.device.aio import IoTHubModuleClient 11 | from azure.iot.device import Message 12 | import logging 13 | 14 | OBJECT_TAG = 'truck' 15 | OBJECT_CONFIDENCE = 0.5 16 | TWIN_CALLBACKS = 0 17 | 18 | 19 | async def main(): 20 | try: 21 | # twin_patch_listener is invoked when the module twin's desired properties are updated. 22 | async def twin_patch_listener(module_client): 23 | global OBJECT_TAG 24 | global OBJECT_CONFIDENCE 25 | global TWIN_CALLBACKS 26 | 27 | while True: 28 | try: 29 | data = await module_client.receive_twin_desired_properties_patch() # blocking call 30 | print('The data in the desired properties patch was: %s' % data) 31 | if 'objectTag' in data: 32 | OBJECT_TAG = data['objectTag'] 33 | if 'objectConfidence' in data: 34 | OBJECT_CONFIDENCE = data['objectConfidence'] 35 | TWIN_CALLBACKS += 1 36 | print('Total calls confirmed: %d\n' % TWIN_CALLBACKS) 37 | except Exception as ex: 38 | print('Unexpected error in twin_patch_listener: %s' % ex) 39 | 40 | async def count_objects_listener(module_client): 41 | global OBJECT_TAG 42 | global OBJECT_CONFIDENCE 43 | 44 | while True: 45 | input_message = await module_client.receive_message_on_input('detectedObjects') # blocking call 46 | 47 | if input_message is not None: 48 | message = input_message.data 49 | message_text = message.decode('utf-8') 50 | count = 0 51 | print(message_text) 52 | # 53 | data = json.loads(message_text) 54 | # 55 | 56 | try: 57 | detected_objects = data['inferences'] 58 | for inference in detected_objects: 59 | entity = inference['entity'] 60 | tag = entity['tag'] 61 | 62 | if (tag['value'] == OBJECT_TAG) and (tag['confidence'] > OBJECT_CONFIDENCE): 63 | count += 1 64 | 65 | if count > 0: 66 | output_message_string = json.dumps(dict({'count': count})) 67 | output_message = Message(output_message_string, content_encoding='utf-8') 68 | 69 | subject = input_message.custom_properties['subject'] 70 | graph_instance_signature = '/livePipelines/' 71 | 72 | if graph_instance_signature in subject: 73 | output_message.custom_properties['eventTime'] = input_message.custom_properties['eventTime'] 74 | await module_client.send_message_to_output(output_message, "objectCountTrigger") 75 | except: 76 | print('No inferences array found for message.') 77 | 78 | def stdin_listener(): 79 | while True: 80 | try: 81 | selection = input("Press Q to quit\n") 82 | if selection == "Q" or selection == "q": 83 | print("Quitting...") 84 | break 85 | except: 86 | time.sleep(10) 87 | 88 | if not sys.version >= "3.5.3": 89 | raise Exception("The sample requires python 3.5.3+. Current version of Python: %s" % sys.version) 90 | print("IoT Hub Client for Python") 91 | 92 | logging.basicConfig( 93 | format='%(asctime)s %(name)-20s %(levelname)-5s %(message)s', 94 | level=logging.DEBUG 95 | ) 96 | 97 | # The client object is used to interact with your Azure IoT hub. 98 | module_client = IoTHubModuleClient.create_from_edge_environment(websockets=True) 99 | 100 | # connect the client. 101 | await module_client.connect() 102 | 103 | # Schedule task for C2D Listener1 104 | listeners = asyncio.gather(count_objects_listener(module_client), twin_patch_listener(module_client)) 105 | 106 | print("The sample is now waiting for messages. ") 107 | 108 | # Run the stdin listener in the event loop 109 | loop = asyncio.get_event_loop() 110 | user_finished = loop.run_in_executor(None, stdin_listener) 111 | 112 | # Wait for user to indicate they are done listening for messages 113 | await user_finished 114 | 115 | # Cancel listening 116 | listeners.cancel() 117 | 118 | # Finally, disconnect 119 | await module_client.disconnect() 120 | 121 | except Exception as e: 122 | print("Unexpected error %s " % e) 123 | raise 124 | 125 | if __name__ == "__main__": 126 | 127 | #loop = asyncio.get_event_loop() 128 | #loop.run_until_complete(main()) 129 | #loop.close() 130 | 131 | # If using Python 3.7 or above, you can use following code instead: 132 | asyncio.run(main()) 133 | -------------------------------------------------------------------------------- /src/edge/deployment.httpExtension.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": { 13 | "$CONTAINER_REGISTRY_USERNAME_myacr": { 14 | "username": "$CONTAINER_REGISTRY_USERNAME_myacr", 15 | "password": "$CONTAINER_REGISTRY_PASSWORD_myacr", 16 | "address": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io" 17 | } 18 | } 19 | } 20 | }, 21 | "systemModules": { 22 | "edgeAgent": { 23 | "type": "docker", 24 | "settings": { 25 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 26 | "createOptions": {} 27 | } 28 | }, 29 | "edgeHub": { 30 | "type": "docker", 31 | "status": "running", 32 | "restartPolicy": "always", 33 | "settings": { 34 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 35 | "createOptions": { 36 | "HostConfig": { 37 | "PortBindings": { 38 | "5671/tcp": [ 39 | { 40 | "HostPort": "5671" 41 | } 42 | ], 43 | "8883/tcp": [ 44 | { 45 | "HostPort": "8883" 46 | } 47 | ], 48 | "443/tcp": [ 49 | { 50 | "HostPort": "443" 51 | } 52 | ] 53 | } 54 | } 55 | } 56 | } 57 | } 58 | }, 59 | "modules": { 60 | "avaedge": { 61 | "version": "1.0", 62 | "type": "docker", 63 | "status": "running", 64 | "restartPolicy": "always", 65 | "settings": { 66 | "image": "mcr.microsoft.com/media/video-analyzer:1", 67 | "createOptions": { 68 | "HostConfig": { 69 | "LogConfig": { 70 | "Type": "", 71 | "Config": { 72 | "max-size": "10m", 73 | "max-file": "10" 74 | } 75 | }, 76 | "Binds": [ 77 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 78 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 79 | ], 80 | "IpcMode": "host", 81 | "ShmSize": 1536870912 82 | } 83 | } 84 | } 85 | }, 86 | "avaextension": { 87 | "version": "1.0", 88 | "type": "docker", 89 | "status": "running", 90 | "restartPolicy": "always", 91 | "settings": { 92 | "image": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io/httpextension:latest", 93 | "createOptions": { 94 | "HostConfig": { 95 | "LogConfig": { 96 | "Type": "", 97 | "Config": { 98 | "max-size": "10m", 99 | "max-file": "10" 100 | } 101 | } 102 | } 103 | } 104 | } 105 | }, 106 | "rtspsim": { 107 | "version": "1.0", 108 | "type": "docker", 109 | "status": "running", 110 | "restartPolicy": "always", 111 | "settings": { 112 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 113 | "createOptions": { 114 | "HostConfig": { 115 | "LogConfig": { 116 | "Type": "", 117 | "Config": { 118 | "max-size": "10m", 119 | "max-file": "10" 120 | } 121 | }, 122 | "Binds": [ 123 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 124 | ] 125 | } 126 | } 127 | } 128 | } 129 | } 130 | } 131 | }, 132 | "$edgeHub": { 133 | "properties.desired": { 134 | "schemaVersion": "1.0", 135 | "routes": { 136 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 137 | }, 138 | "storeAndForwardConfiguration": { 139 | "timeToLiveSecs": 7200 140 | } 141 | } 142 | }, 143 | "avaedge": { 144 | "properties.desired": { 145 | "applicationDataDirectory": "/var/lib/videoanalyzer", 146 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 147 | "diagnosticsEventsOutputName": "diagnostics", 148 | "operationalEventsOutputName": "operational", 149 | "logLevel": "information", 150 | "logCategories": "application, events", 151 | "debugLogsDirectory": "/tmp/logs", 152 | "allowUnsecuredEndpoints": true, 153 | "telemetryOptOut": false 154 | } 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/edge/deployment.openvino.grpc.xpu.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "" 12 | } 13 | }, 14 | "systemModules": { 15 | "edgeAgent": { 16 | "type": "docker", 17 | "settings": { 18 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 19 | "createOptions": "{}" 20 | } 21 | }, 22 | "edgeHub": { 23 | "type": "docker", 24 | "status": "running", 25 | "restartPolicy": "always", 26 | "settings": { 27 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 28 | "createOptions": "{ \"HostConfig\": { \"PortBindings\": { \"5671/tcp\": [ { \"HostPort\": \"5671\" } ], \"8883/tcp\": [ { \"HostPort\": \"8883\" } ], \"443/tcp\": [ { \"HostPort\": \"443\" } ] } } }" 29 | }, 30 | "env": { 31 | "UpstreamProtocol": { 32 | "value": "AMQPWS" 33 | } 34 | } 35 | } 36 | }, 37 | "modules": { 38 | "avaedge": { 39 | "version": "1.0", 40 | "type": "docker", 41 | "status": "running", 42 | "restartPolicy": "always", 43 | "settings": { 44 | "image": "mcr.microsoft.com/media/video-analyzer:1", 45 | "createOptions": { 46 | "HostConfig": { 47 | "LogConfig": { 48 | "Type": "", 49 | "Config": { 50 | "max-size": "10m", 51 | "max-file": "10" 52 | } 53 | }, 54 | "Binds": [ 55 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 56 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 57 | ], 58 | "IpcMode": "host", 59 | "ShmSize": 1536870912 60 | } 61 | } 62 | } 63 | }, 64 | "avaextension": { 65 | "version": "1.0", 66 | "type": "docker", 67 | "status": "running", 68 | "restartPolicy": "always", 69 | "settings": { 70 | "image": "intel/video-analytics-serving:0.5.0-dlstreamer-edge-ai-extension", 71 | "createOptions": { 72 | "ExposedPorts": { 73 | "80/tcp": {}, 74 | "5001/tcp": {} 75 | }, 76 | "HostConfig": { 77 | "Binds": ["/tmp:/tmp", "/dev/bus/usb:/dev/bus/usb"], 78 | "DeviceCgroupRules": ["c 189:* rmw"], 79 | "Devices": [ 80 | { 81 | "PathOnHost": "/dev/dri", 82 | "PathInContainer": "/dev/dri", 83 | "CgroupPermissions": "rwm" 84 | } 85 | ], 86 | "PortBindings": { 87 | "80/tcp": [ 88 | { 89 | "HostPort": "8080" 90 | } 91 | ], 92 | "5001/tcp": [ 93 | { 94 | "HostPort": "5001" 95 | } 96 | ] 97 | }, 98 | "LogConfig": { 99 | "Type": "", 100 | "Config": { 101 | "max-size": "10m", 102 | "max-file": "10" 103 | } 104 | }, 105 | "IpcMode": "host" 106 | } 107 | } 108 | } 109 | }, 110 | "rtspsim": { 111 | "version": "1.0", 112 | "type": "docker", 113 | "status": "running", 114 | "restartPolicy": "always", 115 | "settings": { 116 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 117 | "createOptions": { 118 | "HostConfig": { 119 | "LogConfig": { 120 | "Type": "", 121 | "Config": { 122 | "max-size": "10m", 123 | "max-file": "10" 124 | } 125 | }, 126 | "Binds": [ 127 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 128 | ] 129 | } 130 | } 131 | } 132 | } 133 | } 134 | } 135 | }, 136 | "$edgeHub": { 137 | "properties.desired": { 138 | "schemaVersion": "1.0", 139 | "routes": { 140 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 141 | }, 142 | "storeAndForwardConfiguration": { 143 | "timeToLiveSecs": 7200 144 | } 145 | } 146 | }, 147 | "avaedge": { 148 | "properties.desired": { 149 | "applicationDataDirectory": "/var/lib/videoanalyzer", 150 | "provisioningToken": "$AVA_PROVISIONING_TOKEN", 151 | "diagnosticsEventsOutputName": "diagnostics", 152 | "operationalEventsOutputName": "operational", 153 | "logLevel": "information", 154 | "logCategories": "application, events", 155 | "debugLogsDirectory": "/tmp/logs", 156 | "allowUnsecuredEndpoints": true, 157 | "telemetryOptOut": false 158 | } 159 | } 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /src/edge/deployment.objectCounter.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": { 13 | "$CONTAINER_REGISTRY_USERNAME_myacr": { 14 | "username": "$CONTAINER_REGISTRY_USERNAME_myacr", 15 | "password": "$CONTAINER_REGISTRY_PASSWORD_myacr", 16 | "address": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io" 17 | } 18 | } 19 | } 20 | }, 21 | "systemModules": { 22 | "edgeAgent": { 23 | "type": "docker", 24 | "settings": { 25 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 26 | "createOptions": {} 27 | } 28 | }, 29 | "edgeHub": { 30 | "type": "docker", 31 | "status": "running", 32 | "restartPolicy": "always", 33 | "settings": { 34 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 35 | "createOptions": { 36 | "HostConfig": { 37 | "PortBindings": { 38 | "5671/tcp": [ 39 | { 40 | "HostPort": "5671" 41 | } 42 | ], 43 | "8883/tcp": [ 44 | { 45 | "HostPort": "8883" 46 | } 47 | ], 48 | "443/tcp": [ 49 | { 50 | "HostPort": "443" 51 | } 52 | ] 53 | } 54 | } 55 | } 56 | } 57 | } 58 | }, 59 | "modules": { 60 | "avaedge": { 61 | "version": "1.0", 62 | "type": "docker", 63 | "status": "running", 64 | "restartPolicy": "always", 65 | "settings": { 66 | "image": "mcr.microsoft.com/media/video-analyzer:1", 67 | "createOptions": { 68 | "HostConfig": { 69 | "LogConfig": { 70 | "Type": "", 71 | "Config": { 72 | "max-size": "10m", 73 | "max-file": "10" 74 | } 75 | }, 76 | "Binds": [ 77 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 78 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 79 | ] 80 | } 81 | } 82 | } 83 | }, 84 | "yolov3": { 85 | "version": "1.0", 86 | "type": "docker", 87 | "status": "running", 88 | "restartPolicy": "always", 89 | "settings": { 90 | "image": "mcr.microsoft.com/ava-utilities/avaextension:http-yolov3-onnx-v1.0", 91 | "createOptions": "" 92 | } 93 | }, 94 | "rtspsim": { 95 | "version": "1.0", 96 | "type": "docker", 97 | "status": "running", 98 | "restartPolicy": "always", 99 | "settings": { 100 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 101 | "createOptions": { 102 | "HostConfig": { 103 | "LogConfig": { 104 | "Type": "", 105 | "Config": { 106 | "max-size": "10m", 107 | "max-file": "10" 108 | } 109 | }, 110 | "Binds": [ 111 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 112 | ] 113 | } 114 | } 115 | } 116 | }, 117 | "objectCounter": { 118 | "version": "1.0", 119 | "type": "docker", 120 | "status": "running", 121 | "restartPolicy": "always", 122 | "settings": { 123 | "image": "${MODULES.objectCounter}", 124 | "createOptions": "" 125 | } 126 | } 127 | } 128 | } 129 | }, 130 | "$edgeHub": { 131 | "properties.desired": { 132 | "schemaVersion": "1.0", 133 | "routes": { 134 | "ObjectCounterToIoTHub": "FROM /messages/modules/objectCounter/outputs/* INTO $upstream", 135 | "AVAToIoTHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream", 136 | "AVAToObjectCounter": "FROM /messages/modules/avaedge/outputs/detectedObjects INTO BrokeredEndpoint(\"/modules/objectCounter/inputs/detectedObjects\")", 137 | "ObjectCounterToAVA": "FROM /messages/modules/objectCounter/outputs/* INTO BrokeredEndpoint(\"/modules/avaedge/inputs/recordingTrigger\")" 138 | }, 139 | "storeAndForwardConfiguration": { 140 | "timeToLiveSecs": 7200 141 | } 142 | } 143 | }, 144 | "avaedge": { 145 | "properties.desired": { 146 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 147 | "ApplicationDataDirectory": "/var/lib/videoanalyzer", 148 | "DiagnosticsEventsOutputName": "diagnostics", 149 | "OperationalEventsOutputName": "operational", 150 | "LogLevel": "Verbose", 151 | "LogCategories": "Application,Events,MediaPipeline", 152 | "AllowUnsecuredEndpoints": true, 153 | "TelemetryOptOut": false 154 | } 155 | }, 156 | "objectCounter": { 157 | "properties.desired": { 158 | "objectTag": "truck", 159 | "objectConfidence": 0.3 160 | } 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/edge/deployment.grpc.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": { 13 | "$CONTAINER_REGISTRY_USERNAME_myacr": { 14 | "username": "$CONTAINER_REGISTRY_USERNAME_myacr", 15 | "password": "$CONTAINER_REGISTRY_PASSWORD_myacr", 16 | "address": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io" 17 | } 18 | } 19 | } 20 | }, 21 | "systemModules": { 22 | "edgeAgent": { 23 | "type": "docker", 24 | "settings": { 25 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 26 | "createOptions": {} 27 | } 28 | }, 29 | "edgeHub": { 30 | "type": "docker", 31 | "status": "running", 32 | "restartPolicy": "always", 33 | "settings": { 34 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 35 | "createOptions": { 36 | "HostConfig": { 37 | "PortBindings": { 38 | "5671/tcp": [ 39 | { 40 | "HostPort": "5671" 41 | } 42 | ], 43 | "8883/tcp": [ 44 | { 45 | "HostPort": "8883" 46 | } 47 | ], 48 | "443/tcp": [ 49 | { 50 | "HostPort": "443" 51 | } 52 | ] 53 | } 54 | } 55 | } 56 | } 57 | } 58 | }, 59 | "modules": { 60 | "avaedge": { 61 | "version": "1.0", 62 | "type": "docker", 63 | "status": "running", 64 | "restartPolicy": "always", 65 | "settings": { 66 | "image": "mcr.microsoft.com/media/video-analyzer:1", 67 | "createOptions": { 68 | "HostConfig": { 69 | "LogConfig": { 70 | "Type": "", 71 | "Config": { 72 | "max-size": "10m", 73 | "max-file": "10" 74 | } 75 | }, 76 | "Binds": [ 77 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 78 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 79 | ], 80 | "IpcMode": "host", 81 | "ShmSize": 1536870912 82 | } 83 | } 84 | } 85 | }, 86 | "avaextension": { 87 | "version": "1.0", 88 | "type": "docker", 89 | "status": "running", 90 | "restartPolicy": "always", 91 | "settings": { 92 | "image": "$CONTAINER_REGISTRY_USERNAME_myacr.azurecr.io/grpcextension:latest", 93 | "createOptions": { 94 | "HostConfig": { 95 | "LogConfig": { 96 | "Type": "", 97 | "Config": { 98 | "max-size": "10m", 99 | "max-file": "10" 100 | } 101 | }, 102 | "IpcMode": "container:avaedge", 103 | "Binds": [ 104 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 105 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 106 | ] 107 | }, 108 | "Env": ["grpcBinding=tcp://0.0.0.0:5001", "batchSize=1"] 109 | } 110 | } 111 | }, 112 | "rtspsim": { 113 | "version": "1.0", 114 | "type": "docker", 115 | "status": "running", 116 | "restartPolicy": "always", 117 | "settings": { 118 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 119 | "createOptions": { 120 | "HostConfig": { 121 | "LogConfig": { 122 | "Type": "", 123 | "Config": { 124 | "max-size": "10m", 125 | "max-file": "10" 126 | } 127 | }, 128 | "Binds": [ 129 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 130 | ] 131 | } 132 | } 133 | } 134 | } 135 | } 136 | } 137 | }, 138 | "$edgeHub": { 139 | "properties.desired": { 140 | "schemaVersion": "1.0", 141 | "routes": { 142 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 143 | }, 144 | "storeAndForwardConfiguration": { 145 | "timeToLiveSecs": 7200 146 | } 147 | } 148 | }, 149 | "avaedge": { 150 | "properties.desired": { 151 | "applicationDataDirectory": "/var/lib/videoanalyzer", 152 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 153 | "diagnosticsEventsOutputName": "diagnostics", 154 | "operationalEventsOutputName": "operational", 155 | "logLevel": "information", 156 | "logCategories": "application, events", 157 | "debugLogsDirectory": "/tmp/logs", 158 | "allowUnsecuredEndpoints": true, 159 | "telemetryOptOut": false 160 | } 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/edge/modules/httpExtension/readme.md: -------------------------------------------------------------------------------- 1 | # Http Extension module 2 | 3 | The HTTP extension module enables your own IoT Edge module to accept video frames as an http POST request. 4 | 5 | ## Prerequisites 6 | 7 | 1. [Install Docker](https://docs.docker.com/desktop/#download-and-install) on your machine 8 | 1. [Install IoT Edge Runtime](https://docs.microsoft.com/en-us/azure/iot-edge/how-to-install-iot-edge?tabs=linux) 9 | 10 | ### Design 11 | 12 | This http extension server is a Python terminal application that will house your custom AI and is built to handle http requests. The server reads the video frame, processes it using an Image Processor, and returns inference results as a json response. 13 | 14 | *main.py*: this is the entry point of the application. It is responsible for the configuring and management of the http extension server. 15 | 16 | 17 | ``` 18 | Main() 19 | ``` 20 | In this method we: 21 | 1. Create an instance of http extension server. 22 | 2. Create an instance of the image processor. 23 | 3. Define a route for handling the client requests. 24 | 4. Set the address and port the http extension server will listen on for client requests. 25 | 26 | *imageProcessor.py*: this class is responsible for processing the image. In a nutshell, it reads the raw bytes, converts an image to grayscale and determines if its color intensity is dark or light. You can add your own processor logic by adding a new class and implementing the method: 27 | 28 | ``` 29 | ProcessImages(self, imgBytes): 30 | ``` 31 | Once you've added the new class, you'll have to update the InferenceServer so it instantiates your class and invokes the **ProcessImage** method on it to run your processing logic. 32 | 33 | ### Building, publishing and running the Docker container 34 | 35 | To build the image, use the Docker file named `Dockerfile`. 36 | 37 | First, a couple assumptions 38 | 39 | * We'll be using Azure Container Registry (ACR) to publish our image before distributing it 40 | * Our local Docker container image is already logged into ACR. 41 | * In this sample, our ACR name is "myregistry". Your name may defer, so please update it properly in the following commands. 42 | 43 | > If you're unfamiliar with ACR or have any questions, please follow this [demo on building and pushing an image into ACR](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-docker-cli). 44 | 45 | `cd` onto the http extension's root directory 46 | 47 | ``` 48 | sudo docker build -f .\docker\Dockerfile -t httpextension:latest . 49 | 50 | sudo docker tag httpextension:latest myregistry.azurecr.io/httpextension:1 51 | 52 | sudo docker push myregistry.azurecr.io/httpextension:1 53 | ``` 54 | 55 | Then, from the box where the container should execute, run this command: 56 | 57 | `sudo docker run -d -p 5001:5001 --name httpextension myregistry.azurecr.io/httpextension:1 -p 5001` 58 | 59 | Let's decompose it a bit: 60 | 61 | * `-p 5001:5001`: it's up to you where you'd like to map the containers 5001 port. You can pick whatever port fits your needs. 62 | * `--name`: the name of the running container. 63 | * `registry/image:tag`: replace this with the corresponding location/image:tag where you've pushed the image built from the `Dockerfile` 64 | * `-p`: the port the http extension server will listen on 65 | 66 | ### Updating references into pipelineTopologies, to target the Http inferencing container address 67 | The [pipelineTopology](https://github.com/Azure/video-analyzer/tree/main/pipelines/live/topologies/httpExtension/2.0/topology.json) must define an inferencing URL: 68 | 69 | * Http Extension inferencing URL Parameter 70 | ``` 71 | { 72 | "name": "inferencingUrl", 73 | "type": "String", 74 | "description": "inferencing Url", 75 | "default": "https:///score" 76 | } 77 | ``` 78 | * Note the configuration of the extension processor 79 | ``` 80 | "processors": [ 81 | { 82 | "@type": "#Microsoft.VideoAnalyzer.HttpExtension", 83 | "name": "inferenceClient", 84 | "endpoint": { 85 | "@type": "#Microsoft.VideoAnalyzer.TlsEndpoint", 86 | "url": "${inferencingUrl}", 87 | "credentials": { 88 | "@type": "#Microsoft.VideoAnalyzer.UsernamePasswordCredentials", 89 | "username": "${inferencingUserName}", 90 | "password": "${inferencingPassword}" 91 | } 92 | }, 93 | "samplingOptions": { 94 | "skipSamplesWithoutAnnotation": "false", 95 | "maximumSamplesPerSecond": "5" 96 | }, 97 | "image": { 98 | "scale": 99 | { 100 | "mode": "Pad", 101 | "width": "416", 102 | "height": "416" 103 | }, 104 | "format": { 105 | "@type": "#Microsoft.VideoAnalyzer.ImageFormatJpeg", 106 | "quality": "90" 107 | } 108 | } 109 | } 110 | ] 111 | 112 | ``` 113 | ## Using the http extension container 114 | 115 | Test the container using the following commands 116 | 117 | ### /score 118 | 119 | To get the response of the processed image, use the following command 120 | 121 | ```bash 122 | curl -X POST https:///score -H "Content-Type: image/jpeg" --data-binary @ 123 | ``` 124 | 125 | If successful, you will see JSON printed on your screen that looks something like this 126 | 127 | ```JSON 128 | { 129 | "inferences": [ 130 | { 131 | "type": "classification", 132 | "subType": "colorIntensity", 133 | "classification": { 134 | "confidence": 1, 135 | "value": "dark" 136 | } 137 | } 138 | ] 139 | } 140 | ``` 141 | 142 | Terminate the container using the following Docker commands 143 | 144 | ```bash 145 | docker stop httpextension 146 | docker rm httpextension 147 | ``` 148 | 149 | ## Upload Docker image to Azure container registry 150 | 151 | Follow instructions in [Push and Pull Docker images - Azure Container Registry](http://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-docker-cli) to save your image for later use on another machine. 152 | 153 | ## Deploy as an Azure IoT Edge module 154 | 155 | Follow instruction in [Deploy module from Azure portal](https://docs.microsoft.com/en-us/azure/iot-edge/how-to-deploy-modules-portal) to deploy the container image as an IoT Edge module (use the IoT Edge module option). 156 | -------------------------------------------------------------------------------- /src/edge/deployment.composite.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "modules": { 54 | "avaedge": { 55 | "version": "1.0", 56 | "type": "docker", 57 | "status": "running", 58 | "restartPolicy": "always", 59 | "settings": { 60 | "image": "mcr.microsoft.com/media/video-analyzer:1", 61 | "createOptions": { 62 | "HostConfig": { 63 | "LogConfig": { 64 | "Type": "", 65 | "Config": { 66 | "max-size": "10m", 67 | "max-file": "10" 68 | } 69 | }, 70 | "Binds": [ 71 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 72 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 73 | ], 74 | "IpcMode": "host", 75 | "ShmSize": 1536870912 76 | } 77 | } 78 | } 79 | }, 80 | "yolov3": { 81 | "version": "1.0", 82 | "type": "docker", 83 | "status": "running", 84 | "restartPolicy": "always", 85 | "settings": { 86 | "image": "mcr.microsoft.com/ava-utilities/avaextension:grpc-yolov3-onnx-v1.0", 87 | "createOptions": { 88 | "HostConfig": { 89 | "LogConfig": { 90 | "Type": "", 91 | "Config": { 92 | "max-size": "10m", 93 | "max-file": "10" 94 | } 95 | }, 96 | "IpcMode": "host" 97 | }, 98 | "Cmd": [ 99 | "python", 100 | "server.py", 101 | "-p", 102 | "44000", 103 | "-c", 104 | "0.2", 105 | "-o", 106 | "car" 107 | ] 108 | } 109 | } 110 | }, 111 | "tinyyolov3": { 112 | "version": "1.0", 113 | "type": "docker", 114 | "status": "running", 115 | "restartPolicy": "always", 116 | "settings": { 117 | "image": "mcr.microsoft.com/ava-utilities/avaextension:grpc-yolov3-tiny-onnx-v1.0", 118 | "createOptions": { 119 | "HostConfig": { 120 | "LogConfig": { 121 | "Type": "", 122 | "Config": { 123 | "max-size": "10m", 124 | "max-file": "10" 125 | } 126 | }, 127 | "IpcMode": "host" 128 | }, 129 | "Cmd": [ 130 | "python", 131 | "server.py", 132 | "-p", 133 | "33000" 134 | ] 135 | } 136 | } 137 | }, 138 | "rtspsim": { 139 | "version": "1.0", 140 | "type": "docker", 141 | "status": "running", 142 | "restartPolicy": "always", 143 | "settings": { 144 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 145 | "createOptions": { 146 | "HostConfig": { 147 | "LogConfig": { 148 | "Type": "", 149 | "Config": { 150 | "max-size": "10m", 151 | "max-file": "10" 152 | } 153 | }, 154 | "Binds": [ 155 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 156 | ] 157 | } 158 | } 159 | } 160 | } 161 | } 162 | } 163 | }, 164 | "$edgeHub": { 165 | "properties.desired": { 166 | "schemaVersion": "1.0", 167 | "routes": { 168 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 169 | }, 170 | "storeAndForwardConfiguration": { 171 | "timeToLiveSecs": 7200 172 | } 173 | } 174 | }, 175 | "avaedge": { 176 | "properties.desired": { 177 | "applicationDataDirectory": "/var/lib/videoanalyzer", 178 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 179 | "diagnosticsEventsOutputName": "diagnostics", 180 | "operationalEventsOutputName": "operational", 181 | "logLevel": "information", 182 | "logCategories": "application,events", 183 | "debugLogsDirectory": "/tmp/logs", 184 | "allowUnsecuredEndpoints": true, 185 | "telemetryOptOut": false 186 | } 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/edge/deployment.spatialAnalysis.ase.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | }, 51 | "env": { 52 | "UpstreamProtocol": { 53 | "value": "AMQPWS" 54 | } 55 | } 56 | } 57 | }, 58 | "modules": { 59 | "avaedge": { 60 | "version": "1.0", 61 | "type": "docker", 62 | "status": "running", 63 | "restartPolicy": "always", 64 | "settings": { 65 | "image": "mcr.microsoft.com/media/video-analyzer:1", 66 | "createOptions": { 67 | "HostConfig": { 68 | "LogConfig": { 69 | "Type": "", 70 | "Config": { 71 | "max-size": "10m", 72 | "max-file": "10" 73 | } 74 | }, 75 | "Binds": [ 76 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 77 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 78 | ], 79 | "IpcMode": "host", 80 | "ShmSize": 1536870912 81 | } 82 | } 83 | } 84 | }, 85 | "rtspsim": { 86 | "version": "1.0", 87 | "type": "docker", 88 | "status": "running", 89 | "restartPolicy": "always", 90 | "settings": { 91 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 92 | "createOptions": { 93 | "HostConfig": { 94 | "Mounts": [ 95 | { 96 | "Target": "/live/mediaServer/media", 97 | "Source": "media", 98 | "Type": "volume" 99 | } 100 | ], 101 | "PortBindings": { 102 | "554/tcp": [ 103 | { 104 | "HostPort": "554" 105 | } 106 | ] 107 | } 108 | } 109 | } 110 | } 111 | }, 112 | "spatialanalysis": { 113 | "version": "1.0", 114 | "type": "docker", 115 | "status": "running", 116 | "restartPolicy": "always", 117 | "settings": { 118 | "image": "mcr.microsoft.com/azure-cognitive-services/vision/spatial-analysis:latest", 119 | "createOptions": { 120 | "HostConfig": { 121 | "PortBindings": { 122 | "50051/tcp": [ 123 | { 124 | "HostPort": "50051" 125 | } 126 | ] 127 | }, 128 | "IpcMode": "host", 129 | "Binds": ["/tmp/.X11-unix:/tmp/.X11-unix"], 130 | "Runtime": "nvidia", 131 | "ShmSize": 536870912, 132 | "LogConfig": { 133 | "Type": "json-file", 134 | "Config": { 135 | "max-size": "10m", 136 | "max-file": "200" 137 | } 138 | } 139 | } 140 | } 141 | }, 142 | "env": { 143 | "DISPLAY": { 144 | "value": ":0" 145 | }, 146 | "ARCHON_SHARED_BUFFER_LIMIT": { 147 | "value": "377487360" 148 | }, 149 | "ARCHON_LOG_LEVEL": { 150 | "value": "info" 151 | }, 152 | "QT_X11_NO_MITSHM": { 153 | "value": "1" 154 | }, 155 | "OMP_WAIT_POLICY": { 156 | "value": "PASSIVE" 157 | }, 158 | "EULA": { 159 | "value": "accept" 160 | }, 161 | "ARCHON_TELEMETRY_IOTHUB": { 162 | "value": "true" 163 | }, 164 | "BILLING": { 165 | "value": "" 166 | }, 167 | "APIKEY": { 168 | "value": "" 169 | }, 170 | "LAUNCHER_TYPE": { 171 | "value": "avaBackend" 172 | }, 173 | "ARCHON_GRAPH_READY_TIMEOUT": { 174 | "value": "600" 175 | } 176 | } 177 | } 178 | } 179 | } 180 | }, 181 | "$edgeHub": { 182 | "properties.desired": { 183 | "schemaVersion": "1.0", 184 | "routes": { 185 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 186 | }, 187 | "storeAndForwardConfiguration": { 188 | "timeToLiveSecs": 7200 189 | } 190 | } 191 | }, 192 | "avaedge": { 193 | "properties.desired": { 194 | "applicationDataDirectory": "/var/lib/videoanalyzer", 195 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 196 | "diagnosticsEventsOutputName": "diagnostics", 197 | "operationalEventsOutputName": "operational", 198 | "logLevel": "information", 199 | "logCategories": "application, events", 200 | "debugLogsDirectory": "/tmp/logs", 201 | "allowUnsecuredEndpoints": true, 202 | "telemetryOptOut": false 203 | } 204 | } 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/edge/deployment.spatialAnalysis.generic.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema-template": "2.0.0", 3 | "modulesContent": { 4 | "$edgeAgent": { 5 | "properties.desired": { 6 | "schemaVersion": "1.0", 7 | "runtime": { 8 | "type": "docker", 9 | "settings": { 10 | "minDockerVersion": "v1.25", 11 | "loggingOptions": "", 12 | "registryCredentials": {} 13 | } 14 | }, 15 | "systemModules": { 16 | "edgeAgent": { 17 | "type": "docker", 18 | "settings": { 19 | "image": "mcr.microsoft.com/azureiotedge-agent:1.2", 20 | "createOptions": {} 21 | } 22 | }, 23 | "edgeHub": { 24 | "type": "docker", 25 | "status": "running", 26 | "restartPolicy": "always", 27 | "settings": { 28 | "image": "mcr.microsoft.com/azureiotedge-hub:1.2", 29 | "createOptions": { 30 | "HostConfig": { 31 | "PortBindings": { 32 | "5671/tcp": [ 33 | { 34 | "HostPort": "5671" 35 | } 36 | ], 37 | "8883/tcp": [ 38 | { 39 | "HostPort": "8883" 40 | } 41 | ], 42 | "443/tcp": [ 43 | { 44 | "HostPort": "443" 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | }, 51 | "env": { 52 | "UpstreamProtocol": { 53 | "value": "AMQPWS" 54 | } 55 | } 56 | } 57 | }, 58 | "modules": { 59 | "avaedge": { 60 | "version": "1.0", 61 | "type": "docker", 62 | "status": "running", 63 | "restartPolicy": "always", 64 | "settings": { 65 | "image": "mcr.microsoft.com/media/video-analyzer:1", 66 | "createOptions": { 67 | "HostConfig": { 68 | "LogConfig": { 69 | "Type": "", 70 | "Config": { 71 | "max-size": "10m", 72 | "max-file": "10" 73 | } 74 | }, 75 | "Binds": [ 76 | "$VIDEO_OUTPUT_FOLDER_ON_DEVICE:/var/media/", 77 | "$APPDATA_FOLDER_ON_DEVICE:/var/lib/videoanalyzer" 78 | ], 79 | "IpcMode": "host", 80 | "ShmSize": 1536870912 81 | } 82 | } 83 | } 84 | }, 85 | "rtspsim": { 86 | "version": "1.0", 87 | "type": "docker", 88 | "status": "running", 89 | "restartPolicy": "always", 90 | "settings": { 91 | "image": "mcr.microsoft.com/ava-utilities/rtspsim-live555:1.2", 92 | "createOptions": { 93 | "HostConfig": { 94 | "LogConfig": { 95 | "Type": "", 96 | "Config": { 97 | "max-size": "10m", 98 | "max-file": "10" 99 | } 100 | }, 101 | "Binds": [ 102 | "$VIDEO_INPUT_FOLDER_ON_DEVICE:/live/mediaServer/media" 103 | ], 104 | "PortBindings": { 105 | "554/tcp": [ 106 | { 107 | "HostPort": "554" 108 | } 109 | ] 110 | } 111 | } 112 | } 113 | } 114 | }, 115 | "spatialanalysis": { 116 | "version": "1.0", 117 | "type": "docker", 118 | "status": "running", 119 | "restartPolicy": "always", 120 | "settings": { 121 | "image": "mcr.microsoft.com/azure-cognitive-services/vision/spatial-analysis:latest", 122 | "createOptions": { 123 | "HostConfig": { 124 | "PortBindings": { 125 | "50051/tcp": [ 126 | { 127 | "HostPort": "50051" 128 | } 129 | ] 130 | }, 131 | "IpcMode": "host", 132 | "Binds": ["/tmp/.X11-unix:/tmp/.X11-unix"], 133 | "Runtime": "nvidia", 134 | "ShmSize": 536870912, 135 | "LogConfig": { 136 | "Type": "json-file", 137 | "Config": { 138 | "max-size": "10m", 139 | "max-file": "200" 140 | } 141 | } 142 | } 143 | } 144 | }, 145 | "env": { 146 | "DISPLAY": { 147 | "value": ":0" 148 | }, 149 | "ARCHON_SHARED_BUFFER_LIMIT": { 150 | "value": "377487360" 151 | }, 152 | "ARCHON_LOG_LEVEL": { 153 | "value": "info" 154 | }, 155 | "QT_X11_NO_MITSHM": { 156 | "value": "1" 157 | }, 158 | "OMP_WAIT_POLICY": { 159 | "value": "PASSIVE" 160 | }, 161 | "EULA": { 162 | "value": "accept" 163 | }, 164 | "ARCHON_TELEMETRY_IOTHUB": { 165 | "value": "true" 166 | }, 167 | "BILLING": { 168 | "value": "" 169 | }, 170 | "APIKEY": { 171 | "value": "" 172 | }, 173 | "LAUNCHER_TYPE": { 174 | "value": "avaBackend" 175 | }, 176 | "ARCHON_GRAPH_READY_TIMEOUT": { 177 | "value": "600" 178 | } 179 | } 180 | } 181 | } 182 | } 183 | }, 184 | "$edgeHub": { 185 | "properties.desired": { 186 | "schemaVersion": "1.0", 187 | "routes": { 188 | "AVAToHub": "FROM /messages/modules/avaedge/outputs/* INTO $upstream" 189 | }, 190 | "storeAndForwardConfiguration": { 191 | "timeToLiveSecs": 7200 192 | } 193 | } 194 | }, 195 | "avaedge": { 196 | "properties.desired": { 197 | "applicationDataDirectory": "/var/lib/videoanalyzer", 198 | "ProvisioningToken": "$AVA_PROVISIONING_TOKEN", 199 | "diagnosticsEventsOutputName": "diagnostics", 200 | "operationalEventsOutputName": "operational", 201 | "logLevel": "information", 202 | "logCategories": "Application,Events", 203 | "debugLogsDirectory": "/tmp/logs", 204 | "allowUnsecuredEndpoints": true, 205 | "telemetryOptOut": false 206 | } 207 | } 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Aa][Rr][Mm]/ 27 | [Aa][Rr][Mm]64/ 28 | bld/ 29 | [Bb]in/ 30 | [Oo]bj/ 31 | [Ll]og/ 32 | [Ll]ogs/ 33 | 34 | # Visual Studio 2015/2017 cache/options directory 35 | .vs/ 36 | # Uncomment if you have tasks that create the project's static files in wwwroot 37 | #wwwroot/ 38 | 39 | # Python3 venv environment folder 40 | .avaedge/ 41 | 42 | # Visual Studio 2017 auto generated files 43 | Generated\ Files/ 44 | 45 | # MSTest test Results 46 | [Tt]est[Rr]esult*/ 47 | [Bb]uild[Ll]og.* 48 | 49 | # NUnit 50 | *.VisualState.xml 51 | TestResult.xml 52 | nunit-*.xml 53 | 54 | # Build Results of an ATL Project 55 | [Dd]ebugPS/ 56 | [Rr]eleasePS/ 57 | dlldata.c 58 | 59 | # Benchmark Results 60 | BenchmarkDotNet.Artifacts/ 61 | 62 | # .NET Core 63 | project.lock.json 64 | project.fragment.lock.json 65 | artifacts/ 66 | 67 | # StyleCop 68 | StyleCopReport.xml 69 | 70 | # Files built by Visual Studio 71 | *_i.c 72 | *_p.c 73 | *_h.h 74 | *.ilk 75 | *.meta 76 | *.obj 77 | *.iobj 78 | *.pch 79 | *.pdb 80 | *.ipdb 81 | *.pgc 82 | *.pgd 83 | *.rsp 84 | *.sbr 85 | *.tlb 86 | *.tli 87 | *.tlh 88 | *.tmp 89 | *.tmp_proj 90 | *_wpftmp.csproj 91 | *.log 92 | *.vspscc 93 | *.vssscc 94 | .builds 95 | *.pidb 96 | *.svclog 97 | *.scc 98 | 99 | # Chutzpah Test files 100 | _Chutzpah* 101 | 102 | # Visual C++ cache files 103 | ipch/ 104 | *.aps 105 | *.ncb 106 | *.opendb 107 | *.opensdf 108 | *.sdf 109 | *.cachefile 110 | *.VC.db 111 | *.VC.VC.opendb 112 | 113 | # Visual Studio profiler 114 | *.psess 115 | *.vsp 116 | *.vspx 117 | *.sap 118 | 119 | # Visual Studio Trace Files 120 | *.e2e 121 | 122 | # TFS 2012 Local Workspace 123 | $tf/ 124 | 125 | # Guidance Automation Toolkit 126 | *.gpState 127 | 128 | # ReSharper is a .NET coding add-in 129 | _ReSharper*/ 130 | *.[Rr]e[Ss]harper 131 | *.DotSettings.user 132 | 133 | # TeamCity is a build add-in 134 | _TeamCity* 135 | 136 | # DotCover is a Code Coverage Tool 137 | *.dotCover 138 | 139 | # AxoCover is a Code Coverage Tool 140 | .axoCover/* 141 | !.axoCover/settings.json 142 | 143 | # Visual Studio code coverage results 144 | *.coverage 145 | *.coveragexml 146 | 147 | # NCrunch 148 | _NCrunch_* 149 | .*crunch*.local.xml 150 | nCrunchTemp_* 151 | 152 | # MightyMoose 153 | *.mm.* 154 | AutoTest.Net/ 155 | 156 | # Web workbench (sass) 157 | .sass-cache/ 158 | 159 | # Installshield output folder 160 | [Ee]xpress/ 161 | 162 | # DocProject is a documentation generator add-in 163 | DocProject/buildhelp/ 164 | DocProject/Help/*.HxT 165 | DocProject/Help/*.HxC 166 | DocProject/Help/*.hhc 167 | DocProject/Help/*.hhk 168 | DocProject/Help/*.hhp 169 | DocProject/Help/Html2 170 | DocProject/Help/html 171 | 172 | # Click-Once directory 173 | publish/ 174 | 175 | # Publish Web Output 176 | *.[Pp]ublish.xml 177 | *.azurePubxml 178 | # Note: Comment the next line if you want to checkin your web deploy settings, 179 | # but database connection strings (with potential passwords) will be unencrypted 180 | *.pubxml 181 | *.publishproj 182 | 183 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 184 | # checkin your Azure Web App publish settings, but sensitive information contained 185 | # in these scripts will be unencrypted 186 | PublishScripts/ 187 | 188 | # NuGet Packages 189 | *.nupkg 190 | # NuGet Symbol Packages 191 | *.snupkg 192 | # The packages folder can be ignored because of Package Restore 193 | **/[Pp]ackages/* 194 | # except build/, which is used as an MSBuild target. 195 | !**/[Pp]ackages/build/ 196 | # Uncomment if necessary however generally it will be regenerated when needed 197 | #!**/[Pp]ackages/repositories.config 198 | # NuGet v3's project.json files produces more ignorable files 199 | *.nuget.props 200 | *.nuget.targets 201 | 202 | # Microsoft Azure Build Output 203 | csx/ 204 | *.build.csdef 205 | 206 | # Microsoft Azure Emulator 207 | ecf/ 208 | rcf/ 209 | 210 | # Windows Store app package directories and files 211 | AppPackages/ 212 | BundleArtifacts/ 213 | Package.StoreAssociation.xml 214 | _pkginfo.txt 215 | *.appx 216 | *.appxbundle 217 | *.appxupload 218 | 219 | # Visual Studio cache files 220 | # files ending in .cache can be ignored 221 | *.[Cc]ache 222 | # but keep track of directories ending in .cache 223 | !?*.[Cc]ache/ 224 | 225 | # Others 226 | ClientBin/ 227 | ~$* 228 | *~ 229 | *.dbmdl 230 | *.dbproj.schemaview 231 | *.jfm 232 | *.pfx 233 | *.publishsettings 234 | orleans.codegen.cs 235 | 236 | # Including strong name files can present a security risk 237 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 238 | #*.snk 239 | 240 | # Since there are multiple workflows, uncomment next line to ignore bower_components 241 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 242 | #bower_components/ 243 | 244 | # RIA/Silverlight projects 245 | Generated_Code/ 246 | 247 | # Backup & report files from converting an old project file 248 | # to a newer Visual Studio version. Backup files are not needed, 249 | # because we have git ;-) 250 | _UpgradeReport_Files/ 251 | Backup*/ 252 | UpgradeLog*.XML 253 | UpgradeLog*.htm 254 | ServiceFabricBackup/ 255 | *.rptproj.bak 256 | 257 | # SQL Server files 258 | *.mdf 259 | *.ldf 260 | *.ndf 261 | 262 | # Business Intelligence projects 263 | *.rdl.data 264 | *.bim.layout 265 | *.bim_*.settings 266 | *.rptproj.rsuser 267 | *- [Bb]ackup.rdl 268 | *- [Bb]ackup ([0-9]).rdl 269 | *- [Bb]ackup ([0-9][0-9]).rdl 270 | 271 | # Microsoft Fakes 272 | FakesAssemblies/ 273 | 274 | # GhostDoc plugin setting file 275 | *.GhostDoc.xml 276 | 277 | # Node.js Tools for Visual Studio 278 | .ntvs_analysis.dat 279 | node_modules/ 280 | 281 | # Visual Studio 6 build log 282 | *.plg 283 | 284 | # Visual Studio 6 workspace options file 285 | *.opt 286 | 287 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 288 | *.vbw 289 | 290 | # Visual Studio LightSwitch build output 291 | **/*.HTMLClient/GeneratedArtifacts 292 | **/*.DesktopClient/GeneratedArtifacts 293 | **/*.DesktopClient/ModelManifest.xml 294 | **/*.Server/GeneratedArtifacts 295 | **/*.Server/ModelManifest.xml 296 | _Pvt_Extensions 297 | 298 | # Paket dependency manager 299 | .paket/paket.exe 300 | paket-files/ 301 | 302 | # FAKE - F# Make 303 | .fake/ 304 | 305 | # CodeRush personal settings 306 | .cr/personal 307 | 308 | # Python Tools for Visual Studio (PTVS) 309 | __pycache__/ 310 | *.pyc 311 | 312 | # Cake - Uncomment if you are using it 313 | # tools/** 314 | # !tools/packages.config 315 | 316 | # Tabs Studio 317 | *.tss 318 | 319 | # Telerik's JustMock configuration file 320 | *.jmconfig 321 | 322 | # BizTalk build output 323 | *.btp.cs 324 | *.btm.cs 325 | *.odx.cs 326 | *.xsd.cs 327 | 328 | # OpenCover UI analysis results 329 | OpenCover/ 330 | 331 | # Azure Stream Analytics local run output 332 | ASALocalRun/ 333 | 334 | # MSBuild Binary and Structured Log 335 | *.binlog 336 | 337 | # NVidia Nsight GPU debugger configuration file 338 | *.nvuser 339 | 340 | # MFractors (Xamarin productivity tool) working folder 341 | .mfractor/ 342 | 343 | # Local History for Visual Studio 344 | .localhistory/ 345 | 346 | # BeatPulse healthcheck temp database 347 | healthchecksdb 348 | 349 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 350 | MigrationBackup/ 351 | 352 | # Ionide (cross platform F# VS Code tools) working folder 353 | .ionide/ 354 | 355 | # Dot Env files (scope "all") 356 | .env -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/readme.md: -------------------------------------------------------------------------------- 1 | # gRPC Server 2 | 3 | This gRPC server enables your own IoT Edge module to accept video frames as [protobuf](https://github.com/Azure/video-analyzer/treemainr/contracts/grpc) messages and return results back to AVA using the [inference metadata schema](https://docs.microsoft.com/azure/azure-video-analyzer/video-analyzer-docs/inference-metadata-schema) defined by AVA. 4 | 5 | ## Prerequisites 6 | 7 | 1. [Install Docker](https://docs.docker.com/desktop/#download-and-install) on your machine 8 | 1. [Install IoT Edge Runtime](https://docs.microsoft.com/azure/iot-edge/how-to-install-iot-edge?tabs=linux) 9 | 10 | ### Design 11 | 12 | This gRPC server is a Python terminal application that will house your custom AI and is built to handle the [protobuf](https://github.com/Azure/video-analyzer/tree/main/contracts/grpc) messages sent between AVA and your custom AI. AVA sends a media stream descriptor which defines what information will be sent followed by video frames to the server as a [protobuf](https://github.com/Azure/video-analyzer/tree/main/contracts/grpc) message over the gRPC stream session. The server validates the stream descriptor, analyses the video frame, processes it using an Image Processor, and returns inference results as a [protobuf](https://github.com/Azure/video-analyzer/tree/main/contracts/grpc) message. 13 | The frames can be transferred through shared memory or they can be embedded in the message. The date transfer mode can be configured in the pipelineTopology to determine how frames will be transferred. 14 | 15 | *main.py*: this is the entry point of the application. It is responsible for the configuring and management of the gRPC server. 16 | 17 | 18 | ``` 19 | Main() 20 | ``` 21 | In this method we: 22 | 1. Create an instance of gRPC server. 23 | 2. Create an instance of the service implementation class **InferenceServer**. 24 | 3. Register InferenceServer service implementation by adding its service definition to the Services collection. 25 | 4. Set the address and port the gRPC server will listen on for client requests. 26 | 5. Initialize the gRPC server. 27 | 28 | *inference_server.py*: this class is responsible for handling the [protobuf](https://github.com/Azure/video-analyzer/tree/main/contracts/grpc) messages communication with the AVA client. 29 | 30 | ``` 31 | ProcessMediaStream(self, requestIterator, context) 32 | ``` 33 | The client sends a media stream descriptor followed by video frames to the server as a protobuf message over the gRPC stream session. 34 | 35 | In this method we: 36 | 1. Read and validate the MediaStreamDescriptor (it is the first message sent by the client). 37 | 2. If the media stream descriptor is valid, the gRPC reads and analyzes the sequence of media samples containing the video frame, and returns inference results as a protobuf message. 38 | 39 | **Note:** this method supports batch processing, it uses the batchSize property to specify the number of the batch. 40 | 41 | *batchImageProcessor.py*: this class is responsible for processing the image. In a nutshell, it reads the raw bytes, converts an image to grayscale and determines if its color intensity is dark or light. You can add your own processor logic by adding a new class and implementing the method: 42 | 43 | ``` 44 | ProcessImages(self, mediaStreamMessage, rawBytes, size): 45 | ``` 46 | Once you've added the new class, you'll have to update the InferenceServer so it instantiates your class and invokes the **ProcessImage** method on it to run your processing logic. 47 | 48 | ### Building, publishing and running the Docker container 49 | 50 | To build the image, use the Docker file named `Dockerfile`. 51 | 52 | First, a couple assumptions 53 | 54 | * We'll be using Azure Container Registry (ACR) to publish our image before distributing it 55 | * Our local Docker container image is already loged into ACR. 56 | * In this sample, our ACR name is "myregistry". Your name may defer, so please update it properly in the following commands. 57 | 58 | > If you're unfamiliar with ACR or have any questions, please follow this [demo on building and pushing an image into ACR](https://docs.microsoft.com/azure/container-registry/container-registry-get-started-docker-cli). 59 | 60 | `cd` onto the grpc extension's root directory 61 | 62 | ``` 63 | sudo docker build -f .\docker\Dockerfile -t grpcextension:latest . 64 | 65 | sudo docker tag grpcextension:latest myregistry.azurecr.io/grpcextension:1 66 | 67 | sudo docker push myregistry.azurecr.io/grpcextension:1 68 | ``` 69 | 70 | Then, from the box where the container should execute, run this command: 71 | 72 | `sudo docker run -d -p 5001:5001 --name grpcextension myregistry.azurecr.io/grpcextension:1 -p 5001 -b 1` 73 | 74 | Let's decompose it a bit: 75 | 76 | * `-p 5001:5001`: it's up to you where you'd like to map the containers 5001 port. You can pick whatever port fits your needs. 77 | * `--name`: the name of the running container. 78 | * `registry/image:tag`: replace this with the corresponding location/image:tag where you've pushed the image built from the `Dockerfile` 79 | * `-p`: the port the gRPC server will listen on 80 | * `-b`: the size of the batch 81 | 82 | ### Updating references into pipelineTopologies, to target the gRPC Extension Address 83 | The [pipelineTopology](https://github.com/Azure/video-analyzer/tree/main/pipelines/live/topologies/motion-with-grpcExtension/topology.json) must define an gRPC Extension Address: 84 | 85 | * gRPC Extension Address Parameter 86 | ``` 87 | { 88 | "name": "grpcExtensionAddress", 89 | "type": "String", 90 | "description": "grpc AVA Extension Address", 91 | "default": "tcp://avaextension:5001" 92 | }, 93 | ``` 94 | * Configuration 95 | ``` 96 | { 97 | "@type": "#Microsoft.VideoAnalyzer.GrpcExtension", 98 | "name": "grpcExtension", 99 | "endpoint": { 100 | "@type": "#Microsoft.VideoAnalyzer.UnsecuredEndpoint", 101 | "url": "${grpcExtensionAddress}", 102 | "credentials": { 103 | "@type": "#Microsoft.VideoAnalyzer.UsernamePasswordCredentials", 104 | "username": "${grpcExtensionUserName}", 105 | "password": "${grpcExtensionPassword}" 106 | } 107 | }, 108 | "dataTransfer": { 109 | "mode": "sharedMemory", 110 | "SharedMemorySizeMiB": "5" 111 | }, 112 | "image": { 113 | "scale": { 114 | "mode": "${imageScaleMode}", 115 | "width": "${frameWidth}", 116 | "height": "${frameHeight}" 117 | }, 118 | "format": { 119 | "@type": "#Microsoft.VideoAnalyzer.ImageFormatRaw", 120 | "pixelFormat": "${imageRawFormat}" 121 | } 122 | }, 123 | "inputs": [ 124 | { 125 | "nodeName": "motionDetection" 126 | } 127 | ] 128 | } 129 | ``` 130 | 131 | The frames can be transferred through shared memory or they can be embedded in the message. The data transfer mode can be configured in the pipelineTopology to determine how frames will be transferred. This is achieved by configuring the dataTransfer element of the GrpcExtension as shown below: 132 | 133 | Embedded: 134 | ```JSON 135 | "dataTransfer": { 136 | "mode": "Embedded" 137 | } 138 | ``` 139 | 140 | Shared memory: 141 | ```JSON 142 | "dataTransfer": { 143 | "mode": "sharedMemory", 144 | "SharedMemorySizeMiB": "20" 145 | } 146 | ``` 147 | 148 | **Note:** When communicating over shared memory the AVA container and the gRPC extension module must have its IPC mode set to host. 149 | AVA module: 150 | ```JSON 151 | { 152 | "HostConfig": { 153 | "LogConfig": { 154 | "Config": { 155 | "max-size": "10m", 156 | "max-file": "10" 157 | } 158 | }, 159 | "IpcMode": "host" 160 | } 161 | } 162 | ``` 163 | 164 | gRPC extension module: 165 | ```JSON 166 | { 167 | "HostConfig": { 168 | "LogConfig": { 169 | "Config": { 170 | "max-size": "10m", 171 | "max-file": "10" 172 | } 173 | }, 174 | "IpcMode": "host" 175 | } 176 | } 177 | ``` 178 | 179 | 180 | ## Upload Docker image to Azure container registry 181 | 182 | Follow instructions in [Push and Pull Docker images - Azure Container Registry](http://docs.microsoft.com/azure/container-registry/container-registry-get-started-docker-cli) to save your image for later use on another machine. 183 | 184 | ## Deploy as an Azure IoT Edge module 185 | 186 | Follow instruction in [Deploy module from Azure portal](https://docs.microsoft.com/azure/iot-edge/how-to-deploy-modules-portal) to deploy the container image as an IoT Edge module (use the IoT Edge module option). 187 | 188 | ## gRPC server response 189 | Once the setup is complete and you instantiate the [gRPCExtension topology](https://github.com/Azure/video-analyzer/tree/main/pipelines/live/topologies/grpcExtension/topology.json) using [our VSCode quickstart](https://aka.ms/ava-grpc-quickstart) or via Azure Portal, you will see JSON printed on your screen that looks something like this 190 | 191 | ```JSON 192 | { 193 | "timestamp": 0, 194 | "inferences": [ 195 | { 196 | "type": "classification", 197 | "subtype": "colorIntensity", 198 | "inferenceId": "", 199 | "relatedInferences": [], 200 | "classification": { 201 | "tag": { 202 | "value": "light", 203 | "confidence": 1 204 | }, 205 | "attributes": [] 206 | }, 207 | "extensions": {}, 208 | "valueCase": "classification" 209 | } 210 | ] 211 | } 212 | ``` 213 | ## Terminate the gRPC extension 214 | Terminate the container using the following Docker commands 215 | 216 | ```bash 217 | docker stop grpcextension 218 | docker rm grpcextension 219 | ``` 220 | -------------------------------------------------------------------------------- /src/edge/modules/grpcExtension/app/inference_server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import media_pb2 4 | import extension_pb2 5 | import extension_pb2_grpc 6 | import inferencing_pb2 7 | from batchImageProcessor import BatchImageProcessor 8 | from enum import Enum 9 | from shared_memory import SharedMemoryManager 10 | from exception_handler import PrintGetExceptionDetails 11 | 12 | # Get debug flag from env variable (Returns None if not set) 13 | # Set this environment variables in the IoTEdge Deployment manifest to activate debugging. 14 | # You should also map the DebugOutputFolder on the host machine to write out the debug frames... 15 | DEBUG = os.getenv('Debug') 16 | DEBUG_OUTPUT_FOLDER = os.getenv('DebugOutputFolder') 17 | 18 | class TransferType(Enum): 19 | BYTES = 1 # Embedded Content 20 | REFERENCE = 2 # Shared Memory 21 | HANDLE = 3 # Reserved... 22 | 23 | class State: 24 | def __init__(self, mediaStreamDescriptor): 25 | try: 26 | # media descriptor holding input data format 27 | self._mediaStreamDescriptor = mediaStreamDescriptor 28 | 29 | # Get how data will be transferred 30 | if self._mediaStreamDescriptor.WhichOneof("data_transfer_properties") is None: 31 | self._contentTransferType = TransferType.BYTES 32 | elif self._mediaStreamDescriptor.HasField("shared_memory_buffer_transfer_properties"): 33 | self._contentTransferType = TransferType.REFERENCE 34 | 35 | # Setup if shared mem used 36 | if self._contentTransferType == TransferType.REFERENCE: 37 | # Create shared memory accessor specific to the client 38 | self._sharedMemoryManager = SharedMemoryManager( 39 | name=self._mediaStreamDescriptor.shared_memory_buffer_transfer_properties.handle_name, 40 | size=self._mediaStreamDescriptor.shared_memory_buffer_transfer_properties.length_bytes) 41 | else: 42 | self._sharedMemoryManager = None 43 | 44 | except: 45 | PrintGetExceptionDetails() 46 | raise 47 | 48 | class InferenceServer(extension_pb2_grpc.MediaGraphExtensionServicer): 49 | def __init__(self, batchSize): 50 | self.processor = BatchImageProcessor() 51 | self.batchSize = batchSize 52 | return 53 | 54 | def process_media_sample(self, mediaStreamMessage, imageDetails): 55 | #Get media content bytes. (bytes sent over shared memory buffer, segment or inline to message) 56 | try: 57 | rawBytes, size = imageDetails 58 | return self.processor.process_images(mediaStreamMessage, rawBytes, size) 59 | 60 | except: 61 | PrintGetExceptionDetails() 62 | raise 63 | 64 | def get_image_details(self, clientState, mediaStreamMessageRequest): 65 | #Get media content bytes. (bytes sent over shared memory buffer, segment or inline to message) 66 | try: 67 | # Get reference to raw bytes 68 | if clientState._contentTransferType == TransferType.BYTES: 69 | rawBytes = memoryview(mediaStreamMessageRequest.media_sample.content_bytes.bytes) 70 | elif clientState._contentTransferType == TransferType.REFERENCE: 71 | # Data sent over shared memory buffer 72 | addressOffset = mediaStreamMessageRequest.media_sample.content_reference.address_offset 73 | lengthBytes = mediaStreamMessageRequest.media_sample.content_reference.length_bytes 74 | 75 | # Get memory reference to (in readonly mode) data sent over shared memory 76 | rawBytes = clientState._sharedMemoryManager.ReadBytes(addressOffset, lengthBytes) 77 | 78 | # Get encoding details of the media sent by client 79 | encoding = clientState._mediaStreamDescriptor.media_descriptor.video_frame_sample_format.encoding 80 | 81 | # Handle RAW content (Just place holder for the user to handle each variation...) 82 | if encoding == clientState._mediaStreamDescriptor.media_descriptor.video_frame_sample_format.Encoding.RAW: 83 | width = clientState._mediaStreamDescriptor.media_descriptor.video_frame_sample_format.dimensions.width 84 | height = clientState._mediaStreamDescriptor.media_descriptor.video_frame_sample_format.dimensions.height 85 | 86 | return rawBytes, (width, height) 87 | else: 88 | raise Exception('Sample format is not RAW') 89 | 90 | except: 91 | PrintGetExceptionDetails() 92 | raise 93 | 94 | def ProcessMediaStream(self, requestIterator, context): 95 | # Below logic can be extended into multi-process (per CPU cores, i.e. in case using CPU inferencing) 96 | # For simplicity below, we use single process to handle gRPC clients 97 | 98 | # Auto increment counter. Increases per client requests 99 | responseSeqNum = 1 100 | 101 | # First message from the client is (must be) MediaStreamDescriptor 102 | mediaStreamMessageRequest = next(requestIterator) 103 | 104 | # Extract message IDs 105 | requestSeqNum = mediaStreamMessageRequest.sequence_number 106 | requestAckSeqNum = mediaStreamMessageRequest.ack_sequence_number 107 | 108 | # State object per client 109 | clientState = State(mediaStreamMessageRequest.media_stream_descriptor) 110 | 111 | logging.info('[Received] SeqNum: {0:07d} | AckNum: {1}\nMediaStreamDescriptor:\n{2}'.format(requestSeqNum, requestAckSeqNum, clientState._mediaStreamDescriptor)) 112 | 113 | # First message response ... 114 | mediaStreamMessage = extension_pb2.MediaStreamMessage( 115 | sequence_number = responseSeqNum, 116 | ack_sequence_number = requestSeqNum, 117 | media_stream_descriptor = extension_pb2.MediaStreamDescriptor( 118 | media_descriptor = media_pb2.MediaDescriptor( 119 | timescale = clientState._mediaStreamDescriptor.media_descriptor.timescale 120 | ) 121 | ) 122 | ) 123 | # Send acknowledge message to client 124 | yield mediaStreamMessage 125 | 126 | # Process rest of the MediaStream message sequence 127 | messageCount = 0 128 | imageBatch = [] 129 | for mediaStreamMessageRequest in requestIterator: 130 | try: 131 | # Read request id, sent by client 132 | requestSeqNum = mediaStreamMessageRequest.sequence_number 133 | 134 | logging.info('[Received] SequenceNum: {0:07d}'.format(requestSeqNum)) 135 | 136 | imageDetails = self.get_image_details(clientState, mediaStreamMessageRequest) 137 | # Increment request sequence number 138 | responseSeqNum += 1 139 | 140 | if(messageCount < self.batchSize): 141 | # Add images to batch and create acknowledge message 142 | msg = 'Adding image #{0} to batch.'.format(messageCount+1) 143 | logging.info(msg) 144 | mediaStreamMessage = extension_pb2.MediaStreamMessage( 145 | sequence_number = responseSeqNum, 146 | ack_sequence_number = requestSeqNum 147 | ) 148 | inference = mediaStreamMessage.media_sample.inferences.add() 149 | inference.subtype = 'BatchAggregation' 150 | event = inferencing_pb2.Event(name=msg) 151 | inference.event.CopyFrom(event) 152 | 153 | imageBatch.append(imageDetails) 154 | messageCount += 1 155 | else: 156 | # Process batch 157 | logging.info('Processing batch ({0}).'.format(messageCount)) 158 | mediaStreamMessage = extension_pb2.MediaStreamMessage() 159 | for image in imageBatch: 160 | mediaStreamMessage = self.process_media_sample(mediaStreamMessage, image) 161 | 162 | if(mediaStreamMessage is None): 163 | # Respond with message without inferencing 164 | mediaStreamMessage = extension_pb2.MediaStreamMessage() 165 | responseStatusMessage = "empty message for request seq = " + str(mediaStreamMessage.ack_sequence_number) + " response seq = " + str(responseSeqNum) 166 | else: 167 | responseStatusMessage = "responding for message with request seq = " + str(mediaStreamMessage.ack_sequence_number) + " response seq = " + str(responseSeqNum) 168 | 169 | logging.info(responseStatusMessage) 170 | mediaStreamMessage.sequence_number = responseSeqNum 171 | mediaStreamMessage.ack_sequence_number = mediaStreamMessageRequest.sequence_number 172 | mediaStreamMessage.media_sample.timestamp = mediaStreamMessageRequest.media_sample.timestamp 173 | 174 | # Clear batch 175 | imageBatch.clear() 176 | messageCount = 0 177 | 178 | if context.is_active(): 179 | # yield response 180 | yield mediaStreamMessage 181 | else: 182 | break 183 | except: 184 | PrintGetExceptionDetails() 185 | 186 | logging.info('Done processing messages') -------------------------------------------------------------------------------- /src/edge/readme.md: -------------------------------------------------------------------------------- 1 | # IoT Edge Samples 2 | 3 | This folder contains IoT Edge deployment manifest templates and sample IoT Edge modules. 4 | 5 | ## Deployment manifest templates 6 | 7 | Name | Description | Sample Tutorial | Supported Architecture | Inference Endpoint 8 | :----- | :---- | :---- | :---- | :---- 9 | [deployment.template.json](deployment.template.json) | This file is a deployment manifest template that has the following modules defined in it: