├── .gitattributes ├── .gitignore ├── CONTRIBUTING.md ├── Config.bat ├── LICENSE ├── README.md ├── SECURITY.md ├── ThirdPartyNotices.md ├── cfg ├── imagenet-classes.json └── sample.txt ├── documents ├── images │ ├── _line_drawing_1.png │ ├── _line_drawing_2.png │ ├── _line_drawing_3.png │ └── _line_sample.png └── line_configuration.md ├── eng └── Versions.props ├── media └── sample.mp4 ├── modelOnnx └── coco.names └── src ├── Directory.Build.props ├── Directory.Build.targets └── VAP ├── AML.Client ├── AML.Client.csproj ├── AMLCaller.cs ├── IPredictionServiceClient.cs ├── IScoringRequest.cs ├── ImageRequest.cs ├── ScoringClient.cs └── TensorExtensions.cs ├── AML.tfserving ├── AML.tfserving.csproj ├── tensorflow-serving │ ├── Classification.cs │ ├── GetModelMetadata.cs │ ├── Inference.cs │ ├── Input.cs │ ├── LICENSE │ ├── Model.cs │ ├── Predict.cs │ ├── PredictionService.cs │ ├── PredictionServiceGrpc.cs │ └── Regression.cs └── tensorflow │ ├── AllocationDescription.cs │ ├── AttrValue.cs │ ├── Cluster.cs │ ├── Config.cs │ ├── ControlFlow.cs │ ├── CostGraph.cs │ ├── Debug.cs │ ├── DeviceAttributes.cs │ ├── DeviceProperties.cs │ ├── ErrorCodes.cs │ ├── Example.cs │ ├── Feature.cs │ ├── Function.cs │ ├── Graph.cs │ ├── LICENSE │ ├── Master.cs │ ├── MasterService.cs │ ├── MetaGraph.cs │ ├── NamedTensor.cs │ ├── NodeDef.cs │ ├── OpDef.cs │ ├── QueueRunner.cs │ ├── ResourceHandle.cs │ ├── RewriterConfig.cs │ ├── SavedModel.cs │ ├── Saver.cs │ ├── StepStats.cs │ ├── Tensor.cs │ ├── TensorBundle.cs │ ├── TensorDescription.cs │ ├── TensorShape.cs │ ├── TensorSlice.cs │ ├── TensorflowServer.cs │ ├── Types.cs │ ├── Versions.cs │ ├── Worker.cs │ └── WorkerService.cs ├── BGSObjectDetector ├── BGSObjectDetector.cs ├── BGSObjectDetector.csproj ├── Box.cs └── MOG2.cs ├── DNNDetector ├── CascadedDNNORTYolo.cs ├── Config │ └── DNNConfig.cs ├── DNNDetector.csproj ├── FrameBuffer.cs ├── FrameDNNORTYolo.cs ├── LineTriggeredDNNORTYolo.cs └── Model │ └── Item.cs ├── DarknetDetector ├── CascadedDNNYolo.cs ├── DarknetDetector.csproj ├── FrameBuffer.cs ├── FrameDNNYolo.cs └── LineTriggeredDNNYolo.cs ├── Decoder ├── Decoder.cs └── Decoder.csproj ├── FramePreProcessor ├── FrameDisplay.cs ├── FramePreProcessor.csproj └── PreProcessor.cs ├── LineDetector ├── CascadedLinesDetector.cs ├── DetectionLine.cs ├── Detector.cs ├── FallingEdgeCrossingDetector.cs ├── ICrossingDetector.cs ├── ILaneBasedDetector.cs ├── ISingleLineCrossingDetector.cs ├── LineDetector.csproj ├── LineSets.cs ├── MultiLaneDetector.cs └── SingleLineCrossingDetector.cs ├── ORTWrapper ├── DNNMode.cs ├── IYoloConfiguration.cs ├── ORTItem.cs ├── ORTWrapper.cs ├── ORTWrapper.csproj └── Yolov3BaseConfig.cs ├── PostProcessor ├── AzureBlobProcessor.cs ├── DBClient.cs ├── DataPersistence.cs ├── Model │ ├── clctCamera.cs │ ├── clctConsolidation.cs │ ├── clctDetection.cs │ ├── clctObject.cs │ ├── queryResponse.cs │ └── repo.cs └── PostProcessor.csproj ├── TFDetector ├── FrameBuffer.cs ├── FrameDNNTF.cs ├── LineTriggeredDNNTF.cs └── TFDetector.csproj ├── TFWrapper ├── Common │ ├── CatalogUtil.cs │ └── ImageUtil.cs ├── ImageEditor.cs ├── TFWrapper.cs ├── TFWrapper.csproj ├── packages │ └── TensorFlowSharp.1.12.0 │ │ └── lib │ │ └── TensorFlowSharp.dll └── test_images │ └── input.jpg ├── Utils ├── Config │ └── OutputFolder.cs ├── Utils.cs └── Utils.csproj ├── VAP.sln ├── VideoPipelineCore ├── App.config ├── Program.cs ├── Properties │ └── launchSettings.json └── VideoPipelineCore.csproj └── YoloWrapper ├── App.config ├── ConfigurationDetector.cs ├── DNNMode.cs ├── Dependencies ├── pthreadVC2.dll ├── yolo_cpp_dll_cpu.dll ├── yolo_cpp_dll_gpu_cc.dll └── yolo_cpp_dll_gpu_lt.dll ├── DetectionSystem.cs ├── ImageAnalyzer.cs ├── ImageResizer.cs ├── Model ├── BboxContainer.cs ├── BboxT.cs ├── EnvironmentReport.cs ├── YoloItem.cs └── YoloTrackingItem.cs ├── Yolo.Config ├── YoloV3Coco │ ├── coco.names │ └── yolov3.cfg └── YoloV3TinyCoco │ ├── coco.names │ └── yolov3-tiny.cfg ├── YoloConfiguration.cs ├── YoloTracking.cs ├── YoloWrapper.cs └── YoloWrapper.csproj /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vs/ 2 | obj/ 3 | bin/ 4 | packages/ 5 | *.csproj.user 6 | 7 | /src/VAP/YoloWrapper/Dependencies/opencv_world340.dll 8 | /src/VAP/YoloWrapper/Dependencies/opencv_world340d.dll 9 | /src/VAP/YoloWrapper/Yolo.Config/YoloV3Coco/yolov3.weights 10 | /src/VAP/YoloWrapper/Yolo.Config/YoloV3TinyCoco/yolov3-tiny.weights 11 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to 4 | agree to a Contributor License Agreement (CLA) declaring that you have the right to, 5 | and actually do, grant us the rights to use your contribution. For details, visit 6 | https://cla.microsoft.com. 7 | 8 | When you submit a pull request, a CLA-bot will automatically determine whether you need 9 | to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the 10 | instructions provided by the bot. You will only need to do this once across all repositories using our CLA. 11 | 12 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 13 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 14 | or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -------------------------------------------------------------------------------- /Config.bat: -------------------------------------------------------------------------------- 1 | powershell -Command "New-Item -ItemType directory -Path ./src/VAP/TFWrapper/packages/TensorFlowSharp.1.12.0/runtimes/win7-x64/native/" 2 | powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Microsoft-Rocket-Video-Analytics-Platform-libtensorflow.dll', './src/VAP/TFWrapper/packages/TensorFlowSharp.1.12.0/runtimes/win7-x64/native/libtensorflow.dll')" 3 | 4 | powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Microsoft-Rocket-Video-Analytics-Platform-opencv_world340.dll', './src/VAP/YoloWrapper/Dependencies/opencv_world340.dll')" 5 | 6 | powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Microsoft-Rocket-Video-Analytics-Platform-opencv_world340d.dll', './src/VAP/YoloWrapper/Dependencies/opencv_world340d.dll')" 7 | 8 | powershell -Command "(New-Object Net.WebClient).DownloadFile('https://pjreddie.com/media/files/yolov3.weights', './src/VAP/YoloWrapper/Yolo.Config/YoloV3Coco/yolov3.weights')" 9 | 10 | powershell -Command "(New-Object Net.WebClient).DownloadFile('https://pjreddie.com/media/files/yolov3-tiny.weights', './src/VAP/YoloWrapper/Yolo.Config/YoloV3TinyCoco/yolov3-tiny.weights')" 11 | 12 | powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Microsoft-Rocket-Video-Analytics-Platform-yolov3ort.onnx', './modelOnnx/yolov3ort.onnx')" 13 | 14 | powershell -Command "(New-Object Net.WebClient).DownloadFile('https://aka.ms/Microsoft-Rocket-Video-Analytics-Platform-yolov3tinyort.onnx', './modelOnnx/yolov3tinyort.onnx')" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Microsoft Corporation. 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /ThirdPartyNotices.md: -------------------------------------------------------------------------------- 1 | Microsoft Rocket Video Analytics Platform uses third-party material as listed 2 | below. The attached notices are provided for informational purposes only. 3 | 4 | Notice for opencvsharp 5 | ------------------------------- 6 | BSD 3-Clause License 7 | 8 | Copyright (c) 2019, shimat 9 | All rights reserved. 10 | 11 | Redistribution and use in source and binary forms, with or without 12 | modification, are permitted provided that the following conditions are met: 13 | 14 | * Redistributions of source code must retain the above copyright notice, this 15 | list of conditions and the following disclaimer. 16 | 17 | * Redistributions in binary form must reproduce the above copyright notice, 18 | this list of conditions and the following disclaimer in the documentation 19 | and/or other materials provided with the distribution. 20 | 21 | * Neither the name of the copyright holder nor the names of its 22 | contributors may be used to endorse or promote products derived from 23 | this software without specific prior written permission. 24 | 25 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 26 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 29 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 31 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 32 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 33 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 | 36 | ------------------------------- 37 | 38 | Notice for TensorFlowSharp 39 | ------------------------------- 40 | 41 | Permission is hereby granted, free of charge, to any person obtaining 42 | a copy of this software and associated documentation files (the 43 | "Software"), to deal in the Software without restriction, including 44 | without limitation the rights to use, copy, modify, merge, publish, 45 | distribute, sublicense, and/or sell copies of the Software, and to 46 | permit persons to whom the Software is furnished to do so, subject to 47 | the following conditions: 48 | 49 | The above copyright notice and this permission notice shall be 50 | included in all copies or substantial portions of the Software. 51 | 52 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 53 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 54 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 55 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 56 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 57 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 58 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 59 | 60 | ------------------------------- 61 | 62 | Notice for YOLO 63 | ------------------------------- 64 | 65 | THIS SOFTWARE LICENSE IS PROVIDED "ALL CAPS" SO THAT YOU KNOW IT IS SUPER 66 | SERIOUS AND YOU DON'T MESS AROUND WITH COPYRIGHT LAW BECAUSE YOU WILL GET IN 67 | TROUBLE HERE ARE SOME OTHER BUZZWORDS COMMONLY IN THESE THINGS WARRANTIES 68 | LIABILITY CONTRACT TORT LIABLE CLAIMS RESTRICTION MERCHANTABILITY. NOW HERE'S 69 | THE REAL LICENSE: 70 | 71 | 0. Darknet is public domain. 72 | 1. Do whatever you want with it. 73 | 2. Stop emailing me about it! 74 | ------------------------------- -------------------------------------------------------------------------------- /cfg/sample.txt: -------------------------------------------------------------------------------- 1 | Left-entrance 1 209 101 266 101 0.3 -------------------------------------------------------------------------------- /documents/images/_line_drawing_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/documents/images/_line_drawing_1.png -------------------------------------------------------------------------------- /documents/images/_line_drawing_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/documents/images/_line_drawing_2.png -------------------------------------------------------------------------------- /documents/images/_line_drawing_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/documents/images/_line_drawing_3.png -------------------------------------------------------------------------------- /documents/images/_line_sample.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/documents/images/_line_sample.png -------------------------------------------------------------------------------- /documents/line_configuration.md: -------------------------------------------------------------------------------- 1 | # Configure lines of interest 2 | 3 | Rocket enables counting and alerting when objects cross lines of interest in the frame, e.g., cars entering a driveway or people walking into the shop. Rocket takes a line-file as input for its line-based counting/alerting and cascaded DNN calls. 4 | 5 | ## Line format and specification 6 | 7 | Each line in the file defines a line-of-interest with the following format. The fields in each line are separated by tabs. 8 | 9 | ` ` 10 | 11 | The fields signify the following: 12 | * line_count_name - unique string to name the line. 13 | * number_of_lines - this must be always set to 1. 14 | * x_1, y_1, x_2, y_2 - the X and Y coordinates of the line's endpoints. 15 | * overlap_threshold - this is the fraction of the line that the object is expected to cover for the count/alert to be triggered. 16 | 17 | Here is a [sample line configuration file](https://aka.ms/linesample) manually created for [rocket-sample.mkv](https://aka.ms/lva-rocket-videosample) with two lines of interest. The two lines are also shown in the picture below, which is a frame from the `rocket-sample.mkv` video. The first line in the [sample configuration file](https://aka.ms/linesample), for instance, lists the coordinates of the two endpoints of the line `(209, 101)` and `(266, 101)`. A count will be triggered when the object's overlap with this line is at least `0.3` of the line's length. 18 | sampleline 19 | 20 | An easy way to mark the lines is to take a "snapshot" from the video, say using VLC, mark the lines on the snapshot frame, and then list the coordinates and parameters of the line in the format specified above to create the line-file. We include helpful snapshots below for marking the lines using VLC and Gnome Paint Drawing Editor on Ubuntu. On Windows, similar steps can be taken by VLC and Paint. 21 | 22 | Taking snapshot on VLC 23 | sampleline 24 | 25 | Opening the snapshot from Pictures folder 26 | sampleline 27 | 28 | Drawing lines in Gnome Paint Drawing Editor. Note the coordinates at the bottom as the cursor moves. 29 | sampleline -------------------------------------------------------------------------------- /eng/Versions.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 3.10.0 6 | 2.24.0 7 | 5.3.0.1 8 | 12.0.2 9 | 4.1.1.20191110 10 | 2.6.1.1 11 | 1.2.0 12 | 4.6.0 13 | 4.6.0 14 | 9.3.3 15 | 16 | 17 | 4.1.1.20191023 18 | 4.1.1.20191110 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /media/sample.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/media/sample.mp4 -------------------------------------------------------------------------------- /modelOnnx/coco.names: -------------------------------------------------------------------------------- 1 | person 2 | bicycle 3 | car 4 | motorbike 5 | aeroplane 6 | bus 7 | train 8 | truck 9 | boat 10 | traffic light 11 | fire hydrant 12 | stop sign 13 | parking meter 14 | bench 15 | bird 16 | cat 17 | dog 18 | horse 19 | sheep 20 | cow 21 | elephant 22 | bear 23 | zebra 24 | giraffe 25 | backpack 26 | umbrella 27 | handbag 28 | tie 29 | suitcase 30 | frisbee 31 | skis 32 | snowboard 33 | sports ball 34 | kite 35 | baseball bat 36 | baseball glove 37 | skateboard 38 | surfboard 39 | tennis racket 40 | bottle 41 | wine glass 42 | cup 43 | fork 44 | knife 45 | spoon 46 | bowl 47 | banana 48 | apple 49 | sandwich 50 | orange 51 | broccoli 52 | carrot 53 | hot dog 54 | pizza 55 | donut 56 | cake 57 | chair 58 | sofa 59 | pottedplant 60 | bed 61 | diningtable 62 | toilet 63 | tvmonitor 64 | laptop 65 | mouse 66 | remote 67 | keyboard 68 | cell phone 69 | microwave 70 | oven 71 | toaster 72 | sink 73 | refrigerator 74 | book 75 | clock 76 | vase 77 | scissors 78 | teddy bear 79 | hair drier 80 | toothbrush 81 | -------------------------------------------------------------------------------- /src/Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/Directory.Build.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /src/VAP/AML.Client/AML.Client.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/VAP/AML.Client/AMLCaller.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Model; 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Drawing; 9 | using System.Drawing.Imaging; 10 | using System.Threading.Tasks; 11 | using System.IO; 12 | using System.Linq; 13 | using System.Reflection; 14 | using Utils.Config; 15 | 16 | namespace AML.Client 17 | { 18 | public class AMLCaller 19 | { 20 | static string Host; 21 | static bool UseSSL; 22 | static string Auth; 23 | static string AksServiceName; 24 | static string InputName = "Placeholder:0"; 25 | static string OutputName = "classifier/resnet_v1_50/predictions/Softmax:0"; 26 | 27 | public AMLCaller(string host, bool useSSL, string auth, string aksServiceName) 28 | { 29 | try 30 | { 31 | Host = host; 32 | UseSSL = useSSL; 33 | Auth = auth; 34 | AksServiceName = aksServiceName; 35 | } 36 | catch (Exception e) 37 | { 38 | Console.WriteLine("AML init error:" + e); 39 | } 40 | } 41 | 42 | public static async Task> Run(int frameIndex, List items, HashSet category) 43 | { 44 | //could implement AML triggering criteria here, e.g., confidence 45 | 46 | if (items == null) 47 | { 48 | return null; 49 | } 50 | 51 | var client = new ScoringClient(Host, UseSSL ? 443 : 80, UseSSL, Auth, AksServiceName); 52 | List amlResult = new List(); 53 | 54 | for (int itemIndex = 0; itemIndex < items.Count(); itemIndex++) 55 | { 56 | MemoryStream mStream = new MemoryStream(); 57 | using (Image image = Image.FromStream(new MemoryStream(items[itemIndex].CroppedImageData))) 58 | { 59 | image.Save(mStream, ImageFormat.Png); 60 | mStream.Position = 0; 61 | } 62 | 63 | using (mStream) 64 | { 65 | IScoringRequest request = new ImageRequest(InputName, mStream); 66 | var stopWatch = System.Diagnostics.Stopwatch.StartNew(); 67 | try 68 | { 69 | var result = await client.ScoreAsync(request, output_name: OutputName); 70 | var latency = stopWatch.Elapsed; 71 | for (int i = 0; i < result.GetLength(0); i++) 72 | { 73 | Console.WriteLine($"Latency: {latency}"); 74 | Console.WriteLine($"Batch {i}:"); 75 | var length = result.GetLength(1); 76 | var results = new Dictionary(); 77 | for (int j = 0; j < length; j++) 78 | { 79 | results.Add(j, result[i, j]); 80 | } 81 | 82 | foreach (var kvp in results.Where(x => x.Value > 0.001).OrderByDescending(x => x.Value).Take(5)) 83 | { 84 | Console.WriteLine( 85 | $" {GetLabel(kvp.Key)} {kvp.Value * 100}%"); 86 | char[] delimiterChars = { ' ', ',' }; 87 | foreach (var key in GetLabel(kvp.Key).Split(delimiterChars)) 88 | { 89 | if (category.Count == 0 || category.Contains(key)) 90 | { 91 | amlResult.Add(true); 92 | 93 | // output AML results 94 | string blobName_AML = $@"frame-{frameIndex}-zAML-{key}-{kvp.Value}.jpg"; 95 | string fileName_AML = @OutputFolder.OutputFolderAML + blobName_AML; 96 | File.WriteAllBytes(fileName_AML, items[itemIndex].CroppedImageData); 97 | File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_AML, items[itemIndex].CroppedImageData); 98 | 99 | goto CheckNextItem; 100 | } 101 | } 102 | } 103 | } 104 | } 105 | catch (Exception e) 106 | { 107 | Console.WriteLine("error:" + e); 108 | return null; 109 | } 110 | } 111 | CheckNextItem:; 112 | } 113 | 114 | return amlResult; 115 | } 116 | 117 | private static Dictionary _classes; 118 | 119 | private static string GetLabel(int classId) 120 | { 121 | if (_classes == null) 122 | { 123 | var assembly = typeof(AMLCaller).GetTypeInfo().Assembly; 124 | var result = assembly.GetManifestResourceStream("AML.Client.imagenet-classes.json"); 125 | 126 | var streamReader = new StreamReader(result); 127 | var classesJson = streamReader.ReadToEnd(); 128 | 129 | _classes = JsonConvert.DeserializeObject>(classesJson); 130 | } 131 | 132 | return _classes[classId]; 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/VAP/AML.Client/IPredictionServiceClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Threading.Tasks; 5 | using Tensorflow.Serving; 6 | 7 | namespace AML.Client 8 | { 9 | public interface IPredictionServiceClient 10 | { 11 | Task PredictAsync(PredictRequest predictRequest); 12 | } 13 | 14 | public class PredictionServiceClientWrapper : IPredictionServiceClient 15 | { 16 | private readonly PredictionService.PredictionServiceClient _predictionServiceClient; 17 | 18 | public PredictionServiceClientWrapper(PredictionService.PredictionServiceClient predictionServiceClient) 19 | { 20 | _predictionServiceClient = predictionServiceClient; 21 | } 22 | 23 | public Task PredictAsync(PredictRequest predictRequest) => _predictionServiceClient.PredictAsync(predictRequest).ResponseAsync; 24 | } 25 | } -------------------------------------------------------------------------------- /src/VAP/AML.Client/IScoringRequest.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | using Tensorflow.Serving; 4 | 5 | namespace AML.Client 6 | { 7 | public interface IScoringRequest 8 | { 9 | PredictRequest MakePredictRequest(); 10 | } 11 | } -------------------------------------------------------------------------------- /src/VAP/AML.Client/ImageRequest.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | using System.IO; 4 | using System.Linq; 5 | using Google.Protobuf; 6 | using Tensorflow; 7 | using Tensorflow.Serving; 8 | using DataType = Tensorflow.DataType; 9 | 10 | namespace AML.Client 11 | { 12 | public class ImageRequest : IScoringRequest 13 | { 14 | private readonly ModelSpec _modelSpec; 15 | private readonly TensorProto _proto; 16 | private readonly string _inputName; 17 | 18 | public ImageRequest(string inputName, params Stream[] images) 19 | { 20 | _modelSpec = new ModelSpec(); 21 | _proto = new TensorProto { Dtype = DataType.DtString }; 22 | 23 | var bytes = images.Select(ByteString.FromStream); 24 | _proto.StringVal.AddRange(bytes); 25 | _proto.TensorShape = new TensorShapeProto(); 26 | _proto.TensorShape.Dim.Add(new TensorShapeProto.Types.Dim()); 27 | _proto.TensorShape.Dim[0].Size = images.Length; 28 | _inputName = inputName; 29 | } 30 | 31 | public PredictRequest MakePredictRequest() 32 | { 33 | var request = new PredictRequest { ModelSpec = _modelSpec }; 34 | 35 | request.Inputs[this._inputName] = _proto; 36 | return request; 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /src/VAP/AML.Client/ScoringClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | 4 | using System; 5 | using System.Threading.Tasks; 6 | using Grpc.Core; 7 | using Tensorflow.Serving; 8 | using Grpc.Core.Interceptors; 9 | using System.Collections.Generic; 10 | using System.Linq; 11 | 12 | namespace AML.Client 13 | { 14 | public class ScoringClient 15 | { 16 | private const int RetryCount = 10; 17 | 18 | private readonly IPredictionServiceClient _client; 19 | 20 | public ScoringClient(IPredictionServiceClient client) 21 | { 22 | _client = client; 23 | } 24 | 25 | public ScoringClient(Channel channel) : this(new PredictionServiceClientWrapper(new PredictionService.PredictionServiceClient(channel))) 26 | { 27 | } 28 | 29 | public ScoringClient(string host, int port, bool useSsl = false, string authKey = null, string serviceName = null) 30 | { 31 | ChannelCredentials baseCreds, creds; 32 | baseCreds = useSsl ? new SslCredentials() : ChannelCredentials.Insecure; 33 | if (authKey != null && useSsl) 34 | { 35 | creds = ChannelCredentials.Create(baseCreds, CallCredentials.FromInterceptor( 36 | async (context, metadata) => 37 | { 38 | metadata.Add(new Metadata.Entry("authorization", authKey)); 39 | await Task.CompletedTask; 40 | })); 41 | } 42 | else 43 | { 44 | creds = baseCreds; 45 | } 46 | var channel = new Channel(host, port, creds); 47 | var callInvoker = channel.Intercept( 48 | metadata => 49 | { 50 | metadata.Add( 51 | new Metadata.Entry("x-ms-aml-grpc-service-route", $"/api/v1/service/{serviceName}")); 52 | return metadata; 53 | }); 54 | _client = new PredictionServiceClientWrapper(new PredictionService.PredictionServiceClient(callInvoker)); 55 | } 56 | 57 | public async Task ScoreAsync(IScoringRequest request, int retryCount = RetryCount, string output_name = "output_alias") 58 | { 59 | return await ScoreAsync(request, retryCount, output_name); 60 | } 61 | 62 | public async Task ScoreAsync(IScoringRequest request, int retryCount = RetryCount, string output_name = "output_alias") where T : class 63 | { 64 | return (await this.PredictAsync(request, retryCount))[output_name]; 65 | } 66 | 67 | public async Task> PredictAsync(IScoringRequest request, int retryCount = RetryCount) where T : class 68 | { 69 | var predictRequest = request.MakePredictRequest(); 70 | 71 | return await RetryAsync(async () => 72 | { 73 | var result = await _client.PredictAsync(predictRequest); 74 | return result.Outputs.ToDictionary( 75 | kvp => kvp.Key, kvp => kvp.Value.Convert()); 76 | }, retryCount); 77 | } 78 | 79 | private async Task RetryAsync( 80 | Func> operation, int retryCount = RetryCount 81 | ) 82 | { 83 | while (true) 84 | { 85 | try 86 | { 87 | return await operation(); 88 | } 89 | catch (RpcException rpcException) 90 | { 91 | if (!IsTransient(rpcException) || --retryCount <= 0) 92 | { 93 | throw; 94 | } 95 | } 96 | } 97 | } 98 | 99 | private static bool IsTransient(RpcException rpcException) 100 | { 101 | return 102 | rpcException.Status.StatusCode == StatusCode.DeadlineExceeded || 103 | rpcException.Status.StatusCode == StatusCode.Unavailable || 104 | rpcException.Status.StatusCode == StatusCode.Aborted || 105 | rpcException.Status.StatusCode == StatusCode.Internal; 106 | } 107 | } 108 | } -------------------------------------------------------------------------------- /src/VAP/AML.Client/TensorExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT License. 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using Tensorflow; 7 | 8 | namespace AML.Client 9 | { 10 | public static class TensorExtensions 11 | { 12 | public static T Convert(this TensorProto tensor) where T : class 13 | { 14 | var shape = tensor.TensorShape; 15 | var dimCount = shape.Dim.Count; 16 | 17 | var resultType = typeof(T); 18 | 19 | if (!resultType.IsArray) 20 | { 21 | throw new Exception("Unable to convert tensor into scalar type"); 22 | } 23 | 24 | var arrayRank = typeof(T).GetArrayRank(); 25 | 26 | if (arrayRank != dimCount) 27 | { 28 | throw new Exception($"result tensor was not the expected rank {arrayRank} - was rank {dimCount}"); 29 | } 30 | 31 | var elementType = resultType.GetElementType(); 32 | 33 | Func getItemFunc = null; 34 | 35 | if (elementType == typeof(float)) 36 | { 37 | getItemFunc = (t, i) => t.FloatVal[i]; 38 | } 39 | 40 | if (getItemFunc == null) 41 | { 42 | throw new Exception($"Don't know how to handle type {elementType}"); 43 | } 44 | 45 | var dimSizes = shape.Dim.Select(d => (int)d.Size).ToArray(); 46 | var sysArray = Array.CreateInstance(elementType, dimSizes); 47 | var tensorIndex = 0; 48 | 49 | foreach (var dimArray in GetPermutations(dimSizes)) 50 | { 51 | sysArray.SetValue(getItemFunc(tensor, tensorIndex), dimArray); 52 | tensorIndex++; 53 | } 54 | 55 | return sysArray as T; 56 | } 57 | 58 | public static IEnumerable GetPermutations(this int[] maxValues) 59 | { 60 | return GetPermutations(new int[maxValues.Length], 0, maxValues); 61 | } 62 | 63 | private static IEnumerable GetPermutations(int[] values, int index, int[] maxValues) 64 | { 65 | if (index >= values.Length) 66 | { 67 | return new[] { values }; 68 | } 69 | 70 | var result = new List(); 71 | 72 | for (var i = 0; i < maxValues[index]; i++) 73 | { 74 | var currentValues = values.ToArray(); 75 | currentValues[index] = i; 76 | result.AddRange(GetPermutations(currentValues, index + 1, maxValues)); 77 | } 78 | 79 | return result; 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/VAP/AML.tfserving/AML.tfserving.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/VAP/AML.tfserving/tensorflow-serving/PredictionService.cs: -------------------------------------------------------------------------------- 1 | // Generated by the protocol buffer compiler. DO NOT EDIT! 2 | // source: tensorflow_serving/apis/prediction_service.proto 3 | #pragma warning disable 1591, 0612, 3021 4 | #region Designer generated code 5 | 6 | using pb = global::Google.Protobuf; 7 | using pbc = global::Google.Protobuf.Collections; 8 | using pbr = global::Google.Protobuf.Reflection; 9 | using scg = global::System.Collections.Generic; 10 | namespace Tensorflow.Serving { 11 | 12 | /// Holder for reflection information generated from tensorflow_serving/apis/prediction_service.proto 13 | public static partial class PredictionServiceReflection { 14 | 15 | #region Descriptor 16 | /// File descriptor for tensorflow_serving/apis/prediction_service.proto 17 | public static pbr::FileDescriptor Descriptor { 18 | get { return descriptor; } 19 | } 20 | private static pbr::FileDescriptor descriptor; 21 | 22 | static PredictionServiceReflection() { 23 | byte[] descriptorData = global::System.Convert.FromBase64String( 24 | string.Concat( 25 | "CjB0ZW5zb3JmbG93X3NlcnZpbmcvYXBpcy9wcmVkaWN0aW9uX3NlcnZpY2Uu", 26 | "cHJvdG8SEnRlbnNvcmZsb3cuc2VydmluZxosdGVuc29yZmxvd19zZXJ2aW5n", 27 | "L2FwaXMvY2xhc3NpZmljYXRpb24ucHJvdG8aMHRlbnNvcmZsb3dfc2Vydmlu", 28 | "Zy9hcGlzL2dldF9tb2RlbF9tZXRhZGF0YS5wcm90bxondGVuc29yZmxvd19z", 29 | "ZXJ2aW5nL2FwaXMvaW5mZXJlbmNlLnByb3RvGiV0ZW5zb3JmbG93X3NlcnZp", 30 | "bmcvYXBpcy9wcmVkaWN0LnByb3RvGih0ZW5zb3JmbG93X3NlcnZpbmcvYXBp", 31 | "cy9yZWdyZXNzaW9uLnByb3RvMvwDChFQcmVkaWN0aW9uU2VydmljZRJhCghD", 32 | "bGFzc2lmeRIpLnRlbnNvcmZsb3cuc2VydmluZy5DbGFzc2lmaWNhdGlvblJl", 33 | "cXVlc3QaKi50ZW5zb3JmbG93LnNlcnZpbmcuQ2xhc3NpZmljYXRpb25SZXNw", 34 | "b25zZRJYCgdSZWdyZXNzEiUudGVuc29yZmxvdy5zZXJ2aW5nLlJlZ3Jlc3Np", 35 | "b25SZXF1ZXN0GiYudGVuc29yZmxvdy5zZXJ2aW5nLlJlZ3Jlc3Npb25SZXNw", 36 | "b25zZRJSCgdQcmVkaWN0EiIudGVuc29yZmxvdy5zZXJ2aW5nLlByZWRpY3RS", 37 | "ZXF1ZXN0GiMudGVuc29yZmxvdy5zZXJ2aW5nLlByZWRpY3RSZXNwb25zZRJn", 38 | "Cg5NdWx0aUluZmVyZW5jZRIpLnRlbnNvcmZsb3cuc2VydmluZy5NdWx0aUlu", 39 | "ZmVyZW5jZVJlcXVlc3QaKi50ZW5zb3JmbG93LnNlcnZpbmcuTXVsdGlJbmZl", 40 | "cmVuY2VSZXNwb25zZRJtChBHZXRNb2RlbE1ldGFkYXRhEisudGVuc29yZmxv", 41 | "dy5zZXJ2aW5nLkdldE1vZGVsTWV0YWRhdGFSZXF1ZXN0GiwudGVuc29yZmxv", 42 | "dy5zZXJ2aW5nLkdldE1vZGVsTWV0YWRhdGFSZXNwb25zZUID+AEBYgZwcm90", 43 | "bzM=")); 44 | descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, 45 | new pbr::FileDescriptor[] { global::Tensorflow.Serving.ClassificationReflection.Descriptor, global::Tensorflow.Serving.GetModelMetadataReflection.Descriptor, global::Tensorflow.Serving.InferenceReflection.Descriptor, global::Tensorflow.Serving.PredictReflection.Descriptor, global::Tensorflow.Serving.RegressionReflection.Descriptor, }, 46 | new pbr::GeneratedClrTypeInfo(null, null)); 47 | } 48 | #endregion 49 | 50 | } 51 | } 52 | 53 | #endregion Designer generated code 54 | -------------------------------------------------------------------------------- /src/VAP/AML.tfserving/tensorflow/MasterService.cs: -------------------------------------------------------------------------------- 1 | // Generated by the protocol buffer compiler. DO NOT EDIT! 2 | // source: tensorflow/core/protobuf/master_service.proto 3 | #pragma warning disable 1591, 0612, 3021 4 | #region Designer generated code 5 | 6 | using pb = global::Google.Protobuf; 7 | using pbc = global::Google.Protobuf.Collections; 8 | using pbr = global::Google.Protobuf.Reflection; 9 | using scg = global::System.Collections.Generic; 10 | namespace Tensorflow.Grpc { 11 | 12 | /// Holder for reflection information generated from tensorflow/core/protobuf/master_service.proto 13 | public static partial class MasterServiceReflection { 14 | 15 | #region Descriptor 16 | /// File descriptor for tensorflow/core/protobuf/master_service.proto 17 | public static pbr::FileDescriptor Descriptor { 18 | get { return descriptor; } 19 | } 20 | private static pbr::FileDescriptor descriptor; 21 | 22 | static MasterServiceReflection() { 23 | byte[] descriptorData = global::System.Convert.FromBase64String( 24 | string.Concat( 25 | "Ci10ZW5zb3JmbG93L2NvcmUvcHJvdG9idWYvbWFzdGVyX3NlcnZpY2UucHJv", 26 | "dG8SD3RlbnNvcmZsb3cuZ3JwYxoldGVuc29yZmxvdy9jb3JlL3Byb3RvYnVm", 27 | "L21hc3Rlci5wcm90bzK8BAoNTWFzdGVyU2VydmljZRJUCg1DcmVhdGVTZXNz", 28 | "aW9uEiAudGVuc29yZmxvdy5DcmVhdGVTZXNzaW9uUmVxdWVzdBohLnRlbnNv", 29 | "cmZsb3cuQ3JlYXRlU2Vzc2lvblJlc3BvbnNlElQKDUV4dGVuZFNlc3Npb24S", 30 | "IC50ZW5zb3JmbG93LkV4dGVuZFNlc3Npb25SZXF1ZXN0GiEudGVuc29yZmxv", 31 | "dy5FeHRlbmRTZXNzaW9uUmVzcG9uc2USWgoPUGFydGlhbFJ1blNldHVwEiIu", 32 | "dGVuc29yZmxvdy5QYXJ0aWFsUnVuU2V0dXBSZXF1ZXN0GiMudGVuc29yZmxv", 33 | "dy5QYXJ0aWFsUnVuU2V0dXBSZXNwb25zZRJCCgdSdW5TdGVwEhoudGVuc29y", 34 | "Zmxvdy5SdW5TdGVwUmVxdWVzdBobLnRlbnNvcmZsb3cuUnVuU3RlcFJlc3Bv", 35 | "bnNlElEKDENsb3NlU2Vzc2lvbhIfLnRlbnNvcmZsb3cuQ2xvc2VTZXNzaW9u", 36 | "UmVxdWVzdBogLnRlbnNvcmZsb3cuQ2xvc2VTZXNzaW9uUmVzcG9uc2USTgoL", 37 | "TGlzdERldmljZXMSHi50ZW5zb3JmbG93Lkxpc3REZXZpY2VzUmVxdWVzdBof", 38 | "LnRlbnNvcmZsb3cuTGlzdERldmljZXNSZXNwb25zZRI8CgVSZXNldBIYLnRl", 39 | "bnNvcmZsb3cuUmVzZXRSZXF1ZXN0GhkudGVuc29yZmxvdy5SZXNldFJlc3Bv", 40 | "bnNlQjMKGm9yZy50ZW5zb3JmbG93LmRpc3RydW50aW1lQhNNYXN0ZXJTZXJ2", 41 | "aWNlUHJvdG9zUAFiBnByb3RvMw==")); 42 | descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, 43 | new pbr::FileDescriptor[] { global::Tensorflow.MasterReflection.Descriptor, }, 44 | new pbr::GeneratedClrTypeInfo(null, null)); 45 | } 46 | #endregion 47 | 48 | } 49 | } 50 | 51 | #endregion Designer generated code 52 | -------------------------------------------------------------------------------- /src/VAP/AML.tfserving/tensorflow/Types.cs: -------------------------------------------------------------------------------- 1 | // Generated by the protocol buffer compiler. DO NOT EDIT! 2 | // source: tensorflow/core/framework/types.proto 3 | #pragma warning disable 1591, 0612, 3021 4 | #region Designer generated code 5 | 6 | using pb = global::Google.Protobuf; 7 | using pbc = global::Google.Protobuf.Collections; 8 | using pbr = global::Google.Protobuf.Reflection; 9 | using scg = global::System.Collections.Generic; 10 | namespace Tensorflow { 11 | 12 | /// Holder for reflection information generated from tensorflow/core/framework/types.proto 13 | public static partial class TypesReflection { 14 | 15 | #region Descriptor 16 | /// File descriptor for tensorflow/core/framework/types.proto 17 | public static pbr::FileDescriptor Descriptor { 18 | get { return descriptor; } 19 | } 20 | private static pbr::FileDescriptor descriptor; 21 | 22 | static TypesReflection() { 23 | byte[] descriptorData = global::System.Convert.FromBase64String( 24 | string.Concat( 25 | "CiV0ZW5zb3JmbG93L2NvcmUvZnJhbWV3b3JrL3R5cGVzLnByb3RvEgp0ZW5z", 26 | "b3JmbG93KqoGCghEYXRhVHlwZRIOCgpEVF9JTlZBTElEEAASDAoIRFRfRkxP", 27 | "QVQQARINCglEVF9ET1VCTEUQAhIMCghEVF9JTlQzMhADEgwKCERUX1VJTlQ4", 28 | "EAQSDAoIRFRfSU5UMTYQBRILCgdEVF9JTlQ4EAYSDQoJRFRfU1RSSU5HEAcS", 29 | "EAoMRFRfQ09NUExFWDY0EAgSDAoIRFRfSU5UNjQQCRILCgdEVF9CT09MEAoS", 30 | "DAoIRFRfUUlOVDgQCxINCglEVF9RVUlOVDgQDBINCglEVF9RSU5UMzIQDRIP", 31 | "CgtEVF9CRkxPQVQxNhAOEg0KCURUX1FJTlQxNhAPEg4KCkRUX1FVSU5UMTYQ", 32 | "EBINCglEVF9VSU5UMTYQERIRCg1EVF9DT01QTEVYMTI4EBISCwoHRFRfSEFM", 33 | "RhATEg8KC0RUX1JFU09VUkNFEBQSDgoKRFRfVkFSSUFOVBAVEg0KCURUX1VJ", 34 | "TlQzMhAWEg0KCURUX1VJTlQ2NBAXEhAKDERUX0ZMT0FUX1JFRhBlEhEKDURU", 35 | "X0RPVUJMRV9SRUYQZhIQCgxEVF9JTlQzMl9SRUYQZxIQCgxEVF9VSU5UOF9S", 36 | "RUYQaBIQCgxEVF9JTlQxNl9SRUYQaRIPCgtEVF9JTlQ4X1JFRhBqEhEKDURU", 37 | "X1NUUklOR19SRUYQaxIUChBEVF9DT01QTEVYNjRfUkVGEGwSEAoMRFRfSU5U", 38 | "NjRfUkVGEG0SDwoLRFRfQk9PTF9SRUYQbhIQCgxEVF9RSU5UOF9SRUYQbxIR", 39 | "Cg1EVF9RVUlOVDhfUkVGEHASEQoNRFRfUUlOVDMyX1JFRhBxEhMKD0RUX0JG", 40 | "TE9BVDE2X1JFRhByEhEKDURUX1FJTlQxNl9SRUYQcxISCg5EVF9RVUlOVDE2", 41 | "X1JFRhB0EhEKDURUX1VJTlQxNl9SRUYQdRIVChFEVF9DT01QTEVYMTI4X1JF", 42 | "RhB2Eg8KC0RUX0hBTEZfUkVGEHcSEwoPRFRfUkVTT1VSQ0VfUkVGEHgSEgoO", 43 | "RFRfVkFSSUFOVF9SRUYQeRIRCg1EVF9VSU5UMzJfUkVGEHoSEQoNRFRfVUlO", 44 | "VDY0X1JFRhB7QiwKGG9yZy50ZW5zb3JmbG93LmZyYW1ld29ya0ILVHlwZXNQ", 45 | "cm90b3NQAfgBAWIGcHJvdG8z")); 46 | descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, 47 | new pbr::FileDescriptor[] { }, 48 | new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Tensorflow.DataType), }, null)); 49 | } 50 | #endregion 51 | 52 | } 53 | #region Enums 54 | /// 55 | /// LINT.IfChange 56 | /// 57 | public enum DataType { 58 | /// 59 | /// Not a legal value for DataType. Used to indicate a DataType field 60 | /// has not been set. 61 | /// 62 | [pbr::OriginalName("DT_INVALID")] DtInvalid = 0, 63 | /// 64 | /// Data types that all computation devices are expected to be 65 | /// capable to support. 66 | /// 67 | [pbr::OriginalName("DT_FLOAT")] DtFloat = 1, 68 | [pbr::OriginalName("DT_DOUBLE")] DtDouble = 2, 69 | [pbr::OriginalName("DT_INT32")] DtInt32 = 3, 70 | [pbr::OriginalName("DT_UINT8")] DtUint8 = 4, 71 | [pbr::OriginalName("DT_INT16")] DtInt16 = 5, 72 | [pbr::OriginalName("DT_INT8")] DtInt8 = 6, 73 | [pbr::OriginalName("DT_STRING")] DtString = 7, 74 | /// 75 | /// Single-precision complex 76 | /// 77 | [pbr::OriginalName("DT_COMPLEX64")] DtComplex64 = 8, 78 | [pbr::OriginalName("DT_INT64")] DtInt64 = 9, 79 | [pbr::OriginalName("DT_BOOL")] DtBool = 10, 80 | /// 81 | /// Quantized int8 82 | /// 83 | [pbr::OriginalName("DT_QINT8")] DtQint8 = 11, 84 | /// 85 | /// Quantized uint8 86 | /// 87 | [pbr::OriginalName("DT_QUINT8")] DtQuint8 = 12, 88 | /// 89 | /// Quantized int32 90 | /// 91 | [pbr::OriginalName("DT_QINT32")] DtQint32 = 13, 92 | /// 93 | /// Float32 truncated to 16 bits. Only for cast ops. 94 | /// 95 | [pbr::OriginalName("DT_BFLOAT16")] DtBfloat16 = 14, 96 | /// 97 | /// Quantized int16 98 | /// 99 | [pbr::OriginalName("DT_QINT16")] DtQint16 = 15, 100 | /// 101 | /// Quantized uint16 102 | /// 103 | [pbr::OriginalName("DT_QUINT16")] DtQuint16 = 16, 104 | [pbr::OriginalName("DT_UINT16")] DtUint16 = 17, 105 | /// 106 | /// Double-precision complex 107 | /// 108 | [pbr::OriginalName("DT_COMPLEX128")] DtComplex128 = 18, 109 | [pbr::OriginalName("DT_HALF")] DtHalf = 19, 110 | [pbr::OriginalName("DT_RESOURCE")] DtResource = 20, 111 | /// 112 | /// Arbitrary C++ data types 113 | /// 114 | [pbr::OriginalName("DT_VARIANT")] DtVariant = 21, 115 | [pbr::OriginalName("DT_UINT32")] DtUint32 = 22, 116 | [pbr::OriginalName("DT_UINT64")] DtUint64 = 23, 117 | /// 118 | /// Do not use! These are only for parameters. Every enum above 119 | /// should have a corresponding value below (verified by types_test). 120 | /// 121 | [pbr::OriginalName("DT_FLOAT_REF")] DtFloatRef = 101, 122 | [pbr::OriginalName("DT_DOUBLE_REF")] DtDoubleRef = 102, 123 | [pbr::OriginalName("DT_INT32_REF")] DtInt32Ref = 103, 124 | [pbr::OriginalName("DT_UINT8_REF")] DtUint8Ref = 104, 125 | [pbr::OriginalName("DT_INT16_REF")] DtInt16Ref = 105, 126 | [pbr::OriginalName("DT_INT8_REF")] DtInt8Ref = 106, 127 | [pbr::OriginalName("DT_STRING_REF")] DtStringRef = 107, 128 | [pbr::OriginalName("DT_COMPLEX64_REF")] DtComplex64Ref = 108, 129 | [pbr::OriginalName("DT_INT64_REF")] DtInt64Ref = 109, 130 | [pbr::OriginalName("DT_BOOL_REF")] DtBoolRef = 110, 131 | [pbr::OriginalName("DT_QINT8_REF")] DtQint8Ref = 111, 132 | [pbr::OriginalName("DT_QUINT8_REF")] DtQuint8Ref = 112, 133 | [pbr::OriginalName("DT_QINT32_REF")] DtQint32Ref = 113, 134 | [pbr::OriginalName("DT_BFLOAT16_REF")] DtBfloat16Ref = 114, 135 | [pbr::OriginalName("DT_QINT16_REF")] DtQint16Ref = 115, 136 | [pbr::OriginalName("DT_QUINT16_REF")] DtQuint16Ref = 116, 137 | [pbr::OriginalName("DT_UINT16_REF")] DtUint16Ref = 117, 138 | [pbr::OriginalName("DT_COMPLEX128_REF")] DtComplex128Ref = 118, 139 | [pbr::OriginalName("DT_HALF_REF")] DtHalfRef = 119, 140 | [pbr::OriginalName("DT_RESOURCE_REF")] DtResourceRef = 120, 141 | [pbr::OriginalName("DT_VARIANT_REF")] DtVariantRef = 121, 142 | [pbr::OriginalName("DT_UINT32_REF")] DtUint32Ref = 122, 143 | [pbr::OriginalName("DT_UINT64_REF")] DtUint64Ref = 123, 144 | } 145 | 146 | #endregion 147 | 148 | } 149 | 150 | #endregion Designer generated code 151 | -------------------------------------------------------------------------------- /src/VAP/AML.tfserving/tensorflow/WorkerService.cs: -------------------------------------------------------------------------------- 1 | // Generated by the protocol buffer compiler. DO NOT EDIT! 2 | // source: tensorflow/core/protobuf/worker_service.proto 3 | #pragma warning disable 1591, 0612, 3021 4 | #region Designer generated code 5 | 6 | using pb = global::Google.Protobuf; 7 | using pbc = global::Google.Protobuf.Collections; 8 | using pbr = global::Google.Protobuf.Reflection; 9 | using scg = global::System.Collections.Generic; 10 | namespace Tensorflow.Grpc { 11 | 12 | /// Holder for reflection information generated from tensorflow/core/protobuf/worker_service.proto 13 | public static partial class WorkerServiceReflection { 14 | 15 | #region Descriptor 16 | /// File descriptor for tensorflow/core/protobuf/worker_service.proto 17 | public static pbr::FileDescriptor Descriptor { 18 | get { return descriptor; } 19 | } 20 | private static pbr::FileDescriptor descriptor; 21 | 22 | static WorkerServiceReflection() { 23 | byte[] descriptorData = global::System.Convert.FromBase64String( 24 | string.Concat( 25 | "Ci10ZW5zb3JmbG93L2NvcmUvcHJvdG9idWYvd29ya2VyX3NlcnZpY2UucHJv", 26 | "dG8SD3RlbnNvcmZsb3cuZ3JwYxoldGVuc29yZmxvdy9jb3JlL3Byb3RvYnVm", 27 | "L3dvcmtlci5wcm90bzKZBwoNV29ya2VyU2VydmljZRJICglHZXRTdGF0dXMS", 28 | "HC50ZW5zb3JmbG93LkdldFN0YXR1c1JlcXVlc3QaHS50ZW5zb3JmbG93Lkdl", 29 | "dFN0YXR1c1Jlc3BvbnNlEmYKE0NyZWF0ZVdvcmtlclNlc3Npb24SJi50ZW5z", 30 | "b3JmbG93LkNyZWF0ZVdvcmtlclNlc3Npb25SZXF1ZXN0GicudGVuc29yZmxv", 31 | "dy5DcmVhdGVXb3JrZXJTZXNzaW9uUmVzcG9uc2USZgoTRGVsZXRlV29ya2Vy", 32 | "U2Vzc2lvbhImLnRlbnNvcmZsb3cuRGVsZXRlV29ya2VyU2Vzc2lvblJlcXVl", 33 | "c3QaJy50ZW5zb3JmbG93LkRlbGV0ZVdvcmtlclNlc3Npb25SZXNwb25zZRJU", 34 | "Cg1SZWdpc3RlckdyYXBoEiAudGVuc29yZmxvdy5SZWdpc3RlckdyYXBoUmVx", 35 | "dWVzdBohLnRlbnNvcmZsb3cuUmVnaXN0ZXJHcmFwaFJlc3BvbnNlEloKD0Rl", 36 | "cmVnaXN0ZXJHcmFwaBIiLnRlbnNvcmZsb3cuRGVyZWdpc3RlckdyYXBoUmVx", 37 | "dWVzdBojLnRlbnNvcmZsb3cuRGVyZWdpc3RlckdyYXBoUmVzcG9uc2USRQoI", 38 | "UnVuR3JhcGgSGy50ZW5zb3JmbG93LlJ1bkdyYXBoUmVxdWVzdBocLnRlbnNv", 39 | "cmZsb3cuUnVuR3JhcGhSZXNwb25zZRJRCgxDbGVhbnVwR3JhcGgSHy50ZW5z", 40 | "b3JmbG93LkNsZWFudXBHcmFwaFJlcXVlc3QaIC50ZW5zb3JmbG93LkNsZWFu", 41 | "dXBHcmFwaFJlc3BvbnNlEksKCkNsZWFudXBBbGwSHS50ZW5zb3JmbG93LkNs", 42 | "ZWFudXBBbGxSZXF1ZXN0Gh4udGVuc29yZmxvdy5DbGVhbnVwQWxsUmVzcG9u", 43 | "c2USTQoKUmVjdlRlbnNvchIdLnRlbnNvcmZsb3cuUmVjdlRlbnNvclJlcXVl", 44 | "c3QaHi50ZW5zb3JmbG93LlJlY3ZUZW5zb3JSZXNwb25zZSIAEkIKB0xvZ2dp", 45 | "bmcSGi50ZW5zb3JmbG93LkxvZ2dpbmdSZXF1ZXN0GhsudGVuc29yZmxvdy5M", 46 | "b2dnaW5nUmVzcG9uc2USQgoHVHJhY2luZxIaLnRlbnNvcmZsb3cuVHJhY2lu", 47 | "Z1JlcXVlc3QaGy50ZW5zb3JmbG93LlRyYWNpbmdSZXNwb25zZUIzChpvcmcu", 48 | "dGVuc29yZmxvdy5kaXN0cnVudGltZUITV29ya2VyU2VydmljZVByb3Rvc1AB", 49 | "YgZwcm90bzM=")); 50 | descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, 51 | new pbr::FileDescriptor[] { global::Tensorflow.WorkerReflection.Descriptor, }, 52 | new pbr::GeneratedClrTypeInfo(null, null)); 53 | } 54 | #endregion 55 | 56 | } 57 | } 58 | 59 | #endregion Designer generated code 60 | -------------------------------------------------------------------------------- /src/VAP/BGSObjectDetector/BGSObjectDetector.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | 7 | using OpenCvSharp; 8 | using OpenCvSharp.Blob; 9 | 10 | namespace BGSObjectDetector 11 | { 12 | public class BGSObjectDetector 13 | { 14 | MOG2 bgs; 15 | 16 | Mat blurredFrame = new Mat(); 17 | Mat fgMask = new Mat(); 18 | Mat fgWOShadows = new Mat(); 19 | Mat fgSmoothedMask2 = new Mat(); 20 | Mat fgSmoothedMask3 = new Mat(); 21 | Mat fgSmoothedMask4 = new Mat(); 22 | 23 | Mat regionOfInterest = null; 24 | 25 | int PRE_BGS_BLUR_SIGMA = 2; 26 | int MEDIAN_BLUR_SIZE = 5; 27 | int GAUSSIAN_BLUR_SIGMA = 4; 28 | int GAUSSIAN_BLUR_THRESHOLD = 50; 29 | static int MIN_BLOB_SIZE = 30; 30 | 31 | static SimpleBlobDetector.Params detectorParams = new SimpleBlobDetector.Params 32 | { 33 | //MinDistBetweenBlobs = 10, // 10 pixels between blobs 34 | //MinRepeatability = 1, 35 | 36 | //MinThreshold = 100, 37 | //MaxThreshold = 255, 38 | //ThresholdStep = 5, 39 | 40 | FilterByArea = true, 41 | MinArea = MIN_BLOB_SIZE, 42 | MaxArea = int.MaxValue, 43 | 44 | FilterByCircularity = false, 45 | //FilterByCircularity = true, 46 | //MinCircularity = 0.001f, 47 | 48 | FilterByConvexity = false, 49 | //FilterByConvexity = true, 50 | //MinConvexity = 0.001f, 51 | //MaxConvexity = 10, 52 | 53 | FilterByInertia = false, 54 | //FilterByInertia = true, 55 | //MinInertiaRatio = 0.001f, 56 | 57 | FilterByColor = false 58 | //FilterByColor = true, 59 | //BlobColor = 255 // to extract light blobs 60 | }; 61 | SimpleBlobDetector _blobDetector = SimpleBlobDetector.Create(detectorParams); 62 | 63 | //public BGSObjectDetector(MOG2 bgs) 64 | public BGSObjectDetector() 65 | { 66 | //this.bgs = bgs; 67 | bgs = new MOG2(); 68 | } 69 | 70 | public List DetectObjects(DateTime timestamp, Mat image, int frameIndex, out Mat fg) 71 | { 72 | if (regionOfInterest != null) 73 | bgs.SetRegionOfInterest(regionOfInterest); 74 | 75 | Cv2.GaussianBlur(image, blurredFrame, Size.Zero, PRE_BGS_BLUR_SIGMA); 76 | 77 | // fgMask is the original foreground bitmap returned by opencv MOG2 78 | fgMask = bgs.DetectForeground(blurredFrame, frameIndex); 79 | fg = fgMask; 80 | if (fgMask == null) 81 | return null; 82 | 83 | // pre-processing 84 | Cv2.Threshold(fgMask, fgWOShadows, 200, 255, ThresholdTypes.Binary); 85 | Cv2.MedianBlur(fgWOShadows, fgSmoothedMask2, MEDIAN_BLUR_SIZE); 86 | Cv2.GaussianBlur(fgSmoothedMask2, fgSmoothedMask3, Size.Zero, GAUSSIAN_BLUR_SIGMA); 87 | Cv2.Threshold(fgSmoothedMask3, fgSmoothedMask4, GAUSSIAN_BLUR_THRESHOLD, 255, ThresholdTypes.Binary); 88 | 89 | fg = fgSmoothedMask4; 90 | 91 | CvBlobs blobs = new CvBlobs(); 92 | KeyPoint[] points = _blobDetector.Detect(fgSmoothedMask4); 93 | //blobs.FilterByArea(MIN_BLOB_SIZE, int.MaxValue); 94 | 95 | //// filter overlapping blobs 96 | //HashSet blobIdsToRemove = new HashSet(); 97 | //foreach (var b0 in blobs) 98 | // foreach (var b1 in blobs) 99 | // { 100 | // if (b0.Key == b1.Key) continue; 101 | // if (b0.Value.BoundingBox.Contains(b1.Value.BoundingBox)) 102 | // blobIdsToRemove.Add(b1.Key); 103 | // } 104 | //foreach (uint blobid in blobIdsToRemove) 105 | // blobs.Remove(blobid); 106 | 107 | // adding text to boxes and foreground frame 108 | List newBlobs = new List(); 109 | uint id = 0; 110 | foreach (var point in points) 111 | { 112 | int x = (int)point.Pt.X; 113 | int y = (int)point.Pt.Y; 114 | int size = (int)point.Size; 115 | Box box = new Box("", x - size, x + size, y - size, y + size, frameIndex, id); 116 | id++; 117 | newBlobs.Add(box); 118 | 119 | Cv2.Rectangle(fgSmoothedMask4, new OpenCvSharp.Point(x - size, y - size), new OpenCvSharp.Point(x + size, y + size), new Scalar(255), 1); 120 | Cv2.PutText(fgSmoothedMask4, box.ID.ToString(), new OpenCvSharp.Point(x, y - size), HersheyFonts.HersheyPlain, 1.0, new Scalar(255.0, 255.0, 255.0)); 121 | } 122 | Cv2.PutText(fgSmoothedMask4, "frame: " + frameIndex, new OpenCvSharp.Point(10, 10), HersheyFonts.HersheyPlain, 1, new Scalar(255, 255, 255)); 123 | 124 | newBlobs.ForEach(b => b.Time = timestamp); 125 | newBlobs.ForEach(b => b.Timestamp = frameIndex); 126 | return newBlobs; 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/VAP/BGSObjectDetector/BGSObjectDetector.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/VAP/BGSObjectDetector/Box.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Drawing; 6 | 7 | using OpenCvSharp; 8 | 9 | namespace BGSObjectDetector 10 | { 11 | public class Box 12 | { 13 | public Rectangle Rectangle { get; private set; } 14 | 15 | public uint ID { get; private set; } 16 | public string ObjectCategory { get; private set; } 17 | 18 | public int Timestamp { get; set; } 19 | public DateTime? Time { get; set; } 20 | public int X0 { get { return Rectangle.X; } } 21 | public int X1 { get { return Rectangle.X + Rectangle.Width; } } 22 | public int Y0 { get { return Rectangle.Y; } } 23 | public int Y1 { get { return Rectangle.Y + Rectangle.Height; } } 24 | 25 | public PointF Center { get { return new PointF(X0 + Width / 2, Y0 + Height / 2); } } 26 | public int Width { get { return Rectangle.Width; } } 27 | public int Height { get { return Rectangle.Height; } } 28 | 29 | public int Area { get { return Width * Height; } } 30 | 31 | public bool IsEligibleToStartTrack { get; set; } 32 | 33 | public bool IsPointInterior(int x, int y) 34 | { 35 | System.Diagnostics.Debug.Assert(X0 < X1); 36 | System.Diagnostics.Debug.Assert(Y0 < Y1); 37 | return Rectangle.Contains(x, y); 38 | } 39 | 40 | // for storing histogram value 41 | public float[] hist; 42 | public void SetHist(float[] h) 43 | { 44 | for (int i = 0; i < h.Length; i++) { hist[i] = h[i]; } 45 | } 46 | 47 | // for storing feature values 48 | public Mat desc; 49 | public void SetDesc(Mat d) 50 | { 51 | try { desc = d.Clone(); } 52 | catch (Exception) {; } 53 | } 54 | 55 | //public Box(string objectCategory, int x0, int x1, int y0, int y1, int timestamp, uint id = 0) // old 56 | public Box(string objectCategory, int x0, int x1, int y0, int y1, int timestamp, uint id = 0, DateTime? time = null, bool isEligibleToStartTrack = true) 57 | { 58 | this.ObjectCategory = objectCategory; 59 | 60 | this.Timestamp = timestamp; 61 | this.Time = time; 62 | this.Rectangle = new Rectangle(x0, y0, x1 - x0, y1 - y0); 63 | //this.X0 = x0; 64 | //this.X1 = x1; 65 | //this.Y0 = y0; 66 | //this.Y1 = y1; 67 | 68 | this.ID = id; 69 | 70 | this.IsEligibleToStartTrack = isEligibleToStartTrack; 71 | 72 | // used for different similarity metrics 73 | this.hist = new float[256]; 74 | this.desc = null; 75 | } 76 | 77 | public static double MaxOverlapFraction(Box b0, Box b1) 78 | { 79 | Rectangle intersection = Rectangle.Intersect(b0.Rectangle, b1.Rectangle); 80 | double f0 = (intersection.Width * intersection.Height) / (double)(b0.Width * b0.Height); 81 | double f1 = (intersection.Width * intersection.Height) / (double)(b1.Width * b1.Height); 82 | return Math.Max(f0, f1); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/VAP/BGSObjectDetector/MOG2.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using OpenCvSharp; 5 | 6 | namespace BGSObjectDetector 7 | { 8 | class MOG2 9 | { 10 | BackgroundSubtractorMOG2 fgDetector = BackgroundSubtractorMOG2.Create(500, 10); //try sweeping (also set it higher than 25) 11 | Mat regionOfInterest = null; 12 | Mat fgMask0 = new Mat(); 13 | Mat fgMask = new Mat(); 14 | 15 | int N_FRAMES_TO_LEARN = 120; // Why do we need this? 16 | 17 | public MOG2() 18 | { 19 | regionOfInterest = null; 20 | fgMask0 = new Mat(); 21 | fgMask = new Mat(); 22 | } 23 | 24 | public Mat DetectForeground(Mat image, int nFrames) 25 | { 26 | fgDetector.Apply(image, fgMask0); 27 | 28 | if (regionOfInterest != null) 29 | Cv2.BitwiseAnd(fgMask0, regionOfInterest, fgMask); 30 | 31 | if (nFrames < N_FRAMES_TO_LEARN) 32 | return null; 33 | else if (regionOfInterest != null) 34 | return fgMask; 35 | else 36 | return fgMask0; 37 | } 38 | 39 | public void SetRegionOfInterest(Mat roi) 40 | { 41 | if (roi != null) 42 | { 43 | regionOfInterest = new Mat(); 44 | roi.CopyTo(regionOfInterest); 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/VAP/DNNDetector/CascadedDNNORTYolo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Config; 5 | using DNNDetector.Model; 6 | using OpenCvSharp; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.IO; 10 | using Utils.Config; 11 | using Wrapper.ORT; 12 | 13 | namespace DNNDetector 14 | { 15 | public class CascadedDNNORTYolo 16 | { 17 | FrameDNNOnnxYolo frameDNNOnnxYolo; 18 | 19 | public CascadedDNNORTYolo(List> lines, string modelName) 20 | { 21 | frameDNNOnnxYolo = new FrameDNNOnnxYolo(lines, modelName, DNNMode.CC); 22 | 23 | Utils.Utils.cleanFolder(@OutputFolder.OutputFolderCcDNN); 24 | } 25 | 26 | public List Run(int frameIndex, List ltDNNItemList, List> lines, Dictionary category, ref long teleCountsHeavyDNN, bool savePictures = false) 27 | { 28 | if (ltDNNItemList == null) 29 | { 30 | return null; 31 | } 32 | 33 | List ccDNNItem = new List(); 34 | 35 | foreach (Item ltDNNItem in ltDNNItemList) 36 | { 37 | if (ltDNNItem.Confidence >= DNNConfig.CONFIDENCE_THRESHOLD) 38 | { 39 | ccDNNItem.Add(ltDNNItem); 40 | continue; 41 | } 42 | else 43 | { 44 | List analyzedTrackingItems = null; 45 | 46 | Console.WriteLine("** Calling Heavy DNN **"); 47 | analyzedTrackingItems = frameDNNOnnxYolo.Run(Cv2.ImDecode(ltDNNItem.RawImageData, ImreadModes.Color), frameIndex, category, System.Drawing.Brushes.Yellow, ltDNNItem.TriggerLineID, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_LARGE); 48 | teleCountsHeavyDNN++; 49 | 50 | // object detected by heavy YOLO 51 | if (analyzedTrackingItems != null) 52 | { 53 | foreach (Item item in analyzedTrackingItems) 54 | { 55 | item.RawImageData = ltDNNItem.RawImageData; 56 | item.TriggerLine = ltDNNItem.TriggerLine; 57 | item.TriggerLineID = ltDNNItem.TriggerLineID; 58 | item.Model = "Heavy"; 59 | ccDNNItem.Add(item); 60 | 61 | // output heavy YOLO results 62 | if (savePictures) 63 | { 64 | string blobName_Heavy = $@"frame-{frameIndex}-Heavy-{item.Confidence}.jpg"; 65 | string fileName_Heavy = @OutputFolder.OutputFolderCcDNN + blobName_Heavy; 66 | File.WriteAllBytes(fileName_Heavy, item.TaggedImageData); 67 | File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_Heavy, item.TaggedImageData); 68 | } 69 | 70 | return ccDNNItem; // if we only return the closest object detected by heavy model 71 | } 72 | } 73 | else 74 | { 75 | Console.WriteLine("** Not detected by Heavy DNN **"); 76 | } 77 | } 78 | } 79 | 80 | return ccDNNItem; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/VAP/DNNDetector/Config/DNNConfig.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace DNNDetector.Config 5 | { 6 | public static class DNNConfig 7 | { 8 | public static double CONFIDENCE_THRESHOLD { get; set; } = 0.7; //threshold for calling heavy DNN 9 | 10 | public static int FRAME_SEARCH_RANGE { get; set; } = 50; // frames 11 | 12 | public static int ValidRange { get; set; } = 200; // pixels. Deprecated with overlap checking instead. 13 | 14 | public static double MIN_SCORE_FOR_TFOBJECT_OUTPUT { get; set; } = 0.5; //TFWrapper.cs 15 | 16 | public static double MIN_SCORE_FOR_LINEBBOX_OVERLAP_SMALL { get; set; } = 0; 17 | 18 | public static double MIN_SCORE_FOR_LINEBBOX_OVERLAP_LARGE { get; set; } = 0.2; 19 | } 20 | } -------------------------------------------------------------------------------- /src/VAP/DNNDetector/DNNDetector.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/VAP/DNNDetector/FrameBuffer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Collections.Generic; 5 | 6 | using OpenCvSharp; 7 | 8 | namespace DNNDetector 9 | { 10 | class FrameBuffer 11 | { 12 | Queue frameBuffer; 13 | int bSize; 14 | 15 | public FrameBuffer(int size) 16 | { 17 | bSize = size; 18 | frameBuffer = new Queue(bSize); 19 | } 20 | 21 | public void Buffer(Mat frame) 22 | { 23 | frameBuffer.Enqueue(frame); 24 | if (frameBuffer.Count > bSize) 25 | frameBuffer.Dequeue(); 26 | } 27 | 28 | public Mat[] ToArray() 29 | { 30 | return frameBuffer.ToArray(); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/VAP/DNNDetector/FrameDNNORTYolo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Model; 5 | using OpenCvSharp; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Configuration; 9 | using System.Drawing; 10 | using System.Drawing.Imaging; 11 | using System.IO; 12 | using System.Linq; 13 | using Utils.Config; 14 | using Wrapper.ORT; 15 | 16 | namespace DNNDetector 17 | { 18 | public class FrameDNNOnnxYolo 19 | { 20 | private static int _imageWidth, _imageHeight, _index; 21 | private static List> _lines; 22 | private static Dictionary _category; 23 | 24 | ORTWrapper onnxWrapper; 25 | byte[] imageByteArray; 26 | 27 | public FrameDNNOnnxYolo(List> lines, string modelName, DNNMode mode) 28 | { 29 | _lines = lines; 30 | onnxWrapper = new ORTWrapper($@"..\..\..\..\..\..\modelOnnx\{modelName}ort.onnx", mode); 31 | } 32 | 33 | public List Run(Mat frameOnnx, int frameIndex, Dictionary category, Brush bboxColor, int lineID, double min_score_for_linebbox_overlap, bool savePictures = false) 34 | { 35 | _imageWidth = frameOnnx.Width; 36 | _imageHeight = frameOnnx.Height; 37 | _category = category; 38 | imageByteArray = Utils.Utils.ImageToByteJpeg(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx)); // Todo: feed in bmp 39 | 40 | List boundingBoxes = onnxWrapper.UseApi( 41 | OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx), 42 | _imageHeight, 43 | _imageWidth); 44 | 45 | List preValidItems = new List(); 46 | foreach (ORTItem bbox in boundingBoxes) 47 | { 48 | preValidItems.Add(new Item(bbox)); 49 | } 50 | List validObjects = new List(); 51 | 52 | //run _category and overlap ratio-based validation 53 | if (_lines != null) 54 | { 55 | var overlapItems = preValidItems.Select(o => new { Overlap = Utils.Utils.checkLineBboxOverlapRatio(_lines[lineID].Item2, o.X, o.Y, o.Width, o.Height), Bbox_x = o.X + o.Width, Bbox_y = o.Y + o.Height, Distance = this.Distance(_lines[lineID].Item2, o.Center()), Item = o }) 56 | .Where(o => o.Bbox_x <= _imageWidth && o.Bbox_y <= _imageHeight && o.Overlap >= min_score_for_linebbox_overlap && _category.ContainsKey(o.Item.ObjName)).OrderBy(o => o.Distance); 57 | foreach (var item in overlapItems) 58 | { 59 | item.Item.TaggedImageData = Utils.Utils.DrawImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height, bboxColor); 60 | item.Item.CroppedImageData = Utils.Utils.CropImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height); 61 | item.Item.Index = _index; 62 | item.Item.TriggerLine = _lines[lineID].Item1; 63 | item.Item.TriggerLineID = lineID; 64 | item.Item.Model = "FrameDNN"; 65 | validObjects.Add(item.Item); 66 | _index++; 67 | } 68 | } 69 | else if (min_score_for_linebbox_overlap == 0.0) //frameDNN object detection 70 | { 71 | var overlapItems = preValidItems.Select(o => new { Bbox_x = o.X + o.Width, Bbox_y = o.Y + o.Height, Item = o }) 72 | .Where(o => o.Bbox_x <= _imageWidth && o.Bbox_y <= _imageHeight && _category.ContainsKey(o.Item.ObjName)); 73 | foreach (var item in overlapItems) 74 | { 75 | item.Item.TaggedImageData = Utils.Utils.DrawImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height, bboxColor); 76 | item.Item.CroppedImageData = Utils.Utils.CropImage(imageByteArray, item.Item.X, item.Item.Y, item.Item.Width, item.Item.Height); 77 | item.Item.Index = _index; 78 | item.Item.TriggerLine = ""; 79 | item.Item.TriggerLineID = -1; 80 | item.Item.Model = "FrameDNN"; 81 | validObjects.Add(item.Item); 82 | _index++; 83 | } 84 | } 85 | 86 | //output onnxyolo results 87 | if (savePictures) 88 | { 89 | foreach (Item it in validObjects) 90 | { 91 | using (Image image = Image.FromStream(new MemoryStream(it.TaggedImageData))) 92 | { 93 | 94 | image.Save(@OutputFolder.OutputFolderFrameDNNONNX + $"frame-{frameIndex}-ONNX-{it.Confidence}.jpg", ImageFormat.Jpeg); 95 | image.Save(@OutputFolder.OutputFolderAll + $"frame-{frameIndex}-ONNX-{it.Confidence}.jpg", ImageFormat.Jpeg); 96 | } 97 | } 98 | //byte[] imgBboxes = DrawAllBb(frameIndex, Utils.Utils.ImageToByteBmp(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx)), 99 | // validObjects, Brushes.Pink); 100 | } 101 | 102 | return (validObjects.Count == 0 ? null : validObjects); 103 | } 104 | 105 | private double Distance(int[] line, System.Drawing.Point bboxCenter) 106 | { 107 | System.Drawing.Point p1 = new System.Drawing.Point((int)((line[0] + line[2]) / 2), (int)((line[1] + line[3]) / 2)); 108 | return Math.Sqrt(this.Pow2(bboxCenter.X - p1.X) + Pow2(bboxCenter.Y - p1.Y)); 109 | } 110 | 111 | private double Pow2(double x) 112 | { 113 | return x * x; 114 | } 115 | 116 | public static byte[] DrawAllBb(int frameIndex, byte[] imgByte, List items, Brush bboxColor) 117 | { 118 | byte[] canvas = new byte[imgByte.Length]; 119 | canvas = imgByte; 120 | foreach (var item in items) 121 | { 122 | canvas = Utils.Utils.DrawImage(canvas, item.X, item.Y, item.Width, item.Height, bboxColor); 123 | } 124 | string frameIndexString = frameIndex.ToString("000000.##"); 125 | File.WriteAllBytes(@OutputFolder.OutputFolderFrameDNNONNX + $@"frame{frameIndexString}-Raw.jpg", canvas); 126 | 127 | return canvas; 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/VAP/DNNDetector/LineTriggeredDNNORTYolo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Config; 5 | using DNNDetector.Model; 6 | using OpenCvSharp; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.IO; 10 | using System.Linq; 11 | using Utils.Config; 12 | using Wrapper.ORT; 13 | 14 | namespace DNNDetector 15 | { 16 | //Todo: merge it with LineTriggeredDNNYolo 17 | public class LineTriggeredDNNORTYolo 18 | { 19 | Dictionary counts_prev = new Dictionary(); 20 | 21 | FrameDNNOnnxYolo frameDNNOnnxYolo; 22 | FrameBuffer frameBufferLtDNNOnnxYolo; 23 | 24 | public LineTriggeredDNNORTYolo(List> lines, string modelName) 25 | { 26 | frameBufferLtDNNOnnxYolo = new FrameBuffer(DNNConfig.FRAME_SEARCH_RANGE); 27 | 28 | frameDNNOnnxYolo = new FrameDNNOnnxYolo(lines, modelName, DNNMode.LT); 29 | 30 | Utils.Utils.cleanFolder(@OutputFolder.OutputFolderLtDNN); 31 | Utils.Utils.cleanFolder(@OutputFolder.OutputFolderFrameDNNONNX); 32 | } 33 | 34 | public List Run(Mat frame, int frameIndex, Dictionary counts, List> lines, Dictionary category, ref long teleCountsCheapDNN, bool savePictures = false) 35 | { 36 | // buffer frame 37 | frameBufferLtDNNOnnxYolo.Buffer(frame); 38 | 39 | if (counts_prev.Count != 0) 40 | { 41 | foreach (string lane in counts.Keys) 42 | { 43 | int diff = Math.Abs(counts[lane] - counts_prev[lane]); 44 | if (diff > 0) //object detected by BGS 45 | { 46 | if (frameIndex >= DNNConfig.FRAME_SEARCH_RANGE) 47 | { 48 | // call onnx cheap model for crosscheck 49 | int lineID = Array.IndexOf(counts.Keys.ToArray(), lane); 50 | Mat[] frameBufferArray = frameBufferLtDNNOnnxYolo.ToArray(); 51 | int frameIndexOnnxYolo = frameIndex - 1; 52 | List analyzedTrackingItems = null; 53 | 54 | while (frameIndex - frameIndexOnnxYolo < DNNConfig.FRAME_SEARCH_RANGE) 55 | { 56 | Console.WriteLine("** Calling DNN on " + (DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexOnnxYolo))); 57 | Mat frameOnnx = frameBufferArray[DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexOnnxYolo)]; 58 | 59 | analyzedTrackingItems = frameDNNOnnxYolo.Run(frameOnnx, (DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexOnnxYolo)), category, System.Drawing.Brushes.Pink, lineID, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_LARGE); 60 | teleCountsCheapDNN++; 61 | 62 | // object detected by cheap model 63 | if (analyzedTrackingItems != null) 64 | { 65 | List ltDNNItem = new List(); 66 | foreach (Item item in analyzedTrackingItems) 67 | { 68 | item.RawImageData = Utils.Utils.ImageToByteBmp(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameOnnx)); 69 | item.TriggerLine = lane; 70 | item.TriggerLineID = lineID; 71 | item.Model = "Cheap"; 72 | ltDNNItem.Add(item); 73 | 74 | // output cheap onnx results 75 | if (savePictures) 76 | { 77 | string blobName_Cheap = $@"frame-{frameIndex}-DNN-{item.Confidence}.jpg"; 78 | string fileName_Cheap = @OutputFolder.OutputFolderLtDNN + blobName_Cheap; 79 | File.WriteAllBytes(fileName_Cheap, item.TaggedImageData); 80 | File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_Cheap, item.TaggedImageData); 81 | } 82 | } 83 | updateCount(counts); 84 | return ltDNNItem; 85 | } 86 | frameIndexOnnxYolo--; 87 | } 88 | } 89 | } 90 | } 91 | } 92 | updateCount(counts); 93 | return null; 94 | } 95 | 96 | void updateCount(Dictionary counts) 97 | { 98 | foreach (string dir in counts.Keys) 99 | { 100 | counts_prev[dir] = counts[dir]; 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/VAP/DNNDetector/Model/Item.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Drawing; 6 | 7 | namespace DNNDetector.Model 8 | { 9 | public class Item 10 | { 11 | public string ObjName { get; set; } 12 | public double Confidence { get; set; } 13 | public int X { get; set; } 14 | public int Y { get; set; } 15 | public int Width { get; set; } 16 | public int Height { get; set; } 17 | public int ObjId { get; set; } 18 | public int TrackId { get; set; } 19 | public int Index { get; set; } 20 | public byte[] RawImageData { get; set; } 21 | public byte[] TaggedImageData { get; set; } 22 | public byte[] CroppedImageData { get; set; } 23 | public string TriggerLine { get; set; } 24 | public int TriggerLineID { get; set; } 25 | public string Model { get; set; } 26 | 27 | public Item(int x, int y, int width, int height, int catId, string catName, double confidence, int lineID, string lineName) 28 | { 29 | this.X = x; 30 | this.Y = y; 31 | this.Width = width; 32 | this.Height = height; 33 | this.ObjId = catId; 34 | this.ObjName = catName; 35 | this.Confidence = confidence; 36 | this.TriggerLineID = lineID; 37 | this.TriggerLine = lineName; 38 | } 39 | 40 | public Item(Wrapper.ORT.ORTItem onnxYoloItem) 41 | { 42 | this.X = onnxYoloItem.X; 43 | this.Y = onnxYoloItem.Y; 44 | this.Width = onnxYoloItem.Width; 45 | this.Height = onnxYoloItem.Height; 46 | this.ObjId = onnxYoloItem.ObjId; 47 | this.ObjName = onnxYoloItem.ObjName; 48 | this.Confidence = onnxYoloItem.Confidence; 49 | this.TriggerLineID = onnxYoloItem.TriggerLineID; 50 | this.TriggerLine = onnxYoloItem.TriggerLine; 51 | } 52 | 53 | public Point Center() 54 | { 55 | return new Point(this.X + this.Width / 2, this.Y + this.Height / 2); 56 | } 57 | 58 | public float[] CenterVec() 59 | { 60 | float[] vec = { this.X + this.Width / 2, this.Y + this.Height / 2 }; 61 | return vec; 62 | } 63 | 64 | public void Print() 65 | { 66 | Console.WriteLine("{0} {1,-5} {2} {3,-5} {4} {5,-5} {6} {7,-10} {8} {9,-10:N2}", 67 | "Index:", Index, "ObjID:", ObjId, "TrackID:", TrackId, "Type:", ObjName, "Conf:", Confidence); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/VAP/DarknetDetector/CascadedDNNYolo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Config; 5 | using DNNDetector.Model; 6 | using OpenCvSharp; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.IO; 10 | using Utils.Config; 11 | using Wrapper.Yolo; 12 | using Wrapper.Yolo.Model; 13 | 14 | namespace DarknetDetector 15 | { 16 | public class CascadedDNNDarknet 17 | { 18 | static string YOLOCONFIG = "YoloV3Coco"; // "cheap" yolo config folder name 19 | FrameDNNDarknet frameDNNYolo; 20 | FrameBuffer frameBufferCcDNN; 21 | 22 | public CascadedDNNDarknet(double rFactor) 23 | { 24 | frameBufferCcDNN = new FrameBuffer(DNNConfig.FRAME_SEARCH_RANGE); 25 | 26 | frameDNNYolo = new FrameDNNDarknet(YOLOCONFIG, DNNMode.CC, rFactor); 27 | } 28 | 29 | public CascadedDNNDarknet(List<(string key, (System.Drawing.Point p1, System.Drawing.Point p2) coordinates)> lines) 30 | { 31 | frameBufferCcDNN = new FrameBuffer(DNNConfig.FRAME_SEARCH_RANGE); 32 | 33 | frameDNNYolo = new FrameDNNDarknet(YOLOCONFIG, DNNMode.CC, lines); 34 | } 35 | 36 | public List Run(Mat frame, int frameIndex, List ltDNNItemList, List<(string key, (System.Drawing.Point p1, System.Drawing.Point p2) coordinates)> lines, HashSet category) 37 | { 38 | if (ltDNNItemList == null) 39 | { 40 | return null; 41 | } 42 | 43 | List ccDNNItem = new List(); 44 | 45 | foreach (Item ltDNNItem in ltDNNItemList) 46 | { 47 | if (ltDNNItem.Confidence >= DNNConfig.CONFIDENCE_THRESHOLD) 48 | { 49 | ccDNNItem.Add(ltDNNItem); 50 | continue; 51 | } 52 | else 53 | { 54 | List analyzedTrackingItems = null; 55 | frameDNNYolo.SetTrackingPoint(new System.Drawing.Point((int)((lines[ltDNNItem.TriggerLineID].coordinates.p1.X + lines[ltDNNItem.TriggerLineID].coordinates.p2.X) / 2), 56 | (int)((lines[ltDNNItem.TriggerLineID].coordinates.p1.Y + lines[ltDNNItem.TriggerLineID].coordinates.p2.Y) / 2))); //only needs to check the last line in each row 57 | byte[] imgByte = ltDNNItem.RawImageData; 58 | 59 | Console.WriteLine("** Calling Heavy"); 60 | analyzedTrackingItems = frameDNNYolo.Detect(imgByte, category, ltDNNItem.TriggerLineID, System.Drawing.Brushes.Red, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_SMALL, frameIndex); 61 | 62 | // object detected by heavy YOLO 63 | if (analyzedTrackingItems != null) 64 | { 65 | foreach (YoloTrackingItem yoloTrackingItem in analyzedTrackingItems) 66 | { 67 | Item item = Item(yoloTrackingItem); 68 | item.RawImageData = imgByte; 69 | item.TriggerLine = ltDNNItem.TriggerLine; 70 | item.TriggerLineID = ltDNNItem.TriggerLineID; 71 | item.Model = "Heavy"; 72 | ccDNNItem.Add(item); 73 | 74 | // output heavy YOLO results 75 | string blobName_Heavy = $@"frame-{frameIndex}-Heavy-{yoloTrackingItem.Confidence}.jpg"; 76 | string fileName_Heavy = @OutputFolder.OutputFolderCcDNN + blobName_Heavy; 77 | File.WriteAllBytes(fileName_Heavy, yoloTrackingItem.TaggedImageData); 78 | File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_Heavy, yoloTrackingItem.TaggedImageData); 79 | 80 | return ccDNNItem; // if we only return the closest object detected by heavy model 81 | } 82 | } 83 | else 84 | { 85 | Console.WriteLine("**Not detected by Heavy"); 86 | } 87 | } 88 | } 89 | 90 | return ccDNNItem; 91 | } 92 | 93 | Item Item(YoloTrackingItem yoloTrackingItem) 94 | { 95 | Item item = new Item(yoloTrackingItem.X, yoloTrackingItem.Y, yoloTrackingItem.Width, yoloTrackingItem.Height, 96 | yoloTrackingItem.ObjId, yoloTrackingItem.Type, yoloTrackingItem.Confidence, 0, ""); 97 | 98 | item.TrackId = yoloTrackingItem.TrackId; 99 | item.Index = yoloTrackingItem.Index; 100 | item.TaggedImageData = yoloTrackingItem.TaggedImageData; 101 | item.CroppedImageData = yoloTrackingItem.CroppedImageData; 102 | 103 | return item; 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/VAP/DarknetDetector/DarknetDetector.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /src/VAP/DarknetDetector/FrameBuffer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | using OpenCvSharp; 9 | 10 | namespace DarknetDetector 11 | { 12 | class FrameBuffer 13 | { 14 | Queue frameBuffer; 15 | int bSize; 16 | 17 | public FrameBuffer(int size) 18 | { 19 | bSize = size; 20 | frameBuffer = new Queue(bSize); 21 | } 22 | 23 | public void Buffer(Mat frame) 24 | { 25 | frameBuffer.Enqueue(frame); 26 | if (frameBuffer.Count > bSize) 27 | frameBuffer.Dequeue(); 28 | } 29 | 30 | public Mat[] ToArray() 31 | { 32 | return frameBuffer.ToArray(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/VAP/DarknetDetector/LineTriggeredDNNYolo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Model; 5 | using DNNDetector.Config; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Linq; 9 | using System.IO; 10 | using OpenCvSharp; 11 | using Utils.Config; 12 | using Wrapper.Yolo; 13 | using Wrapper.Yolo.Model; 14 | 15 | 16 | namespace DarknetDetector 17 | { 18 | public class LineTriggeredDNNDarknet 19 | { 20 | static string YOLOCONFIG = "YoloV3TinyCoco"; // "cheap" yolo config folder name 21 | FrameDNNDarknet frameDNNYolo; 22 | FrameBuffer frameBufferLtDNNYolo; 23 | Dictionary counts_prev = new Dictionary(); 24 | 25 | public LineTriggeredDNNDarknet(double rFactor) 26 | { 27 | frameBufferLtDNNYolo = new FrameBuffer(DNNConfig.FRAME_SEARCH_RANGE); 28 | 29 | frameDNNYolo = new FrameDNNDarknet(YOLOCONFIG, DNNMode.LT, rFactor); 30 | } 31 | 32 | public LineTriggeredDNNDarknet(List<(string key, (System.Drawing.Point p1, System.Drawing.Point p2) coordinates)> lines) 33 | { 34 | frameBufferLtDNNYolo = new FrameBuffer(DNNConfig.FRAME_SEARCH_RANGE); 35 | 36 | frameDNNYolo = new FrameDNNDarknet(YOLOCONFIG, DNNMode.LT, lines); 37 | } 38 | 39 | public List Run(Mat frame, int frameIndex, Dictionary counts, List<(string key, (System.Drawing.Point p1, System.Drawing.Point p2) coordinates)> lines, HashSet category) 40 | { 41 | // buffer frame 42 | frameBufferLtDNNYolo.Buffer(frame); 43 | 44 | if (counts_prev.Count != 0) 45 | { 46 | foreach (string lane in counts.Keys) 47 | { 48 | int diff = Math.Abs(counts[lane] - counts_prev[lane]); 49 | if (diff > 0) //object detected by BGS-based counter 50 | { 51 | if (frameIndex >= DNNConfig.FRAME_SEARCH_RANGE) 52 | { 53 | // call yolo for crosscheck 54 | int lineID = Array.IndexOf(counts.Keys.ToArray(), lane); 55 | frameDNNYolo.SetTrackingPoint(new System.Drawing.Point((int)((lines[lineID].coordinates.p1.X + lines[lineID].coordinates.p2.X) / 2), 56 | (int)((lines[lineID].coordinates.p1.Y + lines[lineID].coordinates.p2.Y) / 2))); //only needs to check the last line in each row 57 | Mat[] frameBufferArray = frameBufferLtDNNYolo.ToArray(); 58 | int frameIndexYolo = frameIndex - 1; 59 | DateTime start = DateTime.Now; 60 | List analyzedTrackingItems = null; 61 | 62 | while (frameIndex - frameIndexYolo < DNNConfig.FRAME_SEARCH_RANGE) 63 | { 64 | Console.WriteLine("** Calling Cheap on " + (DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexYolo))); 65 | Mat frameYolo = frameBufferArray[DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexYolo)]; 66 | byte[] imgByte = Utils.Utils.ImageToByteBmp(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameYolo)); 67 | 68 | analyzedTrackingItems = frameDNNYolo.Detect(imgByte, category, lineID, System.Drawing.Brushes.Pink, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_LARGE, frameIndexYolo); 69 | 70 | // object detected by cheap YOLO 71 | if (analyzedTrackingItems != null) 72 | { 73 | List ltDNNItem = new List(); 74 | foreach (YoloTrackingItem yoloTrackingItem in analyzedTrackingItems) 75 | { 76 | Item item = Item(yoloTrackingItem); 77 | item.RawImageData = imgByte; 78 | item.TriggerLine = lane; 79 | item.TriggerLineID = lineID; 80 | item.Model = "Cheap"; 81 | ltDNNItem.Add(item); 82 | 83 | // output cheap YOLO results 84 | string blobName_Cheap = $@"frame-{frameIndex}-Cheap-{yoloTrackingItem.Confidence}.jpg"; 85 | string fileName_Cheap = @OutputFolder.OutputFolderLtDNN + blobName_Cheap; 86 | File.WriteAllBytes(fileName_Cheap, yoloTrackingItem.TaggedImageData); 87 | File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_Cheap, yoloTrackingItem.TaggedImageData); 88 | } 89 | updateCount(counts); 90 | return ltDNNItem; 91 | } 92 | frameIndexYolo--; 93 | } 94 | } 95 | } 96 | } 97 | } 98 | updateCount(counts); 99 | 100 | return null; 101 | } 102 | 103 | Item Item(YoloTrackingItem yoloTrackingItem) 104 | { 105 | Item item = new Item(yoloTrackingItem.X, yoloTrackingItem.Y, yoloTrackingItem.Width, yoloTrackingItem.Height, 106 | yoloTrackingItem.ObjId, yoloTrackingItem.Type, yoloTrackingItem.Confidence, 0, ""); 107 | 108 | item.TrackId = yoloTrackingItem.TrackId; 109 | item.Index = yoloTrackingItem.Index; 110 | item.TaggedImageData = yoloTrackingItem.TaggedImageData; 111 | item.CroppedImageData = yoloTrackingItem.CroppedImageData; 112 | 113 | return item; 114 | } 115 | 116 | void updateCount(Dictionary counts) 117 | { 118 | foreach (string dir in counts.Keys) 119 | { 120 | counts_prev[dir] = counts[dir]; 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/VAP/Decoder/Decoder.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using OpenCvSharp; 6 | 7 | namespace Decoder 8 | { 9 | public class Decoder 10 | { 11 | VideoCapture capture = null; 12 | string inputURL; 13 | 14 | bool toLoop; 15 | 16 | int objTotal, objDirA, objDirB; 17 | 18 | public Decoder(string input, bool loop) 19 | { 20 | capture = new VideoCapture(input); 21 | inputURL = input; 22 | 23 | toLoop = loop; 24 | } 25 | 26 | public Mat getNextFrame() 27 | { 28 | Mat sourceMat = new Mat(); 29 | 30 | try 31 | { 32 | capture.Read(sourceMat); 33 | } 34 | 35 | catch (Exception e) 36 | { 37 | Console.WriteLine(e.ToString()); 38 | Console.WriteLine("********RESET*****"); 39 | 40 | capture = new VideoCapture(inputURL); 41 | 42 | return null; 43 | } 44 | 45 | if (sourceMat == null) 46 | return sourceMat; 47 | 48 | if (toLoop) 49 | { 50 | if (sourceMat.Height == 0 && sourceMat.Width == 0) 51 | { 52 | capture = new VideoCapture(inputURL); 53 | capture.Read(sourceMat); 54 | } 55 | } 56 | 57 | return sourceMat; 58 | } 59 | 60 | public int getTotalFrameNum() 61 | { 62 | int length; 63 | length = (int)Math.Floor(capture.Get(CaptureProperty.FrameCount)); 64 | 65 | return length; 66 | } 67 | 68 | public double getVideoFPS() 69 | { 70 | double framerate; 71 | framerate = capture.Get(CaptureProperty.Fps); 72 | 73 | return framerate; 74 | } 75 | 76 | public void updateObjNum(int[] dirCount) 77 | { 78 | objTotal = dirCount[0] + dirCount[1] + dirCount[2]; 79 | objDirA = dirCount[0]; 80 | objDirB = dirCount[1]; 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/VAP/Decoder/Decoder.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/VAP/FramePreProcessor/FrameDisplay.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | using OpenCvSharp; 9 | 10 | namespace FramePreProcessor 11 | { 12 | public class FrameDisplay 13 | { 14 | static Dictionary displayKVpairs = new Dictionary(); 15 | static Dictionary prev_displayKVpairs = new Dictionary(); 16 | 17 | public static void display(Mat resizedFrame) 18 | { 19 | Mat frameToDisplay = resizedFrame.Clone(); 20 | if (displayKVpairs.Count > 0) 21 | { 22 | double scale = 0.75; 23 | 24 | //Header box configs 25 | int boxHeight = (int)(80 * scale); 26 | int boxWidth = frameToDisplay.Width; 27 | int boxpadding = (int)(15 * scale); 28 | 29 | //Text configs 30 | int leftTextPadding = (int)(50 * scale); 31 | int textPadding = (int)(10 * scale); 32 | int textHeight = (int)(22 * scale); 33 | 34 | string row1Text = "Total:"; 35 | string row2Text = "Network:"; 36 | 37 | int row1Textbox = (int)(20 * row1Text.Length * scale); 38 | int row2Textbox = (int)(20 * row2Text.Length * scale); 39 | 40 | Cv2.Rectangle(frameToDisplay, new Rect(0, boxpadding, boxWidth, boxHeight), new Scalar(255, 255, 240), Cv2.FILLED); 41 | 42 | //Draw row 1: total count 43 | int row1Height = textHeight + textPadding + boxpadding; 44 | 45 | string result = ""; 46 | foreach (string dir in displayKVpairs.Keys) 47 | { 48 | int displayTotal = Int32.Parse(displayKVpairs[dir]); 49 | result += dir + " " + displayTotal + " "; 50 | 51 | } 52 | Cv2.PutText(frameToDisplay, result, new Point(leftTextPadding + row1Textbox, row1Height), HersheyFonts.HersheyPlain, scale, Scalar.Black); 53 | } 54 | 55 | Cv2.ImShow("Raw Frame", frameToDisplay); 56 | Cv2.WaitKey(1); 57 | } 58 | 59 | public static void updateKVPairs(Dictionary kvpairs) 60 | { 61 | foreach(string s in kvpairs.Keys) 62 | { 63 | if (!displayKVpairs.ContainsKey(s)) 64 | displayKVpairs.Add(s, kvpairs[s]); 65 | else 66 | { 67 | int currentVal = Int32.Parse(displayKVpairs[s]); 68 | currentVal += Int32.Parse(kvpairs[s]); 69 | displayKVpairs[s] = currentVal.ToString(); 70 | } 71 | } 72 | } 73 | 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/VAP/FramePreProcessor/FramePreProcessor.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/VAP/FramePreProcessor/PreProcessor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | 6 | using OpenCvSharp; 7 | 8 | namespace FramePreProcessor 9 | { 10 | public class PreProcessor 11 | { 12 | public static Mat returnFrame(Mat sourceMat, int frameIndex, int SAMPLING_FACTOR, double RESOLUTION_FACTOR, bool display) 13 | { 14 | Mat resizedFrame = null; 15 | 16 | if (frameIndex % SAMPLING_FACTOR != 0) return resizedFrame; 17 | 18 | try 19 | { 20 | resizedFrame = sourceMat.Resize(new OpenCvSharp.Size((int)(sourceMat.Size().Width * RESOLUTION_FACTOR), (int)(sourceMat.Size().Height * RESOLUTION_FACTOR))); 21 | if (display) 22 | FrameDisplay.display(resizedFrame); 23 | } 24 | catch (Exception e) 25 | { 26 | Console.WriteLine(e.ToString()); 27 | Console.WriteLine("********RESET RESIZE*****"); 28 | return null; 29 | } 30 | return resizedFrame; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/Detector.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Drawing; 7 | using System.IO; 8 | using OpenCvSharp; 9 | using Utils.Config; 10 | using BGSObjectDetector; 11 | 12 | namespace LineDetector 13 | { 14 | public class Detector 15 | { 16 | bool DISPLAY_BGS; 17 | 18 | /// 19 | /// The initial delay to allow for the background subtractor to kick in, N_FRAMES_TO_LEARN in MOG2.cs 20 | /// 21 | public int START_DELAY = 120; 22 | 23 | 24 | public MultiLaneDetector multiLaneDetector; 25 | 26 | Dictionary counts = new Dictionary(); 27 | Dictionary counts_prev = new Dictionary(); 28 | Dictionary occupancy = new Dictionary(); 29 | Dictionary occupancy_prev = new Dictionary(); 30 | 31 | /// 32 | /// Constructs a object with the provided 33 | /// 34 | /// Sampling rate scaling factor. 35 | /// Resolution scaling factor. 36 | /// A file specifying the lines used for the line-crossing algorithm. 37 | /// True to display a separate image each frame with the current frame number, the lines used, and the changes from the previous frame. 38 | public Detector(int sFactor, double rFactor, string linesFile, bool displayBGS) 39 | { 40 | Dictionary lineBasedDetectors = LineSets.readLineSet_LineDetector_FromTxtFile(linesFile, sFactor, rFactor); 41 | 42 | multiLaneDetector = (lineBasedDetectors != null) ? new MultiLaneDetector(lineBasedDetectors) : null; 43 | 44 | this.DISPLAY_BGS = displayBGS; 45 | Console.WriteLine(linesFile); 46 | } 47 | 48 | /// 49 | /// Checks for items crossing the provided LineSet. 50 | /// 51 | /// The frame to check. 52 | /// The index of the frame given. 53 | /// The foreground mask of the frame. 54 | /// A list of bounding boxes of items in the frame which deviate from the background. 55 | /// 56 | /// Returns a tuple with two Dictionaries. 57 | /// The first dictionary contains the number of items which cross the lines of interest, indexed by line name. 58 | /// The second dictionary contains a boolean for each line indicating whether or not an item is present at that line. 59 | /// 60 | public (Dictionary, Dictionary) updateLineResults(Mat frame, int frameIndex, Mat fgmask, List boxes) 61 | { 62 | if (frameIndex > START_DELAY) 63 | { 64 | Bitmap fgmaskBit = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(fgmask); 65 | 66 | multiLaneDetector.notifyFrameArrival(frameIndex, boxes, fgmaskBit); 67 | 68 | // bgs visualization with lines 69 | if (DISPLAY_BGS) 70 | { 71 | List<(string key, (System.Drawing.Point p1, System.Drawing.Point p2) coordinates)> lines = this.multiLaneDetector.getAllLines(); 72 | for (int i = 0; i < lines.Count; i++) 73 | { 74 | System.Drawing.Point p1 = lines[i].coordinates.p1; 75 | System.Drawing.Point p2 = lines[i].coordinates.p2; 76 | Cv2.Line(fgmask, p1.X, p1.Y, p2.X, p2.Y, new Scalar(255, 0, 255, 255), 5); 77 | } 78 | Cv2.ImShow("BGS Output", fgmask); 79 | //Cv2.WaitKey(1); 80 | } 81 | } 82 | counts = multiLaneDetector.getCounts(); 83 | 84 | if (counts_prev.Count != 0) 85 | { 86 | foreach (string lane in counts.Keys) 87 | { 88 | int diff = Math.Abs(counts[lane] - counts_prev[lane]); 89 | if (diff > 0) //object detected by BGS-based counter 90 | { 91 | Console.WriteLine($"Line: {lane}\tCounts: {counts[lane]}"); 92 | string blobName_BGS = $@"frame-{frameIndex}-BGS-{lane}-{counts[lane]}.jpg"; 93 | string fileName_BGS = @OutputFolder.OutputFolderBGSLine + blobName_BGS; 94 | frame.SaveImage(fileName_BGS); 95 | frame.SaveImage(@OutputFolder.OutputFolderAll + blobName_BGS); 96 | } 97 | } 98 | } 99 | updateCount(counts); 100 | 101 | //occupancy 102 | occupancy = multiLaneDetector.getOccupancy(); 103 | foreach (string lane in occupancy.Keys) 104 | { 105 | //output frames that have line occupied by objects 106 | //if (frameIndex > 1) 107 | //{ 108 | // if (occupancy[lane]) 109 | // { 110 | // string blobName_BGS = $@"frame-{frameIndex}-BGS-{lane}-{occupancy[lane]}.jpg"; 111 | // string fileName_BGS = @OutputFolder.OutputFolderBGSLine + blobName_BGS; 112 | // frame.SaveImage(fileName_BGS); 113 | // frame.SaveImage(@OutputFolder.OutputFolderAll + blobName_BGS); 114 | // } 115 | //} 116 | updateCount(lane, occupancy); 117 | } 118 | 119 | return (counts, occupancy); 120 | } 121 | 122 | bool occupancyChanged(string lane) 123 | { 124 | bool diff = false; 125 | if (occupancy_prev.Count != 0) 126 | { 127 | diff = occupancy[lane] != occupancy_prev[lane]; 128 | } 129 | 130 | return diff; 131 | } 132 | 133 | void updateCount(string lane, Dictionary counts) 134 | { 135 | occupancy_prev[lane] = counts[lane]; 136 | } 137 | 138 | void updateCount(Dictionary counts) 139 | { 140 | foreach (string dir in counts.Keys) 141 | { 142 | counts_prev[dir] = counts[dir]; 143 | } 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/FallingEdgeCrossingDetector.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace LineDetector 5 | { 6 | /// 7 | /// A line crossing detector that fires when an item leaves the line area. 8 | /// 9 | class FallingEdgeCrossingDetector : ICrossingDetector 10 | { 11 | List FrameNoList = new List(); 12 | List OccupancyValueList = new List(); 13 | int UP_STATE_TRANSITION_LENGTH = 4; 14 | int DOWN_STATE_TRANSITION_LENGTH = 10; 15 | int History; 16 | OCCUPANCY_STATE curState = OCCUPANCY_STATE.UNOCCUPIED; 17 | bool debug = false; 18 | public List debug_occupancySequence; 19 | 20 | /// 21 | /// Enables debug mode. Within , this enables logging occupancy values. 22 | /// 23 | /// 24 | /// The occupancy log may be accessed with . 25 | public void setDebug() 26 | { 27 | debug = true; 28 | debug_occupancySequence = new List(); 29 | } 30 | 31 | /// 32 | /// Creates a with the provided frame rate sampling factor. 33 | /// 34 | /// The frame rate sampling factor. Note that particularly large values could behave unexpectedly. 35 | public FallingEdgeCrossingDetector(int sFactor) 36 | { 37 | UP_STATE_TRANSITION_LENGTH = (int)Math.Ceiling((double)UP_STATE_TRANSITION_LENGTH / sFactor); 38 | DOWN_STATE_TRANSITION_LENGTH = (int)Math.Ceiling((double)DOWN_STATE_TRANSITION_LENGTH / sFactor); 39 | History = Math.Max(UP_STATE_TRANSITION_LENGTH, DOWN_STATE_TRANSITION_LENGTH); 40 | } 41 | 42 | private bool CheckForStateTransision() 43 | { 44 | if (OccupancyValueList.Count < History) 45 | { 46 | return false; 47 | } 48 | if (curState == OCCUPANCY_STATE.UNOCCUPIED) 49 | { 50 | for (int i = 0; i < UP_STATE_TRANSITION_LENGTH; i++) 51 | { 52 | if (OccupancyValueList[OccupancyValueList.Count - i - 1] < 0) 53 | { 54 | return false; 55 | } 56 | } 57 | curState = OCCUPANCY_STATE.OCCUPIED; 58 | return false; 59 | } 60 | else 61 | { 62 | for (int i = 0; i < DOWN_STATE_TRANSITION_LENGTH; i++) 63 | { 64 | if (OccupancyValueList[OccupancyValueList.Count - i - 1] > 0) 65 | { 66 | return false; 67 | } 68 | } 69 | curState = OCCUPANCY_STATE.UNOCCUPIED; 70 | return true; 71 | } 72 | } 73 | 74 | 75 | //return true if there was a line crossing event 76 | /// 77 | /// Notifies the detector of a new occupancy state at a given frame. 78 | /// 79 | /// The index of the frame of interest. 80 | /// The occupancy state at that frame. 81 | /// Returns true if an event was detected, and false otherwise. 82 | public bool notifyOccupancy(int frameNo, bool occupancy) 83 | { 84 | while (FrameNoList.Count > 0) 85 | { 86 | if (FrameNoList[0] <= frameNo - History) 87 | { 88 | FrameNoList.RemoveAt(0); 89 | OccupancyValueList.RemoveAt(0); 90 | } 91 | else 92 | { 93 | break; 94 | } 95 | } 96 | 97 | double finalOccupancyValue = -1; 98 | if (occupancy) 99 | { 100 | finalOccupancyValue = 1; 101 | } 102 | 103 | //interpolate for missing frames 104 | if (FrameNoList.Count > 0) 105 | { 106 | int curFrameNo = FrameNoList[FrameNoList.Count - 1]; 107 | int nextFrameNo = frameNo; 108 | int diff = nextFrameNo - curFrameNo; 109 | if (diff > 1) 110 | { 111 | double initialOccupancyValue = OccupancyValueList[OccupancyValueList.Count - 1]; 112 | double occupancyDiff = finalOccupancyValue - initialOccupancyValue; 113 | double ratio = occupancyDiff / (double)diff; 114 | for (int f = curFrameNo + 1; f < nextFrameNo; f++) 115 | { 116 | double value = (f - curFrameNo) * ratio + initialOccupancyValue; 117 | FrameNoList.Add(f); 118 | OccupancyValueList.Add(value); 119 | if (debug) 120 | { 121 | debug_occupancySequence.Add(value); 122 | } 123 | } 124 | } 125 | } 126 | FrameNoList.Add(frameNo); 127 | OccupancyValueList.Add(finalOccupancyValue); 128 | if (debug) 129 | { 130 | debug_occupancySequence.Add(finalOccupancyValue); 131 | } 132 | 133 | //Console.WriteLine("finalOccupancyValue:" + finalOccupancyValue); 134 | return CheckForStateTransision(); 135 | } 136 | 137 | /// 138 | /// Gets the occupancy state of the detector as of the latest frame. 139 | /// 140 | /// 141 | public OCCUPANCY_STATE getState() 142 | { 143 | return curState; 144 | } 145 | 146 | /// 147 | /// Gets a list of all occupancy values observed by the detector while debugging has been enabled. No frame indices are included. 148 | /// 149 | /// 150 | public List getLineOccupancyHistory() 151 | { 152 | return debug_occupancySequence; 153 | } 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/ICrossingDetector.cs: -------------------------------------------------------------------------------- 1 | namespace LineDetector 2 | { 3 | /// 4 | /// Indicates whether a detector is currently occupied or not. 5 | /// 6 | public enum OCCUPANCY_STATE 7 | { 8 | /// 9 | /// Indicates that at least one item is present. 10 | /// 11 | OCCUPIED, 12 | /// 13 | /// Indicates that no items are present. 14 | /// 15 | UNOCCUPIED 16 | }; 17 | 18 | /// 19 | /// Interface for a crossing detector that fires as a function of occupancy. 20 | /// 21 | interface ICrossingDetector 22 | { 23 | /// 24 | /// Notifies the detector of a new occupancy state at a given frame. 25 | /// 26 | /// The index of the frame of interest. 27 | /// The occupancy state at that frame. 28 | /// Returns true if an event was detected, and false otherwise. 29 | bool notifyOccupancy(int frameNo, bool occupancy); 30 | 31 | //TODO(iharwell): Should be a property. 32 | /// 33 | /// Gets the occupancy state of the detector as of the latest frame. 34 | /// 35 | /// 36 | OCCUPANCY_STATE getState(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/ILaneBasedDetector.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Drawing; 7 | 8 | using BGSObjectDetector; 9 | 10 | namespace LineDetector 11 | { 12 | /// 13 | /// A detector that checks for items crossing a line. 14 | /// 15 | public interface ILineBasedDetector 16 | { 17 | /// 18 | /// Processes a frame upon arrival. 19 | /// 20 | /// The index of the frame to process. 21 | /// A list of bounding boxes of items in frame. 22 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 23 | void notifyFrameArrival(int frameNo, List boxes, Bitmap mask); 24 | 25 | /// 26 | /// Processes a frame upon arrival. 27 | /// 28 | /// The index of the frame to process. 29 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 30 | void notifyFrameArrival(int frameNo, Bitmap mask); 31 | 32 | //TODO(iharwell): This should be moved somewhere more appropriate. 33 | /// 34 | /// Activates debug logging. 35 | /// 36 | void setDebug(); 37 | 38 | /// 39 | /// Provides a history of the occupancy of this line detector, with each entry containing a list of occupancy values for each line considered by this detector. 40 | /// 41 | List> getOccupancyHistory(); 42 | 43 | /// 44 | /// Gets the DetectionLine used by this detector. 45 | /// 46 | DetectionLine getDetectionLine(); 47 | 48 | /// 49 | /// Gets the current occupancy state of this detector. This updates when the detector is notified of a frame arrival. 50 | /// 51 | /// Returns true if the line is occupied, and false otherwise. 52 | bool getOccupancy(); 53 | 54 | /// 55 | /// Gets the number of times that this detector has been triggered. 56 | /// 57 | /// 58 | int getCount(); 59 | 60 | //TODO(iharwell): This seems like it should not be part of the interface. 61 | /// 62 | /// Sets the count of this detector. 63 | /// 64 | /// 65 | void setCount(int value); 66 | 67 | /// 68 | /// Gets the bounding box of the line used by this detector. 69 | /// 70 | /// 71 | Box getBbox(); 72 | 73 | /// 74 | /// Gets a Dictionary of the parameters used by this detector, stored by name. 75 | /// 76 | /// 77 | Dictionary getParameters(); 78 | 79 | /// 80 | /// Gets the line segments used by this detector. 81 | /// 82 | /// 83 | List<(Point p1, Point p2)> getLineCoor(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/ISingleLineCrossingDetector.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Drawing; 3 | 4 | using BGSObjectDetector; 5 | 6 | namespace LineDetector 7 | { 8 | //TODO(iharwell): Pull methods into a separate interface for things that overlap with ICrossingDetector and ILineBasedDetector. 9 | /// 10 | /// Interface for a line crossing detector that uses a single line for detection. 11 | /// 12 | interface ISingleLineCrossingDetector 13 | { 14 | //notify the arrival of a frame to the detector 15 | //return true if there is a crossing event detected 16 | /// 17 | /// Processes a frame upon arrival. 18 | /// 19 | /// The index of the frame to process. 20 | /// A list of bounding boxes of items in frame. 21 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 22 | /// 23 | /// Returns a Tuple that contains a boolean indicating whether a crossing was detected, and the bounding box of the crossing item. 24 | /// 25 | (bool crossingResult, Box b) notifyFrameArrival(int frameNo, List boxes, Bitmap mask); 26 | 27 | /// 28 | /// Processes a frame upon arrival. 29 | /// 30 | /// The index of the frame to process. 31 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 32 | /// 33 | /// Returns a boolean indicating whether a crossing was detected. 34 | /// 35 | bool notifyFrameArrival(int frameNo, Bitmap mask); 36 | 37 | //returns occupied or not 38 | /// 39 | /// Gets the occupancy state of this detector as of the latest frame. 40 | /// 41 | /// 42 | OCCUPANCY_STATE getState(); 43 | 44 | /// 45 | /// Enables debug logging. 46 | /// 47 | void setDebug(); 48 | 49 | /// 50 | /// Gets the line occupancy overlap values which are stored while in debug mode. 51 | /// 52 | /// 53 | List getLineOccupancyHistory(); 54 | 55 | /// 56 | /// Gets the used by this detector. 57 | /// 58 | /// 59 | DetectionLine getDetectionLine(); 60 | 61 | /// 62 | /// Gets the occupancy state of this detector as of the latest frame. 63 | /// 64 | /// Returns true if the detector is occupied by one or more items, and false otherwise. 65 | bool getOccupancy(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/LineDetector.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/LineSets.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.IO; 7 | 8 | namespace LineDetector 9 | { 10 | class LineSets 11 | { 12 | /// 13 | /// Reads a line set from the provided CSV text file. 14 | /// 15 | /// The name of the file to read. 16 | /// The 17 | /// 18 | /// Returns a Dictionary with all lines contained in the provided file, indexed by name. 19 | /// 20 | /// The file provided to this function is expected to contain the following values in order, separated by the tab character: 21 | /// 22 | /// The name of the line. (a string) 23 | /// The number of lines in the file. (an integer) 24 | /// The x coordinate of the first point of the line within the video frame. (an integer) 25 | /// The y coordinate of the first point of the line within the video frame. (an integer) 26 | /// The x coordinate of the second point of the line within the video frame. (an integer) 27 | /// The y coordinate of the second point of the line within the video frame. (an integer) 28 | /// The overlap fraction threshold. 29 | /// 30 | /// 31 | public static Dictionary readLineSet_LineDetector_FromTxtFile(string fileName, int sFactor, double imageScaling) 32 | { 33 | Dictionary ret = new Dictionary(); 34 | try 35 | { 36 | StreamReader r = new StreamReader(fileName); 37 | do 38 | { 39 | string line = r.ReadLine(); 40 | if (line == null) 41 | { 42 | break; 43 | } 44 | string[] fields = line.Split('\t'); 45 | string directionName = fields[0]; 46 | int noLines = Convert.ToInt32(fields[1]); 47 | List lineDetectors = new List(); 48 | 49 | for (int i = 0; i < noLines; i++) 50 | { 51 | int x1 = (int)(Convert.ToInt32(fields[2 + 0 * 5]) * imageScaling); 52 | int y1 = (int)(Convert.ToInt32(fields[3 + 0 * 5]) * imageScaling); 53 | int x2 = (int)(Convert.ToInt32(fields[4 + 0 * 5]) * imageScaling); 54 | int y2 = (int)(Convert.ToInt32(fields[5 + 0 * 5]) * imageScaling); 55 | double threshold = Convert.ToDouble(fields[6 + 0 * 5]); 56 | SingleLineCrossingDetector lineDetector = new SingleLineCrossingDetector(x1, y1, x2, y2, threshold, sFactor); 57 | 58 | lineDetectors.Add(lineDetector); 59 | } 60 | List minLags = new List(); 61 | for (int i = 0; i < noLines - 1; i++) 62 | { 63 | int lag = Convert.ToInt32(fields[noLines * 5 + 2 + i]); 64 | minLags.Add(lag); 65 | } 66 | List maxLags = new List(); 67 | for (int i = 0; i < noLines - 1; i++) 68 | { 69 | int lag = Convert.ToInt32(fields[noLines * 6 + 1 + i]); 70 | maxLags.Add(lag); 71 | } 72 | CascadedLinesDetector counter = new CascadedLinesDetector(lineDetectors, minLags, maxLags); 73 | 74 | ret.Add(directionName, counter); 75 | } while (true); 76 | r.Close(); 77 | } 78 | catch (IOException e) 79 | { 80 | Console.WriteLine(e.ToString()); 81 | return null; 82 | } 83 | 84 | return ret; 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/MultiLaneDetector.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using BGSObjectDetector; 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Drawing; 8 | 9 | namespace LineDetector 10 | { 11 | // corresponds to LineCrossingBasedMultiLaneCounter.cs 12 | public class MultiLaneDetector 13 | { 14 | Dictionary laneDetector; 15 | 16 | /// 17 | /// Creates a object using the provided set of named objects. 18 | /// 19 | /// The named set of detectors used by this . 20 | public MultiLaneDetector(Dictionary lineBasedDetector) 21 | { 22 | laneDetector = lineBasedDetector; 23 | } 24 | 25 | /// 26 | /// Processes a frame upon arrival. 27 | /// 28 | /// The index of the frame to process. 29 | /// A list of bounding boxes of items in frame. 30 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 31 | public void notifyFrameArrival(int frameNo, List boxes, Bitmap mask) 32 | { 33 | 34 | foreach (KeyValuePair entry in laneDetector) 35 | { 36 | entry.Value.notifyFrameArrival(frameNo, boxes, mask); 37 | } 38 | } 39 | 40 | /// 41 | /// Processes a frame upon arrival. 42 | /// 43 | /// The index of the frame to process. 44 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 45 | public void notifyFrameArrival(int frameNo, Bitmap mask) 46 | { 47 | 48 | foreach (KeyValuePair entry in laneDetector) 49 | { 50 | entry.Value.notifyFrameArrival(frameNo, mask); 51 | } 52 | } 53 | 54 | /// 55 | /// Gets the detection counts of each line used by this detector as of the latest frame. 56 | /// 57 | /// Returns a Dictionary of all occupancy counters, organized by the name of the lines. 58 | public Dictionary getCounts() 59 | { 60 | Dictionary counts = new Dictionary(); 61 | foreach (KeyValuePair entry in laneDetector) 62 | { 63 | counts.Add(entry.Key, entry.Value.getCount()); 64 | } 65 | return counts; 66 | } 67 | 68 | /// 69 | /// Gets the occupancy state of each line used by this detector as of the latest frame. 70 | /// 71 | /// Returns a Dictionary of all occupancy states, organized by the name of the lines. 72 | public Dictionary getOccupancy() 73 | { 74 | Dictionary occupancy = new Dictionary(); 75 | foreach (KeyValuePair entry in laneDetector) 76 | { 77 | occupancy.Add(entry.Key, entry.Value.getOccupancy()); 78 | } 79 | return occupancy; 80 | } 81 | 82 | /// 83 | /// Gets the center of the bounding box of the requested line. 84 | /// 85 | /// The name of the line to get the center of. 86 | /// Returns a PointF with the value of the center of the requested line if it exists, and null otherwise. 87 | public PointF? getBboxCenter(string laneID) 88 | { 89 | foreach (KeyValuePair entry in laneDetector) 90 | { 91 | if (entry.Key == laneID) 92 | { 93 | return entry.Value.getBbox().Center; 94 | } 95 | } 96 | 97 | return null; 98 | } 99 | 100 | /// 101 | /// Gets all lines used by this detector. 102 | /// 103 | /// 104 | /// Returns a list of Tuples containing the name and coordinates of each line. 105 | /// 106 | public List<(string key, (Point p1, Point p2) coordinates)> getAllLines() 107 | { 108 | List<(string key, (Point p1, Point p2) coordinates)> lines = new List<(string key, (Point p1, Point p2) coordinates)>(); 109 | foreach (KeyValuePair lane in laneDetector) 110 | { 111 | (Point p1, Point p2) coor = lane.Value.getLineCoor()[0]; 112 | lines.Add((lane.Key, coor)); 113 | } 114 | return lines; 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/VAP/LineDetector/SingleLineCrossingDetector.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using BGSObjectDetector; 5 | using System.Collections.Generic; 6 | using System.Drawing; 7 | 8 | namespace LineDetector 9 | { 10 | class SingleLineCrossingDetector : ISingleLineCrossingDetector 11 | { 12 | DetectionLine line; 13 | bool occupancy; 14 | Box bbox; 15 | FallingEdgeCrossingDetector lineCrossingDetector; 16 | bool debug = false; 17 | 18 | /// 19 | /// Creates a using the provided coordinates for the start and end point of the line. 20 | /// 21 | /// The X coordinate of the first point of the line. 22 | /// The Y coordinate of the first point of the line. 23 | /// The X coordinate of the second point of the line. 24 | /// The Y coordinate of the second point of the line. 25 | public SingleLineCrossingDetector(int a, int b, int c, int d) 26 | { 27 | line = new DetectionLine(a, b, c, d); 28 | lineCrossingDetector = new FallingEdgeCrossingDetector(1); 29 | } 30 | 31 | /// 32 | /// Creates a using the provided coordinates for the start and end point of the line. 33 | /// 34 | /// The X coordinate of the first point of the line. 35 | /// The Y coordinate of the first point of the line. 36 | /// The X coordinate of the second point of the line. 37 | /// The Y coordinate of the second point of the line. 38 | /// The overlap fraction threshold for this detector to be considered occupied. 39 | /// The frame rate sampling factor. 40 | public SingleLineCrossingDetector(int a, int b, int c, int d, double threshold, int sFactor) 41 | { 42 | line = new DetectionLine(a, b, c, d, threshold); 43 | lineCrossingDetector = new FallingEdgeCrossingDetector(sFactor); 44 | } 45 | 46 | /// 47 | /// Processes a frame upon arrival. 48 | /// 49 | /// The index of the frame to process. 50 | /// A list of bounding boxes of items in frame. 51 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 52 | /// 53 | /// Returns a Tuple that contains a boolean indicating whether a crossing was detected, and the bounding box of the crossing item. 54 | /// 55 | public (bool crossingResult, Box b) notifyFrameArrival(int frameNo, List boxes, Bitmap mask) 56 | { 57 | (occupancy, bbox) = line.isOccupied(boxes, mask); 58 | bool crossingResult = lineCrossingDetector.notifyOccupancy(frameNo, occupancy); 59 | return (crossingResult, bbox); 60 | } 61 | 62 | /// 63 | /// Processes a frame upon arrival. 64 | /// 65 | /// The index of the frame to process. 66 | /// A mask detailing the precise layout of items in the frame using black to indicate vacant space, and white to indicate occupied space. 67 | /// 68 | /// Returns a boolean indicating whether a crossing was detected. 69 | /// 70 | public bool notifyFrameArrival(int frameNo, Bitmap mask) 71 | { 72 | occupancy = line.isOccupied(mask); 73 | bool crossingResult = lineCrossingDetector.notifyOccupancy(frameNo, occupancy); 74 | return crossingResult; 75 | } 76 | 77 | /// 78 | /// Gets the occupancy state of this detector as of the latest frame. 79 | /// 80 | /// 81 | public OCCUPANCY_STATE getState() 82 | { 83 | return lineCrossingDetector.getState(); 84 | } 85 | 86 | /// 87 | /// Enables debug logging. 88 | /// 89 | public void setDebug() 90 | { 91 | debug = true; 92 | lineCrossingDetector.setDebug(); 93 | } 94 | 95 | /// 96 | /// Gets the line occupancy overlap values which are stored while in debug mode. 97 | /// 98 | /// 99 | public List getLineOccupancyHistory() 100 | { 101 | if (debug) 102 | { 103 | return lineCrossingDetector.getLineOccupancyHistory(); 104 | } 105 | else 106 | { 107 | return null; 108 | } 109 | } 110 | 111 | /// 112 | /// Gets the occupancy state of this detector as of the latest frame. 113 | /// 114 | /// Returns true if the detector is occupied by one or more items, and false otherwise. 115 | public bool getOccupancy() 116 | { 117 | return occupancy; 118 | } 119 | 120 | /// 121 | /// Gets the used by this detector. 122 | /// 123 | /// 124 | public DetectionLine getDetectionLine() 125 | { 126 | return line; 127 | } 128 | 129 | /// 130 | /// Gets the bounding box of this detector's region of interest. 131 | /// 132 | /// 133 | public Box getBbox() 134 | { 135 | return bbox; 136 | } 137 | 138 | /// 139 | /// Gets the coordinates of the line used by this detector. 140 | /// 141 | /// 142 | public (Point p1, Point p2) getLineCoor() 143 | { 144 | return (getDetectionLine().p1, getDetectionLine().p2); 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/VAP/ORTWrapper/DNNMode.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Wrapper.ORT 5 | { 6 | public enum DNNMode 7 | { 8 | Unknown, 9 | Frame, //FrameDNN 10 | LT, //LineTriggered 11 | CC //Cascaded 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/VAP/ORTWrapper/IYoloConfiguration.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Collections.Generic; 5 | 6 | namespace Wrapper.ORT 7 | { 8 | public interface IYoloConfiguration 9 | { 10 | uint ImageHeight { get; } 11 | uint ImageWidth { get; } 12 | string[] Labels { get; } 13 | } 14 | } -------------------------------------------------------------------------------- /src/VAP/ORTWrapper/ORTItem.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | 6 | namespace Wrapper.ORT 7 | { 8 | public class ORTItem 9 | { 10 | public string ObjName { get; set; } 11 | public double Confidence { get; set; } 12 | public int X { get; set; } 13 | public int Y { get; set; } 14 | public int Width { get; set; } 15 | public int Height { get; set; } 16 | public int ObjId { get; set; } 17 | public string TriggerLine { get; set; } 18 | public int TriggerLineID { get; set; } 19 | 20 | public ORTItem(int x, int y, int width, int height, int catId, string catName, double confidence, int lineID, string lineName) 21 | { 22 | this.X = Math.Max(0, x); 23 | this.Y = Math.Max(0, y); 24 | this.Width = width; 25 | this.Height = height; 26 | this.ObjId = catId; 27 | this.ObjName = catName; 28 | this.Confidence = confidence; 29 | this.TriggerLineID = lineID; 30 | this.TriggerLine = lineName; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/VAP/ORTWrapper/ORTWrapper.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | true 9 | 10 | 11 | 12 | false 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/VAP/ORTWrapper/Yolov3BaseConfig.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Wrapper.ORT 5 | { 6 | public class Yolov3BaseConfig : IYoloConfiguration 7 | { 8 | public string[] Labels => new string[] { 9 | "person", 10 | "bicycle", 11 | "car", 12 | "motorbike", 13 | "aeroplane", 14 | "bus", 15 | "train", 16 | "truck", 17 | "boat", 18 | "traffic light", 19 | "fire hydrant", 20 | "stop sign", 21 | "parking meter", 22 | "bench", 23 | "bird", 24 | "cat", 25 | "dog", 26 | "horse", 27 | "sheep", 28 | "cow", 29 | "elephant", 30 | "bear", 31 | "zebra", 32 | "giraffe", 33 | "backpack", 34 | "umbrella", 35 | "handbag", 36 | "tie", 37 | "suitcase", 38 | "frisbee", 39 | "skis", 40 | "snowboard", 41 | "sports ball", 42 | "kite", 43 | "baseball bat", 44 | "baseball glove", 45 | "skateboard", 46 | "surfboard", 47 | "tennis racket", 48 | "bottle", 49 | "wine glass", 50 | "cup", 51 | "fork", 52 | "knife", 53 | "spoon", 54 | "bowl", 55 | "banana", 56 | "apple", 57 | "sandwich", 58 | "orange", 59 | "broccoli", 60 | "carrot", 61 | "hot dog", 62 | "pizza", 63 | "donut", 64 | "cake", 65 | "chair", 66 | "sofa", 67 | "pottedplant", 68 | "bed", 69 | "diningtable", 70 | "toilet", 71 | "tvmonitor", 72 | "laptop", 73 | "mouse", 74 | "remote", 75 | "keyboard", 76 | "cell phone", 77 | "microwave", 78 | "oven", 79 | "toaster", 80 | "sink", 81 | "refrigerator", 82 | "book", 83 | "clock", 84 | "vase", 85 | "scissors", 86 | "teddy bear", 87 | "hair drier", 88 | "toothbrush" 89 | }; 90 | 91 | public uint ImageWidth => 416; 92 | public uint ImageHeight => 416; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/VAP/PostProcessor/AzureBlobProcessor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Configuration; 6 | using System.IO; 7 | using System.Threading.Tasks; 8 | 9 | using Microsoft.WindowsAzure.Storage; 10 | using Microsoft.WindowsAzure.Storage.Blob; 11 | 12 | namespace PostProcessor 13 | { 14 | public class AzureBlobProcessor 15 | { 16 | private static CloudStorageAccount storageAccount; 17 | private static CloudBlobClient cloudBlobClient; 18 | 19 | public AzureBlobProcessor() 20 | { 21 | // Retrieve the connection string from app.config 22 | string storageConnectionString = ConfigurationManager.AppSettings["StorageConnectionString"]; 23 | 24 | if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount)) 25 | { 26 | try 27 | { 28 | // Create the CloudBlobClient that represents the Blob storage endpoint for the storage account. 29 | cloudBlobClient = storageAccount.CreateCloudBlobClient(); 30 | } 31 | catch (StorageException ex) 32 | { 33 | Console.WriteLine("Error returned from the service: {0}", ex.Message); 34 | } 35 | } 36 | } 37 | 38 | public async Task CreateContainerAsync(string container = "") 39 | { 40 | try 41 | { 42 | // Create a container (appending a GUID value to it to make the name unique). 43 | if (container == "") 44 | { 45 | container = Guid.NewGuid().ToString(); 46 | } 47 | CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(container); 48 | await cloudBlobContainer.CreateIfNotExistsAsync(); 49 | Console.WriteLine("Created container '{0}'", cloudBlobContainer.Name); 50 | Console.WriteLine(); 51 | 52 | // Set the permissions so the blobs are public. 53 | BlobContainerPermissions permissions = new BlobContainerPermissions 54 | { 55 | PublicAccess = BlobContainerPublicAccessType.Blob 56 | }; 57 | await cloudBlobContainer.SetPermissionsAsync(permissions); 58 | 59 | } 60 | catch (StorageException ex) 61 | { 62 | Console.WriteLine("Error returned from the service: {0}", ex.Message); 63 | } 64 | 65 | return container; 66 | } 67 | 68 | public async Task DeleteContainerAsync(string containerName) 69 | { 70 | try 71 | { 72 | CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); 73 | if (cloudBlobContainer != null) 74 | { 75 | await cloudBlobContainer.DeleteIfExistsAsync(); 76 | } 77 | } 78 | catch (StorageException ex) 79 | { 80 | Console.WriteLine("Error returned from the service: {0}", ex.Message); 81 | } 82 | } 83 | 84 | public async Task UploadFileAsync(string containerName, string blobName, string sourceFile) 85 | { 86 | string blobUri = null; 87 | try 88 | { 89 | // Get a reference to the blob address, then upload the file to the blob. 90 | CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); 91 | CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(blobName); 92 | await cloudBlockBlob.UploadFromFileAsync(sourceFile); 93 | blobUri = cloudBlobContainer.Uri.AbsoluteUri + "/" + blobName; 94 | } 95 | catch (StorageException ex) 96 | { 97 | Console.WriteLine("Error returned from the service: {0}", ex.Message); 98 | } 99 | 100 | return blobUri; 101 | } 102 | 103 | public async Task DownloadFileAsync(string containerName, string blobName, string destinationFile) 104 | { 105 | try 106 | { 107 | // Download the blob to a local file. 108 | CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); 109 | CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(blobName); 110 | Console.WriteLine("Downloading blob {0} to {1}", blobName, destinationFile); 111 | Console.WriteLine(); 112 | await cloudBlockBlob.DownloadToFileAsync(destinationFile, FileMode.Create); 113 | } 114 | catch (StorageException ex) 115 | { 116 | Console.WriteLine("Error returned from the service: {0}", ex.Message); 117 | } 118 | } 119 | 120 | public async Task ListBlobAsync(string containerName) 121 | { 122 | try 123 | { 124 | // List the blobs in the container. 125 | CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); 126 | Console.WriteLine("Listing blobs in container {0}.", containerName); 127 | BlobContinuationToken blobContinuationToken = null; 128 | do 129 | { 130 | var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken); 131 | // Get the value of the continuation token returned by the listing call. 132 | blobContinuationToken = results.ContinuationToken; 133 | foreach (IListBlobItem item in results.Results) 134 | { 135 | Console.WriteLine(item.Uri); 136 | } 137 | } while (blobContinuationToken != null); // Loop while the continuation token is not null. 138 | Console.WriteLine(); 139 | } 140 | catch (StorageException ex) 141 | { 142 | Console.WriteLine("Error returned from the service: {0}", ex.Message); 143 | } 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/VAP/PostProcessor/DataPersistence.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Model; 5 | using System; 6 | using System.IO; 7 | using System.Collections.Generic; 8 | using System.Runtime.Serialization.Json; 9 | using Utils.Config; 10 | 11 | namespace PostProcessor 12 | { 13 | public enum Position 14 | { 15 | Right, 16 | Left, 17 | Up, 18 | Down, 19 | Unknown 20 | } 21 | 22 | public class DataPersistence 23 | { 24 | //string blobUri_BGS = null; 25 | static AzureBlobProcessor blobProcessor = new AzureBlobProcessor(); 26 | 27 | // force precise initialization 28 | static DataPersistence() { } 29 | 30 | public static void PersistResult(string dbCollectionName, string videoUrl, int cameraID, int frameIndex, List detectionResult, Position[] objDir, string YOLOCONFIG, string YOLOCONFIG_HEAVY, 31 | string azureContainerName) 32 | { 33 | if (detectionResult != null && detectionResult.Count != 0) 34 | { 35 | foreach (Item it in detectionResult) 36 | { 37 | var fileList = Directory.GetFiles(@OutputFolder.OutputFolderAll, $"frame-{frameIndex}*"); 38 | string blobName = Path.GetFileName(fileList[fileList.Length-1]); 39 | //string blobName = it.Model == "Cheap" ? $@"frame-{frameIndex}-Cheap-{it.Confidence}.jpg" : $@"frame-{frameIndex}-Heavy-{it.Confidence}.jpg"; 40 | string blobUri = SendDataToCloud(azureContainerName, blobName, @OutputFolder.OutputFolderAll + blobName); 41 | string serializedResult = SerializeDetectionResult(videoUrl, cameraID, frameIndex, it, objDir, blobUri, YOLOCONFIG, YOLOCONFIG_HEAVY); 42 | WriteDB(dbCollectionName, serializedResult); 43 | } 44 | } 45 | } 46 | 47 | public static string SendDataToCloud(string containerName, string blobName, string sourceFile) 48 | { 49 | return blobProcessor.UploadFileAsync(containerName, blobName, sourceFile).GetAwaiter().GetResult(); 50 | } 51 | 52 | private static string SerializeDetectionResult(string videoUrl, int cameraID, int frameIndex, Item item, Position[] objDir, string imageUri, string YOLOCONFIG, string YOLOCONFIG_HEAVY) 53 | { 54 | Model.Consolidation detectionConsolidation = new Model.Consolidation(); 55 | detectionConsolidation.Key = Guid.NewGuid().ToString(); 56 | detectionConsolidation.CameraID = cameraID; 57 | detectionConsolidation.Frame = frameIndex; 58 | 59 | detectionConsolidation.ObjID = item.ObjId; 60 | detectionConsolidation.ObjName = item.ObjName; 61 | detectionConsolidation.Bbox = new int[] { item.X, item.Y, item.Height, item.Width }; 62 | detectionConsolidation.Prob = item.Confidence; 63 | detectionConsolidation.ObjDir = objDir[0].ToString() + objDir[1].ToString(); 64 | detectionConsolidation.ImageUri = new Uri(imageUri); 65 | 66 | detectionConsolidation.VideoInput = videoUrl; 67 | detectionConsolidation.YoloCheap = YOLOCONFIG; 68 | detectionConsolidation.YoloHeavy = YOLOCONFIG_HEAVY; 69 | detectionConsolidation.Time = DateTime.Now.ToString("yyyy-MM-ddTHH:mm:ss.fffffff"); 70 | 71 | //Create a stream to serialize the object to. 72 | MemoryStream ms = new MemoryStream(); 73 | 74 | // Serializer the User object to the stream. 75 | DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(Model.Consolidation)); 76 | ser.WriteObject(ms, detectionConsolidation); 77 | byte[] json = ms.ToArray(); 78 | ms.Close(); 79 | return System.Text.Encoding.UTF8.GetString(json, 0, json.Length); 80 | } 81 | 82 | private static int WriteDB(string collectionName, string content) 83 | { 84 | //var createCltResult = Client.CreateCollection().Result; 85 | var createDocResult = DBClient.CreateDocument(collectionName, content).Result; 86 | return (int)createDocResult; 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/VAP/PostProcessor/Model/clctCamera.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Runtime.Serialization; 5 | using System; 6 | using System.Globalization; 7 | 8 | namespace PostProcessor.Model 9 | { 10 | [DataContract(Name = "clctCamera")] 11 | public class Camera 12 | { 13 | [DataMember(Name = "_key")] 14 | public string Key { get; set; } 15 | 16 | [DataMember(Name = "_id")] 17 | public string ID { get; set; } 18 | 19 | [DataMember(Name = "_rev")] 20 | public string Rev { get; set; } 21 | 22 | [DataMember(Name = "camera_id")] 23 | public int CameraID { get; set; } 24 | 25 | [DataMember(Name = "frame")] 26 | public int Frame { get; set; } 27 | 28 | [DataMember(Name = "obj_num")] 29 | public int ObjNum { get; set; } 30 | 31 | [DataMember(Name = "VideoInput")] 32 | public Uri VideoInput { get; set; } 33 | 34 | [DataMember(Name = "YOLOCONFIG_CFG")] 35 | public Uri YoloCfg { get; set; } 36 | 37 | [DataMember(Name = "YOLOCONFIG_NAMES")] 38 | public Uri YoloNames { get; set; } 39 | 40 | [DataMember(Name = "YOLOCONFIG_WEIGHTS")] 41 | public Uri YoloWeights { get; set; } 42 | 43 | [DataMember(Name = "time")] 44 | private string Time { get; set; } 45 | 46 | [IgnoreDataMember] 47 | public DateTime RecordTime 48 | { 49 | get 50 | { 51 | return DateTime.ParseExact(Time, "yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); 52 | } 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /src/VAP/PostProcessor/Model/clctConsolidation.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Runtime.Serialization; 5 | using System; 6 | using System.Globalization; 7 | 8 | namespace PostProcessor.Model 9 | { 10 | [DataContract(Name = "clctConsolidation")] 11 | public class Consolidation 12 | { 13 | [DataMember(Name = "_key")] 14 | public string Key { get; set; } 15 | 16 | [DataMember(Name = "_id")] 17 | public string ID { get; set; } 18 | 19 | [DataMember(Name = "_rev")] 20 | public string Rev { get; set; } 21 | 22 | [DataMember(Name = "camera_id")] 23 | public int CameraID { get; set; } 24 | 25 | [DataMember(Name = "frame")] 26 | public int Frame { get; set; } 27 | 28 | // @TODO: support multiple objects 29 | [DataMember(Name = "obj_id")] 30 | public int ObjID { get; set; } 31 | 32 | [DataMember(Name = "obj_name")] 33 | public string ObjName { get; set; } 34 | 35 | [DataMember(Name = "bbox")] 36 | public int[] Bbox { get; set; } 37 | 38 | [DataMember(Name = "prob")] 39 | public double Prob { get; set; } 40 | 41 | [DataMember(Name = "obj_moving_dir")] 42 | public string ObjDir { get; set; } 43 | 44 | [DataMember(Name = "imageUri")] 45 | public Uri ImageUri { get; set; } 46 | 47 | 48 | [DataMember(Name = "time")] 49 | public string Time { get; set; } 50 | 51 | [DataMember(Name = "VideoInput")] 52 | public string VideoInput { get; set; } 53 | 54 | [DataMember(Name = "YOLOCONFIG_CHEAP")] 55 | public string YoloCheap { get; set; } 56 | 57 | [DataMember(Name = "YOLOCONFIG_CHEAP_CFG")] 58 | public Uri YoloCheapCfg { get; set; } 59 | 60 | [DataMember(Name = "YOLOCONFIG_CHEAP_NAMES")] 61 | public Uri YoloCheapNames { get; set; } 62 | 63 | [DataMember(Name = "YOLOCONFIG_CHEAP_WEIGHTS")] 64 | public Uri YoloCheapWeights { get; set; } 65 | 66 | [DataMember(Name = "YOLOCONFIG_HEAVY")] 67 | public string YoloHeavy { get; set; } 68 | 69 | [DataMember(Name = "YOLOCONFIG_HEAVY_CFG")] 70 | public Uri YoloHeavyCfg { get; set; } 71 | 72 | [DataMember(Name = "YOLOCONFIG_HEAVY_NAMES")] 73 | public Uri YoloHeavyNames { get; set; } 74 | 75 | [DataMember(Name = "YOLOCONFIG_HEAVY_WEIGHTS")] 76 | public Uri YoloHeavyWeights { get; set; } 77 | } 78 | } -------------------------------------------------------------------------------- /src/VAP/PostProcessor/Model/clctDetection.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Runtime.Serialization; 5 | using System; 6 | using System.Globalization; 7 | 8 | namespace PostProcessor.Model 9 | { 10 | [DataContract(Name = "clctDetection")] 11 | public class Detection 12 | { 13 | [DataMember(Name = "_key")] 14 | public string Key { get; set; } 15 | 16 | [DataMember(Name = "_id")] 17 | public string ID { get; set; } 18 | 19 | [DataMember(Name = "_rev")] 20 | public string Rev { get; set; } 21 | 22 | [DataMember(Name = "_from")] 23 | public string From { get; set; } 24 | 25 | [DataMember(Name = "_to")] 26 | public string To { get; set; } 27 | 28 | [DataMember(Name = "bbox")] 29 | public int[] Bbox { get; set; } 30 | 31 | [DataMember(Name = "prob")] 32 | public double Prob { get; set; } 33 | 34 | [DataMember(Name = "obj_moving_dir")] 35 | public string ObjDir { get; set; } 36 | 37 | [DataMember(Name = "time")] 38 | private string Time { get; set; } 39 | 40 | [IgnoreDataMember] 41 | public DateTime RecordTime 42 | { 43 | get 44 | { 45 | return DateTime.ParseExact(Time, "yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); 46 | } 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /src/VAP/PostProcessor/Model/clctObject.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Runtime.Serialization; 5 | using System; 6 | using System.Globalization; 7 | 8 | namespace PostProcessor.Model 9 | { 10 | [DataContract(Name = "clctObject")] 11 | public class Object 12 | { 13 | [DataMember(Name = "_key")] 14 | public string Key { get; set; } 15 | 16 | [DataMember(Name = "_id")] 17 | public string ID { get; set; } 18 | 19 | [DataMember(Name = "_rev")] 20 | public string Rev { get; set; } 21 | 22 | [DataMember(Name = "obj_id")] 23 | public int ObjID { get; set; } 24 | 25 | [DataMember(Name = "obj_name")] 26 | public string ObjName { get; set; } 27 | 28 | [DataMember(Name = "camera_id")] 29 | public int CameraID { get; set; } 30 | 31 | [DataMember(Name = "track_id")] 32 | public int TrackID { get; set; } 33 | 34 | [DataMember(Name = "VideoInput")] 35 | public Uri VideoInput { get; set; } 36 | 37 | [DataMember(Name = "YOLOCONFIG_CFG")] 38 | public Uri YoloCfg { get; set; } 39 | 40 | [DataMember(Name = "YOLOCONFIG_NAMES")] 41 | public Uri YoloNames { get; set; } 42 | 43 | [DataMember(Name = "YOLOCONFIG_WEIGHTS")] 44 | public Uri YoloWeights { get; set; } 45 | 46 | [DataMember(Name = "time")] 47 | private string Time { get; set; } 48 | 49 | [IgnoreDataMember] 50 | public DateTime RecordTime 51 | { 52 | get 53 | { 54 | return DateTime.ParseExact(Time, "yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); 55 | } 56 | } 57 | } 58 | } -------------------------------------------------------------------------------- /src/VAP/PostProcessor/Model/queryResponse.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Runtime.Serialization; 5 | using System.Collections.Generic; 6 | 7 | namespace PostProcessor.Model 8 | { 9 | [DataContract(Name = "queryResponse")] 10 | public class QueryRaw 11 | { 12 | [DataMember(Name = "result")] 13 | public List QResult { get; set; } 14 | 15 | [DataMember(Name = "hasmore")] 16 | public bool Hasmore { get; set; } 17 | 18 | [DataMember(Name = "cached")] 19 | public bool Cached { get; set; } 20 | 21 | //[DataMember(Name = "extra")] 22 | //public string Extra { get; set; } 23 | 24 | [DataMember(Name = "error")] 25 | public string Error { get; set; } 26 | 27 | [DataMember(Name = "code")] 28 | public int Code { get; set; } 29 | } 30 | } -------------------------------------------------------------------------------- /src/VAP/PostProcessor/Model/repo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Runtime.Serialization; 5 | using System; 6 | using System.Globalization; 7 | 8 | namespace PostProcessor.Model 9 | { 10 | [DataContract(Name = "repo")] 11 | public class Repository 12 | { 13 | /* universal names */ 14 | [DataMember(Name = "_key")] 15 | public string Key { get; set; } 16 | 17 | [DataMember(Name = "_id")] 18 | public string ID { get; set; } 19 | 20 | [DataMember(Name = "_rev")] 21 | public string Rev { get; set; } 22 | 23 | [DataMember(Name = "time")] 24 | private string Time { get; set; } 25 | 26 | [IgnoreDataMember] 27 | public DateTime RecordTime 28 | { 29 | get 30 | { 31 | return DateTime.ParseExact(Time, "yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); 32 | } 33 | } 34 | 35 | /* camera/object collection names */ 36 | [DataMember(Name = "camera_id")] 37 | public int CameraID { get; set; } 38 | 39 | [DataMember(Name = "VideoInput")] 40 | public Uri VideoInput { get; set; } 41 | 42 | [DataMember(Name = "YOLOCONFIG_CFG")] 43 | public Uri YoloCfg { get; set; } 44 | 45 | [DataMember(Name = "YOLOCONFIG_NAMES")] 46 | public Uri YoloNames { get; set; } 47 | 48 | [DataMember(Name = "YOLOCONFIG_WEIGHTS")] 49 | public Uri YoloWeights { get; set; } 50 | 51 | /* camera collection names */ 52 | [DataMember(Name = "frame")] 53 | public int Frame { get; set; } 54 | 55 | [DataMember(Name = "obj_num")] 56 | public int ObjNum { get; set; } 57 | 58 | /* object collection names */ 59 | [DataMember(Name = "obj_id")] 60 | public int ObjID { get; set; } 61 | 62 | [DataMember(Name = "obj_name")] 63 | public string ObjName { get; set; } 64 | 65 | [DataMember(Name = "track_id")] 66 | public int TrackID { get; set; } 67 | 68 | /* detection collection names */ 69 | [DataMember(Name = "_from")] 70 | public string From { get; set; } 71 | 72 | [DataMember(Name = "_to")] 73 | public string To { get; set; } 74 | 75 | [DataMember(Name = "bbox")] 76 | public int[] Bbox { get; set; } 77 | 78 | [DataMember(Name = "prob")] 79 | public double Prob { get; set; } 80 | 81 | [DataMember(Name = "obj_moving_dir")] 82 | public string ObjDir { get; set; } 83 | } 84 | } -------------------------------------------------------------------------------- /src/VAP/PostProcessor/PostProcessor.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/VAP/TFDetector/FrameBuffer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Text; 7 | 8 | using OpenCvSharp; 9 | 10 | namespace TFDetector 11 | { 12 | class FrameBuffer 13 | { 14 | Queue frameBuffer; 15 | int bSize; 16 | 17 | public FrameBuffer(int size) 18 | { 19 | bSize = size; 20 | frameBuffer = new Queue(bSize); 21 | } 22 | 23 | public void Buffer(Mat frame) 24 | { 25 | frameBuffer.Enqueue(frame); 26 | if (frameBuffer.Count > bSize) 27 | frameBuffer.Dequeue(); 28 | } 29 | 30 | public Mat[] ToArray() 31 | { 32 | return frameBuffer.ToArray(); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/VAP/TFDetector/LineTriggeredDNNTF.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using DNNDetector.Config; 5 | using DNNDetector.Model; 6 | using OpenCvSharp; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.IO; 10 | using System.Linq; 11 | using Utils.Config; 12 | 13 | namespace TFDetector 14 | { 15 | public class LineTriggeredDNNTF 16 | { 17 | //static string TFCONFIG = ""; 18 | FrameDNNTF frameDNNTF; 19 | FrameBuffer frameBufferLtDNNTF; 20 | Dictionary counts_prev = new Dictionary(); 21 | 22 | public LineTriggeredDNNTF(List<(string key, (System.Drawing.Point p1, System.Drawing.Point p2) coordinates)> lines) 23 | { 24 | frameBufferLtDNNTF = new FrameBuffer(DNNConfig.FRAME_SEARCH_RANGE); 25 | 26 | frameDNNTF = new FrameDNNTF(lines); 27 | } 28 | 29 | public List Run(Mat frame, int frameIndex, Dictionary counts, List<(string key, (System.Drawing.Point p1, System.Drawing.Point p2) coordinates)> lines, HashSet category) 30 | { 31 | // buffer frame 32 | frameBufferLtDNNTF.Buffer(frame); 33 | 34 | if (counts_prev.Count != 0) 35 | { 36 | foreach (string lane in counts.Keys) 37 | { 38 | int diff = Math.Abs(counts[lane] - counts_prev[lane]); 39 | if (diff > 0) //object detected by BGS-based counter 40 | { 41 | if (frameIndex >= DNNConfig.FRAME_SEARCH_RANGE) 42 | { 43 | // call tf cheap model for crosscheck 44 | int lineID = Array.IndexOf(counts.Keys.ToArray(), lane); 45 | Mat[] frameBufferArray = frameBufferLtDNNTF.ToArray(); 46 | int frameIndexTF = frameIndex - 1; 47 | DateTime start = DateTime.Now; 48 | List analyzedTrackingItems = null; 49 | 50 | while (frameIndex - frameIndexTF < DNNConfig.FRAME_SEARCH_RANGE) 51 | { 52 | Console.WriteLine("** Calling Cheap on " + (DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexTF))); 53 | Mat frameTF = frameBufferArray[DNNConfig.FRAME_SEARCH_RANGE - (frameIndex - frameIndexTF)]; 54 | 55 | analyzedTrackingItems = frameDNNTF.Run(frameTF, frameIndexTF, category, System.Drawing.Brushes.Pink, DNNConfig.MIN_SCORE_FOR_LINEBBOX_OVERLAP_LARGE, false); 56 | 57 | // object detected by cheap model 58 | if (analyzedTrackingItems != null) 59 | { 60 | List ltDNNItem = new List(); 61 | foreach (Item item in analyzedTrackingItems) 62 | { 63 | item.RawImageData = Utils.Utils.ImageToByteBmp(OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frameTF)); 64 | item.TriggerLine = lane; 65 | item.TriggerLineID = lineID; 66 | item.Model = "Cheap"; 67 | ltDNNItem.Add(item); 68 | 69 | // output cheap TF results 70 | string blobName_Cheap = $@"frame-{frameIndex}-Cheap-{item.Confidence}.jpg"; 71 | string fileName_Cheap = @OutputFolder.OutputFolderLtDNN + blobName_Cheap; 72 | File.WriteAllBytes(fileName_Cheap, item.TaggedImageData); 73 | File.WriteAllBytes(@OutputFolder.OutputFolderAll + blobName_Cheap, item.TaggedImageData); 74 | } 75 | updateCount(counts); 76 | return ltDNNItem; 77 | } 78 | frameIndexTF--; 79 | } 80 | } 81 | } 82 | } 83 | } 84 | updateCount(counts); 85 | 86 | return null; 87 | } 88 | 89 | void updateCount(Dictionary counts) 90 | { 91 | foreach (string dir in counts.Keys) 92 | { 93 | counts_prev[dir] = counts[dir]; 94 | } 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/VAP/TFDetector/TFDetector.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/VAP/TFWrapper/Common/CatalogUtil.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Globalization; 7 | using System.IO; 8 | using System.Text.RegularExpressions; 9 | 10 | namespace Wrapper.TF.Common 11 | { 12 | public static class CatalogUtil 13 | { 14 | // regexes with different new line symbols 15 | private static string CATALOG_ITEM_PATTERN = @"item {{{0} name: ""(?.*)""{0} id: (?\d+){0} display_name: ""(?.*)""{0}}}"; 16 | private static string CATALOG_ITEM_PATTERN_ENV = string.Format(CultureInfo.InvariantCulture, CATALOG_ITEM_PATTERN, Environment.NewLine); 17 | private static string CATALOG_ITEM_PATTERN_UNIX = string.Format (CultureInfo.InvariantCulture, CATALOG_ITEM_PATTERN, "\n"); 18 | 19 | /// 20 | /// Reads catalog of well-known objects from text file. 21 | /// 22 | /// path to the text file 23 | /// collection of items 24 | public static IEnumerable ReadCatalogItems (string file) 25 | { 26 | using (FileStream stream = File.OpenRead (file)) 27 | using (StreamReader reader = new StreamReader (stream)) { 28 | string text = reader.ReadToEnd (); 29 | if (string.IsNullOrWhiteSpace (text)) { 30 | yield break; 31 | } 32 | 33 | Regex regex = new Regex (CATALOG_ITEM_PATTERN_ENV); 34 | var matches = regex.Matches (text); 35 | if(matches.Count == 0) { 36 | regex = new Regex (CATALOG_ITEM_PATTERN_UNIX); 37 | matches = regex.Matches (text); 38 | } 39 | 40 | foreach (Match match in matches) { 41 | var name = match.Groups [1].Value; 42 | var id = int.Parse (match.Groups [2].Value); 43 | var displayName = match.Groups [3].Value; 44 | 45 | yield return new CatalogItem () { 46 | Id = id, 47 | Name = name, 48 | DisplayName = displayName 49 | }; 50 | } 51 | } 52 | } 53 | } 54 | 55 | public class CatalogItem 56 | { 57 | public int Id { get; set; } 58 | public string Name { get; set; } 59 | public string DisplayName { get; set; } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/VAP/TFWrapper/Common/ImageUtil.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Drawing; 5 | using System.IO; 6 | using TensorFlow; 7 | 8 | namespace Wrapper.TF.Common 9 | { 10 | public static class ImageUtil 11 | { 12 | // Convert the image in filename to a Tensor suitable as input to the Inception model. 13 | public static TFTensor CreateTensorFromImageFile(string file, TFDataType destinationDataType = TFDataType.Float) 14 | { 15 | var contents = File.ReadAllBytes (file); 16 | 17 | return CreateTensorFromByteArray(contents, destinationDataType); 18 | } 19 | 20 | // Convert byte[] to a Tensor suitable as input to the Inception model. 21 | public static TFTensor CreateTensorFromByteArray(byte[] imageByteArray, TFDataType destinationDataType = TFDataType.Float) 22 | { 23 | // DecodeJpeg uses a scalar String-valued tensor as input. 24 | var tensor = TFTensor.CreateString(imageByteArray); 25 | 26 | TFOutput input, output; 27 | 28 | // Construct a graph to normalize the image 29 | using (var graph = ConstructGraphToNormalizeImage(out input, out output, destinationDataType)) 30 | { 31 | // Execute that graph to normalize this one image 32 | using (var session = new TFSession(graph)) 33 | { 34 | var normalized = session.Run( 35 | inputs: new[] { input }, 36 | inputValues: new[] { tensor }, 37 | outputs: new[] { output }); 38 | 39 | return normalized[0]; 40 | } 41 | } 42 | } 43 | 44 | // The inception model takes as input the image described by a Tensor in a very 45 | // specific normalized format (a particular image size, shape of the input tensor, 46 | // normalized pixel values etc.). 47 | // 48 | // This function constructs a graph of TensorFlow operations which takes as 49 | // input a JPEG-encoded string and returns a tensor suitable as input to the 50 | // inception model. 51 | private static TFGraph ConstructGraphToNormalizeImage (out TFOutput input, out TFOutput output, TFDataType destinationDataType = TFDataType.Float) 52 | { 53 | // Some constants specific to the pre-trained model at: 54 | // https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip 55 | // 56 | // - The model was trained after with images scaled to 224x224 pixels. 57 | // - The colors, represented as R, G, B in 1-byte each were converted to 58 | // float using (value - Mean)/Scale. 59 | 60 | const int W = 224; 61 | const int H = 224; 62 | const float Mean = 117; 63 | const float Scale = 1; 64 | 65 | var graph = new TFGraph (); 66 | input = graph.Placeholder (TFDataType.String); 67 | 68 | output = graph.Cast (graph.Div ( 69 | x: graph.Sub ( 70 | x: graph.ResizeBilinear ( 71 | images: graph.ExpandDims ( 72 | input: graph.Cast ( 73 | graph.DecodeJpeg (contents: input, channels: 3), DstT: TFDataType.Float), 74 | dim: graph.Const (0, "make_batch")), 75 | size: graph.Const (new int [] { W, H }, "size")), 76 | y: graph.Const (Mean, "mean")), 77 | y: graph.Const (Scale, "scale")), destinationDataType); 78 | 79 | return graph; 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/VAP/TFWrapper/ImageEditor.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Drawing; 6 | 7 | namespace Wrapper.TF 8 | { 9 | /// 10 | /// Allows to add graphic elements to the existing image. 11 | /// 12 | class ImageEditor : IDisposable 13 | { 14 | private Graphics _graphics; 15 | private Image _image; 16 | private string _fontFamily; 17 | private float _fontSize; 18 | private string _outputFile; 19 | 20 | public ImageEditor(string inputFile, string outputFile, string fontFamily = "Ariel", float fontSize = 12) 21 | { 22 | if (string.IsNullOrEmpty(inputFile)) 23 | { 24 | throw new ArgumentNullException(nameof(inputFile)); 25 | } 26 | 27 | if (string.IsNullOrEmpty(outputFile)) 28 | { 29 | throw new ArgumentNullException(nameof(outputFile)); 30 | } 31 | 32 | _fontFamily = fontFamily; 33 | _fontSize = fontSize; 34 | _outputFile = outputFile; 35 | 36 | _image = Bitmap.FromFile(inputFile); 37 | _graphics = Graphics.FromImage(_image); 38 | } 39 | 40 | /// 41 | /// Adds rectangle with a label in particular position of the image 42 | /// 43 | /// 44 | /// 45 | /// 46 | /// 47 | /// 48 | /// 49 | public void AddBox(float xmin, float xmax, float ymin, float ymax, string text = "", string colorName = "red") 50 | { 51 | var left = xmin * _image.Width; 52 | var right = xmax * _image.Width; 53 | var top = ymin * _image.Height; 54 | var bottom = ymax * _image.Height; 55 | 56 | 57 | var imageRectangle = new Rectangle(new Point(0, 0), new Size(_image.Width, _image.Height)); 58 | _graphics.DrawImage(_image, imageRectangle); 59 | 60 | Color color = Color.FromName(colorName); 61 | Brush brush = new SolidBrush(color); 62 | Pen pen = new Pen(brush); 63 | 64 | _graphics.DrawRectangle(pen, left, top, right - left, bottom - top); 65 | var font = new Font(_fontFamily, _fontSize); 66 | SizeF size = _graphics.MeasureString(text, font); 67 | _graphics.DrawString(text, font, brush, new PointF(left, top - size.Height)); 68 | } 69 | 70 | public void Dispose() 71 | { 72 | if (_image != null) 73 | { 74 | _image.Save(_outputFile); 75 | 76 | if (_graphics != null) 77 | { 78 | _graphics.Dispose(); 79 | } 80 | 81 | _image.Dispose(); 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/VAP/TFWrapper/TFWrapper.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | packages\TensorFlowSharp.1.12.0\lib\TensorFlowSharp.dll 17 | 18 | 19 | 20 | 21 | 22 | PreserveNewest 23 | 24 | 25 | 26 | 27 | true 28 | true 29 | 30 | 31 | True 32 | 33 | 34 | 35 | 36 | %(Filename)%(Extension) 37 | PreserveNewest 38 | 39 | 40 | %(Filename)%(Extension) 41 | PreserveNewest 42 | 43 | 44 | %(Filename)%(Extension) 45 | PreserveNewest 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /src/VAP/TFWrapper/packages/TensorFlowSharp.1.12.0/lib/TensorFlowSharp.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/src/VAP/TFWrapper/packages/TensorFlowSharp.1.12.0/lib/TensorFlowSharp.dll -------------------------------------------------------------------------------- /src/VAP/TFWrapper/test_images/input.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/src/VAP/TFWrapper/test_images/input.jpg -------------------------------------------------------------------------------- /src/VAP/Utils/Config/OutputFolder.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Utils.Config 5 | { 6 | public static class OutputFolder 7 | { 8 | public static string OutputFolderAll { get; set; } = "../../output_all/"; 9 | 10 | public static string OutputFolderBGSLine { get; set; } = "../../output_bgsline/"; 11 | 12 | public static string OutputFolderLtDNN { get; set; } = "../../output_ltdnn/"; 13 | 14 | public static string OutputFolderCcDNN { get; set; } = "../../output_ccdnn/"; 15 | 16 | public static string OutputFolderAML { get; set; } = "../../output_aml/"; 17 | 18 | public static string OutputFolderFrameDNNDarknet { get; set; } = "../../output_framednndarknet/"; 19 | 20 | public static string OutputFolderFrameDNNTF { get; set; } = "../../output_framednntf/"; 21 | 22 | public static string OutputFolderFrameDNNONNX { get; set; } = "../../output_framednnonnx/"; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/VAP/Utils/Utils.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Drawing; 7 | using System.Drawing.Imaging; 8 | using System.IO; 9 | using System.Linq; 10 | 11 | namespace Utils 12 | { 13 | public class Utils 14 | { 15 | public static void cleanFolder(string folder) 16 | { 17 | Directory.CreateDirectory(folder); 18 | DirectoryInfo di = new DirectoryInfo(folder); 19 | foreach (FileInfo file in di.GetFiles()) file.Delete(); 20 | } 21 | 22 | public static void cleanFolderAll() 23 | { 24 | cleanFolder(Config.OutputFolder.OutputFolderAll); 25 | cleanFolder(Config.OutputFolder.OutputFolderBGSLine); 26 | cleanFolder(Config.OutputFolder.OutputFolderLtDNN); 27 | cleanFolder(Config.OutputFolder.OutputFolderCcDNN); 28 | cleanFolder(Config.OutputFolder.OutputFolderAML); 29 | cleanFolder(Config.OutputFolder.OutputFolderFrameDNNDarknet); 30 | cleanFolder(Config.OutputFolder.OutputFolderFrameDNNTF); 31 | cleanFolder(Config.OutputFolder.OutputFolderFrameDNNONNX); 32 | } 33 | 34 | public static byte[] ImageToByteBmp(Image imageIn) 35 | { 36 | using (var ms = new MemoryStream()) 37 | { 38 | imageIn.Save(ms, System.Drawing.Imaging.ImageFormat.Bmp); 39 | return ms.ToArray(); 40 | } 41 | } 42 | 43 | public static byte[] ImageToByteJpeg(Image imageIn) 44 | { 45 | using (var ms = new MemoryStream()) 46 | { 47 | imageIn.Save(ms, System.Drawing.Imaging.ImageFormat.Jpeg); 48 | return ms.ToArray(); 49 | } 50 | } 51 | 52 | public static float checkLineBboxOverlapRatio(int[] line, int bbox_x, int bbox_y, int bbox_w, int bbox_h) 53 | { 54 | (Point p1, Point p2) newLine = (new Point(line[0], line[1]), new Point(line[2], line[3])); 55 | return checkLineBboxOverlapRatio(newLine, bbox_x, bbox_y, bbox_w, bbox_h); 56 | } 57 | 58 | public static float checkLineBboxOverlapRatio((Point p1, Point p2) line, int bbox_x, int bbox_y, int bbox_w, int bbox_h) 59 | { 60 | float overlapRatio = 0.0F; 61 | int insidePixels = 0; 62 | 63 | IEnumerable linePixels = EnumerateLineNoDiagonalSteps(line.p1, line.p2); 64 | 65 | foreach(Point pixel in linePixels) 66 | { 67 | if ((pixel.X >= bbox_x) && (pixel.X <= bbox_x + bbox_w) && (pixel.Y >= bbox_y) && (pixel.Y <= bbox_y + bbox_h)) 68 | { 69 | insidePixels++; 70 | } 71 | } 72 | 73 | overlapRatio = (float)insidePixels / linePixels.Count(); 74 | return overlapRatio; 75 | } 76 | 77 | private static IEnumerable EnumerateLineNoDiagonalSteps(Point p0, Point p1) 78 | { 79 | int dx = Math.Abs(p1.X - p0.X), sx = p0.X < p1.X ? 1 : -1; 80 | int dy = -Math.Abs(p1.Y - p0.Y), sy = p0.Y < p1.Y ? 1 : -1; 81 | int err = dx + dy, e2; 82 | 83 | while (true) 84 | { 85 | yield return p0; 86 | 87 | if (p0.X == p1.X && p0.Y == p1.Y) break; 88 | 89 | e2 = 2 * err; 90 | 91 | // EITHER horizontal OR vertical step (but not both!) 92 | if (e2 > dy) 93 | { 94 | err += dy; 95 | p0.X += sx; 96 | } 97 | else if (e2 < dx) 98 | { // <--- this "else" makes the difference 99 | err += dx; 100 | p0.Y += sy; 101 | } 102 | } 103 | } 104 | 105 | public static byte[] DrawImage(byte[] imageData, int x, int y, int w, int h, Brush color, string annotation = "") 106 | { 107 | using (var memoryStream = new MemoryStream(imageData)) 108 | using (var image = Image.FromStream(memoryStream)) 109 | using (var canvas = Graphics.FromImage(image)) 110 | using (var pen = new Pen(color, 3)) 111 | { 112 | canvas.DrawRectangle(pen, x, y, w, h); 113 | canvas.DrawString(annotation, new Font("Arial", 16), color, new PointF(x, y - 20)); 114 | canvas.Flush(); 115 | 116 | using (var memoryStream2 = new MemoryStream()) 117 | { 118 | image.Save(memoryStream2, ImageFormat.Bmp); 119 | return memoryStream2.ToArray(); 120 | } 121 | } 122 | } 123 | 124 | public static byte[] CropImage(byte[] imageData, int x, int y, int w, int h) 125 | { 126 | using (var memoryStream = new MemoryStream(imageData)) 127 | using (var image = Image.FromStream(memoryStream)) 128 | { 129 | Rectangle cropRect = new Rectangle(x, y, Math.Min(image.Width - x, w), Math.Min(image.Height - y, h)); 130 | Bitmap bmpImage = new Bitmap(image); 131 | Image croppedImage = bmpImage.Clone(cropRect, bmpImage.PixelFormat); 132 | 133 | using (var memoryStream2 = new MemoryStream()) 134 | { 135 | croppedImage.Save(memoryStream2, ImageFormat.Bmp); 136 | return memoryStream2.ToArray(); 137 | } 138 | } 139 | } 140 | 141 | public static List> ConvertLines(List<(string key, (Point p1, Point p2) coordinates)> lines) 142 | { 143 | List> newLines = new List>(); 144 | foreach ((string key, (Point p1, Point p2) coordinates) line in lines) 145 | { 146 | int[] coor = new int[] { line.coordinates.p1.X, line.coordinates.p1.Y, line.coordinates.p2.X, line.coordinates.p2.Y }; 147 | Tuple newLine = new Tuple(line.key, coor); 148 | newLines.Add(newLine); 149 | } 150 | return newLines; 151 | } 152 | 153 | public static Dictionary CatHashSet2Dict(HashSet cat) 154 | { 155 | Dictionary catDict = new Dictionary(); 156 | foreach (string c in cat) 157 | { 158 | catDict.Add(c, 0); 159 | } 160 | return catDict; 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/VAP/Utils/Utils.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/VAP/VideoPipelineCore/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/VAP/VideoPipelineCore/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "Sample (cars)": { 4 | "commandName": "Project", 5 | "commandLineArgs": "sample.mp4 sample.txt 1 1 car" 6 | }, 7 | "Sample (all categories)": { 8 | "commandName": "Project", 9 | "commandLineArgs": "sample.mp4 sample.txt 1 1" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/VAP/VideoPipelineCore/VideoPipelineCore.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | netcoreapp2.2 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/ConfigurationDetector.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.IO; 5 | using System.Linq; 6 | 7 | namespace Wrapper.Yolo 8 | { 9 | public class ConfigurationDetector 10 | { 11 | private string configFolder; 12 | 13 | public ConfigurationDetector(string yoloConfig) 14 | { 15 | configFolder = yoloConfig; 16 | } 17 | 18 | public YoloConfiguration Detect() 19 | { 20 | var files = this.GetYoloFiles(); 21 | var yoloConfiguration = this.MapFiles(files); 22 | var configValid = this.AreValidYoloFiles(yoloConfiguration); 23 | 24 | if (configValid) 25 | { 26 | return yoloConfiguration; 27 | } 28 | 29 | throw new FileNotFoundException("Cannot found pre-trained model, check all config files available (.cfg, .weights, .names)"); 30 | } 31 | 32 | private string[] GetYoloFiles() 33 | { 34 | return Directory.GetFiles(@"../../../../YoloWrapper/Yolo.Config/"+configFolder, "*.*", SearchOption.TopDirectoryOnly).Where(o => o.EndsWith(".names") || o.EndsWith(".cfg") || o.EndsWith(".weights")).ToArray(); 35 | //return Directory.GetFiles(@"D:\Projects\McD\src\VAP\YoloWrapper\Yolo.Config\" + configFolder, "*.*", SearchOption.TopDirectoryOnly).Where(o => o.EndsWith(".names") || o.EndsWith(".cfg") || o.EndsWith(".weights")).ToArray(); 36 | } 37 | 38 | private YoloConfiguration MapFiles(string[] files) 39 | { 40 | var configurationFile = files.Where(o => o.EndsWith(".cfg")).FirstOrDefault(); 41 | var weightsFile = files.Where(o => o.EndsWith(".weights")).FirstOrDefault(); 42 | var namesFile = files.Where(o => o.EndsWith(".names")).FirstOrDefault(); 43 | 44 | return new YoloConfiguration(configurationFile, weightsFile, namesFile); 45 | } 46 | 47 | private bool AreValidYoloFiles(YoloConfiguration config) 48 | { 49 | if (string.IsNullOrEmpty(config.ConfigFile) || 50 | string.IsNullOrEmpty(config.WeightsFile) || 51 | string.IsNullOrEmpty(config.NamesFile)) 52 | { 53 | return false; 54 | } 55 | 56 | if (Path.GetFileNameWithoutExtension(config.ConfigFile) != Path.GetFileNameWithoutExtension(config.WeightsFile)) 57 | { 58 | return false; 59 | } 60 | 61 | return true; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/DNNMode.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Wrapper.Yolo 5 | { 6 | public enum DNNMode 7 | { 8 | Unknown, 9 | Frame, //FrameDNN 10 | LT, //LineTriggered 11 | CC //Cascaded 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Dependencies/pthreadVC2.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/src/VAP/YoloWrapper/Dependencies/pthreadVC2.dll -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Dependencies/yolo_cpp_dll_cpu.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/src/VAP/YoloWrapper/Dependencies/yolo_cpp_dll_cpu.dll -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Dependencies/yolo_cpp_dll_gpu_cc.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/src/VAP/YoloWrapper/Dependencies/yolo_cpp_dll_gpu_cc.dll -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Dependencies/yolo_cpp_dll_gpu_lt.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform/d626e0e18bed385d71e72aeefdce404380b101c9/src/VAP/YoloWrapper/Dependencies/yolo_cpp_dll_gpu_lt.dll -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/DetectionSystem.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Wrapper.Yolo 5 | { 6 | public enum DetectionSystem 7 | { 8 | Unknown, 9 | CPU, 10 | GPU 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/ImageAnalyzer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Text; 7 | 8 | namespace Wrapper.Yolo 9 | { 10 | public class ImageAnalyzer 11 | { 12 | private Dictionary _imageFormats = new Dictionary(); 13 | 14 | public ImageAnalyzer() 15 | { 16 | var bmp = Encoding.ASCII.GetBytes("BM"); //BMP 17 | var png = new byte[] { 137, 80, 78, 71 }; //PNG 18 | var jpeg = new byte[] { 255, 216, 255 }; //JPEG 19 | 20 | this._imageFormats.Add("bmp", bmp); 21 | this._imageFormats.Add("png", png); 22 | this._imageFormats.Add("jpeg", jpeg); 23 | } 24 | 25 | public bool IsValidImageFormat(byte[] imageData) 26 | { 27 | if (imageData.Length <= 3) 28 | { 29 | return false; 30 | } 31 | 32 | foreach(var imageFormat in this._imageFormats) 33 | { 34 | if (imageData.Take(imageFormat.Value.Length).SequenceEqual(imageFormat.Value)) 35 | { 36 | return true; 37 | } 38 | } 39 | 40 | return false; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/ImageResizer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Drawing; 5 | using System.Drawing.Drawing2D; 6 | using System.Drawing.Imaging; 7 | using System.IO; 8 | 9 | namespace Wrapper.Yolo 10 | { 11 | public class ImageResizer 12 | { 13 | public byte[] Resize(byte[] imageData, int width, int height) 14 | { 15 | using (var image = this.Byte2Image(imageData)) 16 | using (var resizedImage = this.ResizeImage(image, width, height)) 17 | { 18 | return this.Image2Byte(resizedImage); 19 | } 20 | } 21 | 22 | /// 23 | /// Resize the image to the specified width and height. 24 | /// 25 | /// The image to resize. 26 | /// The width to resize to. 27 | /// The height to resize to. 28 | /// The resized image. 29 | private Bitmap ResizeImage(Image image, int width, int height) 30 | { 31 | var destRect = new Rectangle(0, 0, width, height); 32 | var destImage = new Bitmap(width, height); 33 | 34 | destImage.SetResolution(image.HorizontalResolution, image.VerticalResolution); 35 | 36 | using (var graphics = Graphics.FromImage(destImage)) 37 | { 38 | graphics.CompositingMode = CompositingMode.SourceCopy; 39 | graphics.CompositingQuality = CompositingQuality.HighSpeed; 40 | graphics.InterpolationMode = InterpolationMode.NearestNeighbor; 41 | graphics.SmoothingMode = SmoothingMode.None; 42 | graphics.PixelOffsetMode = PixelOffsetMode.HighQuality; 43 | 44 | using (var wrapMode = new ImageAttributes()) 45 | { 46 | wrapMode.SetWrapMode(WrapMode.TileFlipXY); 47 | graphics.DrawImage(image, destRect, 0, 0, image.Width, image.Height, GraphicsUnit.Pixel, wrapMode); 48 | } 49 | } 50 | 51 | return destImage; 52 | } 53 | 54 | private Image Byte2Image(byte[] imageData) 55 | { 56 | using (var memoryStream = new MemoryStream(imageData)) 57 | { 58 | return Image.FromStream(memoryStream); 59 | } 60 | } 61 | 62 | private byte[] Image2Byte(Image image) 63 | { 64 | using (var memoryStream = new MemoryStream()) 65 | { 66 | image.Save(memoryStream, ImageFormat.Bmp); 67 | return memoryStream.ToArray(); 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Model/BboxContainer.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Runtime.InteropServices; 5 | 6 | namespace Wrapper.Yolo.Model 7 | { 8 | [StructLayout(LayoutKind.Sequential)] 9 | internal struct BboxContainer 10 | { 11 | [MarshalAs(UnmanagedType.ByValArray, SizeConst = YoloWrapper.MaxObjects)] 12 | internal BboxT[] candidates; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Model/BboxT.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | 6 | namespace Wrapper.Yolo.Model 7 | { 8 | internal struct BboxT 9 | { 10 | internal UInt32 x, y, w, h; // (x,y) - top-left corner, (w, h) - width & height of bounded box 11 | internal float prob; // confidence - probability that the object was found correctly 12 | internal UInt32 obj_id; // class of object - from range [0, classes-1] 13 | internal UInt32 track_id; // tracking id for video (0 - untracked, 1 - inf - tracked object) 14 | internal UInt32 frames_counter; 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Model/EnvironmentReport.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Wrapper.Yolo.Model 5 | { 6 | public class EnvironmentReport 7 | { 8 | //Microsoft Visual C++ 2017 Redistributable 9 | public bool MicrosoftVisualCPlusPlus2017RedistributableExists { get; set; } 10 | //Nvida CUDA Toolkit 9.2 11 | public bool CudaExists { get; set; } 12 | //Nvida cuDNN v7.1.4 for CUDA 9.2 13 | public bool CudnnExists { get; set; } 14 | //Graphic device name 15 | public string GraphicDeviceName { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Model/YoloItem.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System.Drawing; 5 | 6 | namespace Wrapper.Yolo.Model 7 | { 8 | public class YoloItem 9 | { 10 | public string Type { get; set; } 11 | public double Confidence { get; set; } 12 | public int X { get; set; } 13 | public int Y { get; set; } 14 | public int Width { get; set; } 15 | public int Height { get; set; } 16 | public int ObjId { get; set; } 17 | public int TrackId { get; set; } 18 | 19 | public Point Center() 20 | { 21 | return new Point(this.X + this.Width / 2, this.Y + this.Height / 2); 22 | } 23 | 24 | public float[] CenterVec() 25 | { 26 | float[] vec = { this.X + this.Width / 2, this.Y + this.Height / 2 }; 27 | return vec; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Model/YoloTrackingItem.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Wrapper.Yolo.Model 5 | { 6 | public class YoloTrackingItem : YoloItem 7 | { 8 | public YoloTrackingItem(YoloItem yoloItem, int index, byte[] taggedImageData, byte[] croppedImageData) 9 | { 10 | this.X = yoloItem.X; 11 | this.Y = yoloItem.Y; 12 | this.Width = yoloItem.Width; 13 | this.Height = yoloItem.Height; 14 | this.Type = yoloItem.Type; 15 | this.Confidence = yoloItem.Confidence; 16 | this.ObjId = yoloItem.ObjId; 17 | this.TrackId = yoloItem.TrackId; 18 | 19 | this.Index = index; 20 | this.TaggedImageData = taggedImageData; 21 | this.CroppedImageData = croppedImageData; 22 | } 23 | 24 | public int Index { get; set; } 25 | public byte[] TaggedImageData { get; set; } 26 | public byte[] CroppedImageData { get; set; } 27 | 28 | public YoloItem GetYoloItem() 29 | { 30 | YoloItem item = new YoloItem(); 31 | item.X = this.X; 32 | item.Y = this.Y; 33 | item.Width = this.Width; 34 | item.Height = this.Height; 35 | item.Type = this.Type; 36 | item.Confidence = this.Confidence; 37 | item.ObjId = this.ObjId; 38 | item.TrackId = this.TrackId; 39 | 40 | return item; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Yolo.Config/YoloV3Coco/coco.names: -------------------------------------------------------------------------------- 1 | person 2 | bicycle 3 | car 4 | motorbike 5 | aeroplane 6 | bus 7 | train 8 | truck 9 | boat 10 | traffic light 11 | fire hydrant 12 | stop sign 13 | parking meter 14 | bench 15 | bird 16 | cat 17 | dog 18 | horse 19 | sheep 20 | cow 21 | elephant 22 | bear 23 | zebra 24 | giraffe 25 | backpack 26 | umbrella 27 | handbag 28 | tie 29 | suitcase 30 | frisbee 31 | skis 32 | snowboard 33 | sports ball 34 | kite 35 | baseball bat 36 | baseball glove 37 | skateboard 38 | surfboard 39 | tennis racket 40 | bottle 41 | wine glass 42 | cup 43 | fork 44 | knife 45 | spoon 46 | bowl 47 | banana 48 | apple 49 | sandwich 50 | orange 51 | broccoli 52 | carrot 53 | hot dog 54 | pizza 55 | donut 56 | cake 57 | chair 58 | sofa 59 | pottedplant 60 | bed 61 | diningtable 62 | toilet 63 | tvmonitor 64 | laptop 65 | mouse 66 | remote 67 | keyboard 68 | cell phone 69 | microwave 70 | oven 71 | toaster 72 | sink 73 | refrigerator 74 | book 75 | clock 76 | vase 77 | scissors 78 | teddy bear 79 | hair drier 80 | toothbrush 81 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Yolo.Config/YoloV3TinyCoco/coco.names: -------------------------------------------------------------------------------- 1 | person 2 | bicycle 3 | car 4 | motorbike 5 | aeroplane 6 | bus 7 | train 8 | truck 9 | boat 10 | traffic light 11 | fire hydrant 12 | stop sign 13 | parking meter 14 | bench 15 | bird 16 | cat 17 | dog 18 | horse 19 | sheep 20 | cow 21 | elephant 22 | bear 23 | zebra 24 | giraffe 25 | backpack 26 | umbrella 27 | handbag 28 | tie 29 | suitcase 30 | frisbee 31 | skis 32 | snowboard 33 | sports ball 34 | kite 35 | baseball bat 36 | baseball glove 37 | skateboard 38 | surfboard 39 | tennis racket 40 | bottle 41 | wine glass 42 | cup 43 | fork 44 | knife 45 | spoon 46 | bowl 47 | banana 48 | apple 49 | sandwich 50 | orange 51 | broccoli 52 | carrot 53 | hot dog 54 | pizza 55 | donut 56 | cake 57 | chair 58 | sofa 59 | pottedplant 60 | bed 61 | diningtable 62 | toilet 63 | tvmonitor 64 | laptop 65 | mouse 66 | remote 67 | keyboard 68 | cell phone 69 | microwave 70 | oven 71 | toaster 72 | sink 73 | refrigerator 74 | book 75 | clock 76 | vase 77 | scissors 78 | teddy bear 79 | hair drier 80 | toothbrush 81 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/Yolo.Config/YoloV3TinyCoco/yolov3-tiny.cfg: -------------------------------------------------------------------------------- 1 | [net] 2 | # Testing 3 | batch=1 4 | subdivisions=1 5 | # Training 6 | # batch=64 7 | # subdivisions=2 8 | width=416 9 | height=416 10 | channels=3 11 | momentum=0.9 12 | decay=0.0005 13 | angle=0 14 | saturation = 1.5 15 | exposure = 1.5 16 | hue=.1 17 | 18 | learning_rate=0.001 19 | burn_in=1000 20 | max_batches = 500200 21 | policy=steps 22 | steps=400000,450000 23 | scales=.1,.1 24 | 25 | [convolutional] 26 | batch_normalize=1 27 | filters=16 28 | size=3 29 | stride=1 30 | pad=1 31 | activation=leaky 32 | 33 | [maxpool] 34 | size=2 35 | stride=2 36 | 37 | [convolutional] 38 | batch_normalize=1 39 | filters=32 40 | size=3 41 | stride=1 42 | pad=1 43 | activation=leaky 44 | 45 | [maxpool] 46 | size=2 47 | stride=2 48 | 49 | [convolutional] 50 | batch_normalize=1 51 | filters=64 52 | size=3 53 | stride=1 54 | pad=1 55 | activation=leaky 56 | 57 | [maxpool] 58 | size=2 59 | stride=2 60 | 61 | [convolutional] 62 | batch_normalize=1 63 | filters=128 64 | size=3 65 | stride=1 66 | pad=1 67 | activation=leaky 68 | 69 | [maxpool] 70 | size=2 71 | stride=2 72 | 73 | [convolutional] 74 | batch_normalize=1 75 | filters=256 76 | size=3 77 | stride=1 78 | pad=1 79 | activation=leaky 80 | 81 | [maxpool] 82 | size=2 83 | stride=2 84 | 85 | [convolutional] 86 | batch_normalize=1 87 | filters=512 88 | size=3 89 | stride=1 90 | pad=1 91 | activation=leaky 92 | 93 | [maxpool] 94 | size=2 95 | stride=1 96 | 97 | [convolutional] 98 | batch_normalize=1 99 | filters=1024 100 | size=3 101 | stride=1 102 | pad=1 103 | activation=leaky 104 | 105 | ########### 106 | 107 | [convolutional] 108 | batch_normalize=1 109 | filters=256 110 | size=1 111 | stride=1 112 | pad=1 113 | activation=leaky 114 | 115 | [convolutional] 116 | batch_normalize=1 117 | filters=512 118 | size=3 119 | stride=1 120 | pad=1 121 | activation=leaky 122 | 123 | [convolutional] 124 | size=1 125 | stride=1 126 | pad=1 127 | filters=255 128 | activation=linear 129 | 130 | 131 | 132 | [yolo] 133 | mask = 3,4,5 134 | anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 135 | classes=80 136 | num=6 137 | jitter=.3 138 | ignore_thresh = .7 139 | truth_thresh = 1 140 | random=1 141 | 142 | [route] 143 | layers = -4 144 | 145 | [convolutional] 146 | batch_normalize=1 147 | filters=128 148 | size=1 149 | stride=1 150 | pad=1 151 | activation=leaky 152 | 153 | [upsample] 154 | stride=2 155 | 156 | [route] 157 | layers = -1, 8 158 | 159 | [convolutional] 160 | batch_normalize=1 161 | filters=256 162 | size=3 163 | stride=1 164 | pad=1 165 | activation=leaky 166 | 167 | [convolutional] 168 | size=1 169 | stride=1 170 | pad=1 171 | filters=255 172 | activation=linear 173 | 174 | [yolo] 175 | mask = 0,1,2 176 | anchors = 10,14, 23,27, 37,58, 81,82, 135,169, 344,319 177 | classes=80 178 | num=6 179 | jitter=.3 180 | ignore_thresh = .7 181 | truth_thresh = 1 182 | random=1 -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/YoloConfiguration.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | namespace Wrapper.Yolo 5 | { 6 | public class YoloConfiguration 7 | { 8 | public string ConfigFile { get; set; } 9 | public string WeightsFile { get; set; } 10 | public string NamesFile { get; set; } 11 | 12 | public YoloConfiguration(string configFile, string weightsFile, string namesFile) 13 | { 14 | this.ConfigFile = configFile; 15 | this.WeightsFile = weightsFile; 16 | this.NamesFile = namesFile; 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/YoloTracking.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT license. 3 | 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Drawing; 7 | using System.Drawing.Imaging; 8 | using System.IO; 9 | using System.Linq; 10 | using Wrapper.Yolo.Model; 11 | 12 | namespace Wrapper.Yolo 13 | { 14 | public class YoloTracking 15 | { 16 | private YoloWrapper _yoloWrapper; 17 | private Point _trackingObject = new Point (0, 0); 18 | private int _maxDistance; 19 | public int _index; 20 | 21 | //distance-based validation 22 | public YoloTracking(YoloWrapper yoloWrapper, int maxDistance = int.MaxValue) 23 | { 24 | this._yoloWrapper = yoloWrapper; 25 | this._maxDistance = maxDistance; 26 | } 27 | 28 | public void SetTrackingObject(Point trackingObject) 29 | { 30 | this._trackingObject = trackingObject; 31 | } 32 | 33 | public List Analyse(byte[] imageData, HashSet category, Brush bboxColor) 34 | { 35 | var items = this._yoloWrapper.Track(imageData); 36 | if (items == null || items.Count() == 0) 37 | { 38 | return null; 39 | } 40 | 41 | var probableObject = this.FindAllMatch(items, this._maxDistance, category); 42 | if (probableObject.Count() == 0) 43 | { 44 | return null; 45 | } 46 | 47 | List validObjects = new List(); 48 | foreach (var obj in probableObject) 49 | { 50 | var taggedImageData = this.DrawImage(imageData, obj, bboxColor); 51 | var croppedImageData = this.CropImage(imageData, obj); 52 | 53 | validObjects.Add(new YoloTrackingItem(obj, this._index, taggedImageData, croppedImageData)); 54 | this._index++; 55 | } 56 | return validObjects; 57 | } 58 | 59 | private YoloItem FindBestMatch(IEnumerable items, int maxDistance) 60 | { 61 | var distanceItems = items.Select(o => new { Distance = this.Distance(o.Center(), this._trackingObject), Item = o }).Where(o => o.Distance <= maxDistance).OrderBy(o => o.Distance); 62 | 63 | var bestMatch = distanceItems.FirstOrDefault(); 64 | return bestMatch?.Item; 65 | } 66 | 67 | //find all match based on distance 68 | private List FindAllMatch(IEnumerable items, int maxDistance, HashSet category) 69 | { 70 | List yItems = new List(); 71 | var distanceItems = items.Select(o => new { Category = o.Type, Distance = this.Distance(o.Center(), this._trackingObject), Item = o }).Where(o => (category.Count == 0 || category.Contains(o.Category)) && o.Distance <= maxDistance).OrderBy(o => o.Distance); 72 | foreach (var item in distanceItems) 73 | { 74 | YoloItem yItem = item.Item; 75 | yItems.Add(yItem); 76 | } 77 | return yItems; 78 | } 79 | 80 | private double Distance(Point p1, Point p2) 81 | { 82 | return Math.Sqrt(this.Pow2(p2.X - p1.X) + Pow2(p2.Y - p1.Y)); 83 | } 84 | 85 | private double Pow2(double x) 86 | { 87 | return x * x; 88 | } 89 | 90 | private byte[] DrawImage(byte[] imageData, YoloItem item, Brush color) 91 | { 92 | using (var memoryStream = new MemoryStream(imageData)) 93 | using (var image = Image.FromStream(memoryStream)) 94 | using (var canvas = Graphics.FromImage(image)) 95 | using (var pen = new Pen(color, 3)) 96 | { 97 | canvas.DrawRectangle(pen, item.X, item.Y, item.Width, item.Height); 98 | canvas.Flush(); 99 | 100 | using (var memoryStream2 = new MemoryStream()) 101 | { 102 | image.Save(memoryStream2, ImageFormat.Bmp); 103 | return memoryStream2.ToArray(); 104 | } 105 | } 106 | } 107 | 108 | private byte[] CropImage(byte[] imageData, YoloItem item) 109 | { 110 | using (var memoryStream = new MemoryStream(imageData)) 111 | using (var image = Image.FromStream(memoryStream)) 112 | { 113 | Rectangle cropRect = new Rectangle(item.X, item.Y, Math.Min(image.Width-item.X, item.Width), Math.Min(image.Height-item.Y, item.Height)); 114 | Bitmap bmpImage = new Bitmap(image); 115 | Image croppedImage = bmpImage.Clone(cropRect, bmpImage.PixelFormat); 116 | 117 | using (var memoryStream2 = new MemoryStream()) 118 | { 119 | croppedImage.Save(memoryStream2, ImageFormat.Bmp); 120 | return memoryStream2.ToArray(); 121 | } 122 | } 123 | } 124 | 125 | public double getDistance(Point p1) 126 | { 127 | return Distance(p1, this._trackingObject); 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/VAP/YoloWrapper/YoloWrapper.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | netcoreapp2.2 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | PreserveNewest 15 | x64\opencv_world340d.dll 16 | 17 | 18 | PreserveNewest 19 | x64\opencv_world340.dll 20 | 21 | 22 | PreserveNewest 23 | x64\yolo_cpp_dll_gpu_cc.dll 24 | 25 | 26 | PreserveNewest 27 | x64\yolo_cpp_dll_gpu_lt.dll 28 | 29 | 30 | PreserveNewest 31 | x64\yolo_cpp_dll_cpu.dll 32 | 33 | 34 | PreserveNewest 35 | x64\pthreadVC2.dll 36 | 37 | 38 | 39 | --------------------------------------------------------------------------------