├── .gitignore ├── LICENSE ├── Makefile ├── Makefile.vars ├── README.md ├── _config.yml ├── inference.py ├── lib └── README.md ├── main.py ├── media ├── Readme.md ├── demo.mp4 ├── speed_detection_infrastructure.jpg ├── speed_detection_model_conversion.jpg ├── speed_detection_modules.jpg ├── speed_detection_top_level_arch.jpg ├── speed_detection_user_interface.jpg └── traffic_viosion.gif ├── models ├── README.md ├── create_yolo_caffemodel.py ├── create_yolo_prototxt.py └── prepareModel.sh ├── speedEstimator.py ├── test_ipcam.py └── yoloOpenVX ├── Makefile ├── README.md ├── __init__.py ├── region.cpp ├── region.h ├── setup.py └── yoloOpenVX.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Rohit Sharma 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ROOT=$(shell pwd) 2 | include Makefile.vars 3 | 4 | all: 5 | @$(MAKE) -C yoloOpenVX $< 6 | 7 | test: FORCE 8 | for vid in $(wildcard media/*.mp4); do \ 9 | echo processing $$vid ;\ 10 | /usr/bin/python ./main.py --video $$vid ;\ 11 | done 12 | 13 | clean: 14 | @$(MAKE) -C yoloOpenVX clean 15 | 16 | FORCE: 17 | 18 | 19 | -------------------------------------------------------------------------------- /Makefile.vars: -------------------------------------------------------------------------------- 1 | OPENVX_MODEL=$(ROOT)/models/openVXModel 2 | 3 | CC=g++ 4 | INCLUDES=-I$(OPENVX_MODEL) -I/opt/rocm/mivisionx/include/ 5 | # python2.7-config --cflags 6 | CFLAGS=-shared -I/usr/include/python2.7 -I/usr/include/x86_64-linux-gnu/python2.7 -fno-strict-aliasing -Wdate-time -D_FORTIFY_SOURCE=2 -g -fstack-protector-strong -Wformat -Werror=format-security -DNDEBUG -g -fwrapv -O2 -Wall 7 | 8 | # python2.7-config --ldflags 9 | LDFLAGS=-fPIC -L/usr/lib/python2.7/config-x86_64-linux-gnu -L/usr/lib -lpython2.7 -lpthread -ldl -lutil -lm -Xlinker -export-dynamic -Wl,-O1 -Wl,-Bsymbolic-functions 10 | 11 | LIB_PATH=$(ROOT)/lib 12 | LIB_EXT=.so 13 | 14 | CP=/bin/cp -pf 15 | 16 | print-% : ; @echo $* = $($*) 17 | #vars : $(foreach var,$(.VARIABLES),$(info $(var) = $($(var)))) 18 | 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Traffic Vision 2 | This app detects cars/buses in a live traffic at a phenomenal **50 frames/sec with HD resolution (1920x1080)** using deep learning network [Yolo-V2](https://pjreddie.com/darknet/yolov2). The model used in the app is optimized for inferencing performnce on AMD-GPUs using [MIVisionX toolkit](https://gpuopen-professionalcompute-libraries.github.io/MIVisionX/). 3 | 4 | [![Traffic Vision Animation](media/traffic_viosion.gif)](https://youtu.be/YASOovwds_A) 5 | 6 | ## Features 7 | 1. Vehicle detection with bounding box 8 | 1. Vehicle direction ((upward, downward) detection 9 | 1. Vehicle speed estimation 10 | 1. Vehicle type: bus/car. 11 | 12 | ## How to Run 13 | 14 | ### Use Model 15 | 16 | 17 | ### Demo 18 | 19 | App starts the demo, if no other option is provided. Demo uses a video stored in the [media/](./media) dir. 20 | ``` 21 | % ./main.py 22 | ('Loaded', 'yoloOpenVX') 23 | OK: loaded 22 kernels from libvx_nn.so 24 | OK: OpenVX using GPU device#0 (gfx900) [OpenCL 1.2 ] [SvmCaps 0 1] 25 | OK: annCreateInference: successful 26 | Processed a total of 102 frames 27 | OK: OpenCL buffer usage: 87771380, 46/46 28 | % 29 | ``` 30 | Here is the [link to YouTube video](https://youtu.be/YASOovwds_A) detecting cars, bounding boxes, car speed, and confidence scores. 31 | ### Other Examples 32 | 33 | **recorded video** 34 | > 1. ./main.py --video /vid.mp4 35 | 36 | **traffic cam ip** 37 | > 2. ./main.py --cam_ip 'http://166.149.104.112:8082/snap.jpg' 38 | 39 | ## Installation 40 | 41 | ### Prerequisites 42 | 43 | 1. GPU: Radeon Instinct or Vega Family of Products with [ROCm](https://rocm.github.io/ROCmInstall.html) and OpenCL development kit 44 | 1. [Install AMD's MIVisionX toolkit](https://gpuopen-professionalcompute-libraries.github.io/MIVisionX/) : AMD's MIVisionX toolkit is a comprehensive computer vision and machine intelligence libraries, utilities 45 | 1. [CMake](http://cmake.org/download/), [Caffe](http://caffe.berkeleyvision.org/installation.html) 46 | 1. [Google's Protobuf](https://github.com/google/protobuf) 47 | 48 | ### Steps 49 | 50 | ``` 51 | % git clone https://github.com/srohit0/trafficVision 52 | ``` 53 | 54 | 55 | **_1. Model Conversion_** 56 | 57 | This steps downloads yolov2-tiny for voc dataset and converts to MIVision's openVX model. 58 | ``` 59 | % cd trafficVision/model 60 | % bash ./prepareModel.sh 61 | ``` 62 | More details on the pre-requisite (like [caffe](http://caffe.berkeleyvision.org/installation.html)) of the model conversion in the [models/](./models) dir. 63 | 64 | **_2. MIVision Model Compilation_** 65 | 66 | ``` 67 | % cd trafficVision 68 | % make 69 | ``` 70 | 71 | **_3. Test App_** 72 | 73 | ``` 74 | % cd trafficVision 75 | % make test 76 | ``` 77 | It'll display detection all videos in media/ dir. 78 | 79 | ## Design 80 | This section is a guide for developers, who would like to port vision and object detections models to AMD's Radeon GPUs from other frameworks including [tensorflow](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md), [caffe](http://caffe.berkeleyvision.org/model_zoo.html) or [pytorch](https://pytorch.org/). 81 | 82 | ### High Level Design 83 | 84 | 85 | ### Lower Level Modules 86 | These lower level modules can be found as python modules (files) or packages (directories) in this repository. 87 | 88 | 89 | ## Development 90 | 91 | ### Model Conversion 92 | Follow model conversion process similar to the one described below. 93 | 94 | 95 | 96 | ### Infrastructure 97 | Make sure you've infrastructure pre-requisites installed before you start porting neural network model for inferencing. 98 | 99 | 100 | ## Developed and Tested on 101 | 1. Hardware 102 | 1. AMD Ryzen Threadripper 1900X 8-Core Processor 103 | 1. Accelerator = Radeon Instinct™ MI25 Accelerator 104 | 1. Software 105 | 1. Ubuntu 16.04 LTS OS 106 | 1. Python 2.7 107 | 1. MIVisionX 1.7.0 108 | 1. AMD OpenVX 0.9.9 109 | 1. GCC 5.4 110 | 111 | ## Credit 112 | * MIVisionX Team 113 | 114 | ## References 115 | 1. [yoloV2 paper](https://arxiv.org/pdf/1612.08242.pdf) 116 | 1. [Tiny Yolo aka Darknet reference network](https://pjreddie.com/darknet/imagenet/#reference) 117 | 1. [MiVisionX Setup](https://github.com/kiritigowda/MIVisionX-setup) 118 | 1. [AMD OpenVX](https://gpuopen.com/compute-product/amd-openvx/) 119 | 1. [Optimization with OpenVX Graphs](http://openaccess.thecvf.com/content_cvpr_workshops_2014/W17/papers/Rainey_Addressing_System-Level_Optimization_2014_CVPR_paper.pdf) 120 | 1. [Measuring Traffic Speed With Deep Learning Object Detection](https://medium.com/datadriveninvestor/measuring-traffic-speed-with-deep-learning-object-detection-efc0bb9a3c57) 121 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018. All rights reserved. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | 12 | 13 | # Description: 14 | # inference the yolo v2 model in openVX format. 15 | # 16 | import numpy as np 17 | import cv2 18 | 19 | import yoloOpenVX 20 | import speedEstimator 21 | 22 | class yoloInferenceNet: 23 | def __init__(self, weights): 24 | # create inference graph. 25 | self.handle = yoloOpenVX.interface.annCreateInference(weights.encode('utf-8')) 26 | 27 | # get graph details for adapting the input. 28 | input_info,output_info = yoloOpenVX.interface.annQueryInference().decode("utf-8").split(';') 29 | self.ni,self.ci,self.hi,self.wi = map(int, input_info.split(',')[2:]) 30 | 31 | self.frame = np.empty(0) 32 | self.offsetX = 0; 33 | self.offsetY = 0; 34 | self.scale = 1; 35 | 36 | self.estimator = speedEstimator.estimate() 37 | 38 | # print details 39 | def __repr__(self): 40 | return "%s " % (self.__class__.__name__, self.ni, self.wi, self.hi, self.scale, self.offsetX, self.offsetY) 41 | 42 | # compute scale factor to convert original 43 | # frame to yonoNet frame. 44 | def scaleFactor (self, ow, oh, nw, nh): 45 | h_ratio = float(nh)/oh 46 | w_ratio = float(nw)/ow 47 | self.scale = 1.0 48 | self.offsetX = 0 49 | self.offsetY = 0 50 | 51 | if (h_ratio>w_ratio): 52 | self.scale = float(nw)/ow; 53 | self.offsetY = int((nh-float(oh*self.scale))/2.0) 54 | else: 55 | self.scale = float(nh)/oh; 56 | self.offsetX = int((nw-float(ow*self.scale))/2.0) 57 | 58 | return (self.scale, int(self.offsetX), int(self.offsetY)) 59 | 60 | # scale frame to yoloNet frame 61 | def yoloInput(self, frame): 62 | 63 | self.frame = frame; # (1080, 1920 ,3) 64 | 65 | # scale frame to yolo size of 416, 416, 3 66 | self.scaleFactor(frame.shape[1], frame.shape[0], self.wi, self.hi) 67 | 68 | new_frame = np.zeros((self.hi,self.wi,self.ci), dtype=np.uint8) 69 | new_frame[self.offsetY:self.hi-self.offsetY, self.offsetX:self.wi-self.offsetX, :] = \ 70 | cv2.resize(frame.copy(), None, fx=self.scale, fy=self.scale, \ 71 | interpolation=cv2.INTER_CUBIC) 72 | 73 | return new_frame 74 | 75 | # add boxes around detected objects. 76 | def addBoxes(self, frame, boxes): 77 | dup_frame = frame.copy() 78 | if boxes == None: 79 | return dup_frame 80 | 81 | boxes = self.estimator.speed(boxes) 82 | for one_box in boxes: 83 | left, top, right, bottom, confidence, ilabel, label, speed, bdir = one_box 84 | if ( frame.shape == self.frame.shape ): 85 | # scale to original frame. 86 | left = int((left-self.offsetX)/self.scale) 87 | top = int((top-self.offsetY)/self.scale) 88 | right = int((right-self.offsetX)/self.scale) 89 | bottom = int((bottom-self.offsetY)/self.scale) 90 | confidence = int(confidence * 100) 91 | if confidence > 20: 92 | color = self.estimator.color(speed); 93 | #print (top+bottom)/2.0, speed, bdir, (color); 94 | cv2.rectangle(dup_frame, (left,top), (right,bottom), color, thickness=5) 95 | size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 1) 96 | #width = size[0][0] + 50 97 | width = abs(right-left-5) 98 | height = size[0][1] 99 | text = str(int(confidence)) + "% " + label 100 | if ( speed > 0 ): 101 | text = str(int(speed)) + "mph " + label 102 | cv2.rectangle(dup_frame, (left+5, (bottom-5) - (height+5)), ((left + width), (bottom-5)),(255,0,0),-1) 103 | cv2.putText(dup_frame,text,((left + 5),(bottom-10)),cv2.FONT_HERSHEY_SIMPLEX,0.6,(255,255,255),1) 104 | return dup_frame 105 | 106 | def destroy(self): 107 | yoloOpenVX.interface.annReleaseInference(self.handle) 108 | 109 | 110 | -------------------------------------------------------------------------------- /lib/README.md: -------------------------------------------------------------------------------- 1 | .so files for openVX model and yolov2 region layer 2 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright (c) 2018. All rights reserved. 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | # copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | # Description: 24 | # Detect Vehicle using AMD MIVisionX Inferencing Engine. 25 | # 26 | from __future__ import print_function 27 | import os, timeit 28 | import argparse 29 | import numpy as np 30 | import cv2 31 | 32 | import yoloOpenVX 33 | import inference 34 | 35 | if __name__ == '__main__' : 36 | 37 | parseHandle = argparse.ArgumentParser(description= 38 | 'Detect Vehicles in static video or a live feed.') 39 | parseHandle.add_argument('--video', dest='video', 40 | type=str, default="./media/demo.mp4", 41 | help='path to video file.') 42 | parseHandle.add_argument('--cam_ip', dest='cam_ip', type=str, 43 | default='', help='IP address for video cam.') 44 | 45 | args = parseHandle.parse_args() 46 | 47 | if ( args.cam_ip ) : 48 | # must be a mjpg or h264 streaming 49 | window_title = args.cam_ip + "- AMD Object Detection on Live Feed" 50 | feed = args.cam_ip 51 | elif ( args.video ) : 52 | window_title = os.path.basename(args.video) + "- AMD Object Detection on Recorded Feed" 53 | feed = args.video 54 | else: 55 | print ("Error: no video source."); 56 | exit(0); 57 | 58 | yoloNet = inference.yoloInferenceNet(yoloOpenVX.weights); 59 | cv2.namedWindow(window_title, cv2.WINDOW_GUI_EXPANDED) 60 | cap = cv2.VideoCapture(feed) 61 | 62 | if ( cap.isOpened() == False ): 63 | print ("Error: could not open video feed", feed); 64 | 65 | startTime = timeit.default_timer() 66 | iframe = 0; 67 | while(cap.isOpened()): 68 | 69 | ret, frame = cap.read() 70 | if ret == False: 71 | break; 72 | 73 | if ( iframe == 0 ): 74 | cv2.resizeWindow(window_title, frame.shape[1], frame.shape[0]) 75 | 76 | iframe = iframe + 1 77 | resized_frame = yoloNet.yoloInput(frame); # image to display 78 | frame_array = np.concatenate((resized_frame[:,:,0], resized_frame[:,:,1], resized_frame[:,:,2]), 0) 79 | boxes = yoloOpenVX.model.detectBoxes(yoloNet.handle, np.ascontiguousarray(frame_array, dtype=np.float32)/(255.0)) 80 | 81 | frame_w_boxes = yoloNet.addBoxes(frame, boxes); 82 | 83 | cv2.imshow(window_title, frame_w_boxes) 84 | key = cv2.waitKey(1) 85 | if key & 0xFF == ord('q'): 86 | break 87 | 88 | if ( os.path.splitext(args.cam_ip)[1] == ".jpg" ) : 89 | cap = cv2.VideoCapture(feed); # required for static jpg like stream in Bosch cams 90 | 91 | elapsedTime = timeit.default_timer() - startTime 92 | print ("Processed a total of ", iframe, "frames in ", elapsedTime, "micro sec with ", iframe/(elapsedTime*1e-6), "fps."); 93 | cap.release() 94 | cv2.destroyAllWindows() 95 | 96 | yoloNet.destroy(); 97 | 98 | 99 | -------------------------------------------------------------------------------- /media/Readme.md: -------------------------------------------------------------------------------- 1 | # Images 2 | -------------------------------------------------------------------------------- /media/demo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srohit0/trafficVision/e58904218d26adfc7da47c9d7614e093f672308d/media/demo.mp4 -------------------------------------------------------------------------------- /media/speed_detection_infrastructure.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srohit0/trafficVision/e58904218d26adfc7da47c9d7614e093f672308d/media/speed_detection_infrastructure.jpg -------------------------------------------------------------------------------- /media/speed_detection_model_conversion.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srohit0/trafficVision/e58904218d26adfc7da47c9d7614e093f672308d/media/speed_detection_model_conversion.jpg -------------------------------------------------------------------------------- /media/speed_detection_modules.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srohit0/trafficVision/e58904218d26adfc7da47c9d7614e093f672308d/media/speed_detection_modules.jpg -------------------------------------------------------------------------------- /media/speed_detection_top_level_arch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srohit0/trafficVision/e58904218d26adfc7da47c9d7614e093f672308d/media/speed_detection_top_level_arch.jpg -------------------------------------------------------------------------------- /media/speed_detection_user_interface.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srohit0/trafficVision/e58904218d26adfc7da47c9d7614e093f672308d/media/speed_detection_user_interface.jpg -------------------------------------------------------------------------------- /media/traffic_viosion.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srohit0/trafficVision/e58904218d26adfc7da47c9d7614e093f672308d/media/traffic_viosion.gif -------------------------------------------------------------------------------- /models/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Model Conversion 3 | ``` 4 | % ./prepareModel.sh 5 | ``` 6 | 7 | 8 | 9 | 10 | ## Pre-Requisites 11 | 12 | 1. curl to download yolo model. ```sudo apt-get install curl``` 13 | 1. [Caffe](http://caffe.berkeleyvision.org/installation.html) 14 | > Make sure to update PYTHONPATH, ```export PYTHONPATH=${CAFFE_ROOT}/python:$PYTHONPATH``` after installation. 15 | 1. [AMD's MIVisionX toolkit](https://gpuopen-professionalcompute-libraries.github.io/MIVisionX/) 16 | 17 | ## Model Conversion Steps 18 | These steps are included in prepareModel.sh. Repeated here for clarification: 19 | 20 | ``` 21 | ################################################### 22 | # Step 1: Download yolo model 23 | ################################################### 24 | mkdir yolomodels && cd yolomodels 25 | curl https://pjreddie.com/media/files/yolov2-tiny-voc.weights -o yolov2-tiny-voc.weights 26 | curl https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2-tiny-voc.cfg -o yolov2-tiny-voc.cfg 27 | cd .. 28 | 29 | ################################################### 30 | # Step 2: Convert to yolo model to caffe model 31 | ################################################### 32 | 33 | mkdir caffemodels 34 | filename=yolov2-tiny-voc 35 | 36 | yolocfg=./yolomodels/$filename.cfg 37 | yoloweight=./yolomodels/$filename.weights 38 | yolocfgcaffe=./caffemodels/$filename.prototxt 39 | yoloweightcaffe=./caffemodels/$filename.caffemodel 40 | 41 | echo "Convert yolo to caffe" 42 | python create_yolo_prototxt.py $yolocfg $yolocfgcaffe 43 | python create_yolo_caffemodel.py -m $yolocfgcaffe -w $yoloweight -o $yoloweightcaffe 44 | 45 | 46 | ################################################### 47 | # Step 3: Convert caffe model to NNIR format 48 | ################################################### 49 | mkdir /nnirModel 50 | python /opt/rocm/mivisionx/model_compiler/python/caffe2nnir.py caffemodels/$filename.caffemodel ./nnirModel --input-dims 1,3,416,416 51 | 52 | ################################################### 53 | # Step 4: Convert NNIR to OpenVX format 54 | ################################################### 55 | python /opt/rocm/mivisionx/model_compiler/python/nnir2openvx.py ./nnirModel ./openVXModel 56 | 57 | ################################################### 58 | # Step 5: Compile openVX model 59 | ################################################### 60 | cd ./openVXModel 61 | mkdir build && cd build 62 | cmake .. 63 | make 64 | cd ../.. 65 | cp ./openVXModel/build/libannmodule.so ../lib 66 | cp ./openVXModel/build/libannpython.so ../lib 67 | ``` 68 | -------------------------------------------------------------------------------- /models/create_yolo_caffemodel.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Fri Apr 29 16:10:21 2016 4 | 5 | @author: xingw 6 | """ 7 | 8 | import caffe 9 | import numpy as np 10 | import sys, getopt 11 | 12 | def Print(name, data, sz, dims): 13 | print "=======",name,"======" 14 | print sz 15 | print dims 16 | print data 17 | 18 | def main(argv): 19 | model_filename = '' 20 | yoloweight_filename = '' 21 | caffemodel_filename = '' 22 | try: 23 | opts, args = getopt.getopt(argv, "hm:w:o:") 24 | print opts 25 | except getopt.GetoptError: 26 | print 'create_yolo_caffemodel.py -m -w -o ' 27 | sys.exit(2) 28 | for opt, arg in opts: 29 | if opt == '-h': 30 | print 'create_yolo_caffemodel.py -m -w -o ' 31 | sys.exit() 32 | elif opt == "-m": 33 | model_filename = arg 34 | elif opt == "-w": 35 | yoloweight_filename = arg 36 | elif opt == "-o": 37 | caffemodel_filename = arg 38 | 39 | print 'model file is ', model_filename 40 | print 'weight file is ', yoloweight_filename 41 | print 'output caffemodel file is ', caffemodel_filename 42 | net = caffe.Net(model_filename, caffe.TEST) 43 | params = net.params.keys() 44 | 45 | # read weights from file and assign to the network 46 | netWeightsInt = np.fromfile(yoloweight_filename, dtype=np.int32) 47 | transFlag = (netWeightsInt[0]>1000 or netWeightsInt[1]>1000) 48 | # transpose flag, the first 4 entries are major, minor, revision and net.seen 49 | start = 4 50 | if (netWeightsInt[0]*10 + netWeightsInt[1]) >= 2: 51 | start = 5 52 | 53 | print transFlag 54 | 55 | netWeightsFloat = np.fromfile(yoloweight_filename, dtype=np.float32) 56 | netWeights = netWeightsFloat[start:] # start from the 5th entry, the first 4 entries are major, minor, revision and net.seen 57 | print netWeights.shape 58 | count = 0 59 | 60 | print "#Total Net Layer", len(net.layers) 61 | 62 | layercnt = 0 63 | for pr in params: 64 | layercnt = layercnt + 1 65 | lidx = list(net._layer_names).index(pr) 66 | layer = net.layers[lidx] 67 | if count == netWeights.shape[0] and (layer.type != 'BatchNorm' and layer.type != 'Scale'): 68 | print "WARNING: no weights left for %s" % pr 69 | break 70 | if layer.type == 'Convolution': 71 | print pr+"(conv)" + "-"+str(layercnt)+"-"+str(len(net.params[pr]) > 1) 72 | # bias 73 | if len(net.params[pr]) > 1: 74 | bias_dim = net.params[pr][1].data.shape 75 | else: 76 | bias_dim = (net.params[pr][0].data.shape[0], ) 77 | biasSize = np.prod(bias_dim) 78 | conv_bias = np.reshape(netWeights[count:count+biasSize], bias_dim) 79 | if len(net.params[pr]) > 1: 80 | assert(bias_dim == net.params[pr][1].data.shape) 81 | net.params[pr][1].data[...] = conv_bias 82 | conv_bias = None 83 | count = count + biasSize 84 | # batch_norm 85 | if lidx+1 < len(net.layers) and net.layers[lidx+1].type == 'BatchNorm': 86 | bn_dims = (3, net.params[pr][0].data.shape[0]) 87 | bnSize = np.prod(bn_dims) 88 | batch_norm = np.reshape(netWeights[count:count+bnSize], bn_dims) 89 | count = count + bnSize 90 | # weights 91 | dims = net.params[pr][0].data.shape 92 | weightSize = np.prod(dims) 93 | net.params[pr][0].data[...] = np.reshape(netWeights[count:count+weightSize], dims) 94 | count = count + weightSize 95 | 96 | if 25 == layercnt: 97 | #print net.params[pr] 98 | #Print("conv", net.params[pr][0].data, weightSize, dims) 99 | #Print("bias", conv_bias, biasSize, bias_dim) 100 | #Print("bnorm", batch_norm, bnSize, bn_dims) 101 | abcd = 10 102 | 103 | elif layer.type == 'InnerProduct': 104 | print pr+"(fc)" 105 | # bias 106 | biasSize = np.prod(net.params[pr][1].data.shape) 107 | net.params[pr][1].data[...] = np.reshape(netWeights[count:count+biasSize], net.params[pr][1].data.shape) 108 | count = count + biasSize 109 | # weights 110 | dims = net.params[pr][0].data.shape 111 | weightSize = np.prod(dims) 112 | if transFlag: 113 | net.params[pr][0].data[...] = np.reshape(netWeights[count:count+weightSize], (dims[1], dims[0])).transpose() 114 | else: 115 | net.params[pr][0].data[...] = np.reshape(netWeights[count:count+weightSize], dims) 116 | count = count + weightSize 117 | elif layer.type == 'BatchNorm': 118 | print pr+"(batchnorm)" 119 | net.params[pr][0].data[...] = batch_norm[1] # mean 120 | net.params[pr][1].data[...] = batch_norm[2] # variance 121 | net.params[pr][2].data[...] = 1.0 # scale factor 122 | elif layer.type == 'Scale': 123 | print pr+"(scale)" 124 | if batch_norm is not None: 125 | net.params[pr][0].data[...] = batch_norm[0] # scale 126 | batch_norm = None 127 | if len(net.params[pr]) > 1: 128 | net.params[pr][1].data[...] = conv_bias # bias 129 | conv_bias = None 130 | else: 131 | print "WARNING: unsupported layer, "+pr 132 | if np.prod(netWeights.shape) != count: 133 | print "ERROR: size mismatch: %d" % count 134 | net.save(caffemodel_filename) 135 | 136 | if __name__=='__main__': 137 | main(sys.argv[1:]) 138 | -------------------------------------------------------------------------------- /models/create_yolo_prototxt.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from ConfigParser import ConfigParser 3 | from collections import OrderedDict 4 | import argparse 5 | import logging 6 | import os 7 | import sys 8 | 9 | class CaffeLayerGenerator(object): 10 | def __init__(self, name, ltype): 11 | self.name = name 12 | self.bottom = [] 13 | self.top = [] 14 | self.type = ltype 15 | def get_template(self): 16 | return """ 17 | layer {{{{ 18 | name: "{}" 19 | type: "{}" 20 | bottom: "{}" 21 | top: "{}"{{}} 22 | }}}}""".format(self.name, self.type, self.bottom[0], self.top[0]) 23 | 24 | class CaffeInputLayer(CaffeLayerGenerator): 25 | def __init__(self, name, channels, width, height): 26 | super(CaffeInputLayer, self).__init__(name, 'Input') 27 | self.channels = channels 28 | self.width = width 29 | self.height = height 30 | def write(self, f): 31 | f.write(""" 32 | input: "{}" 33 | input_shape {{ 34 | dim: 1 35 | dim: {} 36 | dim: {} 37 | dim: {} 38 | }}""".format(self.name, self.channels, self.width, self.height)) 39 | 40 | class CaffeConvolutionLayer(CaffeLayerGenerator): 41 | def __init__(self, name, filters, ksize=None, stride=None, pad=None, bias=True): 42 | super(CaffeConvolutionLayer, self).__init__(name, 'Convolution') 43 | self.filters = filters 44 | self.ksize = ksize 45 | self.stride = stride 46 | self.pad = pad 47 | self.bias = bias 48 | def write(self, f): 49 | opts = [''] 50 | if self.ksize is not None: opts.append('kernel_size: {}'.format(self.ksize)) 51 | if self.stride is not None: opts.append('stride: {}'.format(self.stride)) 52 | if self.pad is not None: opts.append('pad: {}'.format(self.pad)) 53 | if not self.bias: opts.append('bias_term: false') 54 | param_str = """ 55 | convolution_param {{ 56 | num_output: {}{} 57 | }}""".format(self.filters, '\n '.join(opts)) 58 | f.write(self.get_template().format(param_str)) 59 | 60 | class CaffePoolingLayer(CaffeLayerGenerator): 61 | def __init__(self, name, pooltype, ksize=None, stride=None, pad=None, global_pooling=None): 62 | super(CaffePoolingLayer, self).__init__(name, 'Pooling') 63 | self.pooltype = pooltype 64 | self.ksize = ksize 65 | self.stride = stride 66 | self.pad = pad 67 | self.global_pooling = global_pooling 68 | def write(self, f): 69 | opts = [''] 70 | if self.ksize is not None: opts.append('kernel_size: {}'.format(self.ksize)) 71 | if self.stride is not None: opts.append('stride: {}'.format(self.stride)) 72 | if self.pad is not None: opts.append('pad: {}'.format(self.pad)) 73 | if self.global_pooling is not None: opts.append('global_pooling: {}'.format('True' if self.global_pooling else 'False')) 74 | param_str = """ 75 | pooling_param {{ 76 | pool: {}{} 77 | }}""".format(self.pooltype, '\n '.join(opts)) 78 | f.write(self.get_template().format(param_str)) 79 | 80 | class CaffeInnerProductLayer(CaffeLayerGenerator): 81 | def __init__(self, name, num_output): 82 | super(CaffeInnerProductLayer, self).__init__(name, 'InnerProduct') 83 | self.num_output = num_output 84 | def write(self, f): 85 | param_str = """ 86 | inner_product_param {{ 87 | num_output: {} 88 | }}""".format(self.num_output) 89 | f.write(self.get_template().format(param_str)) 90 | 91 | class CaffeBatchNormLayer(CaffeLayerGenerator): 92 | def __init__(self, name): 93 | super(CaffeBatchNormLayer, self).__init__(name, 'BatchNorm') 94 | def write(self, f): 95 | param_str = """ 96 | batch_norm_param { 97 | use_global_stats: true 98 | }""" 99 | f.write(self.get_template().format(param_str)) 100 | 101 | class CaffeScaleLayer(CaffeLayerGenerator): 102 | def __init__(self, name): 103 | super(CaffeScaleLayer, self).__init__(name, 'Scale') 104 | def write(self, f): 105 | param_str = """ 106 | scale_param { 107 | bias_term: true 108 | }""" 109 | f.write(self.get_template().format(param_str)) 110 | 111 | class CaffeReluLayer(CaffeLayerGenerator): 112 | def __init__(self, name, negslope=None): 113 | super(CaffeReluLayer, self).__init__(name, 'ReLU') 114 | self.negslope = negslope 115 | def write(self, f): 116 | param_str = "" 117 | if self.negslope is not None: 118 | param_str = """ 119 | relu_param {{ 120 | negative_slope: {} 121 | }}""".format(self.negslope) 122 | f.write(self.get_template().format(param_str)) 123 | 124 | class CaffeDropoutLayer(CaffeLayerGenerator): 125 | def __init__(self, name, prob): 126 | super(CaffeDropoutLayer, self).__init__(name, 'Dropout') 127 | self.prob = prob 128 | def write(self, f): 129 | param_str = """ 130 | dropout_param {{ 131 | dropout_ratio: {} 132 | }}""".format(self.prob) 133 | f.write(self.get_template().format(param_str)) 134 | 135 | class CaffeSoftmaxLayer(CaffeLayerGenerator): 136 | def __init__(self, name): 137 | super(CaffeSoftmaxLayer, self).__init__(name, 'Softmax') 138 | def write(self, f): 139 | f.write(self.get_template().format("")) 140 | 141 | class CaffeProtoGenerator: 142 | def __init__(self, name): 143 | self.name = name 144 | self.sections = [] 145 | self.lnum = 0 146 | self.layer = None 147 | def add_layer(self, l): 148 | self.sections.append( l ) 149 | def add_input_layer(self, items): 150 | self.lnum = 0 151 | lname = "data" 152 | self.layer = CaffeInputLayer(lname, items['channels'], items['width'], items['height']) 153 | self.layer.top.append( lname ) 154 | self.add_layer( self.layer ) 155 | def update_last_convolution_layer(self): 156 | self.sections[len(self.sections)-1].pad = 0 157 | def add_convolution_layer(self, items): 158 | self.lnum += 1 159 | prev_blob = self.layer.top[0] 160 | lname = "conv"+str(self.lnum) 161 | filters = items['filters'] 162 | ksize = items['size'] if 'size' in items else None 163 | stride = items['stride'] if 'stride' in items else None 164 | pad = items['pad'] if 'pad' in items else None 165 | bias = not bool(items['batch_normalize']) if 'batch_normalize' in items else True 166 | self.layer = CaffeConvolutionLayer( lname, filters, ksize=ksize, stride=stride, pad=pad, bias=bias ) 167 | self.layer.bottom.append( prev_blob ) 168 | self.layer.top.append( lname ) 169 | self.add_layer( self.layer ) 170 | def add_innerproduct_layer(self, items): 171 | self.lnum += 1 172 | prev_blob = self.layer.top[0] 173 | lname = "fc"+str(self.lnum) 174 | num_output = items['output'] 175 | self.layer = CaffeInnerProductLayer( lname, num_output ) 176 | self.layer.bottom.append( prev_blob ) 177 | self.layer.top.append( lname ) 178 | self.add_layer( self.layer ) 179 | def add_pooling_layer(self, ltype, items, global_pooling=None): 180 | prev_blob = self.layer.top[0] 181 | lname = "pool"+str(self.lnum) 182 | ksize = items['size'] if 'size' in items else None 183 | stride = items['stride'] if 'stride' in items else None 184 | pad = items['pad'] if 'pad' in items else None 185 | self.layer = CaffePoolingLayer( lname, ltype, ksize=ksize, stride=stride, pad=pad, global_pooling=global_pooling ) 186 | self.layer.bottom.append( prev_blob ) 187 | self.layer.top.append( lname ) 188 | self.add_layer( self.layer ) 189 | def add_batchnorm_layer(self, items): 190 | prev_blob = self.layer.top[0] 191 | lname = "bn"+str(self.lnum) 192 | self.layer = CaffeBatchNormLayer( lname ) 193 | self.layer.bottom.append( prev_blob ) 194 | self.layer.top.append( lname ) 195 | self.add_layer( self.layer ) 196 | def add_scale_layer(self, items): 197 | prev_blob = self.layer.top[0] 198 | lname = "scale"+str(self.lnum) 199 | self.layer = CaffeScaleLayer( lname ) 200 | self.layer.bottom.append( prev_blob ) 201 | self.layer.top.append( lname ) 202 | self.add_layer( self.layer ) 203 | def add_relu_layer(self, items): 204 | prev_blob = self.layer.top[0] 205 | lname = "relu"+str(self.lnum) 206 | self.layer = CaffeReluLayer( lname , 0.1) 207 | self.layer.bottom.append( prev_blob ) 208 | self.layer.top.append( prev_blob ) # loopback 209 | self.add_layer( self.layer ) 210 | def add_dropout_layer(self, items): 211 | prev_blob = self.layer.top[0] 212 | lname = "drop"+str(self.lnum) 213 | self.layer = CaffeDropoutLayer( lname, items['probability'] ) 214 | self.layer.bottom.append( prev_blob ) 215 | self.layer.top.append( prev_blob ) # loopback 216 | self.add_layer( self.layer ) 217 | def add_softmax_layer(self, items): 218 | prev_blob = self.layer.top[0] 219 | lname = "prob" 220 | self.layer = CaffeSoftmaxLayer( lname ) 221 | self.layer.bottom.append( prev_blob ) 222 | self.layer.top.append( lname ) 223 | self.add_layer( self.layer ) 224 | def finalize(self, name): 225 | self.layer.top[0] = name # replace 226 | def write(self, fname): 227 | with open(fname, 'w') as f: 228 | f.write('name: "{}"'.format(self.name)) 229 | for sec in self.sections: 230 | sec.write(f) 231 | logging.info('{} is generated'.format(fname)) 232 | 233 | ###################################################################33 234 | class uniqdict(OrderedDict): 235 | _unique = 0 236 | def __setitem__(self, key, val): 237 | if isinstance(val, OrderedDict): 238 | self._unique += 1 239 | key += "_"+str(self._unique) 240 | OrderedDict.__setitem__(self, key, val) 241 | 242 | def convert(cfgfile, ptxtfile): 243 | # 244 | parser = ConfigParser(dict_type=uniqdict) 245 | parser.read(cfgfile) 246 | netname = os.path.basename(cfgfile).split('.')[0] 247 | #print netname 248 | gen = CaffeProtoGenerator(netname) 249 | for section in parser.sections(): 250 | _section = section.split('_')[0] 251 | if _section in ["crop", "cost"]: 252 | continue 253 | # 254 | batchnorm_followed = False 255 | relu_followed = False 256 | items = dict(parser.items(section)) 257 | if 'batch_normalize' in items and items['batch_normalize']: 258 | batchnorm_followed = True 259 | if 'activation' in items and items['activation'] != 'linear': 260 | relu_followed = True 261 | # 262 | if _section == 'net': 263 | gen.add_input_layer(items) 264 | elif _section == 'convolutional': 265 | gen.add_convolution_layer(items) 266 | if batchnorm_followed: 267 | gen.add_batchnorm_layer(items) 268 | gen.add_scale_layer(items) 269 | if relu_followed: 270 | gen.add_relu_layer(items) 271 | elif _section == 'connected': 272 | gen.add_innerproduct_layer(items) 273 | if relu_followed: 274 | gen.add_relu_layer(items) 275 | elif _section == 'maxpool': 276 | gen.add_pooling_layer('MAX', items) 277 | elif _section == 'avgpool': 278 | gen.add_pooling_layer('AVE', items, global_pooling=True) 279 | elif _section == 'dropout': 280 | gen.add_dropout_layer(items) 281 | elif _section == 'softmax': 282 | gen.add_softmax_layer(items) 283 | else: 284 | logging.error("{} layer is not supported".format(_section)) 285 | gen.update_last_convolution_layer() 286 | #gen.finalize('result') 287 | gen.write(ptxtfile) 288 | 289 | def main(): 290 | parser = argparse.ArgumentParser(description='Convert YOLO cfg to Caffe prototxt') 291 | parser.add_argument('cfg', type=str, help='YOLO cfg') 292 | parser.add_argument('prototxt', type=str, help='Caffe prototxt') 293 | args = parser.parse_args() 294 | 295 | convert(args.cfg, args.prototxt) 296 | 297 | if __name__ == "__main__": 298 | main() 299 | 300 | # vim:sw=4:ts=4:et 301 | -------------------------------------------------------------------------------- /models/prepareModel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xvf 2 | # 3 | # Copyright (c) 2018. All rights reserved. 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | 24 | hash curl 2>/dev/null || 25 | { 26 | echo >&2 "ERROR: Curl is required to download yolo model. Install curl with command below and try again." 27 | echo >&2 "sudo apt-get install curl" 28 | exit 1; 29 | } 30 | 31 | hash python 2>/dev/null || 32 | { 33 | echo >&2 "ERROR: Python is required to run traffic Vision app. Install python with command below and try again." 34 | echo >&2 "sudo apt-get install python" 35 | exit 1; 36 | } 37 | 38 | hash runvx 2>/dev/null || 39 | { 40 | echo >&2 "ERROR: MIVisionX is required to run traffic Vision app. Install MIVisionX and try again." 41 | echo >&2 "git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX" 42 | exit 1; 43 | } 44 | 45 | 46 | filename=yolov2-tiny-voc 47 | 48 | ################################################### 49 | # Step 1: Download yolo model 50 | ################################################### 51 | yolocfg=yolomodels/$filename.cfg 52 | yoloweight=yolomodels/$filename.weights 53 | 54 | if [[ -f $yolocfg && -f $yoloweight ]]; then 55 | echo "Found yolo model: $yolocfg , $yoloweight" 56 | else 57 | echo "Downloading yolo model: $yolocfg , $yoloweight" 58 | mkdir -p yolomodels && cd yolomodels 59 | curl https://pjreddie.com/media/files/yolov2-tiny-voc.weights -o yolov2-tiny-voc.weights 60 | curl https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2-tiny-voc.cfg -o yolov2-tiny-voc.cfg 61 | cd .. 62 | fi 63 | 64 | ################################################### 65 | # Step 2: Convert to yolo model to caffe model 66 | ################################################### 67 | 68 | mkdir -p caffemodels 69 | 70 | yolocfgcaffe=caffemodels/$filename.prototxt 71 | yoloweightcaffe=caffemodels/$filename.caffemodel 72 | 73 | if [[ -f $yolocfg && -f $yoloweight ]]; then 74 | echo 75 | #echo "Found yolo model: $yolocfg , $yoloweight" 76 | else 77 | echo "Could not locate/download yolo model: $yolocfg , $yoloweight" 78 | exit 1 79 | fi 80 | 81 | echo "Convert yolo to caffe" 82 | python create_yolo_prototxt.py $yolocfg $yolocfgcaffe 83 | python create_yolo_caffemodel.py -m $yolocfgcaffe -w $yoloweight -o $yoloweightcaffe 84 | 85 | 86 | ################################################### 87 | # Step 3: Convert caffe model to NNIR format 88 | ################################################### 89 | mkdir -p nnirModel 90 | python /opt/rocm/mivisionx/model_compiler/python/caffe_to_nnir.py caffemodels/$filename.caffemodel ./nnirModel --input-dims 1,3,416,416 91 | 92 | ################################################### 93 | # Step 4: Convert NNIR to OpenVX format 94 | ################################################### 95 | python /opt/rocm/mivisionx/model_compiler/python/nnir_to_openvx.py ./nnirModel ./openVXModel 96 | 97 | ################################################### 98 | # Step 5: Compile openVX model 99 | ################################################### 100 | cd ./openVXModel 101 | mkdir -p build && cd build 102 | cmake .. 103 | make 104 | cd ../.. 105 | cp ./openVXModel/build/libannmodule.so ../lib 106 | cp ./openVXModel/build/libannpython.so ../lib 107 | -------------------------------------------------------------------------------- /speedEstimator.py: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | # Copyright (c) 2018. All rights reserved. 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | 14 | class estimate: 15 | def __init__(self): 16 | self.speed_limit = 35.0; # miles/hr 17 | self.fps = 30 18 | self.y_range = (160,232) 19 | #self.y_range = (194,200) 20 | #self.y_range = (0,1080) 21 | self.caliberation_factor = 10.0; # specific to camera angle/distance 22 | self.scale_factors = []; # to compute speed beyond y-range. 23 | self.last_boxes = [] 24 | 25 | 26 | # caliberated for media/VID_20180709_111331.mp4 27 | def mph(self, vehicle_center, pix_dist, vdir): 28 | expected_dist = pix_dist; 29 | if ( vdir == "down" ): 30 | expected_dist = 0.25*(vehicle_center-400) 31 | if ( vdir == "up" ): 32 | expected_dist = -0.18*(vehicle_center-580) 33 | 34 | return int(self.speed_limit * (pix_dist/expected_dist)) 35 | 36 | def color(self, speed): 37 | red = 0 38 | green = 255 39 | if ( speed > 2*self.speed_limit ): 40 | red = 255; 41 | green = 0; 42 | elif ( speed > self.speed_limit ): 43 | red = 255 44 | green = 255-int(255*((speed-self.speed_limit)/self.speed_limit)) 45 | return (0, green, red) 46 | 47 | 48 | def speed(self, boxes): 49 | # calibration video = media/VID_20180709_111331.mp4 50 | 51 | if ( len(self.last_boxes) == 0 ): 52 | self.last_boxes = boxes 53 | 54 | new_boxes = [] 55 | for box, last_box in zip(boxes, self.last_boxes): 56 | box_speed = box 57 | box_speed.append(-1.0); # unknown speed 58 | box_speed.append("unknown"); # unknown direction 59 | box_center_y = (box[1] + box[3])/2 60 | if ( last_box != box and 61 | ((box_center_y > self.y_range[0] and box_center_y < self.y_range[1]) or 62 | (box_center_y > self.y_range[0] and box_center_y < self.y_range[1]) ) 63 | ): 64 | 65 | if ( box[3] > last_box[3] ): 66 | pix_dir = "down" 67 | else: 68 | pix_dir = "up" 69 | 70 | pix_distance = last_box[3]-box[3] 71 | 72 | box_speed[7] = self.mph(box_center_y, pix_distance, pix_dir); 73 | box_speed[8] = pix_dir 74 | 75 | new_boxes.append(box_speed) 76 | 77 | last_boxes = new_boxes 78 | return new_boxes 79 | 80 | -------------------------------------------------------------------------------- /test_ipcam.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright (c) 2018. All rights reserved. 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in 13 | 14 | import sys, os 15 | import cv2 16 | 17 | if ( len(sys.argv) < 2 ): 18 | print "\nUsage : ", sys.argv[0], "\n" 19 | print "examples: ", sys.argv[0], "\'http://166.149.104.112:8082/snap.jpg\'" 20 | print " ", sys.argv[0], "\'http://177.72.7.85:8001/mjpg/video.mjpg\'\n" 21 | exit(0); 22 | 23 | cap = cv2.VideoCapture(sys.argv[1]) 24 | if ( cap.isOpened() ): 25 | print "opened stream" 26 | else: 27 | print "failed to open stream" 28 | exit(0) 29 | 30 | cv2.namedWindow (sys.argv[1], cv2.WINDOW_GUI_EXPANDED) 31 | cv2.resizeWindow(sys.argv[1], 480, 270) 32 | 33 | while(cap.isOpened()): 34 | ret, frame = cap.read() 35 | if ret == False: 36 | break 37 | 38 | cv2.imshow(sys.argv[1], frame) 39 | 40 | key = cv2.waitKey(1) 41 | if ( os.path.splitext(sys.argv[1])[1] == ".jpg" ) : 42 | cap = cv2.VideoCapture(sys.argv[1]); # required for static jpg like stream in Bosch cams 43 | if key & 0xFF == ord('q'): 44 | break 45 | 46 | -------------------------------------------------------------------------------- /yoloOpenVX/Makefile: -------------------------------------------------------------------------------- 1 | ROOT=$(shell dirname $(shell pwd)) 2 | include ../Makefile.vars 3 | 4 | TARGET_LIB = libyoloOpenVX$(LIB_EXT) 5 | 6 | SRCS := $(wildcard *.cpp) 7 | OBJS := $(SRCS:%.cpp=%.o) 8 | 9 | .PHONY: all clean 10 | 11 | all: INSTALL 12 | 13 | INSTALL: ${TARGET_LIB} 14 | $(CP) ${TARGET_LIB} $(LIB_PATH) 15 | 16 | $(TARGET_LIB): $(OBJS) 17 | $(CC) ${CFLAGS} ${LDFLAGS} -o $@ $^ $(wildcard $(ROOT)/lib/libann*.so) 18 | @echo done compiling openVX model for yolo network. 19 | 20 | %.o : %.cpp 21 | $(CC) $(CFLAGS) $(LDFLAGS) $(INCLUDES) -c $< -o $@ 22 | 23 | clean: 24 | $(RM) *.o *~ *pyc $(TARGET_LIB) 25 | 26 | 27 | -------------------------------------------------------------------------------- /yoloOpenVX/README.md: -------------------------------------------------------------------------------- 1 | ** Python package for tiny-yolov2 neural network model. 2 | -------------------------------------------------------------------------------- /yoloOpenVX/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018. All rights reserved. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import sys, os 22 | import ctypes 23 | import numpy.ctypeslib as ctl 24 | 25 | root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 26 | weights = os.path.join(root_dir , 'models', 'openVXModel', 'weights.bin'); 27 | libdir = os.path.join(root_dir , 'lib'); 28 | graph = ctl.load_library('libannmodule', libdir) 29 | 30 | interface = ctl.load_library('libannpython', libdir); 31 | interface.annQueryInference.restype = ctypes.c_char_p 32 | interface.annQueryInference.argtypes = [] 33 | interface.annCreateInference.restype = ctypes.c_void_p 34 | interface.annCreateInference.argtypes = [ctypes.c_char_p] 35 | interface.annReleaseInference.restype = ctypes.c_int 36 | interface.annReleaseInference.argtypes = [ctypes.c_void_p] 37 | 38 | 39 | 40 | model = ctl.load_library('libyoloOpenVX', libdir) 41 | model.detectBoxes.restype = ctypes.py_object 42 | model.detectBoxes.argtypes = [ctypes.c_void_p, ctl.ndpointer(ctypes.c_float, flags="C_CONTIGUOUS")]; 43 | 44 | print ("Loaded", os.path.basename(os.path.dirname(__file__))) 45 | -------------------------------------------------------------------------------- /yoloOpenVX/region.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2018. All rights reserved. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | 23 | 24 | #pragma once 25 | 26 | #ifndef _REGION_H 27 | #define _REGION_H 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | struct ibox 38 | { 39 | float x; // x co-ordinate 40 | float y; // y co-ordinate 41 | float w; // box width 42 | float h; // box-height 43 | }; 44 | 45 | struct indexsort 46 | { 47 | int iclass; 48 | int index; 49 | int channel; 50 | float* prob; 51 | }; 52 | 53 | class Region 54 | { 55 | private: 56 | int size; 57 | int totalLength; 58 | int totalObjects; 59 | std::vector output; 60 | std::vector boxes; 61 | std::vector s; 62 | 63 | static const int numBoxesPerGrid ; 64 | static const float biases[]; 65 | static const std::string objectnames[] ; 66 | 67 | protected: 68 | Region(); // prohibited. 69 | 70 | static int indexsort_comparator(const void *pa, const void *pb); 71 | float logistic_activate (float x); 72 | void transpose (float *src, float* tar, int k, int n); 73 | void softmax (float *input, int n, float temp, float *output); 74 | float overlap (float x1, float w1, float x2, float w2); 75 | float box_intersection (ibox a, ibox b); 76 | float box_union (ibox a, ibox b); 77 | float box_iou (ibox a, ibox b); 78 | int max_index (float *a, int n); 79 | 80 | public: 81 | Region (int c, int h, int w, int classes); 82 | 83 | PyObject* GetDetections (float* data, int c, int h, int w, 84 | int classes, int imgw, int imgh, 85 | float thresh, float nms, 86 | int blockwd); 87 | }; 88 | 89 | #endif // _REGION_H 90 | -------------------------------------------------------------------------------- /yoloOpenVX/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018. All rights reserved. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from setuptools import setup 22 | 23 | setup(name='yoloOpenVX', 24 | version='0.1', 25 | description='OpenVX model for the yolo v2 network', 26 | url='http://github.com/srohit0/trafficVision', 27 | author='Rohit Sharma', 28 | author_email='rohit@parip@th.com', 29 | license='MIT', 30 | packages=['yoloOpenVX'], 31 | zip_safe=False) 32 | -------------------------------------------------------------------------------- /yoloOpenVX/yoloOpenVX.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2018. All rights reserved. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | 23 | #include "annpython.h" 24 | #include "region.h" 25 | #include 26 | 27 | #include 28 | 29 | #define RETURN_ON_ERROR(status, gstate) { if(status != VX_SUCCESS) { printf("ERROR: failed with status = (%d) at " __FILE__ "#%d\n", status, __LINE__); PyGILState_Release(gstate); return 0x0; } } 30 | 31 | using namespace std; 32 | 33 | struct yoloDetails { 34 | protected: 35 | vector tokenize(string info_str, char delim) 36 | { 37 | string token; 38 | vector tokens; 39 | stringstream info_stream(info_str); 40 | 41 | // Tokenizing by comma char ',' 42 | while(getline(info_stream, token, delim)) 43 | tokens.push_back(token); 44 | 45 | return tokens; 46 | } 47 | public: 48 | int ni, ci, hi, wi; // network input parameters. 49 | int numBoxesParams, maxBoxWidth, maxBoxHeight; // network output parameters. 50 | int targetBlockwd, classes ; 51 | float threshold, nms ; 52 | 53 | yoloDetails() 54 | { 55 | const char* info_cstr = annQueryInference(); 56 | vector yolo_terminal_info = tokenize(info_cstr, ';'); 57 | vector yolo_inputs = tokenize(yolo_terminal_info[0], ','); 58 | vector yolo_outputs = tokenize(yolo_terminal_info[1], ','); 59 | 60 | istringstream(yolo_inputs[2]) >> ni ; // numInputs = 1 61 | istringstream(yolo_inputs[3]) >> ci ; // channelInputs = 3 62 | istringstream(yolo_inputs[4]) >> hi ; // heightImage = 416 63 | istringstream(yolo_inputs[5]) >> wi ; // widthImage = 416 64 | 65 | istringstream(yolo_outputs[3]) >> numBoxesParams ; // 125 66 | istringstream(yolo_outputs[4]) >> maxBoxWidth ; // 12 67 | istringstream(yolo_outputs[5]) >> maxBoxHeight ; // 12 68 | 69 | targetBlockwd = 13 ; // yolo grid size 70 | classes = 20 ; // yolo number of classes 71 | threshold = 0.18 ; // yolo confidence threshold 72 | nms = 0.4 ; // yolo non-maximum suppression threshold 73 | } 74 | }; 75 | 76 | static yoloDetails yoloInfo ; 77 | 78 | extern "C" PyObject* detectBoxes(pyif_ann_handle handle, float* image) { 79 | 80 | // 1. runInference 81 | // 1a. Check for graph 82 | if ( !handle ) 83 | return 0x0 ; 84 | 85 | PyGILState_STATE gstate = PyGILState_Ensure(); 86 | 87 | size_t inp_size = yoloInfo.hi * yoloInfo.wi * yoloInfo.ci * sizeof(float); 88 | bool is_nhwc=0; 89 | // 1b. Prepare input for graph 90 | vx_status status = annCopyToInferenceInput(handle, image, inp_size, is_nhwc); 91 | RETURN_ON_ERROR(status, gstate) 92 | 93 | // 1c. Process bipertite graph consisting of a set of Nodes and a set of data objects 94 | status = annRunInference(handle, 1); 95 | RETURN_ON_ERROR(status, gstate) 96 | 97 | // 1d. Extract results from the graph outputs. 98 | size_t out_size = yoloInfo.numBoxesParams * yoloInfo.maxBoxWidth * yoloInfo.maxBoxHeight * sizeof(float) ; 99 | static float * out_ptr = (float*) malloc(out_size); 100 | status = annCopyFromInferenceOutput(handle, out_ptr, out_size); 101 | RETURN_ON_ERROR(status, gstate) 102 | 103 | 104 | // 2. DetectBoxes 105 | Region box(yoloInfo.numBoxesParams, yoloInfo.maxBoxHeight, yoloInfo.maxBoxWidth, yoloInfo.classes); 106 | PyObject* objects = box.GetDetections(out_ptr, yoloInfo.numBoxesParams, 107 | yoloInfo.maxBoxHeight, yoloInfo.maxBoxWidth, 108 | yoloInfo.classes, yoloInfo.wi, yoloInfo.hi, 109 | yoloInfo.threshold, yoloInfo.nms, yoloInfo.targetBlockwd); 110 | 111 | PyGILState_Release(gstate); 112 | return objects; 113 | } 114 | 115 | --------------------------------------------------------------------------------