├── .gitattributes ├── .history ├── CMakeLists_20220430100629.txt ├── CMakeLists_20220430195340.txt ├── README_20220428220610.md ├── README_20220430202803.md ├── README_20220430203109.md ├── README_20220430203213.md ├── README_20220430203750.md ├── README_20220430210646.md ├── README_20220430213513.md └── src │ ├── detection_20220430115235.cc │ └── detection_20220430213512.cc ├── .vscode └── settings.json ├── CMakeLists.txt ├── README.md ├── build-linux_RK3588.sh ├── include ├── Hungarian.h ├── KalmanTracker.h └── global.h ├── model ├── PRC_9resize.mp4 ├── labels.txt └── test.rknn └── src ├── Hungarian.cc ├── KalmanTracker.cc ├── detection.cc ├── global.cc ├── main.cc ├── postprocess.cc ├── trackprocess.cc └── video_io.cc /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.history/CMakeLists_20220430100629.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yxhuang7538/rknn_yolov5_3588_hyx/d15899ec61cf29d3b2a727bcd410b1ea23f0d032/.history/CMakeLists_20220430100629.txt -------------------------------------------------------------------------------- /.history/CMakeLists_20220430195340.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.4.1) 2 | 3 | project(rknn_yolov5_3588_hyx) 4 | 5 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,--allow-shlib-undefined") 6 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wl,--allow-shlib-undefined") 7 | 8 | # install target and libraries 9 | set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install/rknn_yolov5_3588_hyx_${CMAKE_SYSTEM_NAME}) 10 | 11 | set(CMAKE_SKIP_INSTALL_RPATH FALSE) 12 | set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) 13 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") 14 | 15 | # rknn api 16 | if(TARGET_SOC STREQUAL "rk356x") 17 | set(RKNN_API_PATH ${CMAKE_SOURCE_DIR}/../../runtime/RK356X/${CMAKE_SYSTEM_NAME}/librknn_api) 18 | elseif(TARGET_SOC STREQUAL "rk3588") 19 | set(RKNN_API_PATH ${CMAKE_SOURCE_DIR}/../../runtime/RK3588/${CMAKE_SYSTEM_NAME}/librknn_api) 20 | else() 21 | message(FATAL_ERROR "TARGET_SOC is not set, ref value: rk356x or rk3588") 22 | endif() 23 | 24 | if (CMAKE_SYSTEM_NAME STREQUAL "Android") 25 | set(RKNN_RT_LIB ${RKNN_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librknnrt.so) 26 | else() 27 | if (CMAKE_C_COMPILER MATCHES "aarch64") 28 | set(LIB_ARCH aarch64) 29 | else() 30 | set(LIB_ARCH armhf) 31 | endif() 32 | set(RKNN_RT_LIB ${RKNN_API_PATH}/${LIB_ARCH}/librknnrt.so) 33 | endif() 34 | include_directories(${RKNN_API_PATH}/include) 35 | 36 | # opencv 37 | if (CMAKE_SYSTEM_NAME STREQUAL "Android") 38 | set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/OpenCV-android-sdk/sdk/native/jni/abi-${CMAKE_ANDROID_ARCH_ABI}) 39 | else() 40 | if(LIB_ARCH STREQUAL "armhf") 41 | set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/opencv-linux-armhf/share/OpenCV) 42 | else() 43 | set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/opencv-linux-aarch64/share/OpenCV) 44 | endif() 45 | endif() 46 | find_package(OpenCV REQUIRED) 47 | 48 | #rga 49 | if(TARGET_SOC STREQUAL "rk356x") 50 | set(RGA_PATH ${CMAKE_SOURCE_DIR}/../3rdparty/rga/RK356X) 51 | elseif(TARGET_SOC STREQUAL "rk3588") 52 | set(RGA_PATH ${CMAKE_SOURCE_DIR}/../3rdparty/rga/RK3588) 53 | else() 54 | message(FATAL_ERROR "TARGET_SOC is not set, ref value: rk356x or rk3588") 55 | endif() 56 | if (CMAKE_SYSTEM_NAME STREQUAL "Android") 57 | set(RGA_LIB ${RGA_PATH}/lib/Android/${CMAKE_ANDROID_ARCH_ABI}/librga.so) 58 | else() 59 | if (CMAKE_C_COMPILER MATCHES "aarch64") 60 | set(LIB_ARCH aarch64) 61 | else() 62 | set(LIB_ARCH armhf) 63 | endif() 64 | set(RGA_LIB ${RGA_PATH}/lib/Linux//${LIB_ARCH}/librga.so) 65 | endif() 66 | include_directories( ${RGA_PATH}/include) 67 | 68 | 69 | 70 | # rknn_yolov5_3588_hyx 71 | include_directories( ${CMAKE_SOURCE_DIR}/include) 72 | add_executable(rknn_yolov5_3588_hyx 73 | src/main.cc 74 | src/postprocess.cc 75 | src/detection.cc 76 | src/video_io.cc 77 | src/trackprocess.cc 78 | ) 79 | 80 | target_link_libraries(rknn_yolov5_3588_hyx 81 | ${RKNN_RT_LIB} 82 | ${RGA_LIB} 83 | ${OpenCV_LIBS} 84 | ) 85 | 86 | install(TARGETS rknn_yolov5_3588_hyx DESTINATION ./) 87 | install(PROGRAMS ${RKNN_RT_LIB} DESTINATION lib) 88 | install(DIRECTORY model DESTINATION ./) 89 | -------------------------------------------------------------------------------- /.history/README_20220428220610.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yxhuang7538/rknn_yolov5_3588_hyx/d15899ec61cf29d3b2a727bcd410b1ea23f0d032/.history/README_20220428220610.md -------------------------------------------------------------------------------- /.history/README_20220430202803.md: -------------------------------------------------------------------------------- 1 | # RK3588上部署YOLOv5-DeepSORT红外目标跟踪模型 2 | 3 | ## 1、结构 4 | - include 5 | - global.h 6 | 声明了所需的全局变量，以及需要使用的全局函数。 7 | 8 | - model 9 | - labels.txt 存放目标类别 10 | - xxx.mp4 需要检测的视频 11 | - xxx.rknn 需要用到的rknn模型 12 | 13 | - src 14 | - detection.cc 实现目标检测 15 | - global.cc 定义全局变量 16 | - main.cc 主函数 17 | - postprocess.cc 实现检测后处理 18 | - trackprocess.cc 实现目标跟踪 19 | - video_io.cc 实现视频读取和存储 20 | 21 | ## 2、快速应用 22 | 23 | 24 | ``` 25 | git clone https://github.com/yxhuang7538/rknn_yolov5_3588_hyx.git 26 | 27 | ``` 28 | 29 | ## 3、进度 30 | - [ ] 实现目标检测 31 | - [ ] 实现目标跟踪 32 | - [ ] 优化目标检测 33 | - [ ] 优化目标跟踪 34 | - [ ] 提高帧率 35 | - [ ] 使用蒸馏模型 36 | - [ ] 去掉Focus层 37 | -------------------------------------------------------------------------------- /.history/README_20220430203109.md: -------------------------------------------------------------------------------- 1 | # RK3588上部署YOLOv5-DeepSORT红外目标跟踪模型 2 | 3 | ## 1、结构 4 | - include 5 | - global.h 6 | 声明了所需的全局变量，以及需要使用的全局函数。 7 | 8 | - model 9 | - labels.txt 存放目标类别 10 | - xxx.mp4 需要检测的视频 11 | - xxx.rknn 需要用到的rknn模型 12 | 13 | - src 14 | - detection.cc 实现目标检测 15 | - global.cc 定义全局变量 16 | - main.cc 主函数 17 | - postprocess.cc 实现检测后处理 18 | - trackprocess.cc 实现目标跟踪 19 | - video_io.cc 实现视频读取和存储 20 | 21 | ## 2、快速应用 22 | 23 | 24 | ``` 25 | git clone https://github.com/yxhuang7538/rknn_yolov5_3588_hyx.git 26 | 27 | ``` 28 | 29 | ## 3、进度 30 | - [x] 实现目标检测 31 | - [ ] 实现目标跟踪 32 | - [ ] 优化目标检测 33 | - [ ] 优化目标跟踪 34 | - [ ] 提高帧率 35 | - [ ] 使用蒸馏模型 36 | - [ ] 去掉Focus层 37 | -------------------------------------------------------------------------------- /.history/README_20220430203213.md: -------------------------------------------------------------------------------- 1 | # RK3588上部署YOLOv5-DeepSORT红外目标跟踪模型 2 | 需要配合rknpu2来使用 3 | 4 | ## 1、结构 5 | - include 6 | - global.h 7 | 声明了所需的全局变量，以及需要使用的全局函数。 8 | 9 | - model 10 | - labels.txt 存放目标类别 11 | - xxx.mp4 需要检测的视频 12 | - xxx.rknn 需要用到的rknn模型 13 | 14 | - src 15 | - detection.cc 实现目标检测 16 | - global.cc 定义全局变量 17 | - main.cc 主函数 18 | - postprocess.cc 实现检测后处理 19 | - trackprocess.cc 实现目标跟踪 20 | - video_io.cc 实现视频读取和存储 21 | 22 | ## 2、快速应用 23 | 24 | 25 | ``` 26 | git clone https://github.com/yxhuang7538/rknn_yolov5_3588_hyx.git 27 | 28 | ``` 29 | 30 | ## 3、进度 31 | - [x] 实现目标检测 32 | - [ ] 实现目标跟踪 33 | - [ ] 优化目标检测 34 | - [ ] 优化目标跟踪 35 | - [ ] 提高帧率 36 | - [ ] 使用蒸馏模型 37 | - [ ] 去掉Focus层 38 | -------------------------------------------------------------------------------- /.history/README_20220430203750.md: -------------------------------------------------------------------------------- 1 | # RK3588上部署YOLOv5-DeepSORT红外目标跟踪模型 2 | 需要配合rknpu2来使用 3 | 4 | ## 1、结构 5 | - include 6 | - global.h 7 | 声明了所需的全局变量，以及需要使用的全局函数。 8 | 9 | - model 10 | - labels.txt 存放目标类别 11 | - xxx.mp4 需要检测的视频 12 | - xxx.rknn 需要用到的rknn模型 13 | 14 | - src 15 | - detection.cc 实现目标检测 16 | - global.cc 定义全局变量 17 | - main.cc 主函数 18 | - postprocess.cc 实现检测后处理 19 | - trackprocess.cc 实现目标跟踪 20 | - video_io.cc 实现视频读取和存储 21 | 22 | ## 2、快速应用 23 | 24 | 25 | ``` 26 | git clone https://github.com/yxhuang7538/rknn_yolov5_3588_hyx.git 27 | 28 | ``` 29 | 30 | ## 3、进度 31 | - [x] 实现目标检测 32 | - [ ] 实现目标跟踪 33 | - [ ] 优化目标检测 34 | - [ ] 优化目标跟踪 35 | - [ ] 修复Resize问题 36 | - [ ] 提高帧率 37 | - [ ] 使用蒸馏模型 38 | - [ ] 去掉Focus层 39 | -------------------------------------------------------------------------------- /.history/README_20220430210646.md: -------------------------------------------------------------------------------- 1 | # RK3588上部署YOLOv5-DeepSORT红外目标跟踪模型 2 | 3 | ## 1、结构 4 | - include 5 | - global.h 6 | 声明了所需的全局变量，以及需要使用的全局函数。 7 | 8 | - model 9 | - labels.txt 存放目标类别 10 | - xxx.mp4 需要检测的视频 11 | - xxx.rknn 需要用到的rknn模型 12 | 13 | - src 14 | - detection.cc 实现目标检测 15 | - global.cc 定义全局变量 16 | - main.cc 主函数 17 | - postprocess.cc 实现检测后处理 18 | - trackprocess.cc 实现目标跟踪 19 | - video_io.cc 实现视频读取和存储 20 | 21 | ## 2、快速应用 22 | ### 2.1 前期准备 23 | 首先需要用RKNN-Toolkit2工具将训练模型转为RKNN模型。 24 | 25 | 得到转换模型后，可以选择rknpu2提供的接口在RK平台进行开发应用。 26 | 27 | 需要准备的库文件 28 | - RKNN API : rknpu2/runtime/librknnrt.so 29 | 30 | ``` 31 | git clone https://github.com/yxhuang7538/rknn_yolov5_3588_hyx.git 32 | 33 | ``` 34 | 35 | ## 3、进度 36 | - [ ] 采用零拷贝API接口框架 37 | - [ ] 采用通用API接口框架 38 | - [x] 实现目标检测 39 | - [ ] 实现目标跟踪 40 | - [ ] 优化目标检测 41 | - [ ] 优化目标跟踪 42 | - [ ] 修复Resize问题 43 | - [ ] 提高帧率 44 | - [ ] 使用蒸馏模型 45 | - [ ] 去掉Focus层 46 | 47 | ## 4、参考 48 | 1. https://github.com/rockchip-linux/rknpu2 49 | -------------------------------------------------------------------------------- /.history/README_20220430213513.md: -------------------------------------------------------------------------------- 1 | # RK3588上部署YOLOv5-DeepSORT红外目标跟踪模型 2 | 3 | ## 1、结构 4 | - include 5 | - global.h 6 | 声明了所需的全局变量，以及需要使用的全局函数。 7 | 8 | - model 9 | - labels.txt 存放目标类别 10 | - xxx.mp4 需要检测的视频 11 | - xxx.rknn 需要用到的rknn模型 12 | 13 | - src 14 | - detection.cc 实现目标检测 15 | - global.cc 定义全局变量 16 | - main.cc 主函数 17 | - postprocess.cc 实现检测后处理 18 | - trackprocess.cc 实现目标跟踪 19 | - video_io.cc 实现视频读取和存储 20 | 21 | ## 2、快速应用 22 | ### 2.1 前期准备 23 | 首先需要用RKNN-Toolkit2工具将训练模型转为RKNN模型。 24 | 25 | 得到转换模型后，可以选择rknpu2提供的接口在RK平台进行开发应用。 26 | 27 | 需要准备的库文件 28 | - RKNN API : rknpu2/runtime/librknnrt.so 29 | 30 | ``` 31 | git clone https://github.com/yxhuang7538/rknn_yolov5_3588_hyx.git 32 | 33 | ``` 34 | 35 | ## 3、进度 36 | - [ ] 采用零拷贝API接口框架 37 | - [ ] 采用通用API接口框架 38 | - [x] 实现目标检测 39 | - [ ] 实现目标跟踪 40 | - [ ] 优化目标检测 41 | - [ ] 优化目标跟踪 42 | - [ ] 修复Resize问题 43 | - [ ] 提高帧率 44 | - [ ] 使用蒸馏模型 45 | - [ ] 去掉Focus层 46 | - [x] int8量化 47 | 48 | ## 4、参考 49 | 1. https://github.com/rockchip-linux/rknpu2 50 | -------------------------------------------------------------------------------- /.history/src/detection_20220430115235.cc: -------------------------------------------------------------------------------- 1 | // 检测 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include // 线程库 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "rknn_api.h" 22 | // #include "rknn_api_1808.h" 23 | #include "im2d.h" 24 | #include "RgaUtils.h" 25 | #include "rga.h" 26 | #include "opencv2/core/core.hpp" 27 | #include "opencv2/imgproc/imgproc.hpp" 28 | #include "opencv2/highgui/highgui.hpp" 29 | #include "opencv2/videoio.hpp" 30 | #include "opencv2/video.hpp" 31 | 32 | #include "global.h" 33 | 34 | using namespace std; 35 | 36 | static void dump_tensor_attr(rknn_tensor_attr *attr) 37 | { 38 | printf(" index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, " 39 | "zp=%d, scale=%f\n", 40 | attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3], 41 | attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type), 42 | get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale); 43 | } 44 | 45 | double __get_us(struct timeval t) { return (t.tv_sec * 1000000 + t.tv_usec); } 46 | 47 | static unsigned char *load_data(FILE *fp, size_t ofst, size_t sz) 48 | { 49 | unsigned char *data; 50 | int ret; 51 | 52 | data = NULL; 53 | 54 | if (NULL == fp) 55 | { 56 | return NULL; 57 | } 58 | 59 | ret = fseek(fp, ofst, SEEK_SET); 60 | if (ret != 0) 61 | { 62 | printf("blob seek failure.\n"); 63 | return NULL; 64 | } 65 | 66 | data = (unsigned char *)malloc(sz); 67 | if (data == NULL) 68 | { 69 | printf("buffer malloc failure.\n"); 70 | return NULL; 71 | } 72 | ret = fread(data, 1, sz, fp); 73 | return data; 74 | } 75 | 76 | static unsigned char *load_model(const char *filename, int *model_size) 77 | { 78 | 79 | FILE *fp; 80 | unsigned char *data; 81 | 82 | fp = fopen(filename, "rb"); 83 | if (NULL == fp) 84 | { 85 | printf("Open file %s failed.\n", filename); 86 | return NULL; 87 | } 88 | 89 | fseek(fp, 0, SEEK_END); 90 | int size = ftell(fp); 91 | 92 | data = load_data(fp, 0, size); 93 | 94 | fclose(fp); 95 | 96 | *model_size = size; 97 | return data; 98 | } 99 | 100 | int detection_process(const char *model_name, int thread_id, int cpuid) 101 | { 102 | /* 103 | model_path : rknn模型位置 104 | thread_id : 进程号 105 | */ 106 | cpu_set_t mask; 107 | CPU_ZERO(&mask); 108 | CPU_SET(cpuid, &mask); // 绑定cpu 109 | 110 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 111 | cerr << "set thread affinity failed" << endl; // 绑定失败 112 | 113 | cout << "NPU进程" << thread_id << "使用 CPU " << cpuid << endl; 114 | 115 | int status = 0; 116 | rknn_context ctx; 117 | size_t actual_size = 0; 118 | int img_width = 0; 119 | int img_height = 0; 120 | int img_channel = 0; 121 | const float nms_threshold = NMS_THRESH; 122 | const float box_conf_threshold = BOX_THRESH; 123 | struct timeval start_time, stop_time; 124 | int ret; 125 | 126 | // init rga context 127 | rga_buffer_t src; 128 | rga_buffer_t dst; 129 | im_rect src_rect; 130 | im_rect dst_rect; 131 | memset(&src_rect, 0, sizeof(src_rect)); 132 | memset(&dst_rect, 0, sizeof(dst_rect)); 133 | memset(&src, 0, sizeof(src)); 134 | memset(&dst, 0, sizeof(dst)); 135 | 136 | // rknn设置 137 | int model_data_size = 0; 138 | unsigned char *model_data = load_model(model_name, &model_data_size); 139 | //ret = rknn_init(&ctx, model_data, model_data_size, 0, NULL); 140 | ret = rknn_init(&ctx, model_data, model_data_size, RKNN_FLAG_COLLECT_PERF_MASK, NULL); 141 | 142 | if (ret < 0) 143 | { 144 | printf("rknn_init error ret=%d\n", ret); 145 | return -1; 146 | } 147 | 148 | rknn_sdk_version version; 149 | ret = rknn_query(ctx, RKNN_QUERY_SDK_VERSION, &version, 150 | sizeof(rknn_sdk_version)); 151 | if (ret < 0) 152 | { 153 | printf("rknn_init error ret=%d\n", ret); 154 | return -1; 155 | } 156 | printf("sdk version: %s driver version: %s\n", version.api_version, 157 | version.drv_version); 158 | 159 | rknn_input_output_num io_num; 160 | ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num)); 161 | if (ret < 0) 162 | { 163 | printf("rknn_init error ret=%d\n", ret); 164 | return -1; 165 | } 166 | printf("model input num: %d, output num: %d\n", io_num.n_input, 167 | io_num.n_output); 168 | 169 | rknn_tensor_attr input_attrs[io_num.n_input]; 170 | memset(input_attrs, 0, sizeof(input_attrs)); 171 | for (int i = 0; i < io_num.n_input; i++) 172 | { 173 | input_attrs[i].index = i; 174 | ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]), 175 | sizeof(rknn_tensor_attr)); 176 | if (ret < 0) 177 | { 178 | printf("rknn_init error ret=%d\n", ret); 179 | return -1; 180 | } 181 | dump_tensor_attr(&(input_attrs[i])); 182 | } 183 | 184 | rknn_tensor_attr output_attrs[io_num.n_output]; 185 | memset(output_attrs, 0, sizeof(output_attrs)); 186 | for (int i = 0; i < io_num.n_output; i++) 187 | { 188 | output_attrs[i].index = i; 189 | ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), 190 | sizeof(rknn_tensor_attr)); 191 | dump_tensor_attr(&(output_attrs[i])); 192 | } 193 | 194 | int channel = 3; 195 | int width = 0; 196 | int height = 0; 197 | if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) 198 | { 199 | printf("model is NCHW input fmt\n"); 200 | channel = input_attrs[0].dims[1]; 201 | width = input_attrs[0].dims[2]; 202 | height = input_attrs[0].dims[3]; 203 | } 204 | else 205 | { 206 | printf("model is NHWC input fmt\n"); 207 | width = input_attrs[0].dims[1]; 208 | height = input_attrs[0].dims[2]; 209 | channel = input_attrs[0].dims[3]; 210 | } 211 | 212 | printf("model input height=%d, width=%d, channel=%d\n", height, width, 213 | channel); 214 | 215 | rknn_input inputs[1]; 216 | memset(inputs, 0, sizeof(inputs)); 217 | inputs[0].index = 0; 218 | inputs[0].type = RKNN_TENSOR_UINT8; 219 | inputs[0].size = width * height * channel; 220 | inputs[0].fmt = RKNN_TENSOR_NHWC; 221 | inputs[0].pass_through = 0; 222 | 223 | void *resize_buf = malloc(height * width * channel); 224 | 225 | // outputs set 226 | rknn_output outputs[io_num.n_output]; 227 | memset(outputs, 0, sizeof(outputs)); 228 | for (int i = 0; i < io_num.n_output; i++) 229 | { 230 | outputs[i].want_float = 0; 231 | out_scales.push_back(output_attrs[i].scale); 232 | out_zps.push_back(output_attrs[i].zp); 233 | } 234 | multi_npu_process_initialized[thread_id] = 1; 235 | printf("%d\n", multi_npu_process_initialized[thread_id]); 236 | while (1) 237 | { 238 | // 加载图片 239 | pair pairIndexImage; 240 | mtxQueueInput.lock(); 241 | // 若输入队列为空则不进入NPU_process 242 | if (queueInput.empty()) 243 | { 244 | mtxQueueInput.unlock(); 245 | usleep(1000); 246 | // 如果queue处于空且 bReading不在可读取状态则销毁 跳出 247 | if (bReading) continue; 248 | else 249 | { 250 | rknn_destroy(ctx); 251 | break; 252 | } 253 | } 254 | 255 | else 256 | { 257 | // 加载图片 258 | cout << "已缓存的图片数： " << queueInput.size() << endl; 259 | pairIndexImage = queueInput.front(); 260 | printf("Idx:%d 图在线程%d中开始处理\n", pairIndexImage.first, thread_id); 261 | queueInput.pop(); 262 | mtxQueueInput.unlock(); 263 | } 264 | cv::Mat img = pairIndexImage.second.clone(); 265 | cv::cvtColor(pairIndexImage.second, img, cv::COLOR_BGR2RGB); // 色彩空间转换 266 | img_width = img.cols; // 输入图片的宽和高 267 | img_height = img.rows; 268 | 269 | // resize 270 | src = wrapbuffer_virtualaddr((void *)img.data, img_width, img_height, RK_FORMAT_RGB_888); 271 | dst = wrapbuffer_virtualaddr((void *)resize_buf, width, height, RK_FORMAT_RGB_888); 272 | ret = imcheck(src, dst, src_rect, dst_rect); 273 | if (IM_STATUS_NOERROR != ret) 274 | { 275 | printf("%d, check error! %s", __LINE__, imStrError((IM_STATUS)ret)); 276 | return -1; 277 | } 278 | IM_STATUS STATUS = imresize(src, dst); 279 | cv::Mat resize_img(cv::Size(width, height), CV_8UC3, resize_buf); 280 | //cv::imwrite("resize_input.jpg", resize_img); 281 | inputs[0].buf = img.data; 282 | rknn_inputs_set(ctx, io_num.n_input, inputs); 283 | 284 | gettimeofday(&start_time, NULL); 285 | ret = rknn_run(ctx, NULL); // 推理 286 | gettimeofday(&stop_time, NULL); 287 | 288 | printf("once run use %f ms\n", 289 | (__get_us(stop_time) - __get_us(start_time)) / 1000); 290 | 291 | ret = rknn_outputs_get(ctx, io_num.n_output, outputs, NULL); 292 | rknn_perf_detail perf_detail; 293 | ret = rknn_query(ctx, RKNN_QUERY_PERF_DETAIL, &perf_detail, sizeof(perf_detail)); 294 | //printf("%s\n",perf_detail.perf_data); 295 | // 后处理(已加入nms) 296 | // TODO 297 | float scale_w = (float)width / img_width; 298 | float scale_h = (float)height / img_height; 299 | detect_result_group_t detect_result_group; 300 | post_process((int8_t *)outputs[0].buf, (int8_t *)outputs[1].buf, (int8_t *)outputs[2].buf, height, width, 301 | box_conf_threshold, nms_threshold, scale_w, scale_h, out_zps, out_scales, &detect_result_group); 302 | 303 | // Draw Objects 304 | char text[256]; 305 | for (int i = 0; i < detect_result_group.count; i++) 306 | { 307 | detect_result_t *det_result = &(detect_result_group.results[i]); 308 | sprintf(text, "%s %.1f%%", det_result->name, det_result->prop * 100); 309 | printf("%s @ (%d %d %d %d) %f\n", 310 | det_result->name, 311 | det_result->box.left, det_result->box.top, det_result->box.right, det_result->box.bottom, 312 | det_result->prop); 313 | int x1 = det_result->box.left; 314 | int y1 = det_result->box.top; 315 | int x2 = det_result->box.right; 316 | int y2 = det_result->box.bottom; 317 | rectangle(pairIndexImage.second, cv::Point(x1, y1), cv::Point(x2, y2), cv::Scalar(255, 0, 0, 255), 3); 318 | putText(pairIndexImage.second, text, cv::Point(x1, y1 + 12), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); 319 | } 320 | 321 | printf("[%4d/%4d] : worked/total\n", pairIndexImage.first, Frame_cnt); 322 | printf("Idx:%d 图在线程%d中处理结束\n", pairIndexImage.first, thread_id); 323 | cv::imwrite("out/out.jpg", pairIndexImage.second); 324 | //rknn_outputs_release(ctx, 3, outputs); 325 | // 将检测结果加入show序列 326 | while(idxDectImage != pairIndexImage.first) 327 | { 328 | usleep(1000); 329 | } 330 | mtxQueueShow.lock(); 331 | idxDectImage++; 332 | queueShow.push(pairIndexImage.second); 333 | mtxQueueShow.unlock(); 334 | if (idxShowImage == Frame_cnt || cv::waitKey(1) == 27) { 335 | //cv::destroyAllWindows(); 336 | bReading = false; 337 | bWriting = false; 338 | break; 339 | } 340 | 341 | } 342 | // release 343 | ret = rknn_destroy(ctx); 344 | 345 | if (model_data) 346 | { 347 | free(model_data); 348 | } 349 | 350 | if (resize_buf) 351 | { 352 | free(resize_buf); 353 | } 354 | return 0; 355 | } 356 | -------------------------------------------------------------------------------- /.history/src/detection_20220430213512.cc: -------------------------------------------------------------------------------- 1 | // 检测 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include // 获取文件属性 13 | #include // 文件夹操作 14 | #include 15 | #include 16 | #include "rknn_api.h" 17 | 18 | #include "im2d.h" 19 | #include "RgaUtils.h" 20 | #include "rga.h" 21 | #include "opencv2/core/core.hpp" 22 | #include "opencv2/imgproc/imgproc.hpp" 23 | #include "opencv2/highgui/highgui.hpp" 24 | #include "opencv2/videoio.hpp" 25 | #include "opencv2/video.hpp" 26 | 27 | #include "global.h" 28 | 29 | using namespace std; 30 | 31 | static void check_ret(int ret, char string(const char* ret_name)): 32 | { 33 | // 检查ret是否正确并输出，ret_name表示哪一步 34 | if (ret < 0) 35 | { 36 | cout << ret_name << " error ret=" << ret << endl; 37 | } 38 | 39 | } 40 | 41 | static void dump_tensor_attr(rknn_tensor_attr *attr) 42 | { 43 | // 打印模型输入和输出的信息 44 | printf(" index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, " 45 | "zp=%d, scale=%f\n", 46 | attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3], 47 | attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type), 48 | get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale); 49 | } 50 | 51 | // t为结构体，存储了时间信息：1、tv_sec 代表多少秒；2、tv_usec 代表多少微秒， 1000000 微秒 = 1秒 52 | double __get_us(struct timeval t) { return (t.tv_sec * 1000000 + t.tv_usec);} 53 | 54 | static unsigned char *load_data(FILE *fp, size_t ofst, size_t sz) 55 | { 56 | unsigned char *data; 57 | int ret; 58 | 59 | data = NULL; 60 | 61 | if (NULL == fp) 62 | { 63 | return NULL; 64 | } 65 | 66 | ret = fseek(fp, ofst, SEEK_SET); 67 | if (ret != 0) 68 | { 69 | printf("blob seek failure.\n"); 70 | return NULL; 71 | } 72 | 73 | data = (unsigned char *)malloc(sz); 74 | if (data == NULL) 75 | { 76 | printf("buffer malloc failure.\n"); 77 | return NULL; 78 | } 79 | ret = fread(data, 1, sz, fp); 80 | return data; 81 | } 82 | 83 | static unsigned char *load_model(const char *filename, int *model_size) 84 | { 85 | /* 86 | 加载rknn模型 87 | filename : rknn模型文件路径 88 | model_size : 模型的大小 89 | */ 90 | FILE *fp; 91 | unsigned char *data; 92 | 93 | fp = fopen(filename, "rb"); 94 | if (NULL == fp) 95 | { 96 | printf("Open rknn model file %s failed.\n", filename); 97 | return NULL; 98 | } 99 | 100 | fseek(fp, 0, SEEK_END); 101 | int size = ftell(fp); 102 | 103 | data = load_data(fp, 0, size); 104 | 105 | fclose(fp); 106 | 107 | *model_size = size; 108 | return data; 109 | } 110 | 111 | int detection_process(const char *model_name, int thread_id, int cpuid) 112 | { 113 | /* 114 | model_path : rknn模型位置 115 | thread_id : 进程号 116 | cpuid : 使用的cpu 117 | */ 118 | 119 | /********************绑定cpu*********************/ 120 | cpu_set_t mask; 121 | CPU_ZERO(&mask); 122 | CPU_SET(cpuid, &mask); // 绑定cpu 123 | 124 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 125 | cerr << "set thread affinity failed" << endl; // 绑定失败 126 | 127 | cout << "NPU进程" << thread_id << "使用 CPU " << cpuid << endl; 128 | 129 | /********************rknn init*********************/ 130 | string ret_name = "rknn_init" // 表示rknn的步骤名称 131 | rknn_context ctx; // 创建rknn_context对象 132 | int model_data_size = 0; // 模型的大小 133 | unsigned char *model_data = load_model(model_name, &model_data_size); // 加载RKNN模型 134 | /* 初始化参数flag 135 | RKNN_FLAG_COLLECT_PERF_MASK：用于运行时查询网络各层时间。 136 | RKNN_FLAG_MEM_ALLOC_OUTSIDE：用于表示模型输入、输出、权重、中间 tensor 内存全部由用户分配。 137 | */ 138 | int ret = rknn_init(&ctx, model_data, model_data_size, RKNN_FLAG_COLLECT_PERF_MASK, NULL); // 初始化RKNN 139 | check_ret(ret, "rknn_init") 140 | int status = 0; 141 | size_t actual_size = 0; 142 | int img_width = 0; 143 | int img_height = 0; 144 | int img_channel = 0; 145 | const float nms_threshold = NMS_THRESH; 146 | const float box_conf_threshold = BOX_THRESH; 147 | struct timeval start_time, stop_time; 148 | 149 | // init rga context 150 | rga_buffer_t src; 151 | rga_buffer_t dst; 152 | im_rect src_rect; 153 | im_rect dst_rect; 154 | memset(&src_rect, 0, sizeof(src_rect)); 155 | memset(&dst_rect, 0, sizeof(dst_rect)); 156 | memset(&src, 0, sizeof(src)); 157 | memset(&dst, 0, sizeof(dst)); 158 | 159 | // rknn设置 160 | 161 | if (ret < 0) 162 | { 163 | printf("rknn_init error ret=%d\n", ret); 164 | return -1; 165 | } 166 | 167 | rknn_sdk_version version; 168 | ret = rknn_query(ctx, RKNN_QUERY_SDK_VERSION, &version, 169 | sizeof(rknn_sdk_version)); 170 | if (ret < 0) 171 | { 172 | printf("rknn_init error ret=%d\n", ret); 173 | return -1; 174 | } 175 | printf("sdk version: %s driver version: %s\n", version.api_version, 176 | version.drv_version); 177 | 178 | rknn_input_output_num io_num; 179 | ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num)); 180 | if (ret < 0) 181 | { 182 | printf("rknn_init error ret=%d\n", ret); 183 | return -1; 184 | } 185 | printf("model input num: %d, output num: %d\n", io_num.n_input, 186 | io_num.n_output); 187 | 188 | rknn_tensor_attr input_attrs[io_num.n_input]; 189 | memset(input_attrs, 0, sizeof(input_attrs)); 190 | for (int i = 0; i < io_num.n_input; i++) 191 | { 192 | input_attrs[i].index = i; 193 | ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]), 194 | sizeof(rknn_tensor_attr)); 195 | if (ret < 0) 196 | { 197 | printf("rknn_init error ret=%d\n", ret); 198 | return -1; 199 | } 200 | dump_tensor_attr(&(input_attrs[i])); 201 | } 202 | 203 | rknn_tensor_attr output_attrs[io_num.n_output]; 204 | memset(output_attrs, 0, sizeof(output_attrs)); 205 | for (int i = 0; i < io_num.n_output; i++) 206 | { 207 | output_attrs[i].index = i; 208 | ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), 209 | sizeof(rknn_tensor_attr)); 210 | dump_tensor_attr(&(output_attrs[i])); 211 | } 212 | 213 | int channel = 3; 214 | int width = 0; 215 | int height = 0; 216 | if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) 217 | { 218 | printf("model is NCHW input fmt\n"); 219 | channel = input_attrs[0].dims[1]; 220 | width = input_attrs[0].dims[2]; 221 | height = input_attrs[0].dims[3]; 222 | } 223 | else 224 | { 225 | printf("model is NHWC input fmt\n"); 226 | width = input_attrs[0].dims[1]; 227 | height = input_attrs[0].dims[2]; 228 | channel = input_attrs[0].dims[3]; 229 | } 230 | 231 | printf("model input height=%d, width=%d, channel=%d\n", height, width, 232 | channel); 233 | 234 | rknn_input inputs[1]; 235 | memset(inputs, 0, sizeof(inputs)); 236 | inputs[0].index = 0; 237 | inputs[0].type = RKNN_TENSOR_UINT8; 238 | inputs[0].size = width * height * channel; 239 | inputs[0].fmt = RKNN_TENSOR_NHWC; 240 | inputs[0].pass_through = 0; 241 | 242 | void *resize_buf = malloc(height * width * channel); 243 | 244 | // outputs set 245 | rknn_output outputs[io_num.n_output]; 246 | memset(outputs, 0, sizeof(outputs)); 247 | for (int i = 0; i < io_num.n_output; i++) 248 | { 249 | outputs[i].want_float = 0; 250 | out_scales.push_back(output_attrs[i].scale); 251 | out_zps.push_back(output_attrs[i].zp); 252 | } 253 | multi_npu_process_initialized[thread_id] = 1; 254 | printf("%d\n", multi_npu_process_initialized[thread_id]); 255 | while (1) 256 | { 257 | // 加载图片 258 | pair pairIndexImage; 259 | mtxQueueInput.lock(); 260 | // 若输入队列为空则不进入NPU_process 261 | if (queueInput.empty()) 262 | { 263 | mtxQueueInput.unlock(); 264 | usleep(1000); 265 | // 如果queue处于空且 bReading不在可读取状态则销毁 跳出 266 | if (bReading) continue; 267 | else 268 | { 269 | rknn_destroy(ctx); 270 | break; 271 | } 272 | } 273 | 274 | else 275 | { 276 | // 加载图片 277 | cout << "已缓存的图片数： " << queueInput.size() << endl; 278 | pairIndexImage = queueInput.front(); 279 | printf("Idx:%d 图在线程%d中开始处理\n", pairIndexImage.first, thread_id); 280 | queueInput.pop(); 281 | mtxQueueInput.unlock(); 282 | } 283 | cv::Mat img = pairIndexImage.second.clone(); 284 | cv::cvtColor(pairIndexImage.second, img, cv::COLOR_BGR2RGB); // 色彩空间转换 285 | img_width = img.cols; // 输入图片的宽和高 286 | img_height = img.rows; 287 | 288 | // resize 289 | src = wrapbuffer_virtualaddr((void *)img.data, img_width, img_height, RK_FORMAT_RGB_888); 290 | dst = wrapbuffer_virtualaddr((void *)resize_buf, width, height, RK_FORMAT_RGB_888); 291 | ret = imcheck(src, dst, src_rect, dst_rect); 292 | if (IM_STATUS_NOERROR != ret) 293 | { 294 | printf("%d, check error! %s", __LINE__, imStrError((IM_STATUS)ret)); 295 | return -1; 296 | } 297 | IM_STATUS STATUS = imresize(src, dst); 298 | cv::Mat resize_img(cv::Size(width, height), CV_8UC3, resize_buf); 299 | //cv::imwrite("resize_input.jpg", resize_img); 300 | inputs[0].buf = img.data; 301 | rknn_inputs_set(ctx, io_num.n_input, inputs); 302 | 303 | gettimeofday(&start_time, NULL); 304 | ret = rknn_run(ctx, NULL); // 推理 305 | gettimeofday(&stop_time, NULL); 306 | 307 | printf("once run use %f ms\n", 308 | (__get_us(stop_time) - __get_us(start_time)) / 1000); 309 | 310 | ret = rknn_outputs_get(ctx, io_num.n_output, outputs, NULL); 311 | rknn_perf_detail perf_detail; 312 | ret = rknn_query(ctx, RKNN_QUERY_PERF_DETAIL, &perf_detail, sizeof(perf_detail)); 313 | //printf("%s\n",perf_detail.perf_data); 314 | // 后处理(已加入nms) 315 | // TODO 316 | float scale_w = (float)width / img_width; 317 | float scale_h = (float)height / img_height; 318 | detect_result_group_t detect_result_group; 319 | post_process((int8_t *)outputs[0].buf, (int8_t *)outputs[1].buf, (int8_t *)outputs[2].buf, height, width, 320 | box_conf_threshold, nms_threshold, scale_w, scale_h, out_zps, out_scales, &detect_result_group); 321 | 322 | // Draw Objects 323 | char text[256]; 324 | for (int i = 0; i < detect_result_group.count; i++) 325 | { 326 | detect_result_t *det_result = &(detect_result_group.results[i]); 327 | sprintf(text, "%s %.1f%%", det_result->name, det_result->prop * 100); 328 | printf("%s @ (%d %d %d %d) %f\n", 329 | det_result->name, 330 | det_result->box.left, det_result->box.top, det_result->box.right, det_result->box.bottom, 331 | det_result->prop); 332 | int x1 = det_result->box.left; 333 | int y1 = det_result->box.top; 334 | int x2 = det_result->box.right; 335 | int y2 = det_result->box.bottom; 336 | rectangle(pairIndexImage.second, cv::Point(x1, y1), cv::Point(x2, y2), cv::Scalar(255, 0, 0, 255), 3); 337 | putText(pairIndexImage.second, text, cv::Point(x1, y1 + 12), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); 338 | } 339 | 340 | printf("[%4d/%4d] : worked/total\n", pairIndexImage.first, Frame_cnt); 341 | printf("Idx:%d 图在线程%d中处理结束\n", pairIndexImage.first, thread_id); 342 | cv::imwrite("out/out.jpg", pairIndexImage.second); 343 | //rknn_outputs_release(ctx, 3, outputs); 344 | // 将检测结果加入show序列 345 | while(idxDectImage != pairIndexImage.first) 346 | { 347 | usleep(1000); 348 | } 349 | mtxQueueShow.lock(); 350 | idxDectImage++; 351 | queueShow.push(pairIndexImage.second); 352 | mtxQueueShow.unlock(); 353 | if (idxShowImage == Frame_cnt || cv::waitKey(1) == 27) { 354 | //cv::destroyAllWindows(); 355 | bReading = false; 356 | bWriting = false; 357 | break; 358 | } 359 | 360 | } 361 | // release 362 | ret = rknn_destroy(ctx); 363 | 364 | if (model_data) 365 | { 366 | free(model_data); 367 | } 368 | 369 | if (resize_buf) 370 | { 371 | free(resize_buf); 372 | } 373 | return 0; 374 | } 375 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "C_Cpp.errorSquiggles": "Disabled", 3 | "files.associations": { 4 | "iostream": "cpp", 5 | "istream": "cpp", 6 | "memory": "cpp", 7 | "tuple": "cpp", 8 | "type_traits": "cpp", 9 | "utility": "cpp", 10 | "xlocnum": "cpp", 11 | "xutility": "cpp", 12 | "queue": "cpp", 13 | "vector": "cpp", 14 | "fstream": "cpp", 15 | "atomic": "cpp", 16 | "thread": "cpp", 17 | "xstring": "cpp", 18 | "xtree": "cpp", 19 | "mutex": "cpp" 20 | }, 21 | "cmake.configureOnOpen": false 22 | } -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.4.1) 2 | 3 | project(rknn_yolov5_3588_hyx) 4 | 5 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,--allow-shlib-undefined") 6 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wl,--allow-shlib-undefined") 7 | 8 | # install target and libraries 9 | set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install/rknn_yolov5_3588_hyx_${CMAKE_SYSTEM_NAME}) 10 | 11 | set(CMAKE_SKIP_INSTALL_RPATH FALSE) 12 | set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) 13 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") 14 | 15 | # rknn api 16 | if(TARGET_SOC STREQUAL "rk356x") 17 | set(RKNN_API_PATH ${CMAKE_SOURCE_DIR}/../../runtime/RK356X/${CMAKE_SYSTEM_NAME}/librknn_api) 18 | elseif(TARGET_SOC STREQUAL "rk3588") 19 | set(RKNN_API_PATH ${CMAKE_SOURCE_DIR}/../../runtime/RK3588/${CMAKE_SYSTEM_NAME}/librknn_api) 20 | else() 21 | message(FATAL_ERROR "TARGET_SOC is not set, ref value: rk356x or rk3588") 22 | endif() 23 | 24 | if (CMAKE_SYSTEM_NAME STREQUAL "Android") 25 | set(RKNN_RT_LIB ${RKNN_API_PATH}/${CMAKE_ANDROID_ARCH_ABI}/librknnrt.so) 26 | else() 27 | #if (CMAKE_C_COMPILER MATCHES "aarch64") 28 | set(LIB_ARCH aarch64) 29 | #else() 30 | #set(LIB_ARCH armhf) 31 | #endif() 32 | set(RKNN_RT_LIB ${RKNN_API_PATH}/${LIB_ARCH}/librknnrt.so) 33 | endif() 34 | include_directories(${RKNN_API_PATH}/include) 35 | 36 | # opencv 37 | if (CMAKE_SYSTEM_NAME STREQUAL "Android") 38 | set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/OpenCV-android-sdk/sdk/native/jni/abi-${CMAKE_ANDROID_ARCH_ABI}) 39 | else() 40 | if(LIB_ARCH STREQUAL "armhf") 41 | set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/opencv-linux-armhf/share/OpenCV) 42 | else() 43 | set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/usr/local/share/opencv4) 44 | endif() 45 | endif() 46 | find_package(OpenCV REQUIRED) 47 | 48 | #rga 49 | if(TARGET_SOC STREQUAL "rk356x") 50 | set(RGA_PATH ${CMAKE_SOURCE_DIR}/../3rdparty/rga/RK356X) 51 | elseif(TARGET_SOC STREQUAL "rk3588") 52 | set(RGA_PATH ${CMAKE_SOURCE_DIR}/../3rdparty/rga/RK3588) 53 | else() 54 | message(FATAL_ERROR "TARGET_SOC is not set, ref value: rk356x or rk3588") 55 | endif() 56 | if (CMAKE_SYSTEM_NAME STREQUAL "Android") 57 | set(RGA_LIB ${RGA_PATH}/lib/Android/${CMAKE_ANDROID_ARCH_ABI}/librga.so) 58 | else() 59 | #if (CMAKE_C_COMPILER MATCHES "aarch64") 60 | set(LIB_ARCH aarch64) 61 | #else() 62 | #set(LIB_ARCH armhf) 63 | #endif() 64 | set(RGA_LIB ${RGA_PATH}/lib/Linux//${LIB_ARCH}/librga.so) 65 | endif() 66 | include_directories( ${RGA_PATH}/include) 67 | 68 | 69 | # rknn_yolov5_demo 70 | include_directories( ${CMAKE_SOURCE_DIR}/include) 71 | include_directories(${PROJECT_SOURCE_DIR}/include) 72 | add_executable(rknn_yolov5_3588_hyx 73 | src/main.cc 74 | src/global.cc 75 | src/postprocess.cc 76 | src/video_io.cc 77 | src/detection.cc 78 | src/trackprocess.cc 79 | src/KalmanTracker.cc 80 | src/Hungarian.cc 81 | ) 82 | set(link_libs pthread) 83 | 84 | target_link_libraries(rknn_yolov5_3588_hyx 85 | ${RKNN_RT_LIB} 86 | ${RGA_LIB} 87 | ${OpenCV_LIBS} 88 | ${link_libs} 89 | ) 90 | 91 | install(TARGETS rknn_yolov5_3588_hyx DESTINATION ./) 92 | install(PROGRAMS ${RKNN_RT_LIB} DESTINATION lib) 93 | install(DIRECTORY model DESTINATION ./) 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RK3588上部署YOLOv5-DeepSORT红外目标跟踪模型 2 | 3 | ## 1、结构 4 | - include 5 | - global.h 6 | 声明了所需的全局变量，以及需要使用的全局函数。 7 | 8 | - model 9 | - labels.txt 存放目标类别 10 | - xxx.mp4 需要检测的视频 11 | - xxx.rknn 需要用到的rknn模型 12 | 13 | - src 14 | - detection.cc 实现目标检测 15 | - global.cc 定义全局变量 16 | - main.cc 主函数 17 | - postprocess.cc 实现检测后处理 18 | - trackprocess.cc 实现目标跟踪 19 | - video_io.cc 实现视频读取和存储 20 | 21 | ## 2、快速应用 22 | ### 2.1 前期准备 23 | 首先需要用RKNN-Toolkit2工具将训练模型转为RKNN模型。 24 | 25 | 得到转换模型后，可以选择rknpu2提供的接口在RK平台进行开发应用。 26 | 27 | 需要准备的库文件 28 | - RKNN API : rknpu2/runtime/librknnrt.so 29 | 30 | ``` 31 | git clone https://github.com/rockchip-linux/rknpu2.git 32 | cd rknpu2/examples 33 | git clone https://github.com/yxhuang7538/rknn_yolov5_3588_hyx.git 34 | cd rknn_yolov5_3588_hyx 35 | # 修改CMakeLists.txt中你的opencv的路径 36 | # 修改build-linux_RK3588.sh中编译器路径 37 | ./build-linux_RK3588.sh 38 | cd install/rknn_yolov5_3588_hyx_linux 39 | ./rknn_yolov5_3588_hyx v ./model/xxx.rknn ./model/xxx.mp4 40 | ``` 41 | 42 | ## 3、进度 43 | - [ ] 采用零拷贝API接口框架 44 | - [x] 采用通用API接口框架 45 | - [x] 实现目标检测 46 | - [x] 实现目标跟踪 47 | - [ ] 修复Resize问题 48 | - [ ] 提高帧率 49 | - [ ] 使用蒸馏模型 50 | - [x] 去掉Focus层 51 | - [x] int8量化 52 | - [ ] 多线程权重复用 53 | 54 | ## 4、参考 55 | 1. https://github.com/rockchip-linux/rknpu2 56 | -------------------------------------------------------------------------------- /build-linux_RK3588.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | TARGET_SOC="rk3588" 4 | export TOOL_CHAIN=/usr/bin 5 | #export TOOL_CHAIN=~/opt/tool_chain/gcc-9.3.0-x86_64_aarch64-linux-gnu/host 6 | 7 | if [ x"${TOOL_CHAIN}" == x"" ] 8 | then 9 | echo "Please set TOOL_CHAIN!" 10 | exit 1 11 | fi 12 | 13 | # for aarch64 14 | #if [ -f "${TOOL_CHAIN}/bin/aarch64-linux-gcc" ] 15 | #then 16 | # GCC_COMPILER=${TOOL_CHAIN}/bin/aarch64-linux 17 | #else 18 | # GCC_COMPILER=${TOOL_CHAIN}/bin/aarch64-linux-gnu 19 | #fi 20 | 21 | export LD_LIBRARY_PATH=${TOOL_CHAIN}/lib64:$LD_LIBRARY_PATH 22 | #export CC=${GCC_COMPILER}-gcc 23 | #export CXX=${GCC_COMPILER}-g++ 24 | export CC=gcc 25 | export cXX=g++ 26 | 27 | ROOT_PWD=$( cd "$( dirname $0 )" && cd -P "$( dirname "$SOURCE" )" && pwd ) 28 | 29 | # build 30 | BUILD_DIR=${ROOT_PWD}/build/build_linux_aarch64 31 | 32 | if [[ ! -d "${BUILD_DIR}" ]]; then 33 | mkdir -p ${BUILD_DIR} 34 | fi 35 | 36 | cd ${BUILD_DIR} 37 | cmake ../.. -DCMAKE_SYSTEM_NAME=Linux -DTARGET_SOC=${TARGET_SOC} 38 | make -j4 39 | make install 40 | cd - 41 | -------------------------------------------------------------------------------- /include/Hungarian.h: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Hungarian.h: Header file for Class HungarianAlgorithm. 3 | // 4 | // This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren. 5 | // The original implementation is a few mex-functions for use in MATLAB, found here: 6 | // http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem 7 | // 8 | // Both this code and the orignal code are published under the BSD license. 9 | // by Cong Ma, 2016 10 | // 11 | 12 | #include 13 | #include 14 | 15 | using namespace std; 16 | 17 | 18 | class HungarianAlgorithm 19 | { 20 | public: 21 | HungarianAlgorithm(); 22 | ~HungarianAlgorithm(); 23 | double Solve(vector>& DistMatrix, vector& Assignment); 24 | 25 | private: 26 | void assignmentoptimal(int *assignment, double *cost, double *distMatrix, int nOfRows, int nOfColumns); 27 | void buildassignmentvector(int *assignment, bool *starMatrix, int nOfRows, int nOfColumns); 28 | void computeassignmentcost(int *assignment, double *cost, double *distMatrix, int nOfRows); 29 | void step2a(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); 30 | void step2b(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); 31 | void step3(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); 32 | void step4(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim, int row, int col); 33 | void step5(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim); 34 | }; 35 | -------------------------------------------------------------------------------- /include/KalmanTracker.h: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // KalmanTracker.h: KalmanTracker Class Declaration 3 | 4 | #ifndef KALMAN_H 5 | #define KALMAN_H 2 6 | 7 | #include 8 | #include 9 | 10 | using namespace std; 11 | 12 | #define StateType cv::Rect_ 13 | 14 | 15 | // This class represents the internel state of individual tracked objects observed as bounding box. 16 | class KalmanTracker 17 | { 18 | public: 19 | KalmanTracker() 20 | { 21 | init_kf(StateType()); 22 | m_time_since_update = 0; 23 | m_hits = 0; 24 | m_hit_streak = 0; 25 | m_age = 0; 26 | m_id = kf_count; 27 | //kf_count++; 28 | } 29 | KalmanTracker(StateType initRect) 30 | { 31 | init_kf(initRect); // Kalman Filter Init 32 | m_time_since_update = 0; 33 | m_hits = 0; 34 | m_hit_streak = 0; 35 | m_age = 0; 36 | m_id = kf_count; 37 | kf_count++; 38 | } 39 | 40 | ~KalmanTracker() 41 | { 42 | m_history.clear(); 43 | } 44 | 45 | StateType predict(); 46 | void update(StateType stateMat); 47 | 48 | StateType get_state(); 49 | StateType get_rect_xysr(float cx, float cy, float s, float r); 50 | 51 | static int kf_count; // 类的静态成员与类本身直接相关 52 | 53 | int m_time_since_update; 54 | int m_hits; 55 | int m_hit_streak; 56 | int m_age; 57 | int m_id; 58 | 59 | private: 60 | void init_kf(StateType stateMat); 61 | 62 | cv::KalmanFilter kf; 63 | cv::Mat measurement; 64 | 65 | vector m_history; 66 | }; 67 | 68 | 69 | 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /include/global.h: -------------------------------------------------------------------------------- 1 | // 声明所有需要用到的全局变量 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include // 线程库 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "rknn_api.h" 22 | // #include "rknn_api_1808.h" 23 | #include "im2d.h" 24 | #include "RgaUtils.h" 25 | #include "rga.h" 26 | #include "opencv2/core/core.hpp" 27 | #include "opencv2/imgproc/imgproc.hpp" 28 | #include "opencv2/highgui/highgui.hpp" 29 | #include "opencv2/videoio.hpp" 30 | #include "opencv2/video.hpp" 31 | 32 | #define OBJ_NAME_MAX_SIZE 16 33 | #define OBJ_NUMB_MAX_SIZE 64 34 | #define OBJ_CLASS_NUM 2 35 | #define NMS_THRESH 0.6 36 | #define BOX_THRESH 0.5 37 | #define IOU_THRESH 0.3 // 跟踪用 38 | #define PROP_BOX_SIZE (5+OBJ_CLASS_NUM) 39 | #define LABEL_NALE_TXT_PATH "./model/labels.txt" 40 | #define SAVE_PATH "output.avi" 41 | #define COLORS_NUMBER 20 // 20个随机颜色 42 | 43 | using namespace std; 44 | 45 | typedef struct _BOX_RECT 46 | { 47 | int left; 48 | int right; 49 | int top; 50 | int bottom; 51 | cv::Rect_ bbox; 52 | } BOX_RECT; // box格式 左上 右下 点坐标 53 | 54 | typedef struct __detect_result_t 55 | { 56 | char name[OBJ_NAME_MAX_SIZE]; // 物体类别名字 57 | BOX_RECT box; // 目标box 58 | float prop; // 类别概率 59 | int color; // 目标对应类别的颜色 60 | int track_id; // 跟踪的时候确定目标实例id 61 | } detect_result_t; 62 | 63 | typedef struct _detect_result_group_t // 多个检测结果组 64 | { 65 | int id; // 类别id 66 | int count; // 一张图框的总数 67 | int frame_id; // 第几帧 68 | cv::Mat img; // 原图 69 | detect_result_t results[OBJ_NUMB_MAX_SIZE]; 70 | } detect_result_group_t; 71 | 72 | class paircomp { 73 | public: 74 | bool operator()(const detect_result_group_t &n1, const detect_result_group_t &n2) const { 75 | return n1.frame_id > n2.frame_id; 76 | } 77 | }; 78 | 79 | extern mutex mtxQueueInput; // 输入队列mutex 80 | extern mutex mtxQueueOutput; // 输出队列mutex 81 | extern mutex mtxQueueShow; // 展示队列mutex 82 | extern queue> queueInput; // 输入队列 83 | extern queue queueOutput; // 输出队列 <图片> 84 | extern priority_queue, paircomp> queueShow; 85 | extern int Frame_cnt; // 帧的计数 86 | extern int Fps; // 帧率 87 | extern int Video_width; // 视频的输入宽度 88 | extern int Video_height; // 视频的输入高度 89 | 90 | extern int multi_npu_process_initialized[4]; // npu初始化完成标志，1为完成，0为未完成 91 | 92 | extern int idxInputImage; // 输入视频的帧的id 93 | extern int idxDectImage; // 要检测的下一帧id 94 | extern int idxShowImage; // 要显示的下一帧的id 95 | extern bool bReading; // flag of input 96 | extern bool bWriting; // flag of output 97 | extern double Time_video; // 整个视频(包括画图)所花的时间 98 | extern double Time_track; // 整个视频追踪所花的时间 99 | 100 | extern cv::Scalar_ randColor[COLORS_NUMBER]; //随机颜色 101 | extern cv::RNG rng; 102 | 103 | 104 | int detection_process(const char *model_name, int thread_id, int cpuid); 105 | 106 | int post_process(int8_t *input0, int8_t *input1, int8_t *input2, int model_in_h, int model_in_w, 107 | float conf_threshold, float nms_threshold, float scale_w, float scale_h, 108 | int32_t *qnt_zps, float *qnt_scales, 109 | detect_result_group_t *group); 110 | 111 | void videoRead(const char *video_path, int cpuid); 112 | 113 | void videoWrite(int cpuid); 114 | void track_process(int cpuid); 115 | double __get_us(struct timeval t); 116 | -------------------------------------------------------------------------------- /model/PRC_9resize.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yxhuang7538/rknn_yolov5_3588_hyx/d15899ec61cf29d3b2a727bcd410b1ea23f0d032/model/PRC_9resize.mp4 -------------------------------------------------------------------------------- /model/labels.txt: -------------------------------------------------------------------------------- 1 | person 2 | car -------------------------------------------------------------------------------- /model/test.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yxhuang7538/rknn_yolov5_3588_hyx/d15899ec61cf29d3b2a727bcd410b1ea23f0d032/model/test.rknn -------------------------------------------------------------------------------- /src/Hungarian.cc: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Hungarian.cpp: Implementation file for Class HungarianAlgorithm. 3 | // 4 | // This is a C++ wrapper with slight modification of a hungarian algorithm implementation by Markus Buehren. 5 | // The original implementation is a few mex-functions for use in MATLAB, found here: 6 | // http://www.mathworks.com/matlabcentral/fileexchange/6543-functions-for-the-rectangular-assignment-problem 7 | // 8 | // Both this code and the orignal code are published under the BSD license. 9 | // by Cong Ma, 2016 10 | // 11 | #include 12 | #include "Hungarian.h" 13 | //#include 14 | 15 | HungarianAlgorithm::HungarianAlgorithm(){} 16 | HungarianAlgorithm::~HungarianAlgorithm(){} 17 | 18 | 19 | //********************************************************// 20 | // A single function wra pper for solving assignment problem. 21 | //********************************************************// 22 | double HungarianAlgorithm::Solve(vector>& DistMatrix, vector& Assignment) 23 | { 24 | unsigned int nRows = DistMatrix.size(); 25 | unsigned int nCols = DistMatrix[0].size();//??? 26 | 27 | double *distMatrixIn = new double[nRows * nCols]; 28 | int *assignment = new int[nRows]; 29 | double cost = 0.0; 30 | 31 | // Fill in the distMatrixIn. Mind the index is "i + nRows * j". 32 | // Here the cost matrix of size MxN is defined as a double precision array of N*M elements. 33 | // In the solving functions matrices are seen to be saved MATLAB-internally in row-order. 34 | // (i.e. the matrix [1 2; 3 4] will be stored as a vector [1 3 2 4], NOT [1 2 3 4]). 35 | for (unsigned int i = 0; i < nRows; i++) 36 | for (unsigned int j = 0; j < nCols; j++) 37 | distMatrixIn[i + nRows * j] = DistMatrix[i][j]; 38 | 39 | // call solving function 40 | assignmentoptimal(assignment, &cost, distMatrixIn, nRows, nCols); 41 | 42 | Assignment.clear(); 43 | for (unsigned int r = 0; r < nRows; r++) 44 | Assignment.push_back(assignment[r]); 45 | 46 | delete[] distMatrixIn; 47 | delete[] assignment; 48 | return cost; 49 | } 50 | 51 | 52 | //********************************************************// 53 | // Solve optimal solution for assignment problem using Munkres algorithm, also known as Hungarian Algorithm. 54 | //********************************************************// 55 | void HungarianAlgorithm::assignmentoptimal(int *assignment, double *cost, double *distMatrixIn, int nOfRows, int nOfColumns) 56 | { 57 | double *distMatrix, *distMatrixTemp, *distMatrixEnd, *columnEnd, value, minValue; 58 | bool *coveredColumns, *coveredRows, *starMatrix, *newStarMatrix, *primeMatrix; 59 | int nOfElements, minDim, row, col; 60 | 61 | /* initialization */ 62 | *cost = 0; 63 | for (row = 0; row nOfColumns) */ 128 | { 129 | minDim = nOfColumns; 130 | 131 | for (col = 0; col= 0) 211 | *cost += distMatrix[row + nOfRows*col]; 212 | } 213 | } 214 | 215 | /********************************************************/ 216 | void HungarianAlgorithm::step2a(int *assignment, double *distMatrix, bool *starMatrix, bool *newStarMatrix, bool *primeMatrix, bool *coveredColumns, bool *coveredRows, int nOfRows, int nOfColumns, int minDim) 217 | { 218 | bool *starMatrixTemp, *columnEnd; 219 | int col; 220 | 221 | /* cover every column containing a starred zero */ 222 | for (col = 0; col(stateNum, stateNum) << 20 | 1, 0, 0, 0, 1, 0, 0, 21 | 0, 1, 0, 0, 0, 1, 0, 22 | 0, 0, 1, 0, 0, 0, 1, 23 | 0, 0, 0, 1, 0, 0, 0, 24 | 0, 0, 0, 0, 1, 0, 0, 25 | 0, 0, 0, 0, 0, 1, 0, 26 | 0, 0, 0, 0, 0, 0, 1); 27 | 28 | // Identity只在对角线上置初值 29 | setIdentity(kf.measurementMatrix); //!< measurement matrix (H) 30 | setIdentity(kf.processNoiseCov, cv::Scalar::all(1e-2)); //!< process noise covariance matrix (Q) 31 | setIdentity(kf.measurementNoiseCov, cv::Scalar::all(1e-1)); //!< measurement noise covariance matrix (R) 32 | setIdentity(kf.errorCovPost, cv::Scalar::all(1)); //!< posteriori error estimate covariance matrix (P(k)): P(k)=(I-K(k)*H)*P'(k) 33 | 34 | // initialize state vector with bounding box in [cx,cy,s,r] style 35 | kf.statePost.at(0, 0) = stateMat.x + stateMat.width / 2; 36 | kf.statePost.at(1, 0) = stateMat.y + stateMat.height / 2; 37 | kf.statePost.at(2, 0) = stateMat.area(); 38 | kf.statePost.at(3, 0) = stateMat.width / stateMat.height; 39 | } 40 | 41 | 42 | // Predict the estimated bounding box. 43 | StateType KalmanTracker::predict() 44 | { 45 | // predict 46 | cv::Mat p = kf.predict(); 47 | m_age += 1; 48 | 49 | if (m_time_since_update > 0) 50 | m_hit_streak = 0; 51 | m_time_since_update += 1; 52 | 53 | StateType predictBox = get_rect_xysr(p.at(0, 0), p.at(1, 0), p.at(2, 0), p.at(3, 0)); 54 | 55 | m_history.push_back(predictBox); 56 | return m_history.back(); 57 | } 58 | 59 | 60 | // Update the state vector with observed bounding box. 61 | void KalmanTracker::update(StateType stateMat) 62 | { 63 | m_time_since_update = 0; 64 | m_history.clear(); 65 | m_hits += 1; 66 | m_hit_streak += 1; 67 | 68 | // measurement 69 | measurement.at(0, 0) = stateMat.x + stateMat.width / 2; 70 | measurement.at(1, 0) = stateMat.y + stateMat.height / 2; 71 | measurement.at(2, 0) = stateMat.area(); 72 | measurement.at(3, 0) = stateMat.width / stateMat.height; 73 | 74 | // update 75 | kf.correct(measurement); 76 | } 77 | 78 | 79 | // Return the current state vector 80 | StateType KalmanTracker::get_state() 81 | { 82 | cv::Mat s = kf.statePost; 83 | return get_rect_xysr(s.at(0, 0), s.at(1, 0), s.at(2, 0), s.at(3, 0)); 84 | } 85 | 86 | 87 | // Convert bounding box from [cx,cy,s,r] to [x,y,w,h] style. 88 | StateType KalmanTracker::get_rect_xysr(float cx, float cy, float s, float r) 89 | { 90 | float w = sqrt(s * r); 91 | float h = s / w; 92 | float x = (cx - w / 2); 93 | float y = (cy - h / 2); 94 | 95 | if (x < 0 && cx > 0) 96 | x = 0; 97 | if (y < 0 && cy > 0) 98 | y = 0; 99 | 100 | return StateType(x, y, w, h); 101 | } 102 | -------------------------------------------------------------------------------- /src/detection.cc: -------------------------------------------------------------------------------- 1 | // 检测 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include // 获取文件属性 13 | #include // 文件夹操作 14 | #include 15 | #include 16 | #include "rknn_api.h" 17 | 18 | #include "im2d.h" 19 | //#include "RgaUtils.h" 20 | //#include "rga.h" 21 | #include "opencv2/core/core.hpp" 22 | #include "opencv2/imgproc/imgproc.hpp" 23 | #include "opencv2/highgui/highgui.hpp" 24 | #include "opencv2/videoio.hpp" 25 | #include "opencv2/video.hpp" 26 | 27 | #include "global.h" 28 | 29 | using namespace std; 30 | 31 | static void check_ret(int ret, string ret_name) 32 | { 33 | // 检查ret是否正确并输出，ret_name表示哪一步 34 | if (ret < 0) 35 | { 36 | cout << ret_name << " error ret=" << ret << endl; 37 | } 38 | 39 | } 40 | 41 | static void dump_tensor_attr(rknn_tensor_attr *attr) 42 | { 43 | // 打印模型输入和输出的信息 44 | printf(" index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, " 45 | "zp=%d, scale=%f\n", 46 | attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3], 47 | attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type), 48 | get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale); 49 | } 50 | 51 | // t为结构体，存储了时间信息：1、tv_sec 代表多少秒；2、tv_usec 代表多少微秒， 1000000 微秒 = 1秒 52 | double __get_us(struct timeval t) { return (t.tv_sec * 1000000 + t.tv_usec);} 53 | 54 | static unsigned char *load_data(FILE *fp, size_t ofst, size_t sz) 55 | { 56 | unsigned char *data; 57 | int ret; 58 | 59 | data = NULL; 60 | 61 | if (NULL == fp) 62 | { 63 | return NULL; 64 | } 65 | 66 | ret = fseek(fp, ofst, SEEK_SET); 67 | if (ret != 0) 68 | { 69 | printf("blob seek failure.\n"); 70 | return NULL; 71 | } 72 | 73 | data = (unsigned char *)malloc(sz); 74 | if (data == NULL) 75 | { 76 | printf("buffer malloc failure.\n"); 77 | return NULL; 78 | } 79 | ret = fread(data, 1, sz, fp); 80 | return data; 81 | } 82 | 83 | static unsigned char *load_model(const char *filename, int *model_size) 84 | { 85 | /* 86 | 加载rknn模型 87 | filename : rknn模型文件路径 88 | model_size : 模型的大小 89 | */ 90 | FILE *fp; 91 | unsigned char *data; 92 | 93 | fp = fopen(filename, "rb"); 94 | if (NULL == fp) 95 | { 96 | printf("Open rknn model file %s failed.\n", filename); 97 | return NULL; 98 | } 99 | 100 | fseek(fp, 0, SEEK_END); 101 | int size = ftell(fp); 102 | 103 | data = load_data(fp, 0, size); 104 | 105 | fclose(fp); 106 | 107 | *model_size = size; 108 | return data; 109 | } 110 | 111 | int detection_process(const char *model_name, int thread_id, int cpuid) 112 | { 113 | /* 114 | model_path : rknn模型位置 115 | thread_id : 进程号 116 | cpuid : 使用的cpu 117 | */ 118 | 119 | /********************初始参数*********************/ 120 | struct timeval start_time, stop_time; // 用于计时 121 | int img_width = 0; 122 | int img_height = 0; 123 | int img_channel = 0; 124 | const float nms_threshold = NMS_THRESH; 125 | const float box_conf_threshold = BOX_THRESH; 126 | 127 | /********************绑定cpu*********************/ 128 | cpu_set_t mask; 129 | CPU_ZERO(&mask); 130 | CPU_SET(cpuid, &mask); // 绑定cpu 131 | 132 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 133 | cerr << "set thread affinity failed" << endl; // 绑定失败 134 | 135 | cout << "NPU进程" << thread_id << "使用 CPU " << cpuid << endl; 136 | 137 | /********************rknn init*********************/ 138 | string ret_name; 139 | ret_name = "rknn_init"; // 表示rknn的步骤名称 140 | rknn_context ctx; // 创建rknn_context对象 141 | int model_data_size = 0; // 模型的大小 142 | unsigned char *model_data = load_model(model_name, &model_data_size); // 加载RKNN模型 143 | /* 初始化参数flag 144 | RKNN_FLAG_COLLECT_PERF_MASK：用于运行时查询网络各层时间。 145 | RKNN_FLAG_MEM_ALLOC_OUTSIDE：用于表示模型输入、输出、权重、中间 tensor 内存全部由用户分配。 146 | */ 147 | int ret = rknn_init(&ctx, model_data, model_data_size, RKNN_FLAG_COLLECT_PERF_MASK, NULL); // 初始化RKNN 148 | check_ret(ret, ret_name); 149 | // 设置NPU核心为自动调度 150 | rknn_core_mask core_mask = RKNN_NPU_CORE_AUTO; 151 | ret = rknn_set_core_mask(ctx, core_mask); 152 | 153 | /********************rknn query*********************/ 154 | // rknn_query 函数能够查询获取到模型输入输出信息、逐层运行时间、模型推理的总时间、 155 | // SDK 版本、内存占用信息、用户自定义字符串等信息。 156 | // 版本信息 157 | ret_name = "rknn_query"; 158 | rknn_sdk_version version; // SDK版本信息结构体 159 | ret = rknn_query(ctx, RKNN_QUERY_SDK_VERSION, &version, sizeof(rknn_sdk_version)); 160 | check_ret(ret, ret_name); 161 | printf("sdk api version: %s\n", version.api_version); 162 | printf("driver version: %s\n", version.drv_version); 163 | 164 | // 输入输出信息 165 | rknn_input_output_num io_num; 166 | ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num)); 167 | check_ret(ret, ret_name); 168 | printf("model input num: %d, output num: %d\n", io_num.n_input, io_num.n_output); 169 | 170 | // 输入输出Tensor属性 171 | rknn_tensor_attr input_attrs[io_num.n_input]; 172 | memset(input_attrs, 0, sizeof(input_attrs)); // 初始化内存 173 | for (int i = 0; i < io_num.n_input; i++) 174 | { 175 | input_attrs[i].index = i; // 输入的索引位置 176 | ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]), sizeof(rknn_tensor_attr)); 177 | check_ret(ret, ret_name); 178 | dump_tensor_attr(&(input_attrs[i])); 179 | } 180 | 181 | rknn_tensor_attr output_attrs[io_num.n_output]; 182 | memset(output_attrs, 0, sizeof(output_attrs)); 183 | for (int i = 0; i < io_num.n_output; i++) 184 | { 185 | output_attrs[i].index = i; 186 | ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr)); 187 | check_ret(ret, ret_name); 188 | dump_tensor_attr(&(output_attrs[i])); 189 | } 190 | 191 | // 模型输入信息 192 | int channel = 3; 193 | int width = 0; 194 | int height = 0; 195 | if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) 196 | { 197 | printf("model is NCHW input fmt\n"); 198 | channel = input_attrs[0].dims[1]; 199 | width = input_attrs[0].dims[2]; 200 | height = input_attrs[0].dims[3]; 201 | } 202 | else 203 | { 204 | printf("model is NHWC input fmt\n"); 205 | width = input_attrs[0].dims[1]; 206 | height = input_attrs[0].dims[2]; 207 | channel = input_attrs[0].dims[3]; 208 | } 209 | 210 | multi_npu_process_initialized[thread_id] = 1; // 进程设置完成标志置1 211 | /********************update frame(user)*********************/ 212 | while(1) 213 | { 214 | // 从输入队列加载图片 215 | pair frame; // <图的id，图> 216 | mtxQueueInput.lock(); 217 | // 如果queue处于空且 bReading 不在可读取状态则销毁跳出 218 | if (queueInput.empty()) 219 | { 220 | mtxQueueInput.unlock(); 221 | usleep(1000); 222 | if (bReading || bWriting) continue; 223 | else 224 | { 225 | // 释放内存rknn_context 226 | /********************rknn_destroy****************************/ 227 | ret_name = "rknn_destroy"; 228 | int ret = rknn_destroy(ctx); 229 | check_ret(ret, ret_name); 230 | break; 231 | } 232 | } 233 | 234 | // 读取到了图片，进行检测 235 | else 236 | { 237 | // 加载图片 238 | cout << "已缓存的图片数： " << queueInput.size() << endl; 239 | frame = queueInput.front(); 240 | printf("Idx:%d 图在线程%d中开始处理\n", frame.first, thread_id); 241 | queueInput.pop(); 242 | mtxQueueInput.unlock(); 243 | cv::Mat img = frame.second.clone(); 244 | cv::cvtColor(frame.second, img, cv::COLOR_BGR2RGB); // 色彩空间转换 245 | img_width = img.cols; // 输入图片的宽、高和通道数 246 | img_height = img.rows; 247 | img_channel = 3; 248 | // Resize (TODO) 249 | 250 | /********************rknn inputs set*********************/ 251 | ret_name = "rknn_inputs_set"; 252 | rknn_input inputs[1]; 253 | memset(inputs, 0, sizeof(inputs)); 254 | 255 | inputs[0].index = 0; // 输入的索引位置 256 | inputs[0].type = RKNN_TENSOR_UINT8; // 输入数据类型 采用INT8 257 | inputs[0].size = width * height * channel; // 这里用的是模型的 258 | inputs[0].fmt = input_attrs[0].fmt; // 输入格式，NHWC 259 | inputs[0].pass_through = 0; // 为0代表需要进行预处理 260 | inputs[0].buf = img.data; // 未进行resize，进行resize需要改为resize的data 261 | 262 | ret = rknn_inputs_set(ctx, io_num.n_input, inputs); 263 | check_ret(ret, ret_name); 264 | 265 | /********************rknn run****************************/ 266 | ret_name = "rknn_run"; 267 | gettimeofday(&start_time, NULL); 268 | ret = rknn_run(ctx, NULL); // 推理 269 | gettimeofday(&stop_time, NULL); 270 | check_ret(ret, ret_name); 271 | printf("once run use %f ms\n", (__get_us(stop_time) - __get_us(start_time)) / 1000); 272 | 273 | /********************rknn outputs get****************************/ 274 | ret_name = "rknn_outputs_get"; 275 | float out_scales[3] = {0, 0, 0}; // 存储scales 和 zp 276 | int32_t out_zps[3] = {0, 0, 0}; 277 | // 创建rknn_output对象 278 | rknn_output outputs[io_num.n_output]; 279 | memset(outputs, 0, sizeof(outputs)); 280 | for (int i = 0; i < io_num.n_output; i++) 281 | { 282 | outputs[i].index = i; // 输出索引 283 | outputs[i].is_prealloc = 0; // 由rknn来分配输出的buf，指向输出数据 284 | outputs[i].want_float = 0; 285 | out_scales[i] = output_attrs[i].scale; 286 | out_zps[i] = output_attrs[i].zp; 287 | } 288 | ret = rknn_outputs_get(ctx, io_num.n_output, outputs, NULL); 289 | 290 | /********************是否打印推理时间细节****************************/ 291 | ret_name = "rknn_perf_detail_display"; 292 | rknn_perf_detail perf_detail; 293 | ret = rknn_query(ctx, RKNN_QUERY_PERF_DETAIL, &perf_detail, sizeof(perf_detail)); 294 | check_ret(ret, ret_name); 295 | //printf("%s\n",perf_detail.perf_data); 296 | 297 | /********************postprocess_cpu****************************/ 298 | float scale_w = (float)width / img_width; // 图片缩放尺度 resize需要 299 | float scale_h = (float)height / img_height; 300 | detect_result_group_t detect_result_group; 301 | post_process((int8_t *)outputs[0].buf, (int8_t *)outputs[1].buf, (int8_t *)outputs[2].buf, height, width, 302 | box_conf_threshold, nms_threshold, scale_w, scale_h, out_zps, out_scales, &detect_result_group); 303 | 304 | /*# 测试检测结果 305 | // 绘制目标检测结果到原frame 306 | char text[256]; 307 | for (int i = 0; i < detect_result_group.count; i++) 308 | { 309 | detect_result_t *det_result = &(detect_result_group.results[i]); 310 | sprintf(text, "%s %.1f%%", det_result->name, det_result->prop * 100); 311 | //printf("%s @ (%d %d %d %d) %f\n", 312 | // det_result->name, 313 | // det_result->box.left, det_result->box.top, det_result->box.right, det_result->box.bottom, 314 | // det_result->prop); 315 | int x1 = det_result->box.left; 316 | int y1 = det_result->box.top; 317 | int x2 = det_result->box.right; 318 | int y2 = det_result->box.bottom; 319 | int color = det_result->color; 320 | rectangle(frame.second, cv::Point(x1, y1), cv::Point(x2, y2), randColor[color], 3); 321 | putText(frame.second, text, cv::Point(x1, y1 + 12), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0)); 322 | } 323 | */ 324 | detect_result_group.img = frame.second.clone(); // 保存原图 325 | detect_result_group.frame_id = frame.first; // 第几帧 326 | printf("[%4d/%4d] : worked/total\n", frame.first, Frame_cnt); 327 | printf("Idx:%d 图在线程%d中处理结束\n", frame.first, thread_id); 328 | // 将检测结果加入quequeShow队列进行展示或保存为视频 329 | while(idxDectImage != frame.first) usleep(1000); // 避免多个进程冲突，保证检测顺序正确 330 | mtxQueueShow.lock(); 331 | idxDectImage++; 332 | queueShow.push(detect_result_group); 333 | mtxQueueShow.unlock(); 334 | } 335 | } 336 | 337 | if (model_data) free(model_data); 338 | // if (resize_buf) free(resize_buf); 339 | printf("over%d\n", thread_id); 340 | return 0; 341 | } 342 | -------------------------------------------------------------------------------- /src/global.cc: -------------------------------------------------------------------------------- 1 | // 声明所有需要用到的全局变量 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include // 线程库 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "rknn_api.h" 22 | 23 | #include "im2d.h" 24 | #include "opencv2/core/core.hpp" 25 | #include "global.h" 26 | 27 | mutex mtxQueueInput; // 输入队列mutex 28 | mutex mtxQueueOutput; // 输出队列mutex 29 | mutex mtxQueueShow; // 展示队列mutex 30 | queue> queueInput; // 输入队列 31 | queue queueOutput; // 输出队列 <图片> 32 | priority_queue, paircomp> queueShow; 33 | int Frame_cnt = 0; // 帧的计数 34 | int Fps = 0; // 帧率 35 | int Video_width = 0; // 视频的输入宽度 36 | int Video_height = 0; // 视频的输入高度 37 | 38 | int multi_npu_process_initialized[4] = {0, 0, 0, 0}; // npu初始化完成标志，1为完成，0为未完成 39 | 40 | int idxInputImage = 0; // 输入视频的帧的id 41 | int idxDectImage = 0; // 要检测的下一帧id 42 | int idxShowImage = 0; // 要显示的下一帧的id 43 | bool bReading = true; // flag of input 44 | bool bWriting = true; // flag of output 45 | double Time_video = 0; // 整个视频(包括画图)所花的时间 46 | double Time_track = 0; // 整个视频追踪所花的时间 47 | 48 | cv::Scalar_ randColor[COLORS_NUMBER]; // 随机颜色 49 | cv::RNG rng(0xFFFFFFFF); //RNG类是opencv里C++的随机数产生器 50 | 51 | -------------------------------------------------------------------------------- /src/main.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include // 线程库 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include "rknn_api.h" 21 | #include "global.h" 22 | #include "Hungarian.h" 23 | 24 | using namespace std; 25 | int main(const int argc, const char **argv) 26 | { 27 | // 主函数，主要为各个模块分配进程 28 | int cpus = sysconf(_SC_NPROCESSORS_CONF); // 获取cpu核的数量 29 | array threads; 30 | struct timeval start_time, stop_time; 31 | 32 | // 随机初始化一些颜色 33 | for (int i = 0; i < COLORS_NUMBER; i++) 34 | rng.fill(randColor[i], cv::RNG::UNIFORM, 0, 256); 35 | // 分配进程 36 | gettimeofday(&start_time, NULL); 37 | if (argv[1][0] == 'v') 38 | { 39 | // 检测视频 40 | threads = { 41 | thread(videoRead, argv[3], 0), 42 | thread(videoWrite, 1), 43 | thread(track_process, 2), 44 | //thread(detection_process, argv[2], 0, 3), 45 | thread(detection_process, argv[2], 0, 4), 46 | thread(detection_process, argv[2], 1, 5), 47 | thread(detection_process, argv[2], 2, 6), 48 | thread(detection_process, argv[2], 3, 7) 49 | }; 50 | for (int i = 0; i < 7; i++) threads[i].join(); // join进程 51 | 52 | // 结果显示 53 | gettimeofday(&stop_time, NULL); 54 | Time_video = (__get_us(stop_time) - __get_us(start_time)) / 1000000; 55 | cout << "总耗时:" << Time_video << " 总帧数:" << Frame_cnt < 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "global.h" 10 | 11 | static char *labels[OBJ_CLASS_NUM]; 12 | const int anchor0[6] = {10, 13, 16, 30, 33, 23}; 13 | const int anchor1[6] = {30, 61, 62, 45, 59, 119}; 14 | const int anchor2[6] = {116, 90, 156, 198, 373, 326}; 15 | 16 | inline static int clamp(float val, int min, int max) 17 | { 18 | return val > min ? (val < max ? val : max) : min; 19 | } 20 | 21 | char *readLine(FILE *fp, char *buffer, int *len) 22 | { 23 | int ch; 24 | int i = 0; 25 | size_t buff_len = 0; 26 | 27 | buffer = (char *)malloc(buff_len + 1); 28 | if (!buffer) 29 | return NULL; // Out of memory 30 | 31 | while ((ch = fgetc(fp)) != '\n' && ch != EOF) 32 | { 33 | buff_len++; 34 | void *tmp = realloc(buffer, buff_len + 1); 35 | if (tmp == NULL) 36 | { 37 | free(buffer); 38 | return NULL; // Out of memory 39 | } 40 | buffer = (char *)tmp; 41 | 42 | buffer[i] = (char)ch; 43 | i++; 44 | } 45 | buffer[i] = '\0'; 46 | 47 | *len = buff_len; 48 | 49 | // Detect end 50 | if (ch == EOF && (i == 0 || ferror(fp))) 51 | { 52 | free(buffer); 53 | return NULL; 54 | } 55 | return buffer; 56 | } 57 | 58 | int readLines(const char *fileName, char *lines[], int max_line) 59 | { 60 | FILE *file = fopen(fileName, "r"); 61 | char *s; 62 | int i = 0; 63 | int n = 0; 64 | while ((s = readLine(file, s, &n)) != NULL) 65 | { 66 | lines[i++] = s; 67 | if (i >= max_line) 68 | break; 69 | } 70 | return i; 71 | } 72 | 73 | int loadLabelName(const char *locationFilename, char *label[]) 74 | { 75 | printf("loadLabelName %s\n", locationFilename); 76 | readLines(locationFilename, label, OBJ_CLASS_NUM); 77 | return 0; 78 | } 79 | 80 | static float CalculateOverlap(float xmin0, float ymin0, float xmax0, float ymax0, float xmin1, float ymin1, float xmax1, float ymax1) 81 | { 82 | float w = fmax(0.f, fmin(xmax0, xmax1) - fmax(xmin0, xmin1) + 1.0); 83 | float h = fmax(0.f, fmin(ymax0, ymax1) - fmax(ymin0, ymin1) + 1.0); 84 | float i = w * h; 85 | float u = (xmax0 - xmin0 + 1.0) * (ymax0 - ymin0 + 1.0) + (xmax1 - xmin1 + 1.0) * (ymax1 - ymin1 + 1.0) - i; 86 | return u <= 0.f ? 0.f : (i / u); 87 | } 88 | 89 | static int nms(int validCount, std::vector &outputLocations, std::vector classIds, std::vector &order,int filterId, float threshold) 90 | { 91 | for (int i = 0; i < validCount; ++i) 92 | { 93 | if (order[i] == -1|| classIds[i] != filterId) 94 | { 95 | continue; 96 | } 97 | int n = order[i]; 98 | for (int j = i + 1; j < validCount; ++j) 99 | { 100 | int m = order[j]; 101 | if (m == -1 || classIds[i] != filterId) 102 | { 103 | continue; 104 | } 105 | float xmin0 = outputLocations[n * 4 + 0]; 106 | float ymin0 = outputLocations[n * 4 + 1]; 107 | float xmax0 = outputLocations[n * 4 + 0] + outputLocations[n * 4 + 2]; 108 | float ymax0 = outputLocations[n * 4 + 1] + outputLocations[n * 4 + 3]; 109 | 110 | float xmin1 = outputLocations[m * 4 + 0]; 111 | float ymin1 = outputLocations[m * 4 + 1]; 112 | float xmax1 = outputLocations[m * 4 + 0] + outputLocations[m * 4 + 2]; 113 | float ymax1 = outputLocations[m * 4 + 1] + outputLocations[m * 4 + 3]; 114 | 115 | float iou = CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1, xmax1, ymax1); 116 | 117 | if (iou > threshold) 118 | { 119 | order[j] = -1; 120 | } 121 | } 122 | } 123 | return 0; 124 | } 125 | 126 | static int quick_sort_indice_inverse( 127 | std::vector &input, 128 | int left, 129 | int right, 130 | std::vector &indices) 131 | { 132 | float key; 133 | int key_index; 134 | int low = left; 135 | int high = right; 136 | if (left < right) 137 | { 138 | key_index = indices[left]; 139 | key = input[left]; 140 | while (low < high) 141 | { 142 | while (low < high && input[high] <= key) 143 | { 144 | high--; 145 | } 146 | input[low] = input[high]; 147 | indices[low] = indices[high]; 148 | while (low < high && input[low] >= key) 149 | { 150 | low++; 151 | } 152 | input[high] = input[low]; 153 | indices[high] = indices[low]; 154 | } 155 | input[low] = key; 156 | indices[low] = key_index; 157 | quick_sort_indice_inverse(input, left, low - 1, indices); 158 | quick_sort_indice_inverse(input, low + 1, right, indices); 159 | } 160 | return low; 161 | } 162 | 163 | static float sigmoid(float x) 164 | { 165 | return 1.0 / (1.0 + expf(-x)); 166 | } 167 | 168 | static float unsigmoid(float y) 169 | { 170 | return -1.0 * logf((1.0 / y) - 1.0); 171 | } 172 | 173 | inline static int32_t __clip(float val, float min, float max) 174 | { 175 | float f = val <= min ? min : (val >= max ? max : val); 176 | return f; 177 | } 178 | 179 | static int8_t qnt_f32_to_affine(float f32, int32_t zp, float scale) 180 | { 181 | float dst_val = (f32 / scale) + zp; 182 | int8_t res = (int8_t)__clip(dst_val, -128, 127); 183 | return res; 184 | } 185 | 186 | static float deqnt_affine_to_f32(int8_t qnt, int32_t zp, float scale) 187 | { 188 | return ((float)qnt - (float)zp) * scale; 189 | } 190 | 191 | int process(int8_t *input, int *anchor, int grid_h, int grid_w, int height, int width, int stride, 192 | std::vector &boxes, std::vector &objProbs, std::vector &classId, 193 | float threshold, int32_t zp, float scale) 194 | { 195 | 196 | int validCount = 0; 197 | int grid_len = grid_h * grid_w; 198 | float thres = unsigmoid(threshold); 199 | int8_t thres_i8 = qnt_f32_to_affine(thres, zp, scale); 200 | for (int a = 0; a < 3; a++) 201 | { 202 | for (int i = 0; i < grid_h; i++) 203 | { 204 | for (int j = 0; j < grid_w; j++) 205 | { 206 | int8_t box_confidence = input[(PROP_BOX_SIZE * a + 4) * grid_len + i * grid_w + j]; 207 | if (box_confidence >= thres_i8) 208 | { 209 | int offset = (PROP_BOX_SIZE * a) * grid_len + i * grid_w + j; 210 | int8_t *in_ptr = input + offset; 211 | float box_x = sigmoid(deqnt_affine_to_f32(*in_ptr, zp, scale)) * 2.0 - 0.5; 212 | float box_y = sigmoid(deqnt_affine_to_f32(in_ptr[grid_len], zp, scale)) * 2.0 - 0.5; 213 | float box_w = sigmoid(deqnt_affine_to_f32(in_ptr[2 * grid_len], zp, scale)) * 2.0; 214 | float box_h = sigmoid(deqnt_affine_to_f32(in_ptr[3 * grid_len], zp, scale)) * 2.0; 215 | box_x = (box_x + j) * (float)stride; 216 | box_y = (box_y + i) * (float)stride; 217 | box_w = box_w * box_w * (float)anchor[a * 2]; 218 | box_h = box_h * box_h * (float)anchor[a * 2 + 1]; 219 | box_x -= (box_w / 2.0); 220 | box_y -= (box_h / 2.0); 221 | boxes.push_back(box_x); 222 | boxes.push_back(box_y); 223 | boxes.push_back(box_w); 224 | boxes.push_back(box_h); 225 | 226 | int8_t maxClassProbs = in_ptr[5 * grid_len]; 227 | int maxClassId = 0; 228 | for (int k = 1; k < OBJ_CLASS_NUM; ++k) 229 | { 230 | int8_t prob = in_ptr[(5 + k) * grid_len]; 231 | if (prob > maxClassProbs) 232 | { 233 | maxClassId = k; 234 | maxClassProbs = prob; 235 | } 236 | } 237 | objProbs.push_back(sigmoid(deqnt_affine_to_f32(maxClassProbs, zp, scale))); 238 | classId.push_back(maxClassId); 239 | validCount++; 240 | } 241 | } 242 | } 243 | } 244 | return validCount; 245 | } 246 | 247 | int post_process(int8_t *input0, int8_t *input1, int8_t *input2, int model_in_h, int model_in_w, 248 | float conf_threshold, float nms_threshold, float scale_w, float scale_h, 249 | int32_t *qnt_zps, float *qnt_scales, 250 | detect_result_group_t *group) 251 | { 252 | static int init = -1; 253 | if (init == -1) 254 | { 255 | int ret = 0; 256 | ret = loadLabelName(LABEL_NALE_TXT_PATH, labels); 257 | if (ret < 0) 258 | { 259 | return -1; 260 | } 261 | 262 | init = 0; 263 | } 264 | memset(group, 0, sizeof(detect_result_group_t)); 265 | 266 | std::vector filterBoxes; 267 | std::vector objProbs; 268 | std::vector classId; 269 | 270 | // stride 8 271 | int stride0 = 8; 272 | int grid_h0 = model_in_h / stride0; 273 | int grid_w0 = model_in_w / stride0; 274 | int validCount0 = 0; 275 | validCount0 = process(input0, (int *)anchor0, grid_h0, grid_w0, model_in_h, model_in_w, 276 | stride0, filterBoxes, objProbs, classId, conf_threshold, qnt_zps[0], qnt_scales[0]); 277 | 278 | // stride 16 279 | int stride1 = 16; 280 | int grid_h1 = model_in_h / stride1; 281 | int grid_w1 = model_in_w / stride1; 282 | int validCount1 = 0; 283 | validCount1 = process(input1, (int *)anchor1, grid_h1, grid_w1, model_in_h, model_in_w, 284 | stride1, filterBoxes, objProbs, classId, conf_threshold, qnt_zps[1], qnt_scales[1]); 285 | 286 | // stride 32 287 | int stride2 = 32; 288 | int grid_h2 = model_in_h / stride2; 289 | int grid_w2 = model_in_w / stride2; 290 | int validCount2 = 0; 291 | validCount2 = process(input2, (int *)anchor2, grid_h2, grid_w2, model_in_h, model_in_w, 292 | stride2, filterBoxes, objProbs, classId, conf_threshold, qnt_zps[2], qnt_scales[2]); 293 | 294 | int validCount = validCount0 + validCount1 + validCount2; 295 | // no object detect 296 | if (validCount <= 0) 297 | { 298 | return 0; 299 | } 300 | 301 | std::vector indexArray; 302 | for (int i = 0; i < validCount; ++i) 303 | { 304 | indexArray.push_back(i); 305 | } 306 | 307 | quick_sort_indice_inverse(objProbs, 0, validCount - 1, indexArray); 308 | 309 | std::set class_set(std::begin(classId),std::end(classId)); 310 | 311 | for(auto c : class_set){ 312 | nms(validCount, filterBoxes, classId, indexArray, c, nms_threshold); 313 | } 314 | 315 | int last_count = 0; 316 | group->count = 0; 317 | /* box valid detect target */ 318 | for (int i = 0; i < validCount; ++i) 319 | { 320 | 321 | if (indexArray[i] == -1 || last_count >= OBJ_NUMB_MAX_SIZE) 322 | { 323 | continue; 324 | } 325 | int n = indexArray[i]; 326 | 327 | float x1 = filterBoxes[n * 4 + 0]; 328 | float y1 = filterBoxes[n * 4 + 1]; 329 | float x2 = x1 + filterBoxes[n * 4 + 2]; 330 | float y2 = y1 + filterBoxes[n * 4 + 3]; 331 | int id = classId[n]; 332 | float obj_conf = objProbs[i]; 333 | 334 | group->results[last_count].box.left = (int)(clamp(x1, 0, model_in_w) / scale_w); 335 | group->results[last_count].box.top = (int)(clamp(y1, 0, model_in_h) / scale_h); 336 | group->results[last_count].box.right = (int)(clamp(x2, 0, model_in_w) / scale_w); 337 | group->results[last_count].box.bottom = (int)(clamp(y2, 0, model_in_h) / scale_h); 338 | group->results[last_count].prop = obj_conf; 339 | char *label = labels[id]; 340 | group->results[last_count].color = id; 341 | strncpy(group->results[last_count].name, label, OBJ_NAME_MAX_SIZE); 342 | 343 | // 增加Rect_ bbox 344 | group->results[last_count].box.bbox = cv::Rect_( 345 | group->results[last_count].box.left, 346 | group->results[last_count].box.top, 347 | group->results[last_count].box.right - group->results[last_count].box.left, 348 | group->results[last_count].box.bottom - group->results[last_count].box.top 349 | ); 350 | 351 | // printf("result %2d: (%4d, %4d, %4d, %4d), %s\n", i, group->results[last_count].box.left, group->results[last_count].box.top, 352 | // group->results[last_count].box.right, group->results[last_count].box.bottom, label); 353 | last_count++; 354 | } 355 | group->count = last_count; 356 | 357 | return 0; 358 | } 359 | -------------------------------------------------------------------------------- /src/trackprocess.cc: -------------------------------------------------------------------------------- 1 | // 跟踪 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include // 获取文件属性 13 | #include // 文件夹操作 14 | #include 15 | #include 16 | #include "rknn_api.h" 17 | 18 | #include "im2d.h" 19 | //#include "RgaUtils.h" 20 | //#include "rga.h" 21 | #include "opencv2/core/core.hpp" 22 | #include "opencv2/imgproc/imgproc.hpp" 23 | #include "opencv2/highgui/highgui.hpp" 24 | #include "opencv2/videoio.hpp" 25 | #include "opencv2/video.hpp" 26 | 27 | #include "global.h" 28 | #include "KalmanTracker.h" 29 | #include "Hungarian.h" 30 | 31 | using namespace std; 32 | 33 | double box_iou(cv::Rect_ bb_test, cv::Rect_ bb_gt) 34 | { 35 | float in = (bb_test & bb_gt).area(); 36 | float un = bb_test.area() + bb_gt.area() - in; 37 | 38 | if (un < DBL_EPSILON) 39 | return 0; 40 | 41 | return (double)(in / un); 42 | } 43 | 44 | void track_process(int cpuid) 45 | { 46 | cpu_set_t mask; 47 | CPU_ZERO(&mask); 48 | CPU_SET(cpuid, &mask); 49 | 50 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 51 | cerr << "set thread affinity failed" << endl; 52 | printf("Bind Track process to CPU %d\n", cpuid); 53 | 54 | vector trackers; 55 | KalmanTracker::kf_count = 0; // 跟踪目标id，在初始时置为0 56 | 57 | vector frameTrackingResult; // 每帧的跟踪结果 58 | cout << "跟踪初始化结束！" << endl; 59 | 60 | while (1) 61 | { 62 | mtxQueueShow.lock(); 63 | if (queueShow.empty()) 64 | { 65 | mtxQueueShow.unlock(); 66 | usleep(1000); // 等待检测结果 67 | } 68 | 69 | // 如果下一个要显示的图片已经被处理好了 则进行追踪任务 70 | else if (idxShowImage == queueShow.top().frame_id) 71 | { 72 | cout << "待追踪的图片数： " << queueShow.size() << endl; 73 | detect_result_group_t group = queueShow.top(); // 取出图片检测结果group 74 | cv::Mat img = group.img.clone(); 75 | queueShow.pop(); 76 | mtxQueueShow.unlock(); 77 | if (trackers.size() == 0) // 追踪器是空（第一帧 或者 跟踪目标丢失） 78 | { 79 | //用第一帧的检测结果初始化跟踪器 80 | for (int i = 0; i < group.count; i++) 81 | { 82 | detect_result_t *det_result = &(group.results[i]); 83 | KalmanTracker trk = KalmanTracker(det_result->box.bbox); 84 | trackers.push_back(trk); 85 | } 86 | } 87 | else 88 | { 89 | // 预测已有跟踪器在当前帧的bbox 90 | vector> predictedBoxes; 91 | for (auto it = trackers.begin(); it != trackers.end();) 92 | { 93 | cv::Rect_ pBox = (*it).predict(); 94 | if (pBox.x >= 0 && pBox.y >= 0) 95 | { 96 | predictedBoxes.push_back(pBox); 97 | it++; 98 | } 99 | else it = trackers.erase(it); //bb不合理的tracker会被清除 100 | } 101 | 102 | // 匈牙利算法进行匹配 103 | vector> iouMatrix; 104 | iouMatrix.clear(); 105 | unsigned int trkNum = 0; 106 | unsigned int detNum = 0; 107 | trkNum = predictedBoxes.size(); //由上一帧预测出来的结果 108 | detNum = group.count; //当前帧的所有检测结果的 视作传感器的结果 109 | iouMatrix.resize(trkNum, vector(detNum, 0)); //提前开好空间 避免频繁重定位 110 | for (unsigned int i = 0; i < trkNum; i++) // 计算IOU矩阵 111 | { 112 | for (unsigned int j = 0; j < detNum; j++) 113 | { 114 | iouMatrix[i][j] = 1 - box_iou(predictedBoxes[i], group.results[j].box.bbox); 115 | } 116 | } 117 | 118 | HungarianAlgorithm HungAlgo; 119 | vector assignment; //匹配结果 给每一个trk找一个det 120 | assignment.clear(); 121 | if(trkNum!=0) HungAlgo.Solve(iouMatrix, assignment);//匈牙利算法核心 122 | 123 | // 寻找匹配 未匹配检测 未匹配预测 124 | set unmatchedDetections; // 没有被配对的检测框 说明有新目标出现 125 | set unmatchedTrajectories; // 没有被配对的追踪器 说明有目标消失 126 | set allItems; 127 | set matchedItems; 128 | vector matchedPairs; // 最终配对结果 trk-det 129 | unmatchedTrajectories.clear(); 130 | unmatchedDetections.clear(); 131 | allItems.clear(); 132 | matchedItems.clear(); 133 | 134 | if (detNum > trkNum) // 检测框的数量 大于 现存追踪器的数量 135 | { 136 | for (unsigned int n = 0; n < detNum; n++) allItems.insert(n); 137 | for (unsigned int i = 0; i < trkNum; ++i) matchedItems.insert(assignment[i]); 138 | /* 139 | set_difference, 求集合1与集合2的差集 即可以找到没有被追踪的 det 140 | 参数：第一个集合的开始位置，第一个集合的结束位置， 141 | 第二个参数的开始位置，第二个参数的结束位置， 142 | 结果集合的插入迭代器。 143 | */ 144 | set_difference( 145 | allItems.begin(), allItems.end(), 146 | matchedItems.begin(), matchedItems.end(), 147 | insert_iterator>(unmatchedDetections, unmatchedDetections.begin()) 148 | ); 149 | } 150 | 151 | else if (detNum < trkNum) // 检测框的数量 小于 现存追踪器的数量; 追踪目标暂时消失 152 | { 153 | for (unsigned int i = 0; i < trkNum; ++i) 154 | if (assignment[i] == -1) // unassigned label will be set as -1 in the assignment algorithm 155 | unmatchedTrajectories.insert(i); 156 | } 157 | 158 | else; // 两者数量相等不做操作 159 | 160 | // 过滤掉低IOU的匹配 161 | matchedPairs.clear(); 162 | for (unsigned int i = 0; i < trkNum; ++i) 163 | { 164 | if (assignment[i] == -1) continue; 165 | if (1 - iouMatrix[i][assignment[i]] < IOU_THRESH) 166 | { 167 | unmatchedTrajectories.insert(i); 168 | unmatchedDetections.insert(assignment[i]); 169 | } 170 | else 171 | matchedPairs.push_back(cv::Point(i, assignment[i])); // 符合条件 成功配对 172 | } 173 | 174 | // 更新跟踪器 175 | int detIdx, trkIdx; 176 | for (unsigned int i = 0; i < matchedPairs.size(); i++) 177 | { 178 | trkIdx = matchedPairs[i].x; 179 | detIdx = matchedPairs[i].y; 180 | trackers[trkIdx].update(group.results[detIdx].box.bbox); 181 | } 182 | 183 | // 给未匹配到的检测框创建和初始化跟踪器 184 | // unmatchedTrajectories没有操作 所以有必要保存unmatchedTrajectories吗?(maybe not) 185 | for (auto umd : unmatchedDetections) 186 | { 187 | KalmanTracker tracker = KalmanTracker(group.results[umd].box.bbox); 188 | trackers.push_back(tracker); 189 | } 190 | } 191 | 192 | // 获得跟踪器输出 193 | int max_age = 1; 194 | int min_hits = 3; 195 | //m_time_since_update：tracker距离上次匹配成功间隔的帧数 196 | //m_hit_streak：tracker连续匹配成功的帧数 197 | frameTrackingResult.clear(); 198 | for (auto it = trackers.begin(); it != trackers.end();) 199 | { 200 | // 输出条件：当前帧和前面2帧（连续3帧）匹配成功才记录 201 | if (((*it).m_time_since_update < 1) && 202 | ((*it).m_hit_streak >= min_hits || idxShowImage <= min_hits))//河狸 203 | { 204 | detect_result_t res; 205 | res.box.bbox = (*it).get_state(); 206 | res.track_id = (*it).m_id + 1; 207 | frameTrackingResult.push_back(res); 208 | it++; 209 | } 210 | else 211 | it++; 212 | if (it != trackers.end() && (*it).m_time_since_update > max_age)//连续3帧还没匹配到，清除 213 | it = trackers.erase(it); 214 | } 215 | 216 | // 绘图 217 | for (auto tb : frameTrackingResult) 218 | { 219 | cv::rectangle(img, tb.box.bbox, randColor[tb.track_id % COLORS_NUMBER], 2, 8, 0); 220 | cv::putText(img, "Id:"+to_string(tb.track_id), cv::Point(tb.box.bbox.x, tb.box.bbox.y), 1, 1, randColor[tb.track_id % COLORS_NUMBER], 1); 221 | } 222 | 223 | imshow("rk3588",img); 224 | mtxQueueOutput.lock(); 225 | queueOutput.push(img); 226 | mtxQueueOutput.unlock(); 227 | idxShowImage++; 228 | 229 | // 因为此时一定允许过至少一次videoRead 因此frame_cnt一定不为0 230 | if (idxShowImage == Frame_cnt || cv::waitKey(1) == 27) { 231 | cv::destroyAllWindows(); 232 | bReading = false; 233 | bWriting = false; 234 | break; 235 | } 236 | } 237 | 238 | else mtxQueueShow.unlock(); 239 | } 240 | 241 | } 242 | -------------------------------------------------------------------------------- /src/video_io.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include // 线程库 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include "rknn_api.h" 21 | #include "opencv2/core/core.hpp" 22 | #include "opencv2/imgproc/imgproc.hpp" 23 | #include "opencv2/highgui/highgui.hpp" 24 | #include "opencv2/videoio.hpp" 25 | #include "opencv2/video.hpp" 26 | 27 | #include "global.h" 28 | 29 | void videoRead(const char *video_path, int cpuid) 30 | { 31 | /* 32 | video_path : 视频路径 33 | cpuid : 视频读取使用的cpu号 34 | */ 35 | 36 | cpu_set_t mask; 37 | CPU_ZERO(&mask); 38 | CPU_SET(cpuid, &mask); // 绑定cpu 39 | 40 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 41 | cerr << "set thread affinity failed" << endl; // 绑定失败 42 | 43 | cout << "视频读取使用 CPU " << cpuid << endl; 44 | 45 | cv::VideoCapture video; 46 | video.open(video_path); 47 | if (!video.open(video_path)) 48 | { 49 | cout << "Fail to open " << video_path << endl; 50 | return; 51 | } 52 | 53 | // 获取视频参数 54 | Frame_cnt = video.get(cv::CAP_PROP_FRAME_COUNT); 55 | Fps = video.get(cv::CAP_PROP_FPS); 56 | Video_width = video.get(cv::CAP_PROP_FRAME_WIDTH); 57 | Video_height = video.get(cv::CAP_PROP_FRAME_HEIGHT); 58 | 59 | // 等待npu_process全部完成 60 | while(1) 61 | { 62 | int initialization_finished = 1; // npu_process完成标志 63 | for (int i = 0; i < sizeof(multi_npu_process_initialized) / sizeof(int); i++) 64 | if (multi_npu_process_initialized[i] == 0) initialization_finished = 0; 65 | 66 | if (initialization_finished == 1) break; 67 | } 68 | 69 | // npu_process全部完成 70 | //cv::Mat img1 = cv::imread("infrared_640.jpg", 1); 71 | while(1) 72 | { 73 | usleep(10); 74 | cv::Mat img; 75 | if (queueInput.size() < 30) 76 | { 77 | // 如果读不到图片，或者bReading不在读取状态，或者读取图像的id大于总帧数则退出读视频 78 | if (!bReading || !video.read(img) || idxInputImage >= Frame_cnt) 79 | { 80 | cout << "读取视频结束！" << endl; 81 | video.release(); 82 | break; 83 | } 84 | // 否则将读取到的图片放入输入队列 85 | mtxQueueInput.lock(); 86 | queueInput.push(make_pair(idxInputImage++, img)); 87 | mtxQueueInput.unlock(); 88 | } 89 | } 90 | 91 | bReading = false; 92 | cout << "读取视频结束！" << endl; 93 | } 94 | 95 | void videoWrite(int cpuid) 96 | { 97 | /* 98 | cpuid : 结果视频生成使用的cpu号 99 | */ 100 | cpu_set_t mask; 101 | CPU_ZERO(&mask); 102 | CPU_SET(cpuid, &mask); // 绑定cpu 103 | 104 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 105 | cerr << "set thread affinity failed" << endl; // 绑定失败 106 | 107 | cout << "写入视频使用 CPU " << cpuid << endl; 108 | 109 | cv::VideoWriter vid_writer; 110 | 111 | while(1) 112 | { 113 | if (Video_width != 0) 114 | { 115 | vid_writer = cv::VideoWriter(SAVE_PATH, cv::VideoWriter::fourcc('M','P','E','G'), Fps, cv::Size(Video_width, Video_height)); 116 | break; 117 | } 118 | } 119 | while(1) 120 | { 121 | usleep(100); 122 | cv::Mat img; 123 | 124 | // 如果输出队列存在元素，就一直写入视频 125 | if (queueOutput.size() > 0) { 126 | mtxQueueOutput.lock(); 127 | img = queueOutput.front(); 128 | queueOutput.pop(); 129 | mtxQueueOutput.unlock(); 130 | vid_writer.write(img); // Save-video 131 | } 132 | 133 | /* 134 | if (queueShow.size() > 0) 135 | { 136 | // 目前用来做检测 137 | mtxQueueShow.lock(); 138 | cv::Mat img = queueShow.front(); 139 | //imshow("RK3588", img); 140 | queueShow.pop(); 141 | mtxQueueShow.unlock(); 142 | //vid_writer.write(img); // Save-video 143 | idxShowImage++; 144 | 145 | } 146 | if (idxShowImage == Frame_cnt || cv::waitKey(1) == 27) 147 | { 148 | printf("*******************************************"); 149 | cv::destroyAllWindows(); 150 | bReading = false; 151 | bWriting = false; 152 | break; 153 | } 154 | */ 155 | if(!bWriting) 156 | { 157 | //vid_writer.release(); 158 | break; 159 | } 160 | } 161 | cout << "视频生成！" << endl; 162 | } 163 | --------------------------------------------------------------------------------