├── .gitignore ├── 3rdparty ├── librknn_api │ ├── aarch64 │ │ ├── librknn_api.so │ │ ├── librknn_api.so.backup │ │ ├── librknnrt.so │ │ └── librknnrt.so.backup │ └── include │ │ ├── rknn_api.h │ │ └── rknn_api.h.backup └── rga │ ├── include │ ├── RgaUtils.h │ ├── im2d.h │ ├── im2d.hpp │ └── rga.h │ └── lib │ └── librga.so ├── Bytetrack.gif ├── CMakeLists.txt ├── README.md ├── SORT.gif ├── build └── .gitkeep ├── data ├── .gitkeep └── DJI_0001_S_cut.mp4 ├── deepsort.gif ├── deepsort ├── CMakeLists.txt ├── include │ ├── MyKalmanFilter.h │ ├── datatype.h │ ├── deepsort.h │ ├── featuretensor.h │ ├── hungarianoper.h │ ├── linear_assignment.h │ ├── matrix.h │ ├── model.hpp │ ├── munkres.h │ ├── nn_matching.h │ ├── track.h │ └── tracker.h └── src │ ├── deepsort.cpp │ ├── featuretensor.cpp │ ├── hungarianoper.cpp │ ├── kalmanfilter.cpp │ ├── linear_assignment.cpp │ ├── nn_matching.cpp │ ├── track.cpp │ └── tracker.cpp ├── detect.gif ├── include ├── box.h ├── common.h ├── mytime.h ├── resize.h └── rknn_fp.h ├── model ├── best.rknn ├── best_nofocus_hyx.rknn ├── best_nofocus_hyx_1920.rknn ├── best_nofocus_relu.rknn ├── hongwai_2_labels_list.txt ├── osnet_x0_25_market.rknn ├── osnet_x1_0_imagenet.rknn ├── yolov5.rknn ├── yolov5_1920.rknn ├── yolov5_nofocus.rknn └── yolov5_nofocus_1920.rknn ├── src ├── mytime.cpp ├── resize.cpp └── rknn_fp.cpp ├── yolov5 ├── include │ ├── decode.h │ ├── detect.h │ └── videoio.h └── src │ ├── decode.cpp │ ├── detect.cpp │ └── videoio.cpp └── yolov5_deepsort.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | !build/.gitkeep 3 | data/* 4 | !data/DJI_0001_S_cut.mp4 5 | !data/.gitkeep 6 | 7 | .vscode 8 | make -------------------------------------------------------------------------------- /3rdparty/librknn_api/aarch64/librknn_api.so: -------------------------------------------------------------------------------- 1 | librknnrt.so -------------------------------------------------------------------------------- /3rdparty/librknn_api/aarch64/librknn_api.so.backup: -------------------------------------------------------------------------------- 1 | IntxLNKlibrknnrt.so -------------------------------------------------------------------------------- /3rdparty/librknn_api/aarch64/librknnrt.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/3rdparty/librknn_api/aarch64/librknnrt.so -------------------------------------------------------------------------------- /3rdparty/librknn_api/aarch64/librknnrt.so.backup: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/3rdparty/librknn_api/aarch64/librknnrt.so.backup -------------------------------------------------------------------------------- /3rdparty/librknn_api/include/rknn_api.h: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * 3 | * Copyright (c) 2017 - 2022 by Rockchip Corp. All rights reserved. 4 | * 5 | * The material in this file is confidential and contains trade secrets 6 | * of Rockchip Corporation. This is proprietary information owned by 7 | * Rockchip Corporation. No part of this work may be disclosed, 8 | * reproduced, copied, transmitted, or used in any way for any purpose, 9 | * without the express written permission of Rockchip Corporation. 10 | * 11 | *****************************************************************************/ 12 | 13 | 14 | #ifndef _RKNN_API_H 15 | #define _RKNN_API_H 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | #include 22 | 23 | /* 24 | Definition of extended flag for rknn_init. 25 | */ 26 | /* set high priority context. */ 27 | #define RKNN_FLAG_PRIOR_HIGH 0x00000000 28 | 29 | /* set medium priority context */ 30 | #define RKNN_FLAG_PRIOR_MEDIUM 0x00000001 31 | 32 | /* set low priority context. */ 33 | #define RKNN_FLAG_PRIOR_LOW 0x00000002 34 | 35 | /* asynchronous mode. 36 | when enable, rknn_outputs_get will not block for too long because it directly retrieves the result of 37 | the previous frame which can increase the frame rate on single-threaded mode, but at the cost of 38 | rknn_outputs_get not retrieves the result of the current frame. 39 | in multi-threaded mode you do not need to turn this mode on. */ 40 | #define RKNN_FLAG_ASYNC_MASK 0x00000004 41 | 42 | /* collect performance mode. 43 | when enable, you can get detailed performance reports via rknn_query(ctx, RKNN_QUERY_PERF_DETAIL, ...), 44 | but it will reduce the frame rate. */ 45 | #define RKNN_FLAG_COLLECT_PERF_MASK 0x00000008 46 | 47 | /* allocate all memory in outside, includes weight/internal/inputs/outputs */ 48 | #define RKNN_FLAG_MEM_ALLOC_OUTSIDE 0x00000010 49 | 50 | /* weight sharing with the same network structure */ 51 | #define RKNN_FLAG_SHARE_WEIGHT_MEM 0x00000020 52 | 53 | /* send fence fd from outside */ 54 | #define RKNN_FLAG_FENCE_IN_OUTSIDE 0x00000040 55 | 56 | /* get fence fd from inside */ 57 | #define RKNN_FLAG_FENCE_OUT_OUTSIDE 0x00000080 58 | 59 | /* 60 | Error code returned by the RKNN API. 61 | */ 62 | #define RKNN_SUCC 0 /* execute succeed. */ 63 | #define RKNN_ERR_FAIL -1 /* execute failed. */ 64 | #define RKNN_ERR_TIMEOUT -2 /* execute timeout. */ 65 | #define RKNN_ERR_DEVICE_UNAVAILABLE -3 /* device is unavailable. */ 66 | #define RKNN_ERR_MALLOC_FAIL -4 /* memory malloc fail. */ 67 | #define RKNN_ERR_PARAM_INVALID -5 /* parameter is invalid. */ 68 | #define RKNN_ERR_MODEL_INVALID -6 /* model is invalid. */ 69 | #define RKNN_ERR_CTX_INVALID -7 /* context is invalid. */ 70 | #define RKNN_ERR_INPUT_INVALID -8 /* input is invalid. */ 71 | #define RKNN_ERR_OUTPUT_INVALID -9 /* output is invalid. */ 72 | #define RKNN_ERR_DEVICE_UNMATCH -10 /* the device is unmatch, please update rknn sdk 73 | and npu driver/firmware. */ 74 | #define RKNN_ERR_INCOMPATILE_PRE_COMPILE_MODEL -11 /* This RKNN model use pre_compile mode, but not compatible with current driver. */ 75 | #define RKNN_ERR_INCOMPATILE_OPTIMIZATION_LEVEL_VERSION -12 /* This RKNN model set optimization level, but not compatible with current driver. */ 76 | #define RKNN_ERR_TARGET_PLATFORM_UNMATCH -13 /* This RKNN model set target platform, but not compatible with current platform. */ 77 | 78 | /* 79 | Definition for tensor 80 | */ 81 | #define RKNN_MAX_DIMS 16 /* maximum dimension of tensor. */ 82 | #define RKNN_MAX_NUM_CHANNEL 15 /* maximum channel number of input tensor. */ 83 | #define RKNN_MAX_NAME_LEN 256 /* maximum name lenth of tensor. */ 84 | 85 | 86 | #ifdef __arm__ 87 | typedef uint32_t rknn_context; 88 | #else 89 | typedef uint64_t rknn_context; 90 | #endif 91 | 92 | 93 | /* 94 | The query command for rknn_query 95 | */ 96 | typedef enum _rknn_query_cmd { 97 | RKNN_QUERY_IN_OUT_NUM = 0, /* query the number of input & output tensor. */ 98 | RKNN_QUERY_INPUT_ATTR = 1, /* query the attribute of input tensor. */ 99 | RKNN_QUERY_OUTPUT_ATTR = 2, /* query the attribute of output tensor. */ 100 | RKNN_QUERY_PERF_DETAIL = 3, /* query the detail performance, need set 101 | RKNN_FLAG_COLLECT_PERF_MASK when call rknn_init, 102 | this query needs to be valid after rknn_outputs_get. */ 103 | RKNN_QUERY_PERF_RUN = 4, /* query the time of run, 104 | this query needs to be valid after rknn_outputs_get. */ 105 | RKNN_QUERY_SDK_VERSION = 5, /* query the sdk & driver version */ 106 | 107 | RKNN_QUERY_MEM_SIZE = 6, /* query the weight & internal memory size */ 108 | RKNN_QUERY_CUSTOM_STRING = 7, /* query the custom string */ 109 | 110 | RKNN_QUERY_NATIVE_INPUT_ATTR = 8, /* query the attribute of native input tensor. */ 111 | RKNN_QUERY_NATIVE_OUTPUT_ATTR = 9, /* query the attribute of native output tensor. */ 112 | 113 | RKNN_QUERY_NATIVE_NC1HWC2_INPUT_ATTR = 8, /* query the attribute of native input tensor. */ 114 | RKNN_QUERY_NATIVE_NC1HWC2_OUTPUT_ATTR = 9, /* query the attribute of native output tensor. */ 115 | 116 | RKNN_QUERY_NATIVE_NHWC_INPUT_ATTR = 10, /* query the attribute of native input tensor. */ 117 | RKNN_QUERY_NATIVE_NHWC_OUTPUT_ATTR = 11, /* query the attribute of native output tensor. */ 118 | 119 | RKNN_QUERY_CMD_MAX 120 | } rknn_query_cmd; 121 | 122 | /* 123 | the tensor data type. 124 | */ 125 | typedef enum _rknn_tensor_type { 126 | RKNN_TENSOR_FLOAT32 = 0, /* data type is float32. */ 127 | RKNN_TENSOR_FLOAT16, /* data type is float16. */ 128 | RKNN_TENSOR_INT8, /* data type is int8. */ 129 | RKNN_TENSOR_UINT8, /* data type is uint8. */ 130 | RKNN_TENSOR_INT16, /* data type is int16. */ 131 | RKNN_TENSOR_UINT16, /* data type is uint16. */ 132 | RKNN_TENSOR_INT32, /* data type is int32. */ 133 | RKNN_TENSOR_UINT32, /* data type is uint32. */ 134 | RKNN_TENSOR_INT64, /* data type is int64. */ 135 | RKNN_TENSOR_BOOL, 136 | 137 | RKNN_TENSOR_TYPE_MAX 138 | } rknn_tensor_type; 139 | 140 | inline const char* get_type_string(rknn_tensor_type type) 141 | { 142 | switch(type) { 143 | case RKNN_TENSOR_FLOAT32: return "FP32"; 144 | case RKNN_TENSOR_FLOAT16: return "FP16"; 145 | case RKNN_TENSOR_INT8: return "INT8"; 146 | case RKNN_TENSOR_UINT8: return "UINT8"; 147 | case RKNN_TENSOR_INT16: return "INT16"; 148 | case RKNN_TENSOR_UINT16: return "UINT16"; 149 | case RKNN_TENSOR_INT32: return "INT32"; 150 | case RKNN_TENSOR_UINT32: return "UINT32"; 151 | case RKNN_TENSOR_INT64: return "UINT64"; 152 | case RKNN_TENSOR_BOOL: return "BOOL"; 153 | default: return "UNKNOW"; 154 | } 155 | } 156 | 157 | /* 158 | the quantitative type. 159 | */ 160 | typedef enum _rknn_tensor_qnt_type { 161 | RKNN_TENSOR_QNT_NONE = 0, /* none. */ 162 | RKNN_TENSOR_QNT_DFP, /* dynamic fixed point. */ 163 | RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC, /* asymmetric affine. */ 164 | 165 | RKNN_TENSOR_QNT_MAX 166 | } rknn_tensor_qnt_type; 167 | 168 | inline const char* get_qnt_type_string(rknn_tensor_qnt_type type) 169 | { 170 | switch(type) { 171 | case RKNN_TENSOR_QNT_NONE: return "NONE"; 172 | case RKNN_TENSOR_QNT_DFP: return "DFP"; 173 | case RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC: return "AFFINE"; 174 | default: return "UNKNOW"; 175 | } 176 | } 177 | 178 | /* 179 | the tensor data format. 180 | */ 181 | typedef enum _rknn_tensor_format { 182 | RKNN_TENSOR_NCHW = 0, /* data format is NCHW. */ 183 | RKNN_TENSOR_NHWC, /* data format is NHWC. */ 184 | RKNN_TENSOR_NC1HWC2, /* data format is NC1HWC2. */ 185 | RKNN_TENSOR_UNDEFINED, 186 | 187 | RKNN_TENSOR_FORMAT_MAX 188 | } rknn_tensor_format; 189 | 190 | /* 191 | the mode of running on target NPU core. 192 | */ 193 | typedef enum _rknn_core_mask { 194 | RKNN_NPU_CORE_AUTO = 0, /* default, run on NPU core randomly. */ 195 | RKNN_NPU_CORE_0 = 1, /* run on NPU core 0. */ 196 | RKNN_NPU_CORE_1 = 2, /* run on NPU core 1. */ 197 | RKNN_NPU_CORE_2 = 4, /* run on NPU core 2. */ 198 | RKNN_NPU_CORE_0_1 = 3, /* run on NPU core 1 and core 2. */ 199 | RKNN_NPU_CORE_0_1_2 = 7, /* run on NPU core 1 and core 2 and core 3. */ 200 | 201 | RKNN_NPU_CORE_UNDEFINED, 202 | } rknn_core_mask; 203 | 204 | inline const char* get_format_string(rknn_tensor_format fmt) 205 | { 206 | switch(fmt) { 207 | case RKNN_TENSOR_NCHW: return "NCHW"; 208 | case RKNN_TENSOR_NHWC: return "NHWC"; 209 | case RKNN_TENSOR_NC1HWC2: return "NC1HWC2"; 210 | case RKNN_TENSOR_UNDEFINED: return "UNDEFINED"; 211 | default: return "UNKNOW"; 212 | } 213 | } 214 | 215 | /* 216 | the information for RKNN_QUERY_IN_OUT_NUM. 217 | */ 218 | typedef struct _rknn_input_output_num { 219 | uint32_t n_input; /* the number of input. */ 220 | uint32_t n_output; /* the number of output. */ 221 | } rknn_input_output_num; 222 | 223 | /* 224 | the information for RKNN_QUERY_INPUT_ATTR / RKNN_QUERY_OUTPUT_ATTR. 225 | */ 226 | typedef struct _rknn_tensor_attr { 227 | uint32_t index; /* input parameter, the index of input/output tensor, 228 | need set before call rknn_query. */ 229 | 230 | uint32_t n_dims; /* the number of dimensions. */ 231 | uint32_t dims[RKNN_MAX_DIMS]; /* the dimensions array. */ 232 | char name[RKNN_MAX_NAME_LEN]; /* the name of tensor. */ 233 | 234 | uint32_t n_elems; /* the number of elements. */ 235 | uint32_t size; /* the bytes size of tensor. */ 236 | 237 | rknn_tensor_format fmt; /* the data format of tensor. */ 238 | rknn_tensor_type type; /* the data type of tensor. */ 239 | rknn_tensor_qnt_type qnt_type; /* the quantitative type of tensor. */ 240 | int8_t fl; /* fractional length for RKNN_TENSOR_QNT_DFP. */ 241 | int32_t zp; /* zero point for RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC. */ 242 | float scale; /* scale for RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC. */ 243 | 244 | uint32_t w_stride; /* the stride of tensor along the width dimention of input, 245 | Note: it is read-only, 0 means equal to width. */ 246 | uint32_t size_with_stride; /* the bytes size of tensor with stride. */ 247 | 248 | uint8_t pass_through; /* pass through mode, for rknn_set_io_mem interface. 249 | if TRUE, the buf data is passed directly to the input node of the rknn model 250 | without any conversion. the following variables do not need to be set. 251 | if FALSE, the buf data is converted into an input consistent with the model 252 | according to the following type and fmt. so the following variables 253 | need to be set.*/ 254 | uint32_t h_stride; /* the stride along the height dimention of input, 255 | Note: it is write-only, if it was set to 0, h_stride = height. */ 256 | } rknn_tensor_attr; 257 | 258 | /* 259 | the information for RKNN_QUERY_PERF_DETAIL. 260 | */ 261 | typedef struct _rknn_perf_detail { 262 | char* perf_data; /* the string pointer of perf detail. don't need free it by user. */ 263 | uint64_t data_len; /* the string length. */ 264 | } rknn_perf_detail; 265 | 266 | /* 267 | the information for RKNN_QUERY_PERF_RUN. 268 | */ 269 | typedef struct _rknn_perf_run { 270 | int64_t run_duration; /* real inference time (us) */ 271 | } rknn_perf_run; 272 | 273 | /* 274 | the information for RKNN_QUERY_SDK_VERSION. 275 | */ 276 | typedef struct _rknn_sdk_version { 277 | char api_version[256]; /* the version of rknn api. */ 278 | char drv_version[256]; /* the version of rknn driver. */ 279 | } rknn_sdk_version; 280 | 281 | /* 282 | the information for RKNN_QUERY_MEM_SIZE. 283 | */ 284 | typedef struct _rknn_mem_size { 285 | uint32_t total_weight_size; /* the weight memory size */ 286 | uint32_t total_internal_size; /* the internal memory size, exclude inputs/outputs */ 287 | } rknn_mem_size; 288 | 289 | /* 290 | the information for RKNN_QUERY_CUSTOM_STRING. 291 | */ 292 | typedef struct _rknn_custom_string { 293 | char string[1024]; /* the string of custom, lengths max to 1024 bytes */ 294 | } rknn_custom_string; 295 | 296 | /* 297 | The flags of rknn_tensor_mem. 298 | */ 299 | typedef enum _rknn_tensor_mem_flags { 300 | RKNN_TENSOR_MEMORY_FLAGS_ALLOC_INSIDE = 1, /*Used to mark in rknn_destroy_mem() whether it is necessary to release the "mem" pointer itself. 301 | If the flag RKNN_TENSOR_MEMORY_FLAGS_ALLOC_INSIDE is set, rknn_destroy_mem() will call free(mem).*/ 302 | RKNN_TENSOR_MEMORY_FLAGS_UNKNOWN 303 | } rknn_tensor_mem_flags; 304 | 305 | /* 306 | the memory information of tensor. 307 | */ 308 | typedef struct _rknn_tensor_memory { 309 | void* virt_addr; /* the virtual address of tensor buffer. */ 310 | uint64_t phys_addr; /* the physical address of tensor buffer. */ 311 | int32_t fd; /* the fd of tensor buffer. */ 312 | int32_t offset; /* indicates the offset of the memory. */ 313 | uint32_t size; /* the size of tensor buffer. */ 314 | uint32_t flags; /* the flags of tensor buffer, reserved */ 315 | void * priv_data; /* the private data of tensor buffer. */ 316 | } rknn_tensor_mem; 317 | 318 | /* 319 | the input information for rknn_input_set. 320 | */ 321 | typedef struct _rknn_input { 322 | uint32_t index; /* the input index. */ 323 | void* buf; /* the input buf for index. */ 324 | uint32_t size; /* the size of input buf. */ 325 | uint8_t pass_through; /* pass through mode. 326 | if TRUE, the buf data is passed directly to the input node of the rknn model 327 | without any conversion. the following variables do not need to be set. 328 | if FALSE, the buf data is converted into an input consistent with the model 329 | according to the following type and fmt. so the following variables 330 | need to be set.*/ 331 | rknn_tensor_type type; /* the data type of input buf. */ 332 | rknn_tensor_format fmt; /* the data format of input buf. 333 | currently the internal input format of NPU is NCHW by default. 334 | so entering NCHW data can avoid the format conversion in the driver. */ 335 | } rknn_input; 336 | 337 | /* 338 | the output information for rknn_outputs_get. 339 | */ 340 | typedef struct _rknn_output { 341 | uint8_t want_float; /* want transfer output data to float */ 342 | uint8_t is_prealloc; /* whether buf is pre-allocated. 343 | if TRUE, the following variables need to be set. 344 | if FALSE, the following variables do not need to be set. */ 345 | uint32_t index; /* the output index. */ 346 | void* buf; /* the output buf for index. 347 | when is_prealloc = FALSE and rknn_outputs_release called, 348 | this buf pointer will be free and don't use it anymore. */ 349 | uint32_t size; /* the size of output buf. */ 350 | } rknn_output; 351 | 352 | /* 353 | the extend information for rknn_init. 354 | */ 355 | typedef struct _rknn_init_extend { 356 | rknn_context ctx; /* rknn context */ 357 | uint8_t reserved[128]; /* reserved */ 358 | } rknn_init_extend; 359 | 360 | /* 361 | the extend information for rknn_run. 362 | */ 363 | typedef struct _rknn_run_extend { 364 | uint64_t frame_id; /* output parameter, indicate current frame id of run. */ 365 | int32_t non_block; /* block flag of run, 0 is block else 1 is non block */ 366 | int32_t timeout_ms; /* timeout for block mode, in milliseconds */ 367 | int32_t fence_fd; /* fence fd from other unit */ 368 | } rknn_run_extend; 369 | 370 | /* 371 | the extend information for rknn_outputs_get. 372 | */ 373 | typedef struct _rknn_output_extend { 374 | uint64_t frame_id; /* output parameter, indicate the frame id of outputs, corresponds to 375 | struct rknn_run_extend.frame_id.*/ 376 | } rknn_output_extend; 377 | 378 | 379 | /* rknn_init 380 | 381 | initial the context and load the rknn model. 382 | 383 | input: 384 | rknn_context* context the pointer of context handle. 385 | void* model if size > 0, pointer to the rknn model, if size = 0, filepath to the rknn model. 386 | uint32_t size the size of rknn model. 387 | uint32_t flag extend flag, see the define of RKNN_FLAG_XXX_XXX. 388 | rknn_init_extend* extend the extend information of init. 389 | return: 390 | int error code. 391 | */ 392 | int rknn_init(rknn_context* context, void* model, uint32_t size, uint32_t flag, rknn_init_extend* extend); 393 | 394 | /* rknn_dup_context 395 | 396 | initial the context and load the rknn model. 397 | 398 | input: 399 | rknn_context* context_in the pointer of context in handle. 400 | rknn_context* context_out the pointer of context out handle. 401 | return: 402 | int error code. 403 | */ 404 | int rknn_dup_context(rknn_context* context_in, rknn_context* context_out); 405 | 406 | /* rknn_destroy 407 | 408 | unload the rknn model and destroy the context. 409 | 410 | input: 411 | rknn_context context the handle of context. 412 | return: 413 | int error code. 414 | */ 415 | int rknn_destroy(rknn_context context); 416 | 417 | 418 | /* rknn_query 419 | 420 | query the information about model or others. see rknn_query_cmd. 421 | 422 | input: 423 | rknn_context context the handle of context. 424 | rknn_query_cmd cmd the command of query. 425 | void* info the buffer point of information. 426 | uint32_t size the size of information. 427 | return: 428 | int error code. 429 | */ 430 | int rknn_query(rknn_context context, rknn_query_cmd cmd, void* info, uint32_t size); 431 | 432 | 433 | /* rknn_inputs_set 434 | 435 | set inputs information by input index of rknn model. 436 | inputs information see rknn_input. 437 | 438 | input: 439 | rknn_context context the handle of context. 440 | uint32_t n_inputs the number of inputs. 441 | rknn_input inputs[] the arrays of inputs information, see rknn_input. 442 | return: 443 | int error code 444 | */ 445 | int rknn_inputs_set(rknn_context context, uint32_t n_inputs, rknn_input inputs[]); 446 | 447 | /* 448 | rknn_set_batch_core_num 449 | 450 | set rknn batch core_num. 451 | 452 | input: 453 | rknn_context context the handle of context. 454 | int core_num the core number. 455 | return: 456 | int error code. 457 | 458 | */ 459 | int rknn_set_batch_core_num(rknn_context context, int core_num); 460 | 461 | /* rknn_set_core_mask 462 | 463 | set rknn core mask. 464 | 465 | RKNN_NPU_CORE_AUTO: auto mode, default value 466 | RKNN_NPU_CORE_0: core 0 mode 467 | RKNN_NPU_CORE_1: core 1 mode 468 | RKNN_NPU_CORE_2: core 2 mode 469 | RKNN_NPU_CORE_0_1: combine core 0/1 mode 470 | RKNN_NPU_CORE_0_1_2: combine core 0/1/2 mode 471 | 472 | input: 473 | rknn_context context the handle of context. 474 | rknn_core_mask core_mask the core mask. 475 | return: 476 | int error code. 477 | */ 478 | int rknn_set_core_mask(rknn_context context, rknn_core_mask core_mask); 479 | 480 | /* rknn_run 481 | 482 | run the model to execute inference. 483 | 484 | input: 485 | rknn_context context the handle of context. 486 | rknn_run_extend* extend the extend information of run. 487 | return: 488 | int error code. 489 | */ 490 | int rknn_run(rknn_context context, rknn_run_extend* extend); 491 | 492 | 493 | /* rknn_wait 494 | 495 | wait the model after execute inference. 496 | 497 | input: 498 | rknn_context context the handle of context. 499 | rknn_run_extend* extend the extend information of run. 500 | return: 501 | int error code. 502 | */ 503 | int rknn_wait(rknn_context context, rknn_run_extend* extend); 504 | 505 | 506 | /* rknn_outputs_get 507 | 508 | wait the inference to finish and get the outputs. 509 | this function will block until inference finish. 510 | the results will set to outputs[]. 511 | 512 | input: 513 | rknn_context context the handle of context. 514 | uint32_t n_outputs the number of outputs. 515 | rknn_output outputs[] the arrays of output, see rknn_output. 516 | rknn_output_extend* the extend information of output. 517 | return: 518 | int error code. 519 | */ 520 | int rknn_outputs_get(rknn_context context, uint32_t n_outputs, rknn_output outputs[], rknn_output_extend* extend); 521 | 522 | 523 | /* rknn_outputs_release 524 | 525 | release the outputs that get by rknn_outputs_get. 526 | after called, the rknn_output[x].buf get from rknn_outputs_get will 527 | also be free when rknn_output[x].is_prealloc = FALSE. 528 | 529 | input: 530 | rknn_context context the handle of context. 531 | uint32_t n_ouputs the number of outputs. 532 | rknn_output outputs[] the arrays of output. 533 | return: 534 | int error code 535 | */ 536 | int rknn_outputs_release(rknn_context context, uint32_t n_ouputs, rknn_output outputs[]); 537 | 538 | 539 | /* new api for zero copy */ 540 | 541 | /* rknn_create_mem_from_phys (memory allocated outside) 542 | 543 | initialize tensor memory from physical address. 544 | 545 | input: 546 | rknn_context ctx the handle of context. 547 | uint64_t phys_addr physical address. 548 | void *virt_addr virtual address. 549 | uint32_t size the size of tensor buffer. 550 | return: 551 | rknn_tensor_mem the pointer of tensor memory information. 552 | */ 553 | rknn_tensor_mem* rknn_create_mem_from_phys(rknn_context ctx, uint64_t phys_addr, void *virt_addr, uint32_t size); 554 | 555 | 556 | /* rknn_create_mem_from_fd (memory allocated outside) 557 | 558 | initialize tensor memory from file description. 559 | 560 | input: 561 | rknn_context ctx the handle of context. 562 | int32_t fd file description. 563 | void *virt_addr virtual address. 564 | uint32_t size the size of tensor buffer. 565 | int32_t offset indicates the offset of the memory (virt_addr without offset). 566 | return: 567 | rknn_tensor_mem the pointer of tensor memory information. 568 | */ 569 | rknn_tensor_mem* rknn_create_mem_from_fd(rknn_context ctx, int32_t fd, void *virt_addr, uint32_t size, int32_t offset); 570 | 571 | 572 | /* rknn_create_mem_from_mb_blk (memory allocated outside) 573 | 574 | create tensor memory from mb_blk. 575 | 576 | input: 577 | rknn_context ctx the handle of context. 578 | void *mb_blk mb_blk allocate from system api. 579 | int32_t offset indicates the offset of the memory. 580 | return: 581 | rknn_tensor_mem the pointer of tensor memory information. 582 | */ 583 | rknn_tensor_mem* rknn_create_mem_from_mb_blk(rknn_context ctx, void *mb_blk, int32_t offset); 584 | 585 | 586 | /* rknn_create_mem (memory allocated inside) 587 | 588 | create tensor memory. 589 | 590 | input: 591 | rknn_context ctx the handle of context. 592 | uint32_t size the size of tensor buffer. 593 | return: 594 | rknn_tensor_mem the pointer of tensor memory information. 595 | */ 596 | rknn_tensor_mem* rknn_create_mem(rknn_context ctx, uint32_t size); 597 | 598 | 599 | /* rknn_destroy_mem (support allocate inside and outside) 600 | 601 | destroy tensor memory. 602 | 603 | input: 604 | rknn_context ctx the handle of context. 605 | rknn_tensor_mem *mem the pointer of tensor memory information. 606 | return: 607 | int error code 608 | */ 609 | int rknn_destroy_mem(rknn_context ctx, rknn_tensor_mem *mem); 610 | 611 | 612 | /* rknn_set_weight_mem 613 | 614 | set the weight memory. 615 | 616 | input: 617 | rknn_context ctx the handle of context. 618 | rknn_tensor_mem *mem the array of tensor memory information 619 | return: 620 | int error code. 621 | */ 622 | int rknn_set_weight_mem(rknn_context ctx, rknn_tensor_mem *mem); 623 | 624 | 625 | /* rknn_set_internal_mem 626 | 627 | set the internal memory. 628 | 629 | input: 630 | rknn_context ctx the handle of context. 631 | rknn_tensor_mem *mem the array of tensor memory information 632 | return: 633 | int error code. 634 | */ 635 | int rknn_set_internal_mem(rknn_context ctx, rknn_tensor_mem *mem); 636 | 637 | 638 | /* rknn_set_io_mem 639 | 640 | set the input and output tensors buffer. 641 | 642 | input: 643 | rknn_context ctx the handle of context. 644 | rknn_tensor_mem *mem the array of tensor memory information. 645 | rknn_tensor_attr *attr the attribute of input or output tensor buffer. 646 | return: 647 | int error code. 648 | */ 649 | int rknn_set_io_mem(rknn_context ctx, rknn_tensor_mem *mem, rknn_tensor_attr *attr); 650 | 651 | 652 | #ifdef __cplusplus 653 | } //extern "C" 654 | #endif 655 | 656 | #endif //_RKNN_API_H 657 | -------------------------------------------------------------------------------- /3rdparty/librknn_api/include/rknn_api.h.backup: -------------------------------------------------------------------------------- 1 | /**************************************************************************** 2 | * 3 | * Copyright (c) 2017 - 2022 by Rockchip Corp. All rights reserved. 4 | * 5 | * The material in this file is confidential and contains trade secrets 6 | * of Rockchip Corporation. This is proprietary information owned by 7 | * Rockchip Corporation. No part of this work may be disclosed, 8 | * reproduced, copied, transmitted, or used in any way for any purpose, 9 | * without the express written permission of Rockchip Corporation. 10 | * 11 | *****************************************************************************/ 12 | 13 | 14 | #ifndef _RKNN_API_H 15 | #define _RKNN_API_H 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | #include 22 | 23 | /* 24 | Definition of extended flag for rknn_init. 25 | */ 26 | /* set high priority context. */ 27 | #define RKNN_FLAG_PRIOR_HIGH 0x00000000 28 | 29 | /* set medium priority context */ 30 | #define RKNN_FLAG_PRIOR_MEDIUM 0x00000001 31 | 32 | /* set low priority context. */ 33 | #define RKNN_FLAG_PRIOR_LOW 0x00000002 34 | 35 | /* asynchronous mode. 36 | when enable, rknn_outputs_get will not block for too long because it directly retrieves the result of 37 | the previous frame which can increase the frame rate on single-threaded mode, but at the cost of 38 | rknn_outputs_get not retrieves the result of the current frame. 39 | in multi-threaded mode you do not need to turn this mode on. */ 40 | #define RKNN_FLAG_ASYNC_MASK 0x00000004 41 | 42 | /* collect performance mode. 43 | when enable, you can get detailed performance reports via rknn_query(ctx, RKNN_QUERY_PERF_DETAIL, ...), 44 | but it will reduce the frame rate. */ 45 | #define RKNN_FLAG_COLLECT_PERF_MASK 0x00000008 46 | 47 | /* allocate all memory in outside, includes weight/internal/inputs/outputs */ 48 | #define RKNN_FLAG_MEM_ALLOC_OUTSIDE 0x00000010 49 | 50 | /* weight sharing with the same network structure */ 51 | #define RKNN_FLAG_SHARE_WEIGHT_MEM 0x00000020 52 | 53 | /* send fence fd from outside */ 54 | #define RKNN_FLAG_FENCE_IN_OUTSIDE 0x00000040 55 | 56 | /* get fence fd from inside */ 57 | #define RKNN_FLAG_FENCE_OUT_OUTSIDE 0x00000080 58 | 59 | /* 60 | Error code returned by the RKNN API. 61 | */ 62 | #define RKNN_SUCC 0 /* execute succeed. */ 63 | #define RKNN_ERR_FAIL -1 /* execute failed. */ 64 | #define RKNN_ERR_TIMEOUT -2 /* execute timeout. */ 65 | #define RKNN_ERR_DEVICE_UNAVAILABLE -3 /* device is unavailable. */ 66 | #define RKNN_ERR_MALLOC_FAIL -4 /* memory malloc fail. */ 67 | #define RKNN_ERR_PARAM_INVALID -5 /* parameter is invalid. */ 68 | #define RKNN_ERR_MODEL_INVALID -6 /* model is invalid. */ 69 | #define RKNN_ERR_CTX_INVALID -7 /* context is invalid. */ 70 | #define RKNN_ERR_INPUT_INVALID -8 /* input is invalid. */ 71 | #define RKNN_ERR_OUTPUT_INVALID -9 /* output is invalid. */ 72 | #define RKNN_ERR_DEVICE_UNMATCH -10 /* the device is unmatch, please update rknn sdk 73 | and npu driver/firmware. */ 74 | #define RKNN_ERR_INCOMPATILE_PRE_COMPILE_MODEL -11 /* This RKNN model use pre_compile mode, but not compatible with current driver. */ 75 | #define RKNN_ERR_INCOMPATILE_OPTIMIZATION_LEVEL_VERSION -12 /* This RKNN model set optimization level, but not compatible with current driver. */ 76 | #define RKNN_ERR_TARGET_PLATFORM_UNMATCH -13 /* This RKNN model set target platform, but not compatible with current platform. */ 77 | 78 | /* 79 | Definition for tensor 80 | */ 81 | #define RKNN_MAX_DIMS 16 /* maximum dimension of tensor. */ 82 | #define RKNN_MAX_NUM_CHANNEL 15 /* maximum channel number of input tensor. */ 83 | #define RKNN_MAX_NAME_LEN 256 /* maximum name lenth of tensor. */ 84 | 85 | 86 | #ifdef __arm__ 87 | typedef uint32_t rknn_context; 88 | #else 89 | typedef uint64_t rknn_context; 90 | #endif 91 | 92 | 93 | /* 94 | The query command for rknn_query 95 | */ 96 | typedef enum _rknn_query_cmd { 97 | RKNN_QUERY_IN_OUT_NUM = 0, /* query the number of input & output tensor. */ 98 | RKNN_QUERY_INPUT_ATTR, /* query the attribute of input tensor. */ 99 | RKNN_QUERY_OUTPUT_ATTR, /* query the attribute of output tensor. */ 100 | RKNN_QUERY_PERF_DETAIL, /* query the detail performance, need set 101 | RKNN_FLAG_COLLECT_PERF_MASK when call rknn_init, 102 | this query needs to be valid after rknn_outputs_get. */ 103 | RKNN_QUERY_PERF_RUN, /* query the time of run, 104 | this query needs to be valid after rknn_outputs_get. */ 105 | RKNN_QUERY_SDK_VERSION, /* query the sdk & driver version */ 106 | 107 | RKNN_QUERY_MEM_SIZE, /* query the weight & internal memory size */ 108 | RKNN_QUERY_CUSTOM_STRING, /* query the custom string */ 109 | 110 | RKNN_QUERY_NATIVE_INPUT_ATTR, /* query the attribute of native input tensor. */ 111 | RKNN_QUERY_NATIVE_OUTPUT_ATTR, /* query the attribute of native output tensor. */ 112 | 113 | RKNN_QUERY_CMD_MAX 114 | } rknn_query_cmd; 115 | 116 | /* 117 | the tensor data type. 118 | */ 119 | typedef enum _rknn_tensor_type { 120 | RKNN_TENSOR_FLOAT32 = 0, /* data type is float32. */ 121 | RKNN_TENSOR_FLOAT16, /* data type is float16. */ 122 | RKNN_TENSOR_INT8, /* data type is int8. */ 123 | RKNN_TENSOR_UINT8, /* data type is uint8. */ 124 | RKNN_TENSOR_INT16, /* data type is int16. */ 125 | RKNN_TENSOR_UINT16, /* data type is uint16. */ 126 | RKNN_TENSOR_INT32, /* data type is int32. */ 127 | RKNN_TENSOR_UINT32, /* data type is uint32. */ 128 | RKNN_TENSOR_INT64, /* data type is int64. */ 129 | RKNN_TENSOR_BOOL, 130 | 131 | RKNN_TENSOR_TYPE_MAX 132 | } rknn_tensor_type; 133 | 134 | inline const char* get_type_string(rknn_tensor_type type) 135 | { 136 | switch(type) { 137 | case RKNN_TENSOR_FLOAT32: return "FP32"; 138 | case RKNN_TENSOR_FLOAT16: return "FP16"; 139 | case RKNN_TENSOR_INT8: return "INT8"; 140 | case RKNN_TENSOR_UINT8: return "UINT8"; 141 | case RKNN_TENSOR_INT16: return "INT16"; 142 | case RKNN_TENSOR_UINT16: return "UINT16"; 143 | case RKNN_TENSOR_INT32: return "INT32"; 144 | case RKNN_TENSOR_UINT32: return "UINT32"; 145 | case RKNN_TENSOR_INT64: return "UINT64"; 146 | case RKNN_TENSOR_BOOL: return "BOOL"; 147 | default: return "UNKNOW"; 148 | } 149 | } 150 | 151 | /* 152 | the quantitative type. 153 | */ 154 | typedef enum _rknn_tensor_qnt_type { 155 | RKNN_TENSOR_QNT_NONE = 0, /* none. */ 156 | RKNN_TENSOR_QNT_DFP, /* dynamic fixed point. */ 157 | RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC, /* asymmetric affine. */ 158 | 159 | RKNN_TENSOR_QNT_MAX 160 | } rknn_tensor_qnt_type; 161 | 162 | inline const char* get_qnt_type_string(rknn_tensor_qnt_type type) 163 | { 164 | switch(type) { 165 | case RKNN_TENSOR_QNT_NONE: return "NONE"; 166 | case RKNN_TENSOR_QNT_DFP: return "DFP"; 167 | case RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC: return "AFFINE"; 168 | default: return "UNKNOW"; 169 | } 170 | } 171 | 172 | /* 173 | the tensor data format. 174 | */ 175 | typedef enum _rknn_tensor_format { 176 | RKNN_TENSOR_NCHW = 0, /* data format is NCHW. */ 177 | RKNN_TENSOR_NHWC, /* data format is NHWC. */ 178 | RKNN_TENSOR_NC1HWC2, /* data format is NC1HWC2. */ 179 | RKNN_TENSOR_UNDEFINED, 180 | 181 | RKNN_TENSOR_FORMAT_MAX 182 | } rknn_tensor_format; 183 | 184 | /* 185 | the mode of running on target NPU core. 186 | */ 187 | typedef enum _rknn_core_mask { 188 | RKNN_NPU_CORE_AUTO = 0, /* default, run on NPU core randomly. */ 189 | RKNN_NPU_CORE_0 = 1, /* run on NPU core 0. */ 190 | RKNN_NPU_CORE_1 = 2, /* run on NPU core 1. */ 191 | RKNN_NPU_CORE_2 = 4, /* run on NPU core 2. */ 192 | RKNN_NPU_CORE_0_1 = 3, /* run on NPU core 1 and core 2. */ 193 | RKNN_NPU_CORE_0_1_2 = 7, /* run on NPU core 1 and core 2 and core 3. */ 194 | 195 | RKNN_NPU_CORE_UNDEFINED, 196 | } rknn_core_mask; 197 | 198 | inline const char* get_format_string(rknn_tensor_format fmt) 199 | { 200 | switch(fmt) { 201 | case RKNN_TENSOR_NCHW: return "NCHW"; 202 | case RKNN_TENSOR_NHWC: return "NHWC"; 203 | case RKNN_TENSOR_NC1HWC2: return "NC1HWC2"; 204 | case RKNN_TENSOR_UNDEFINED: return "UNDEFINED"; 205 | default: return "UNKNOW"; 206 | } 207 | } 208 | 209 | /* 210 | the information for RKNN_QUERY_IN_OUT_NUM. 211 | */ 212 | typedef struct _rknn_input_output_num { 213 | uint32_t n_input; /* the number of input. */ 214 | uint32_t n_output; /* the number of output. */ 215 | } rknn_input_output_num; 216 | 217 | /* 218 | the information for RKNN_QUERY_INPUT_ATTR / RKNN_QUERY_OUTPUT_ATTR. 219 | */ 220 | typedef struct _rknn_tensor_attr { 221 | uint32_t index; /* input parameter, the index of input/output tensor, 222 | need set before call rknn_query. */ 223 | 224 | uint32_t n_dims; /* the number of dimensions. */ 225 | uint32_t dims[RKNN_MAX_DIMS]; /* the dimensions array. */ 226 | char name[RKNN_MAX_NAME_LEN]; /* the name of tensor. */ 227 | 228 | uint32_t n_elems; /* the number of elements. */ 229 | uint32_t size; /* the bytes size of tensor. */ 230 | 231 | rknn_tensor_format fmt; /* the data format of tensor. */ 232 | rknn_tensor_type type; /* the data type of tensor. */ 233 | rknn_tensor_qnt_type qnt_type; /* the quantitative type of tensor. */ 234 | int8_t fl; /* fractional length for RKNN_TENSOR_QNT_DFP. */ 235 | int32_t zp; /* zero point for RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC. */ 236 | float scale; /* scale for RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC. */ 237 | 238 | uint32_t stride; /* the stride of tensor, 0 means equal to width. */ 239 | uint32_t size_with_stride; /* the bytes size of tensor with stride. */ 240 | 241 | uint8_t pass_through; /* pass through mode, for rknn_set_io_mem interface. 242 | if TRUE, the buf data is passed directly to the input node of the rknn model 243 | without any conversion. the following variables do not need to be set. 244 | if FALSE, the buf data is converted into an input consistent with the model 245 | according to the following type and fmt. so the following variables 246 | need to be set.*/ 247 | } rknn_tensor_attr; 248 | 249 | /* 250 | the information for RKNN_QUERY_PERF_DETAIL. 251 | */ 252 | typedef struct _rknn_perf_detail { 253 | char* perf_data; /* the string pointer of perf detail. don't need free it by user. */ 254 | uint64_t data_len; /* the string length. */ 255 | } rknn_perf_detail; 256 | 257 | /* 258 | the information for RKNN_QUERY_PERF_RUN. 259 | */ 260 | typedef struct _rknn_perf_run { 261 | int64_t run_duration; /* real inference time (us) */ 262 | } rknn_perf_run; 263 | 264 | /* 265 | the information for RKNN_QUERY_SDK_VERSION. 266 | */ 267 | typedef struct _rknn_sdk_version { 268 | char api_version[256]; /* the version of rknn api. */ 269 | char drv_version[256]; /* the version of rknn driver. */ 270 | } rknn_sdk_version; 271 | 272 | /* 273 | the information for RKNN_QUERY_MEM_SIZE. 274 | */ 275 | typedef struct _rknn_mem_size { 276 | uint32_t total_weight_size; /* the weight memory size */ 277 | uint32_t total_internal_size; /* the internal memory size, exclude inputs/outputs */ 278 | } rknn_mem_size; 279 | 280 | /* 281 | the information for RKNN_QUERY_CUSTOM_STRING. 282 | */ 283 | typedef struct _rknn_custom_string { 284 | char string[1024]; /* the string of custom, lengths max to 1024 bytes */ 285 | } rknn_custom_string; 286 | 287 | /* 288 | The flags of rknn_tensor_mem. 289 | */ 290 | typedef enum _rknn_tensor_mem_flags { 291 | RKNN_TENSOR_MEMORY_FLAGS_ALLOC_INSIDE = 1, /*Used to mark in rknn_destory_mem() whether it is necessary to release the "mem" pointer itself. 292 | If the flag RKNN_TENSOR_MEMORY_FLAGS_ALLOC_INSIDE is set, rknn_destory_mem() will call free(mem).*/ 293 | RKNN_TENSOR_MEMORY_FLAGS_UNKNOWN 294 | } rknn_tensor_mem_flags; 295 | 296 | /* 297 | the memory information of tensor. 298 | */ 299 | typedef struct _rknn_tensor_memory { 300 | void* virt_addr; /* the virtual address of tensor buffer. */ 301 | uint64_t phys_addr; /* the physical address of tensor buffer. */ 302 | int32_t fd; /* the fd of tensor buffer. */ 303 | int32_t offset; /* indicates the offset of the memory. */ 304 | uint32_t size; /* the size of tensor buffer. */ 305 | uint32_t flags; /* the flags of tensor buffer, reserved */ 306 | void * priv_data; /* the private data of tensor buffer. */ 307 | } rknn_tensor_mem; 308 | 309 | /* 310 | the input information for rknn_input_set. 311 | */ 312 | typedef struct _rknn_input { 313 | uint32_t index; /* the input index. */ 314 | void* buf; /* the input buf for index. */ 315 | uint32_t size; /* the size of input buf. */ 316 | uint8_t pass_through; /* pass through mode. 317 | if TRUE, the buf data is passed directly to the input node of the rknn model 318 | without any conversion. the following variables do not need to be set. 319 | if FALSE, the buf data is converted into an input consistent with the model 320 | according to the following type and fmt. so the following variables 321 | need to be set.*/ 322 | rknn_tensor_type type; /* the data type of input buf. */ 323 | rknn_tensor_format fmt; /* the data format of input buf. 324 | currently the internal input format of NPU is NCHW by default. 325 | so entering NCHW data can avoid the format conversion in the driver. */ 326 | } rknn_input; 327 | 328 | /* 329 | the output information for rknn_outputs_get. 330 | */ 331 | typedef struct _rknn_output { 332 | uint8_t want_float; /* want transfer output data to float */ 333 | uint8_t is_prealloc; /* whether buf is pre-allocated. 334 | if TRUE, the following variables need to be set. 335 | if FALSE, the following variables do not need to be set. */ 336 | uint32_t index; /* the output index. */ 337 | void* buf; /* the output buf for index. 338 | when is_prealloc = FALSE and rknn_outputs_release called, 339 | this buf pointer will be free and don't use it anymore. */ 340 | uint32_t size; /* the size of output buf. */ 341 | } rknn_output; 342 | 343 | /* 344 | the extend information for rknn_init. 345 | */ 346 | typedef struct _rknn_init_extend { 347 | rknn_context ctx; /* rknn context */ 348 | uint8_t reserved[128]; /* reserved */ 349 | } rknn_init_extend; 350 | 351 | /* 352 | the extend information for rknn_run. 353 | */ 354 | typedef struct _rknn_run_extend { 355 | uint64_t frame_id; /* output parameter, indicate current frame id of run. */ 356 | int32_t non_block; /* block flag of run, 0 is block else 1 is non block */ 357 | int32_t timeout_ms; /* timeout for block mode, in milliseconds */ 358 | int32_t fence_fd; /* fence fd from other unit */ 359 | } rknn_run_extend; 360 | 361 | /* 362 | the extend information for rknn_outputs_get. 363 | */ 364 | typedef struct _rknn_output_extend { 365 | uint64_t frame_id; /* output parameter, indicate the frame id of outputs, corresponds to 366 | struct rknn_run_extend.frame_id.*/ 367 | } rknn_output_extend; 368 | 369 | 370 | /* rknn_init 371 | 372 | initial the context and load the rknn model. 373 | 374 | input: 375 | rknn_context* context the pointer of context handle. 376 | void* model if size > 0, pointer to the rknn model, if size = 0, filepath to the rknn model. 377 | uint32_t size the size of rknn model. 378 | uint32_t flag extend flag, see the define of RKNN_FLAG_XXX_XXX. 379 | rknn_init_extend* extend the extend information of init. 380 | return: 381 | int error code. 382 | */ 383 | int rknn_init(rknn_context* context, void* model, uint32_t size, uint32_t flag, rknn_init_extend* extend); 384 | 385 | /* rknn_dup_context 386 | 387 | initial the context and load the rknn model. 388 | 389 | input: 390 | rknn_context* context_in the pointer of context in handle. 391 | rknn_context* context_out the pointer of context out handle. 392 | return: 393 | int error code. 394 | */ 395 | int rknn_dup_context(rknn_context* context_in, rknn_context* context_out); 396 | 397 | /* rknn_destroy 398 | 399 | unload the rknn model and destroy the context. 400 | 401 | input: 402 | rknn_context context the handle of context. 403 | return: 404 | int error code. 405 | */ 406 | int rknn_destroy(rknn_context context); 407 | 408 | 409 | /* rknn_query 410 | 411 | query the information about model or others. see rknn_query_cmd. 412 | 413 | input: 414 | rknn_context context the handle of context. 415 | rknn_query_cmd cmd the command of query. 416 | void* info the buffer point of information. 417 | uint32_t size the size of information. 418 | return: 419 | int error code. 420 | */ 421 | int rknn_query(rknn_context context, rknn_query_cmd cmd, void* info, uint32_t size); 422 | 423 | 424 | /* rknn_inputs_set 425 | 426 | set inputs information by input index of rknn model. 427 | inputs information see rknn_input. 428 | 429 | input: 430 | rknn_context context the handle of context. 431 | uint32_t n_inputs the number of inputs. 432 | rknn_input inputs[] the arrays of inputs information, see rknn_input. 433 | return: 434 | int error code 435 | */ 436 | int rknn_inputs_set(rknn_context context, uint32_t n_inputs, rknn_input inputs[]); 437 | 438 | /* rknn_set_core_mask 439 | 440 | set rknn core mask. 441 | 442 | RKNN_NPU_CORE_AUTO: auto mode, default value 443 | RKNN_NPU_CORE_0: core 0 mode 444 | RKNN_NPU_CORE_1: core 1 mode 445 | RKNN_NPU_CORE_2: core 2 mode 446 | RKNN_NPU_CORE_0_1: combine core 0/1 mode 447 | RKNN_NPU_CORE_0_1_2: combine core 0/1/2 mode 448 | 449 | input: 450 | rknn_context context the handle of context. 451 | rknn_core_mask core_mask the core mask. 452 | return: 453 | int error code. 454 | */ 455 | int rknn_set_core_mask(rknn_context context, rknn_core_mask core_mask); 456 | 457 | /* rknn_run 458 | 459 | run the model to execute inference. 460 | 461 | input: 462 | rknn_context context the handle of context. 463 | rknn_run_extend* extend the extend information of run. 464 | return: 465 | int error code. 466 | */ 467 | int rknn_run(rknn_context context, rknn_run_extend* extend); 468 | 469 | 470 | /* rknn_wait 471 | 472 | wait the model after execute inference. 473 | 474 | input: 475 | rknn_context context the handle of context. 476 | rknn_run_extend* extend the extend information of run. 477 | return: 478 | int error code. 479 | */ 480 | int rknn_wait(rknn_context context, rknn_run_extend* extend); 481 | 482 | 483 | /* rknn_outputs_get 484 | 485 | wait the inference to finish and get the outputs. 486 | this function will block until inference finish. 487 | the results will set to outputs[]. 488 | 489 | input: 490 | rknn_context context the handle of context. 491 | uint32_t n_outputs the number of outputs. 492 | rknn_output outputs[] the arrays of output, see rknn_output. 493 | rknn_output_extend* the extend information of output. 494 | return: 495 | int error code. 496 | */ 497 | int rknn_outputs_get(rknn_context context, uint32_t n_outputs, rknn_output outputs[], rknn_output_extend* extend); 498 | 499 | 500 | /* rknn_outputs_release 501 | 502 | release the outputs that get by rknn_outputs_get. 503 | after called, the rknn_output[x].buf get from rknn_outputs_get will 504 | also be free when rknn_output[x].is_prealloc = FALSE. 505 | 506 | input: 507 | rknn_context context the handle of context. 508 | uint32_t n_ouputs the number of outputs. 509 | rknn_output outputs[] the arrays of output. 510 | return: 511 | int error code 512 | */ 513 | int rknn_outputs_release(rknn_context context, uint32_t n_ouputs, rknn_output outputs[]); 514 | 515 | 516 | /* new api for zero copy */ 517 | 518 | /* rknn_create_mem_from_phys (memory allocated outside) 519 | 520 | initialize tensor memory from physical address. 521 | 522 | input: 523 | rknn_context ctx the handle of context. 524 | uint64_t phys_addr physical address. 525 | void *virt_addr virtual address. 526 | uint32_t size the size of tensor buffer. 527 | return: 528 | rknn_tensor_mem the pointer of tensor memory information. 529 | */ 530 | rknn_tensor_mem* rknn_create_mem_from_phys(rknn_context ctx, uint64_t phys_addr, void *virt_addr, uint32_t size); 531 | 532 | 533 | /* rknn_create_mem_from_fd (memory allocated outside) 534 | 535 | initialize tensor memory from file description. 536 | 537 | input: 538 | rknn_context ctx the handle of context. 539 | int32_t fd file description. 540 | void *virt_addr virtual address. 541 | uint32_t size the size of tensor buffer. 542 | int32_t offset indicates the offset of the memory (virt_addr without offset). 543 | return: 544 | rknn_tensor_mem the pointer of tensor memory information. 545 | */ 546 | rknn_tensor_mem* rknn_create_mem_from_fd(rknn_context ctx, int32_t fd, void *virt_addr, uint32_t size, int32_t offset); 547 | 548 | 549 | /* rknn_create_mem_from_mb_blk (memory allocated outside) 550 | 551 | create tensor memory from mb_blk. 552 | 553 | input: 554 | rknn_context ctx the handle of context. 555 | void *mb_blk mb_blk allocate from system api. 556 | int32_t offset indicates the offset of the memory. 557 | return: 558 | rknn_tensor_mem the pointer of tensor memory information. 559 | */ 560 | rknn_tensor_mem* rknn_create_mem_from_mb_blk(rknn_context ctx, void *mb_blk, int32_t offset); 561 | 562 | 563 | /* rknn_create_mem (memory allocated inside) 564 | 565 | create tensor memory. 566 | 567 | input: 568 | rknn_context ctx the handle of context. 569 | uint32_t size the size of tensor buffer. 570 | return: 571 | rknn_tensor_mem the pointer of tensor memory information. 572 | */ 573 | rknn_tensor_mem* rknn_create_mem(rknn_context ctx, uint32_t size); 574 | 575 | 576 | /* rknn_destory_mem (support allocate inside and outside) 577 | 578 | destory tensor memory. 579 | 580 | input: 581 | rknn_context ctx the handle of context. 582 | rknn_tensor_mem *mem the pointer of tensor memory information. 583 | return: 584 | int error code 585 | */ 586 | int rknn_destory_mem(rknn_context ctx, rknn_tensor_mem *mem); 587 | 588 | 589 | /* rknn_set_weight_mem 590 | 591 | set the weight memory. 592 | 593 | input: 594 | rknn_context ctx the handle of context. 595 | rknn_tensor_mem *mem the array of tensor memory information 596 | return: 597 | int error code. 598 | */ 599 | int rknn_set_weight_mem(rknn_context ctx, rknn_tensor_mem *mem); 600 | 601 | 602 | /* rknn_set_internal_mem 603 | 604 | set the internal memory. 605 | 606 | input: 607 | rknn_context ctx the handle of context. 608 | rknn_tensor_mem *mem the array of tensor memory information 609 | return: 610 | int error code. 611 | */ 612 | int rknn_set_internal_mem(rknn_context ctx, rknn_tensor_mem *mem); 613 | 614 | 615 | /* rknn_set_io_mem 616 | 617 | set the input and output tensors buffer. 618 | 619 | input: 620 | rknn_context ctx the handle of context. 621 | rknn_tensor_mem *mem the array of tensor memory information. 622 | rknn_tensor_attr *attr the attribute of input or output tensor buffer. 623 | return: 624 | int error code. 625 | */ 626 | int rknn_set_io_mem(rknn_context ctx, rknn_tensor_mem *mem, rknn_tensor_attr *attr); 627 | 628 | 629 | #ifdef __cplusplus 630 | } //extern "C" 631 | #endif 632 | 633 | #endif //_RKNN_API_H 634 | -------------------------------------------------------------------------------- /3rdparty/rga/include/RgaUtils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Rockchip Electronics Co., Ltd. 3 | * Authors: 4 | * Zhiqin Wei 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _rga_utils_h_ 20 | #define _rga_utils_h_ 21 | 22 | // ------------------------------------------------------------------------------- 23 | float get_bpp_from_format(int format); 24 | int get_buf_from_file(void *buf, int f, int sw, int sh, int index); 25 | int output_buf_data_to_file(void *buf, int f, int sw, int sh, int index); 26 | const char *translate_format_str(int format); 27 | int get_buf_from_file_AFBC(void *buf, int f, int sw, int sh, int index); 28 | int output_buf_data_to_file_AFBC(void *buf, int f, int sw, int sh, int index); 29 | #endif 30 | 31 | -------------------------------------------------------------------------------- /3rdparty/rga/include/im2d.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2020 Rockchip Electronics Co., Ltd. 3 | * Authors: 4 | * PutinLee 5 | * Cerf Yu 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | #ifndef _im2d_hpp_ 20 | #define _im2d_hpp_ 21 | 22 | #include "im2d.h" 23 | #include "RgaUtils.h" 24 | 25 | #ifdef ANDROID 26 | 27 | #include 28 | 29 | using namespace android; 30 | 31 | IM_API rga_buffer_t wrapbuffer_handle(buffer_handle_t hnd); 32 | IM_API rga_buffer_t wrapbuffer_GraphicBuffer(sp buf); 33 | 34 | #if USE_AHARDWAREBUFFER 35 | #include 36 | IM_API rga_buffer_t wrapbuffer_AHardwareBuffer(AHardwareBuffer *buf); 37 | 38 | #endif /* USE_AHARDWAREBUFFER */ 39 | #endif /* ANDROID */ 40 | #endif /* _im2d_hpp_ */ 41 | 42 | -------------------------------------------------------------------------------- /3rdparty/rga/include/rga.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Rockchip Electronics Co., Ltd. 3 | * Authors: 4 | * Zhiqin Wei 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _RGA_DRIVER_H_ 20 | #define _RGA_DRIVER_H_ 21 | 22 | #ifdef __cplusplus 23 | extern "C" 24 | { 25 | #endif 26 | 27 | #include 28 | 29 | #define RGA_BLIT_SYNC 0x5017 30 | #define RGA_BLIT_ASYNC 0x5018 31 | #define RGA_FLUSH 0x5019 32 | #define RGA_GET_RESULT 0x501a 33 | #define RGA_GET_VERSION 0x501b 34 | 35 | #define RGA2_BLIT_SYNC 0x6017 36 | #define RGA2_BLIT_ASYNC 0x6018 37 | #define RGA2_FLUSH 0x6019 38 | #define RGA2_GET_RESULT 0x601a 39 | #define RGA2_GET_VERSION 0x601b 40 | #define RGA2_GET_VERSION 0x601b 41 | 42 | #define RGA_REG_CTRL_LEN 0x8 /* 8 */ 43 | #define RGA_REG_CMD_LEN 0x1c /* 28 */ 44 | #define RGA_CMD_BUF_SIZE 0x700 /* 16*28*4 */ 45 | 46 | #ifndef ENABLE 47 | #define ENABLE 1 48 | #endif 49 | 50 | #ifndef DISABLE 51 | #define DISABLE 0 52 | #endif 53 | 54 | /* RGA process mode enum */ 55 | enum { 56 | bitblt_mode = 0x0, 57 | color_palette_mode = 0x1, 58 | color_fill_mode = 0x2, 59 | line_point_drawing_mode = 0x3, 60 | blur_sharp_filter_mode = 0x4, 61 | pre_scaling_mode = 0x5, 62 | update_palette_table_mode = 0x6, 63 | update_patten_buff_mode = 0x7, 64 | }; 65 | 66 | 67 | enum { 68 | rop_enable_mask = 0x2, 69 | dither_enable_mask = 0x8, 70 | fading_enable_mask = 0x10, 71 | PD_enbale_mask = 0x20, 72 | }; 73 | 74 | enum { 75 | yuv2rgb_mode0 = 0x0, /* BT.601 MPEG */ 76 | yuv2rgb_mode1 = 0x1, /* BT.601 JPEG */ 77 | yuv2rgb_mode2 = 0x2, /* BT.709 */ 78 | 79 | rgb2yuv_601_full = 0x1 << 8, 80 | rgb2yuv_709_full = 0x2 << 8, 81 | yuv2yuv_601_limit_2_709_limit = 0x3 << 8, 82 | yuv2yuv_601_limit_2_709_full = 0x4 << 8, 83 | yuv2yuv_709_limit_2_601_limit = 0x5 << 8, 84 | yuv2yuv_709_limit_2_601_full = 0x6 << 8, //not support 85 | yuv2yuv_601_full_2_709_limit = 0x7 << 8, 86 | yuv2yuv_601_full_2_709_full = 0x8 << 8, //not support 87 | yuv2yuv_709_full_2_601_limit = 0x9 << 8, //not support 88 | yuv2yuv_709_full_2_601_full = 0xa << 8, //not support 89 | full_csc_mask = 0xf00, 90 | }; 91 | 92 | /* RGA rotate mode */ 93 | enum { 94 | rotate_mode0 = 0x0, /* no rotate */ 95 | rotate_mode1 = 0x1, /* rotate */ 96 | rotate_mode2 = 0x2, /* x_mirror */ 97 | rotate_mode3 = 0x3, /* y_mirror */ 98 | }; 99 | 100 | enum { 101 | color_palette_mode0 = 0x0, /* 1K */ 102 | color_palette_mode1 = 0x1, /* 2K */ 103 | color_palette_mode2 = 0x2, /* 4K */ 104 | color_palette_mode3 = 0x3, /* 8K */ 105 | }; 106 | 107 | enum { 108 | BB_BYPASS = 0x0, /* no rotate */ 109 | BB_ROTATE = 0x1, /* rotate */ 110 | BB_X_MIRROR = 0x2, /* x_mirror */ 111 | BB_Y_MIRROR = 0x3 /* y_mirror */ 112 | }; 113 | 114 | enum { 115 | nearby = 0x0, /* no rotate */ 116 | bilinear = 0x1, /* rotate */ 117 | bicubic = 0x2, /* x_mirror */ 118 | }; 119 | 120 | #define RGA_SCHED_PRIORITY_DEFAULT 0 121 | #define RGA_SCHED_PRIORITY_MAX 6 122 | 123 | enum { 124 | RGA3_SCHEDULER_CORE0 = 1 << 0, 125 | RGA3_SCHEDULER_CORE1 = 1 << 1, 126 | RGA2_SCHEDULER_CORE0 = 1 << 2, 127 | }; 128 | 129 | /* 130 | // Alpha Red Green Blue 131 | { 4, 32, {{32,24, 8, 0, 16, 8, 24,16 }}, GGL_RGBA }, // RK_FORMAT_RGBA_8888 132 | { 4, 24, {{ 0, 0, 8, 0, 16, 8, 24,16 }}, GGL_RGB }, // RK_FORMAT_RGBX_8888 133 | { 3, 24, {{ 0, 0, 8, 0, 16, 8, 24,16 }}, GGL_RGB }, // RK_FORMAT_RGB_888 134 | { 4, 32, {{32,24, 24,16, 16, 8, 8, 0 }}, GGL_BGRA }, // RK_FORMAT_BGRA_8888 135 | { 2, 16, {{ 0, 0, 16,11, 11, 5, 5, 0 }}, GGL_RGB }, // RK_FORMAT_RGB_565 136 | { 2, 16, {{ 1, 0, 16,11, 11, 6, 6, 1 }}, GGL_RGBA }, // RK_FORMAT_RGBA_5551 137 | { 2, 16, {{ 4, 0, 16,12, 12, 8, 8, 4 }}, GGL_RGBA }, // RK_FORMAT_RGBA_4444 138 | { 3, 24, {{ 0, 0, 24,16, 16, 8, 8, 0 }}, GGL_BGR }, // RK_FORMAT_BGB_888 139 | 140 | */ 141 | /* In order to be compatible with RK_FORMAT_XX and HAL_PIXEL_FORMAT_XX, 142 | * RK_FORMAT_XX is shifted to the left by 8 bits to distinguish. */ 143 | typedef enum _Rga_SURF_FORMAT { 144 | RK_FORMAT_RGBA_8888 = 0x0 << 8, 145 | RK_FORMAT_RGBX_8888 = 0x1 << 8, 146 | RK_FORMAT_RGB_888 = 0x2 << 8, 147 | RK_FORMAT_BGRA_8888 = 0x3 << 8, 148 | RK_FORMAT_RGB_565 = 0x4 << 8, 149 | RK_FORMAT_RGBA_5551 = 0x5 << 8, 150 | RK_FORMAT_RGBA_4444 = 0x6 << 8, 151 | RK_FORMAT_BGR_888 = 0x7 << 8, 152 | 153 | RK_FORMAT_YCbCr_422_SP = 0x8 << 8, 154 | RK_FORMAT_YCbCr_422_P = 0x9 << 8, 155 | RK_FORMAT_YCbCr_420_SP = 0xa << 8, 156 | RK_FORMAT_YCbCr_420_P = 0xb << 8, 157 | 158 | RK_FORMAT_YCrCb_422_SP = 0xc << 8, 159 | RK_FORMAT_YCrCb_422_P = 0xd << 8, 160 | RK_FORMAT_YCrCb_420_SP = 0xe << 8, 161 | RK_FORMAT_YCrCb_420_P = 0xf << 8, 162 | 163 | RK_FORMAT_BPP1 = 0x10 << 8, 164 | RK_FORMAT_BPP2 = 0x11 << 8, 165 | RK_FORMAT_BPP4 = 0x12 << 8, 166 | RK_FORMAT_BPP8 = 0x13 << 8, 167 | 168 | RK_FORMAT_Y4 = 0x14 << 8, 169 | RK_FORMAT_YCbCr_400 = 0x15 << 8, 170 | 171 | RK_FORMAT_BGRX_8888 = 0x16 << 8, 172 | 173 | RK_FORMAT_YVYU_422 = 0x18 << 8, 174 | RK_FORMAT_YVYU_420 = 0x19 << 8, 175 | RK_FORMAT_VYUY_422 = 0x1a << 8, 176 | RK_FORMAT_VYUY_420 = 0x1b << 8, 177 | RK_FORMAT_YUYV_422 = 0x1c << 8, 178 | RK_FORMAT_YUYV_420 = 0x1d << 8, 179 | RK_FORMAT_UYVY_422 = 0x1e << 8, 180 | RK_FORMAT_UYVY_420 = 0x1f << 8, 181 | 182 | RK_FORMAT_YCbCr_420_SP_10B = 0x20 << 8, 183 | RK_FORMAT_YCrCb_420_SP_10B = 0x21 << 8, 184 | RK_FORMAT_YCbCr_422_10b_SP = 0x22 << 8, 185 | RK_FORMAT_YCrCb_422_10b_SP = 0x23 << 8, 186 | 187 | RK_FORMAT_BGR_565 = 0x24 << 8, 188 | RK_FORMAT_BGRA_5551 = 0x25 << 8, 189 | RK_FORMAT_BGRA_4444 = 0x26 << 8, 190 | 191 | RK_FORMAT_ARGB_8888 = 0x28 << 8, 192 | RK_FORMAT_XRGB_8888 = 0x29 << 8, 193 | RK_FORMAT_ARGB_5551 = 0x2a << 8, 194 | RK_FORMAT_ARGB_4444 = 0x2b << 8, 195 | RK_FORMAT_ABGR_8888 = 0x2c << 8, 196 | RK_FORMAT_XBGR_8888 = 0x2d << 8, 197 | RK_FORMAT_ABGR_5551 = 0x2e << 8, 198 | RK_FORMAT_ABGR_4444 = 0x2f << 8, 199 | 200 | RK_FORMAT_UNKNOWN = 0x100 << 8, 201 | } RgaSURF_FORMAT; 202 | 203 | /* RGA3 rd_mode */ 204 | enum 205 | { 206 | raster_mode = 0x1 << 0, 207 | afbc_mode = 0x1 << 1, 208 | tile_mode = 0x1 << 2, 209 | }; 210 | 211 | typedef struct rga_img_info_t { 212 | uint64_t yrgb_addr; /* yrgb mem addr */ 213 | uint64_t uv_addr; /* cb/cr mem addr */ 214 | uint64_t v_addr; /* cr mem addr */ 215 | 216 | uint32_t format; //definition by RK_FORMAT 217 | uint16_t act_w; 218 | uint16_t act_h; 219 | uint16_t x_offset; 220 | uint16_t y_offset; 221 | 222 | uint16_t vir_w; 223 | uint16_t vir_h; 224 | 225 | uint16_t endian_mode; //for BPP 226 | uint16_t alpha_swap; 227 | 228 | //used by RGA3 229 | uint16_t rotate_mode; 230 | uint16_t rd_mode; 231 | 232 | uint16_t is_10b_compact; 233 | uint16_t is_10b_endian; 234 | 235 | uint16_t enable; 236 | } 237 | rga_img_info_t; 238 | 239 | typedef struct POINT { 240 | uint16_t x; 241 | uint16_t y; 242 | } 243 | POINT; 244 | 245 | typedef struct RECT { 246 | uint16_t xmin; 247 | uint16_t xmax; // width - 1 248 | uint16_t ymin; 249 | uint16_t ymax; // height - 1 250 | } RECT; 251 | 252 | typedef struct MMU { 253 | uint8_t mmu_en; 254 | uint64_t base_addr; 255 | uint32_t mmu_flag; /* [0] mmu enable [1] src_flush [2] dst_flush [3] CMD_flush [4~5] page size*/ 256 | } MMU; 257 | 258 | typedef struct COLOR_FILL { 259 | int16_t gr_x_a; 260 | int16_t gr_y_a; 261 | int16_t gr_x_b; 262 | int16_t gr_y_b; 263 | int16_t gr_x_g; 264 | int16_t gr_y_g; 265 | int16_t gr_x_r; 266 | int16_t gr_y_r; 267 | //u8 cp_gr_saturation; 268 | } 269 | COLOR_FILL; 270 | 271 | typedef struct FADING { 272 | uint8_t b; 273 | uint8_t g; 274 | uint8_t r; 275 | uint8_t res; 276 | } 277 | FADING; 278 | 279 | typedef struct line_draw_t { 280 | POINT start_point; /* LineDraw_start_point */ 281 | POINT end_point; /* LineDraw_end_point */ 282 | uint32_t color; /* LineDraw_color */ 283 | uint32_t flag; /* (enum) LineDrawing mode sel */ 284 | uint32_t line_width; /* range 1~16 */ 285 | } 286 | line_draw_t; 287 | 288 | /* color space convert coefficient. */ 289 | typedef struct csc_coe_t { 290 | int16_t r_v; 291 | int16_t g_y; 292 | int16_t b_u; 293 | int32_t off; 294 | } csc_coe_t; 295 | 296 | typedef struct full_csc_t { 297 | uint8_t flag; 298 | csc_coe_t coe_y; 299 | csc_coe_t coe_u; 300 | csc_coe_t coe_v; 301 | } full_csc_t; 302 | 303 | struct rga_req { 304 | uint8_t render_mode; /* (enum) process mode sel */ 305 | 306 | rga_img_info_t src; /* src image info */ 307 | rga_img_info_t dst; /* dst image info */ 308 | rga_img_info_t pat; /* patten image info */ 309 | 310 | uint64_t rop_mask_addr; /* rop4 mask addr */ 311 | uint64_t LUT_addr; /* LUT addr */ 312 | 313 | RECT clip; /* dst clip window default value is dst_vir */ 314 | /* value from [0, w-1] / [0, h-1]*/ 315 | 316 | int32_t sina; /* dst angle default value 0 16.16 scan from table */ 317 | int32_t cosa; /* dst angle default value 0 16.16 scan from table */ 318 | 319 | uint16_t alpha_rop_flag; /* alpha rop process flag */ 320 | /* ([0] = 1 alpha_rop_enable) */ 321 | /* ([1] = 1 rop enable) */ 322 | /* ([2] = 1 fading_enable) */ 323 | /* ([3] = 1 PD_enable) */ 324 | /* ([4] = 1 alpha cal_mode_sel) */ 325 | /* ([5] = 1 dither_enable) */ 326 | /* ([6] = 1 gradient fill mode sel) */ 327 | /* ([7] = 1 AA_enable) */ 328 | /* ([8] = 1 nn_quantize) */ 329 | /* ([9] = 1 Real color mode) */ 330 | 331 | uint8_t scale_mode; /* 0 nearst / 1 bilnear / 2 bicubic */ 332 | 333 | uint32_t color_key_max; /* color key max */ 334 | uint32_t color_key_min; /* color key min */ 335 | 336 | uint32_t fg_color; /* foreground color */ 337 | uint32_t bg_color; /* background color */ 338 | 339 | COLOR_FILL gr_color; /* color fill use gradient */ 340 | 341 | line_draw_t line_draw_info; 342 | 343 | FADING fading; 344 | 345 | uint8_t PD_mode; /* porter duff alpha mode sel */ 346 | 347 | uint8_t alpha_global_value; /* global alpha value */ 348 | 349 | uint16_t rop_code; /* rop2/3/4 code scan from rop code table*/ 350 | 351 | uint8_t bsfilter_flag; /* [2] 0 blur 1 sharp / [1:0] filter_type*/ 352 | 353 | uint8_t palette_mode; /* (enum) color palatte 0/1bpp, 1/2bpp 2/4bpp 3/8bpp*/ 354 | 355 | uint8_t yuv2rgb_mode; /* (enum) BT.601 MPEG / BT.601 JPEG / BT.709 */ 356 | 357 | uint8_t endian_mode; /* 0/big endian 1/little endian*/ 358 | 359 | uint8_t rotate_mode; /* (enum) rotate mode */ 360 | /* 0x0, no rotate */ 361 | /* 0x1, rotate */ 362 | /* 0x2, x_mirror */ 363 | /* 0x3, y_mirror */ 364 | 365 | uint8_t color_fill_mode; /* 0 solid color / 1 patten color */ 366 | 367 | MMU mmu_info; /* mmu information */ 368 | 369 | uint8_t alpha_rop_mode; /* ([0~1] alpha mode) */ 370 | /* ([2~3] rop mode) */ 371 | /* ([4] zero mode en) */ 372 | /* ([5] dst alpha mode) (RGA1) */ 373 | 374 | uint8_t src_trans_mode; 375 | 376 | uint8_t dither_mode; 377 | 378 | full_csc_t full_csc; /* full color space convert */ 379 | 380 | int32_t in_fence_fd; 381 | uint8_t core; 382 | uint8_t priority; 383 | int32_t out_fence_fd; 384 | 385 | uint8_t reservr[128]; 386 | }; 387 | 388 | #ifdef __cplusplus 389 | } 390 | #endif 391 | 392 | #endif /*_RK29_IPP_DRIVER_H_*/ 393 | -------------------------------------------------------------------------------- /3rdparty/rga/lib/librga.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/3rdparty/rga/lib/librga.so -------------------------------------------------------------------------------- /Bytetrack.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/Bytetrack.gif -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0.0) 2 | project(yolov5_deepsort VERSION 0.1.0) 3 | 4 | add_subdirectory(deepsort) 5 | 6 | set(OpenCV_DIR /home/wjp/opencv-4.6.0/install/lib/cmake/opencv4) # 填入OpenCVConfig.cmake 7 | find_package(OpenCV 4 REQUIRED) 8 | include_directories(${OpenCV_INCLUDE_DIRS}) 9 | 10 | set( 11 | include_libs 12 | "${PROJECT_SOURCE_DIR}/include" 13 | "${PROJECT_SOURCE_DIR}/yolov5/include" 14 | "${PROJECT_SOURCE_DIR}/deepsort/include" 15 | ${PROJECT_SOURCE_DIR}/3rdparty/librknn_api/include 16 | ${PROJECT_SOURCE_DIR}/3rdparty/rga/include 17 | ) 18 | 19 | include_directories(${include_libs}) 20 | 21 | aux_source_directory(${PROJECT_SOURCE_DIR}/yolov5/src YOLO_SRC_DIR) 22 | aux_source_directory(${PROJECT_SOURCE_DIR}/src YOLO_SRC_DIR) 23 | add_executable(yolov5_deepsort yolov5_deepsort.cpp ${YOLO_SRC_DIR}) 24 | 25 | # 添加动态链接库 26 | set( 27 | dynamic_libs pthread 28 | ${PROJECT_SOURCE_DIR}/3rdparty/librknn_api/aarch64/librknnrt.so 29 | ${PROJECT_SOURCE_DIR}/3rdparty/rga/lib/librga.so 30 | ) 31 | 32 | # 开启调试选项 33 | add_definitions("-g") 34 | 35 | target_link_libraries(yolov5_deepsort ${OpenCV_LIBS}) 36 | target_link_libraries(yolov5_deepsort ${dynamic_libs} deepsort) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Yolov5_DeepSORT_rknn 2 | 3 | **** 4 | # 改动: 5 | 6 | ## 本仓库在原仓库的基础上: 7 | 1. 改善了边界框漂移, 完善了当图中没有目标等其他情形出现的bug, 增加了对cost matrix出现nan时的处理 8 | 2. 加入了隔帧检测的功能. 设置方法: 9 | > 在`./yolov5/include/detect.h`中 将 10 | > `const int det_interval = 1;`改成期望的数值, 例如3, 表示每隔3帧检测一次, 这样可以**显著提升速度**. 11 | > 同时, 也需要更改`./deepsort/include/deepsort.h`中`line 39`的`const int track_interval = 1; `, 数值要和检测的保持一致. 12 | 3. 加入Re-ID多线程的功能 13 | > 如果您不希望使用多线程, 则在`./deepsort/src/deepsort.cpp`中`line 144`的`if (numOfDetections < 2)` 14 | > 改成`if (true)` 15 | 16 | 17 | 自己使用时, 除了更改OpenCV的路径外, 要在`./include/common.h`中修改`IMG_WIDTH, IMG_HEIGHT, IMG_PAD, OBJ_CLASS_NUM` 18 | 在`./yolov5/src/decode.cpp`中修改`LABEL_NALE_TXT_PATH`. 19 | **** 20 | 21 | [![standard-readme compliant](https://img.shields.io/badge/readme%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) 22 | 23 | Yolov5_DeepSORT_rknn是基于瑞芯微Rockchip Neural Network(RKNN)开发的目标跟踪部署仓库，除了DeepSORT还支持SORT算法，可以根据不同的嵌入式平台选择合适的跟踪算法。本仓库中的DeepSORT在Rk3588上测试通过，SORT和ByteTrack应该在Rk3588和Rk3399Pro上都可运行。 24 | 25 | 下面是我们的演示视频 具有强烈的抖动，进一步提高目标检测模型精度并且在视频无抖动情况下追踪性能应该会很Nice。 26 | 27 |
28 |     29 |
30 | Detect 31 |          32 |          33 |          34 | DeepSORT 35 |
36 |
37 | 38 |
39 |     40 |
41 | SORT 42 |          43 |          44 |          45 | Bytetrack 46 |
47 |
48 | 49 | DeepSORT、SORT和ByteTrack已经上线，放在三个分支里！除了这三个算法之外，可能还会更新其他SOTA跟踪算法，多多关注~！ 50 | 51 | 我是野生程序猿，如果在代码编写上存在不规范的情况，请多多见谅。 52 | 53 | ## 文档内容 54 | 55 | - [文件目录结构描述](#文件目录结构描述) 56 | - [安装及使用](#安装及使用) 57 | - [数据说明](#数据说明) 58 | - [性能测试](#性能测试) 59 | - [参考仓库](#参考仓库) 60 | 61 | ## 文件目录结构描述 62 | 63 | ``` 64 | ├── Readme.md // help 65 | ├── data // 数据 66 | ├── model // 模型 67 | ├── build 68 | ├── CMakeLists.txt // 编译Yolov5_DeepSORT 69 | ├── include // 通用头文件 70 | ├── src 71 | ├── 3rdparty 72 | │ ├── linrknn_api // rknn 动态链接库 73 | │ ├── rga // rga 动态链接库 74 | │ ├── opencv // opencv 动态链接库(自行编译并在CmakeLists.txt中设置相应路径) 75 | ├── yolov5 76 | │ └── include 77 | │ └── decode.h // 解码 78 | │ └── detect.h // 推理 79 | │ └── videoio.h // 视频IO 80 | │ └── src 81 | │ └── decode.cpp 82 | │ └── ... 83 | ├── deepsort 84 | │ └── include 85 | │ └── deepsort.h // class DeepSort 86 | │ └── featuretensor.h // Reid推理 87 | │ └── ... 88 | │ └── src 89 | │ └── deepsort.cpp 90 | │ └── ... 91 | │ └── CMakeLists.txt // 编译deepsort子模块 92 | 93 | ``` 94 | 95 | ## 安装及使用 96 | 97 | + RKNN-Toolkit 98 | 99 | 这个项目需要使用RKNN-Toolkit2(Rk3588)或者RKNN-Toolkit1(Rk3399Pro)，请确保librknnrt.so和rknn_server正常运行。可以先运行瑞芯微仓库中的Demo来测试。 100 | 101 | ``` 102 | rknpu2 103 | https://github.com/rockchip-linux/rknpu2 104 | ``` 105 | 106 | + opencv的编译安装 107 | 108 | 可以选择直接在板子上编译，Rk3588编译速度很快，不到十分钟。 109 | 也可以选择交叉编译，我使用的交叉编译工具链：gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu 110 | 111 | ``` 112 | 注意！！！ 113 | 请根据自己的OpenCV路径修改CMakeLists.txt文件 114 | 115 | 搜索 set(OpenCV_DIR /home/linaro/workspace/opencv/lib/cmake/opencv4) 116 | 将路径替换成你的OpenCVConfig.cmake所在的文件夹 117 | 118 | 本项目中有两个CMakeLists.txt请同时修改！！！ 119 | ``` 120 | 121 | + DeepSort选用的模型是TorchReID中的osnet_x0_25 ，输入尺寸是256x512 122 | 123 | 目前还没有针对于自己的数据集重新训练 124 | 125 | ``` 126 | Torchreid 127 | https://kaiyangzhou.github.io/deep-person-reid/MODEL_ZOO 128 | ``` 129 | 130 | + 项目编译与运行 131 | 132 | 参考Cmake的使用，会在build目录下生成可执行文件。 133 | 134 | ## 数据说明 135 | 136 | + 使用的是红外车辆、行人数据集，自行拍摄的，暂时不公开。 137 | 138 | + 测试的视频存在严重的抖动，影响了跟踪性能，不代表DeepSORT的跟踪性能。 139 | 140 | ## 性能测试 141 | 142 | 目前只对模型速度进行了测试，首先送上瑞芯微官方的benchmark 143 | 144 | + 瑞芯微rknn_model_zoo 145 | 146 | | platform（fps） | yolov5s-relu | yolov5s-silu | yolov5m-relu | yolov5m-silu | 147 | | ------------------------ | ------------ | ------------ | ------------ | ------------ | 148 | | rk1808 - u8 | 35.24 | 26.41 | 16.27 | 12.60 | 149 | | rv1109 - u8 | 19.58 | 13.33 | 8.11 | 5.45 | 150 | | rv1126 - u8 | 27.54 | 19.29 | 11.69 | 7.86 | 151 | | rk3566 - u8 | 15.16 | 10.60 | 8.65 | 6.61 | 152 | | rk3588 - u8(single core) | 53.73 | 33.24 | 22.31 | 14.74 | 153 | 154 | + Ours 155 | 156 | DeepSORT的ReID网络单次推理耗时约3ms，但是由于每个检测框都需要推理一次网络故受目标个数影响很大。SORT和ByteTrack由于没有ReID网络，在目标不是很多的情况下跟踪时间约等于目标检测，但是ID切换现象会更明显更严重一些。 157 | 158 | | platform（ms） | yolov5s-relu | yolov5s-relu+Deepsort |yolov5s-relu+Sort |yolov5s-relu+ByteTrack | 159 | | :-------------------------: | :--: | :--: | :--: | :--: | 160 | | rk3588 - u8(single core) | 24 | - | - | - | 161 | | rk3588 - u8(double core) | 12 | 33.24(infulenced)| 12 | 12 | 162 | | rk3399Pro - u8(single core) | - | - | - | - | 163 | 164 | ## 参考仓库 165 | 166 | 本项目参考了大量前人的优秀工作，在最后放上一些有用的仓库链接。 167 | 168 | 1. https://github.com/ultralytics/yolov5 169 | 2. https://github.com/airockchip/rknn_model_zoo 170 | 3. https://github.com/airockchip/librga 171 | 4. https://github.com/RichardoMrMu/yolov5-deepsort-tensorrt 172 | 5. https://github.com/KaiyangZhou/deep-person-reid 173 | -------------------------------------------------------------------------------- /SORT.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/SORT.gif -------------------------------------------------------------------------------- /build/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/build/.gitkeep -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/data/.gitkeep -------------------------------------------------------------------------------- /data/DJI_0001_S_cut.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/data/DJI_0001_S_cut.mp4 -------------------------------------------------------------------------------- /deepsort.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/deepsort.gif -------------------------------------------------------------------------------- /deepsort/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0.0) 2 | 3 | set(OpenCV_DIR /home/wjp/opencv-4.6.0/install/lib/cmake/opencv4) # 填入OpenCVConfig.cmake 4 | find_package(OpenCV 4 REQUIRED) 5 | include_directories(${OpenCV_INCLUDE_DIRS}) 6 | 7 | set( 8 | include_libs 9 | "${PROJECT_SOURCE_DIR}/include" 10 | "${PROJECT_SOURCE_DIR}/deepsort/include" 11 | ${PROJECT_SOURCE_DIR}/3rdparty/librknn_api/include 12 | ${PROJECT_SOURCE_DIR}/3rdparty/rga/include 13 | ) 14 | 15 | set( 16 | dynamic_libs 17 | ${PROJECT_SOURCE_DIR}/3rdparty/librknn_api/aarch64/librknnrt.so 18 | ${PROJECT_SOURCE_DIR}/3rdparty/rga/lib/librga.so 19 | ) 20 | 21 | include_directories(${include_libs}) 22 | aux_source_directory(${PROJECT_SOURCE_DIR}/deepsort/src DEEPSORT_SRC_DIR) 23 | aux_source_directory(${PROJECT_SOURCE_DIR}/src DEEPSORT_SRC_DIR) 24 | 25 | add_library(deepsort ${DEEPSORT_SRC_DIR}) 26 | 27 | target_link_libraries(deepsort ${OpenCV_LIBS}) 28 | target_link_libraries(deepsort ${dynamic_libs}) -------------------------------------------------------------------------------- /deepsort/include/MyKalmanFilter.h: -------------------------------------------------------------------------------- 1 | #ifndef MyKalmanFilter_H 2 | #define MyKalmanFilter_H 3 | 4 | #include "datatype.h" 5 | 6 | class MyKalmanFilter { 7 | public: 8 | static const double chi2inv95[10]; 9 | MyKalmanFilter(); 10 | KAL_DATA initiate(const DETECTBOX& measurement); 11 | void predict(KAL_MEAN& mean, KAL_COVA& covariance); 12 | KAL_HDATA project(const KAL_MEAN& mean, const KAL_COVA& covariance); 13 | KAL_DATA update(const KAL_MEAN& mean, 14 | const KAL_COVA& covariance, 15 | const DETECTBOX& measurement); 16 | 17 | Eigen::Matrix gating_distance( 18 | const KAL_MEAN& mean, 19 | const KAL_COVA& covariance, 20 | const std::vector& measurements, 21 | bool only_position = false); 22 | 23 | private: 24 | Eigen::Matrix _motion_mat; 25 | Eigen::Matrix _update_mat; 26 | float _std_weight_position; 27 | float _std_weight_velocity; 28 | }; 29 | 30 | #endif // MyKalmanFilter_H 31 | -------------------------------------------------------------------------------- /deepsort/include/datatype.h: -------------------------------------------------------------------------------- 1 | #ifndef BOX_H 2 | #include "box.h" 3 | #define BOX_H 4 | #endif // BOX_H 5 | 6 | #ifndef DEEPSORTDATATYPE_H 7 | #define DEEPSORTDATATYPE_H 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | typedef struct CLSCONF { 14 | CLSCONF() { 15 | this->cls = -1; 16 | this->conf = -1; 17 | } 18 | CLSCONF(int cls, float conf) { 19 | this->cls = cls; 20 | this->conf = conf; 21 | } 22 | int cls; 23 | float conf; 24 | } CLSCONF; 25 | 26 | typedef Eigen::Matrix DETECTBOX; 27 | typedef Eigen::Matrix DETECTBOXSS; 28 | typedef Eigen::Matrix FEATURE; 29 | typedef Eigen::Matrix FEATURESS; 30 | //typedef std::vector FEATURESS; 31 | 32 | //MyKalmanFilter 33 | //typedef Eigen::Matrix KAL_FILTER; 34 | typedef Eigen::Matrix KAL_MEAN; 35 | typedef Eigen::Matrix KAL_COVA; 36 | typedef Eigen::Matrix KAL_HMEAN; 37 | typedef Eigen::Matrix KAL_HCOVA; 38 | using KAL_DATA = std::pair; 39 | using KAL_HDATA = std::pair; 40 | 41 | //main 42 | using RESULT_DATA = std::pair; 43 | 44 | //tracker: 45 | using TRACKER_DATA = std::pair; 46 | using MATCH_DATA = std::pair; 47 | typedef struct t{ 48 | std::vector matches; 49 | std::vector unmatched_tracks; 50 | std::vector unmatched_detections; 51 | }TRACHER_MATCHD; 52 | 53 | //linear_assignment: 54 | typedef Eigen::Matrix DYNAMICM; 55 | 56 | #endif //DEEPSORTDATATYPE_H -------------------------------------------------------------------------------- /deepsort/include/deepsort.h: -------------------------------------------------------------------------------- 1 | #ifndef DEEPSORT_H 2 | #define DEEPSORT_H 3 | 4 | #include 5 | #include "featuretensor.h" 6 | #include "tracker.h" 7 | #include "datatype.h" 8 | #include "model.hpp" 9 | #include 10 | 11 | using std::vector; 12 | 13 | class DeepSort { 14 | public: 15 | DeepSort(std::string modelPath, int batchSize, int featureDim, int cpu_id, rknn_core_mask npu_id); 16 | ~DeepSort(); 17 | 18 | public: 19 | void sort(cv::Mat& frame, vector& dets); 20 | void sort_interval(cv::Mat& frame, vector& dets); 21 | int track_process(); 22 | void showDetection(cv::Mat& img, std::vector& boxes); 23 | 24 | private: 25 | void sort(cv::Mat& frame, DETECTIONS& detections); 26 | void sort(cv::Mat& frame, DETECTIONSV2& detectionsv2); 27 | void init(); 28 | 29 | private: 30 | std::string enginePath; 31 | int batchSize; 32 | int featureDim; 33 | cv::Size imgShape; 34 | float confThres; 35 | float nmsThres; 36 | int maxBudget; 37 | float maxCosineDist; 38 | 39 | const int track_interval = 1; 40 | private: 41 | vector result; 42 | vector> results; 43 | tracker* objTracker; 44 | FeatureTensor* featureExtractor1; 45 | FeatureTensor* featureExtractor2; 46 | rknn_core_mask npu_id; 47 | int cpu_id; 48 | }; 49 | 50 | #endif //deepsort.h 51 | -------------------------------------------------------------------------------- /deepsort/include/featuretensor.h: -------------------------------------------------------------------------------- 1 | #ifndef FEATURETENSOR_H 2 | #define FEATURETENSOR_H 3 | 4 | #include 5 | #include 6 | #include "model.hpp" 7 | #include "datatype.h" 8 | #include "rknn_fp.h" 9 | #include "resize.h" 10 | 11 | using std::vector; 12 | 13 | class FeatureTensor :public rknn_fp{ 14 | public: 15 | using rknn_fp::rknn_fp; 16 | void init(cv::Size, int, int); 17 | bool getRectsFeature(const cv::Mat& img, DETECTIONS& det); 18 | void doInference(vector& imgMats, DETECTIONS& det); 19 | 20 | public: 21 | cv::Size imgShape; 22 | int featureDim; 23 | PreResize pre_do; 24 | }; 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /deepsort/include/hungarianoper.h: -------------------------------------------------------------------------------- 1 | #ifndef HUNGARIANOPER_H 2 | #define HUNGARIANOPER_H 3 | 4 | #include "munkres.h" 5 | #include "datatype.h" 6 | 7 | 8 | class HungarianOper { 9 | public: 10 | static Eigen::Matrix Solve(const DYNAMICM &cost_matrix); 11 | }; 12 | 13 | #endif // HUNGARIANOPER_H 14 | -------------------------------------------------------------------------------- /deepsort/include/linear_assignment.h: -------------------------------------------------------------------------------- 1 | #ifndef LINEAR_ASSIGNMENT_H 2 | #define LINEAR_ASSIGNMENT_H 3 | 4 | #include "datatype.h" 5 | #include "tracker.h" 6 | 7 | #define INFTY_COST 1e5 8 | class tracker; 9 | //for matching; 10 | class linear_assignment 11 | { 12 | linear_assignment(); 13 | linear_assignment(const linear_assignment& ); 14 | linear_assignment& operator=(const linear_assignment&); 15 | static linear_assignment* instance; 16 | 17 | public: 18 | static linear_assignment* getInstance(); 19 | TRACHER_MATCHD matching_cascade(tracker* distance_metric, 20 | tracker::GATED_METRIC_FUNC distance_metric_func, 21 | float max_distance, 22 | int cascade_depth, 23 | std::vector& tracks, 24 | const DETECTIONS& detections, 25 | std::vector &track_indices, 26 | std::vector detection_indices = std::vector()); 27 | TRACHER_MATCHD min_cost_matching( 28 | tracker* distance_metric, 29 | tracker::GATED_METRIC_FUNC distance_metric_func, 30 | float max_distance, 31 | std::vector& tracks, 32 | const DETECTIONS& detections, 33 | std::vector& track_indices, 34 | std::vector& detection_indices); 35 | DYNAMICM gate_cost_matrix( 36 | MyKalmanFilter* kf, 37 | DYNAMICM& cost_matrix, 38 | std::vector& tracks, 39 | const DETECTIONS& detections, 40 | const std::vector& track_indices, 41 | const std::vector& detection_indices, 42 | float gated_cost = INFTY_COST, 43 | bool only_position = false); 44 | }; 45 | 46 | #endif // LINEAR_ASSIGNMENT_H 47 | -------------------------------------------------------------------------------- /deepsort/include/matrix.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007 John Weaver 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation; either version 2 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program; if not, write to the Free Software 16 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 | */ 18 | 19 | #ifndef _MATRIX_H_ 20 | #define _MATRIX_H_ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #define XYZMIN(x, y) (x)<(y)?(x):(y) 30 | #define XYZMAX(x, y) (x)>(y)?(x):(y) 31 | 32 | template 33 | class Matrix { 34 | public: 35 | Matrix(){ 36 | m_rows = 0; 37 | m_columns = 0; 38 | m_matrix = nullptr; 39 | } 40 | Matrix(const size_t rows, const size_t columns) { 41 | m_matrix = nullptr; 42 | resize(rows, columns); 43 | } 44 | Matrix(const std::initializer_list> init) { 45 | m_matrix = nullptr; 46 | m_rows = init.size(); 47 | if ( m_rows == 0 ) { 48 | m_columns = 0; 49 | } else { 50 | m_columns = init.begin()->size(); 51 | if ( m_columns > 0 ) { 52 | resize(m_rows, m_columns); 53 | } 54 | } 55 | 56 | size_t i = 0, j; 57 | for ( auto row = init.begin() ; row != init.end() ; ++row, ++i ) { 58 | assert ( row->size() == m_columns && "All rows must have the same number of columns." ); 59 | j = 0; 60 | for ( auto value = row->begin() ; value != row->end() ; ++value, ++j ) { 61 | m_matrix[i][j] = *value; 62 | } 63 | } 64 | } 65 | Matrix(const Matrix &other) { 66 | if ( other.m_matrix != nullptr ) { 67 | // copy arrays 68 | m_matrix = nullptr; 69 | resize(other.m_rows, other.m_columns); 70 | for ( size_t i = 0 ; i < m_rows ; i++ ) { 71 | for ( size_t j = 0 ; j < m_columns ; j++ ) { 72 | m_matrix[i][j] = other.m_matrix[i][j]; 73 | } 74 | } 75 | } else { 76 | m_matrix = nullptr; 77 | m_rows = 0; 78 | m_columns = 0; 79 | } 80 | } 81 | Matrix & operator= (const Matrix &other){ 82 | if ( other.m_matrix != nullptr ) { 83 | // copy arrays 84 | resize(other.m_rows, other.m_columns); 85 | for ( size_t i = 0 ; i < m_rows ; i++ ) { 86 | for ( size_t j = 0 ; j < m_columns ; j++ ) { 87 | m_matrix[i][j] = other.m_matrix[i][j]; 88 | } 89 | } 90 | } else { 91 | // free arrays 92 | for ( size_t i = 0 ; i < m_columns ; i++ ) { 93 | delete [] m_matrix[i]; 94 | } 95 | 96 | delete [] m_matrix; 97 | 98 | m_matrix = nullptr; 99 | m_rows = 0; 100 | m_columns = 0; 101 | } 102 | 103 | return *this; 104 | } 105 | ~Matrix(){ 106 | if ( m_matrix != nullptr ) { 107 | // free arrays 108 | for ( size_t i = 0 ; i < m_rows ; i++ ) { 109 | delete [] m_matrix[i]; 110 | } 111 | 112 | delete [] m_matrix; 113 | } 114 | m_matrix = nullptr; 115 | } 116 | // all operations modify the matrix in-place. 117 | void resize(const size_t rows, const size_t columns, const T default_value = 0) { 118 | assert ( rows > 0 && columns > 0 && "Columns and rows must exist." ); 119 | 120 | if ( m_matrix == nullptr ) { 121 | // alloc arrays 122 | m_matrix = new T*[rows]; // rows 123 | for ( size_t i = 0 ; i < rows ; i++ ) { 124 | m_matrix[i] = new T[columns]; // columns 125 | } 126 | 127 | m_rows = rows; 128 | m_columns = columns; 129 | clear(); 130 | } else { 131 | // save array pointer 132 | T **new_matrix; 133 | // alloc new arrays 134 | new_matrix = new T*[rows]; // rows 135 | for ( size_t i = 0 ; i < rows ; i++ ) { 136 | new_matrix[i] = new T[columns]; // columns 137 | for ( size_t j = 0 ; j < columns ; j++ ) { 138 | new_matrix[i][j] = default_value; 139 | } 140 | } 141 | 142 | // copy data from saved pointer to new arrays 143 | size_t minrows = XYZMIN(rows, m_rows); 144 | size_t mincols = XYZMIN(columns, m_columns); 145 | for ( size_t x = 0 ; x < minrows ; x++ ) { 146 | for ( size_t y = 0 ; y < mincols ; y++ ) { 147 | new_matrix[x][y] = m_matrix[x][y]; 148 | } 149 | } 150 | 151 | // delete old arrays 152 | if ( m_matrix != nullptr ) { 153 | for ( size_t i = 0 ; i < m_rows ; i++ ) { 154 | delete [] m_matrix[i]; 155 | } 156 | 157 | delete [] m_matrix; 158 | } 159 | 160 | m_matrix = new_matrix; 161 | } 162 | 163 | m_rows = rows; 164 | m_columns = columns; 165 | } 166 | void clear() { 167 | assert( m_matrix != nullptr ); 168 | 169 | for ( size_t i = 0 ; i < m_rows ; i++ ) { 170 | for ( size_t j = 0 ; j < m_columns ; j++ ) { 171 | m_matrix[i][j] = 0; 172 | } 173 | } 174 | } 175 | T& operator () (const size_t x, const size_t y) { 176 | assert ( x < m_rows ); 177 | assert ( y < m_columns ); 178 | assert ( m_matrix != nullptr ); 179 | return m_matrix[x][y]; 180 | } 181 | 182 | const T& operator () (const size_t x, const size_t y) const { 183 | assert ( x < m_rows ); 184 | assert ( y < m_columns ); 185 | assert ( m_matrix != nullptr ); 186 | return m_matrix[x][y]; 187 | } 188 | const T mmin() const { 189 | assert( m_matrix != nullptr ); 190 | assert ( m_rows > 0 ); 191 | assert ( m_columns > 0 ); 192 | T min = m_matrix[0][0]; 193 | 194 | for ( size_t i = 0 ; i < m_rows ; i++ ) { 195 | for ( size_t j = 0 ; j < m_columns ; j++ ) { 196 | min = std::min(min, m_matrix[i][j]); 197 | } 198 | } 199 | 200 | return min; 201 | } 202 | 203 | const T mmax() const { 204 | assert( m_matrix != nullptr ); 205 | assert ( m_rows > 0 ); 206 | assert ( m_columns > 0 ); 207 | T max = m_matrix[0][0]; 208 | 209 | for ( size_t i = 0 ; i < m_rows ; i++ ) { 210 | for ( size_t j = 0 ; j < m_columns ; j++ ) { 211 | max = std::max(max, m_matrix[i][j]); 212 | } 213 | } 214 | 215 | return max; 216 | } 217 | inline size_t minsize() { return ((m_rows < m_columns) ? m_rows : m_columns); } 218 | inline size_t columns() const { return m_columns;} 219 | inline size_t rows() const { return m_rows;} 220 | 221 | friend std::ostream& operator<<(std::ostream& os, const Matrix &matrix) 222 | { 223 | os << "Matrix:" << std::endl; 224 | for (size_t row = 0 ; row < matrix.rows() ; row++ ) 225 | { 226 | for (size_t col = 0 ; col < matrix.columns() ; col++ ) 227 | { 228 | os.width(8); 229 | os << matrix(row, col) << ","; 230 | } 231 | os << std::endl; 232 | } 233 | return os; 234 | } 235 | 236 | private: 237 | T **m_matrix; 238 | size_t m_rows; 239 | size_t m_columns; 240 | }; 241 | 242 | //#ifndef USE_EXPORT_KEYWORD 243 | //#include "matrix.cpp" 244 | ////#define export /*export*/ 245 | //#endif 246 | 247 | #endif /* !defined(_MATRIX_H_) */ 248 | 249 | -------------------------------------------------------------------------------- /deepsort/include/model.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MODEL_HPP 2 | #define MODEL_HPP 3 | 4 | #include 5 | #include "datatype.h" 6 | 7 | 8 | // * Each rect's data structure. 9 | // * tlwh: topleft point & (w,h) 10 | // * confidence: detection confidence. 11 | // * feature: the rect's 512d feature. 12 | // */ 13 | 14 | const float kRatio=0.5; 15 | enum DETECTBOX_IDX {IDX_X = 0, IDX_Y, IDX_W, IDX_H }; 16 | 17 | class DETECTION_ROW { 18 | public: 19 | DETECTBOX tlwh; 20 | float confidence; 21 | FEATURE feature; 22 | 23 | DETECTBOX to_xyah() const { 24 | //(centerx, centery, ration, h) 25 | DETECTBOX ret = tlwh; 26 | ret(0, IDX_X) += (ret(0, IDX_W)*kRatio); 27 | ret(0, IDX_Y) += (ret(0, IDX_H)*kRatio); 28 | ret(0, IDX_W) /= ret(0, IDX_H); 29 | return ret; 30 | } 31 | DETECTBOX to_tlbr() const { 32 | //(x,y,xx,yy) 33 | DETECTBOX ret = tlwh; 34 | ret(0, IDX_X) += ret(0, IDX_W); 35 | ret(0, IDX_Y) += ret(0, IDX_H); 36 | return ret; 37 | } 38 | 39 | void updateFeature(FEATURE& feature_) { 40 | // only update feature 41 | this->feature = feature_; 42 | } 43 | }; 44 | 45 | typedef std::vector DETECTIONS; 46 | typedef std::pair, DETECTIONS> DETECTIONSV2; 47 | 48 | #endif // MODEL_HPP 49 | -------------------------------------------------------------------------------- /deepsort/include/munkres.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2007 John Weaver 3 | * Copyright (c) 2015 Miroslav Krajicek 4 | * 5 | * This program is free software; you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation; either version 2 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program; if not, write to the Free Software 17 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 | */ 19 | 20 | #if !defined(_MUNKRES_H_) 21 | #define _MUNKRES_H_ 22 | 23 | #include "matrix.h" 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | 32 | template class Munkres 33 | { 34 | static constexpr int NORMAL = 0; 35 | static constexpr int STAR = 1; 36 | static constexpr int PRIME = 2; 37 | public: 38 | 39 | /* 40 | * 41 | * Linear assignment problem solution 42 | * [modifies matrix in-place.] 43 | * matrix(row,col): row major format assumed. 44 | * 45 | * Assignments are remaining 0 values 46 | * (extra 0 values are replaced with -1) 47 | * 48 | */ 49 | void solve(Matrix &m) { 50 | const size_t rows = m.rows(), 51 | columns = m.columns(), 52 | size = XYZMAX(rows, columns); 53 | 54 | #ifdef DEBUG 55 | std::cout << "Munkres input: " << m << std::endl; 56 | #endif 57 | // Copy input matrix 58 | this->matrix = m; 59 | 60 | if ( rows != columns ) { 61 | // If the input matrix isn't square, make it square 62 | // and fill the empty values with the largest value present 63 | // in the matrix. 64 | matrix.resize(size, size, matrix.mmax()); 65 | } 66 | 67 | 68 | // STAR == 1 == starred, PRIME == 2 == primed 69 | mask_matrix.resize(size, size); 70 | 71 | row_mask = new bool[size]; 72 | col_mask = new bool[size]; 73 | for ( size_t i = 0 ; i < size ; i++ ) { 74 | row_mask[i] = false; 75 | } 76 | 77 | for ( size_t i = 0 ; i < size ; i++ ) { 78 | col_mask[i] = false; 79 | } 80 | 81 | // Prepare the matrix values... 82 | 83 | // If there were any infinities, replace them with a value greater 84 | // than the maximum value in the matrix. 85 | replace_infinites(matrix); 86 | 87 | minimize_along_direction(matrix, rows >= columns); 88 | minimize_along_direction(matrix, rows < columns); 89 | 90 | // Follow the steps 91 | int step = 1; 92 | while ( step ) { 93 | switch ( step ) { 94 | case 1: 95 | step = step1(); 96 | // step is always 2 97 | break; 98 | case 2: 99 | step = step2(); 100 | // step is always either 0 or 3 101 | break; 102 | case 3: 103 | step = step3(); 104 | // step in [3, 4, 5] 105 | break; 106 | case 4: 107 | step = step4(); 108 | // step is always 2 109 | break; 110 | case 5: 111 | step = step5(); 112 | // step is always 3 113 | break; 114 | } 115 | } 116 | 117 | // Store results 118 | for ( size_t row = 0 ; row < size ; row++ ) { 119 | for ( size_t col = 0 ; col < size ; col++ ) { 120 | if ( mask_matrix(row, col) == STAR ) { 121 | matrix(row, col) = 0; 122 | } else { 123 | matrix(row, col) = -1; 124 | } 125 | } 126 | } 127 | #ifdef DEBUG 128 | std::cout << "Munkres output: " << matrix << std::endl; 129 | #endif 130 | // Remove the excess rows or columns that we added to fit the 131 | // input to a square matrix. 132 | matrix.resize(rows, columns); 133 | 134 | m = matrix; 135 | 136 | delete [] row_mask; 137 | delete [] col_mask; 138 | } 139 | 140 | static void replace_infinites(Matrix &matrix) { 141 | const size_t rows = matrix.rows(), 142 | columns = matrix.columns(); 143 | //assert( rows > 0 && columns > 0 ); 144 | double max = matrix(0, 0); 145 | constexpr auto infinity = std::numeric_limits::infinity(); 146 | 147 | // Find the greatest value in the matrix that isn't infinity. 148 | for ( size_t row = 0 ; row < rows ; row++ ) { 149 | for ( size_t col = 0 ; col < columns ; col++ ) { 150 | if ( matrix(row, col) != infinity ) { 151 | if ( max == infinity ) { 152 | max = matrix(row, col); 153 | } else { 154 | max = XYZMAX(max, matrix(row, col)); 155 | } 156 | } 157 | } 158 | } 159 | 160 | // a value higher than the maximum value present in the matrix. 161 | if ( max == infinity ) { 162 | // This case only occurs when all values are infinite. 163 | max = 0; 164 | } else { 165 | max++; 166 | } 167 | 168 | for ( size_t row = 0 ; row < rows ; row++ ) { 169 | for ( size_t col = 0 ; col < columns ; col++ ) { 170 | if ( matrix(row, col) == infinity ) { 171 | matrix(row, col) = max; 172 | } 173 | } 174 | } 175 | 176 | } 177 | static void minimize_along_direction(Matrix &matrix, const bool over_columns) { 178 | const size_t outer_size = over_columns ? matrix.columns() : matrix.rows(), 179 | inner_size = over_columns ? matrix.rows() : matrix.columns(); 180 | 181 | // Look for a minimum value to subtract from all values along 182 | // the "outer" direction. 183 | for ( size_t i = 0 ; i < outer_size ; i++ ) { 184 | double min = over_columns ? matrix(0, i) : matrix(i, 0); 185 | 186 | // As long as the current minimum is greater than zero, 187 | // keep looking for the minimum. 188 | // Start at one because we already have the 0th value in min. 189 | for ( size_t j = 1 ; j < inner_size && min > 0 ; j++ ) { 190 | min = XYZMIN( 191 | min, 192 | over_columns ? matrix(j, i) : matrix(i, j)); 193 | } 194 | 195 | if ( min > 0 ) { 196 | for ( size_t j = 0 ; j < inner_size ; j++ ) { 197 | if ( over_columns ) { 198 | matrix(j, i) -= min; 199 | } else { 200 | matrix(i, j) -= min; 201 | } 202 | } 203 | } 204 | } 205 | } 206 | 207 | private: 208 | 209 | inline bool find_uncovered_in_matrix(const double item, size_t &row, size_t &col) const { 210 | const size_t rows = matrix.rows(), 211 | columns = matrix.columns(); 212 | 213 | for ( row = 0 ; row < rows ; row++ ) { 214 | if ( !row_mask[row] ) { 215 | for ( col = 0 ; col < columns ; col++ ) { 216 | if ( !col_mask[col] ) { 217 | if ( matrix(row,col) == item ) { 218 | return true; 219 | } 220 | } 221 | } 222 | } 223 | } 224 | 225 | return false; 226 | } 227 | 228 | bool pair_in_list(const std::pair &needle, const std::list > &haystack) { 229 | for ( std::list >::const_iterator i = haystack.begin() ; i != haystack.end() ; i++ ) { 230 | if ( needle == *i ) { 231 | return true; 232 | } 233 | } 234 | 235 | return false; 236 | } 237 | 238 | int step1() { 239 | const size_t rows = matrix.rows(), 240 | columns = matrix.columns(); 241 | 242 | for ( size_t row = 0 ; row < rows ; row++ ) { 243 | for ( size_t col = 0 ; col < columns ; col++ ) { 244 | if ( 0 == matrix(row, col) ) { 245 | for ( size_t nrow = 0 ; nrow < row ; nrow++ ) 246 | if ( STAR == mask_matrix(nrow,col) ) 247 | goto next_column; 248 | 249 | mask_matrix(row,col) = STAR; 250 | goto next_row; 251 | } 252 | next_column:; 253 | } 254 | next_row:; 255 | } 256 | 257 | return 2; 258 | } 259 | 260 | int step2() { 261 | const size_t rows = matrix.rows(), 262 | columns = matrix.columns(); 263 | size_t covercount = 0; 264 | 265 | for ( size_t row = 0 ; row < rows ; row++ ) 266 | for ( size_t col = 0 ; col < columns ; col++ ) 267 | if ( STAR == mask_matrix(row, col) ) { 268 | col_mask[col] = true; 269 | covercount++; 270 | } 271 | 272 | if ( covercount >= matrix.minsize() ) { 273 | #ifdef DEBUG 274 | std::cout << "Final cover count: " << covercount << std::endl; 275 | #endif 276 | return 0; 277 | } 278 | 279 | #ifdef DEBUG 280 | std::cout << "Munkres matrix has " << covercount << " of " << matrix.minsize() << " Columns covered:" << std::endl; 281 | std::cout << matrix << std::endl; 282 | #endif 283 | 284 | 285 | return 3; 286 | } 287 | 288 | int step3() { 289 | /* 290 | Main Zero Search 291 | 292 | 1. Find an uncovered Z in the distance matrix and prime it. If no such zero exists, go to Step 5 293 | 2. If No Z* exists in the row of the Z', go to Step 4. 294 | 3. If a Z* exists, cover this row and uncover the column of the Z*. Return to Step 3.1 to find a new Z 295 | */ 296 | if ( find_uncovered_in_matrix(0, saverow, savecol) ) { 297 | mask_matrix(saverow,savecol) = PRIME; // prime it. 298 | } else { 299 | return 5; 300 | } 301 | 302 | for ( size_t ncol = 0 ; ncol < matrix.columns() ; ncol++ ) { 303 | if ( mask_matrix(saverow,ncol) == STAR ) { 304 | row_mask[saverow] = true; //cover this row and 305 | col_mask[ncol] = false; // uncover the column containing the starred zero 306 | return 3; // repeat 307 | } 308 | } 309 | 310 | return 4; // no starred zero in the row containing this primed zero 311 | } 312 | 313 | int step4() { 314 | const size_t rows = matrix.rows(), 315 | columns = matrix.columns(); 316 | 317 | // seq contains pairs of row/column values where we have found 318 | // either a star or a prime that is part of the ``alternating sequence``. 319 | std::list > seq; 320 | // use saverow, savecol from step 3. 321 | std::pair z0(saverow, savecol); 322 | seq.insert(seq.end(), z0); 323 | 324 | // We have to find these two pairs: 325 | std::pair z1(-1, -1); 326 | std::pair z2n(-1, -1); 327 | 328 | size_t row, col = savecol; 329 | /* 330 | Increment Set of Starred Zeros 331 | 332 | 1. Construct the ``alternating sequence'' of primed and starred zeros: 333 | 334 | Z0 : Unpaired Z' from Step 4.2 335 | Z1 : The Z* in the column of Z0 336 | Z[2N] : The Z' in the row of Z[2N-1], if such a zero exists 337 | Z[2N+1] : The Z* in the column of Z[2N] 338 | 339 | The sequence eventually terminates with an unpaired Z' = Z[2N] for some N. 340 | */ 341 | bool madepair; 342 | do { 343 | madepair = false; 344 | for ( row = 0 ; row < rows ; row++ ) { 345 | if ( mask_matrix(row,col) == STAR ) { 346 | z1.first = row; 347 | z1.second = col; 348 | if ( pair_in_list(z1, seq) ) { 349 | continue; 350 | } 351 | 352 | madepair = true; 353 | seq.insert(seq.end(), z1); 354 | break; 355 | } 356 | } 357 | 358 | if ( !madepair ) 359 | break; 360 | 361 | madepair = false; 362 | 363 | for ( col = 0 ; col < columns ; col++ ) { 364 | if ( mask_matrix(row, col) == PRIME ) { 365 | z2n.first = row; 366 | z2n.second = col; 367 | if ( pair_in_list(z2n, seq) ) { 368 | continue; 369 | } 370 | madepair = true; 371 | seq.insert(seq.end(), z2n); 372 | break; 373 | } 374 | } 375 | } while ( madepair ); 376 | 377 | for ( std::list >::iterator i = seq.begin() ; 378 | i != seq.end() ; 379 | i++ ) { 380 | // 2. Unstar each starred zero of the sequence. 381 | if ( mask_matrix(i->first,i->second) == STAR ) 382 | mask_matrix(i->first,i->second) = NORMAL; 383 | 384 | // 3. Star each primed zero of the sequence, 385 | // thus increasing the number of starred zeros by one. 386 | if ( mask_matrix(i->first,i->second) == PRIME ) 387 | mask_matrix(i->first,i->second) = STAR; 388 | } 389 | 390 | // 4. Erase all primes, uncover all columns and rows, 391 | for ( size_t row = 0 ; row < mask_matrix.rows() ; row++ ) { 392 | for ( size_t col = 0 ; col < mask_matrix.columns() ; col++ ) { 393 | if ( mask_matrix(row,col) == PRIME ) { 394 | mask_matrix(row,col) = NORMAL; 395 | } 396 | } 397 | } 398 | 399 | for ( size_t i = 0 ; i < rows ; i++ ) { 400 | row_mask[i] = false; 401 | } 402 | 403 | for ( size_t i = 0 ; i < columns ; i++ ) { 404 | col_mask[i] = false; 405 | } 406 | 407 | // and return to Step 2. 408 | return 2; 409 | } 410 | 411 | int step5() { 412 | const size_t rows = matrix.rows(), 413 | columns = matrix.columns(); 414 | /* 415 | New Zero Manufactures 416 | 417 | 1. Let h be the smallest uncovered entry in the (modified) distance matrix. 418 | 2. Add h to all covered rows. 419 | 3. Subtract h from all uncovered columns 420 | 4. Return to Step 3, without altering stars, primes, or covers. 421 | */ 422 | double h = 100000;//xyzoylz std::numeric_limits::max(); 423 | for ( size_t row = 0 ; row < rows ; row++ ) { 424 | if ( !row_mask[row] ) { 425 | for ( size_t col = 0 ; col < columns ; col++ ) { 426 | if ( !col_mask[col] ) { 427 | if ( h > matrix(row, col) && matrix(row, col) != 0 ) { 428 | h = matrix(row, col); 429 | } 430 | } 431 | } 432 | } 433 | } 434 | 435 | for ( size_t row = 0 ; row < rows ; row++ ) { 436 | if ( row_mask[row] ) { 437 | for ( size_t col = 0 ; col < columns ; col++ ) { 438 | matrix(row, col) += h; 439 | } 440 | } 441 | } 442 | 443 | for ( size_t col = 0 ; col < columns ; col++ ) { 444 | if ( !col_mask[col] ) { 445 | for ( size_t row = 0 ; row < rows ; row++ ) { 446 | matrix(row, col) -= h; 447 | } 448 | } 449 | } 450 | 451 | return 3; 452 | } 453 | 454 | Matrix mask_matrix; 455 | Matrix matrix; 456 | bool *row_mask; 457 | bool *col_mask; 458 | size_t saverow = 0, savecol = 0; 459 | }; 460 | 461 | 462 | #endif /* !defined(_MUNKRES_H_) */ 463 | -------------------------------------------------------------------------------- /deepsort/include/nn_matching.h: -------------------------------------------------------------------------------- 1 | #ifndef NN_MATCHING_H 2 | #define NN_MATCHING_H 3 | 4 | #include "datatype.h" 5 | 6 | #include 7 | 8 | //A tool to calculate distance; 9 | class NearNeighborDisMetric{ 10 | public: 11 | enum METRIC_TYPE{euclidean=1, cosine}; 12 | NearNeighborDisMetric(METRIC_TYPE metric, 13 | float matching_threshold, 14 | int budget); 15 | DYNAMICM distance(const FEATURESS& features, const std::vector &targets); 16 | // void partial_fit(FEATURESS& features, std::vector targets, std::vector active_targets); 17 | void partial_fit(std::vector& tid_feats, std::vector& active_targets); 18 | float mating_threshold; 19 | 20 | private: 21 | typedef Eigen::VectorXf (NearNeighborDisMetric::*PTRFUN)(const FEATURESS&, const FEATURESS&); 22 | Eigen::VectorXf _nncosine_distance(const FEATURESS& x, const FEATURESS& y); 23 | Eigen::VectorXf _nneuclidean_distance(const FEATURESS& x, const FEATURESS& y); 24 | 25 | Eigen::MatrixXf _pdist(const FEATURESS& x, const FEATURESS& y); 26 | Eigen::MatrixXf _cosine_distance(const FEATURESS & a, const FEATURESS& b, bool data_is_normalized = false); 27 | private: 28 | PTRFUN _metric; 29 | int budget; 30 | std::map samples; 31 | }; 32 | 33 | #endif // NN_MATCHING_H 34 | -------------------------------------------------------------------------------- /deepsort/include/track.h: -------------------------------------------------------------------------------- 1 | #ifndef TRACK_H 2 | #define TRACK_H 3 | 4 | 5 | #include "MyKalmanFilter.h" 6 | #include "datatype.h" 7 | #include "model.hpp" 8 | 9 | class Track 10 | { 11 | /*""" 12 | A single target track with state space `(x, y, a, h)` and associated 13 | velocities, where `(x, y)` is the center of the bounding box, `a` is the 14 | aspect ratio and `h` is the height. 15 | 16 | Parameters 17 | ---------- 18 | mean : ndarray 19 | Mean vector of the initial state distribution. 20 | covariance : ndarray 21 | Covariance matrix of the initial state distribution. 22 | track_id : int 23 | A unique track identifier. 24 | n_init : int 25 | Number of consecutive detections before the track is confirmed. The 26 | track state is set to `Deleted` if a miss occurs within the first 27 | `n_init` frames. 28 | max_age : int 29 | The maximum number of consecutive misses before the track state is 30 | set to `Deleted`. 31 | feature : Optional[ndarray] 32 | Feature vector of the detection this track originates from. If not None, 33 | this feature is added to the `features` cache. 34 | 35 | Attributes 36 | ---------- 37 | mean : ndarray 38 | Mean vector of the initial state distribution. 39 | covariance : ndarray 40 | Covariance matrix of the initial state distribution. 41 | track_id : int 42 | A unique track identifier. 43 | hits : int 44 | Total number of measurement updates. 45 | age : int 46 | Total number of frames since first occurance. 47 | time_since_update : int 48 | Total number of frames since last measurement update. 49 | state : TrackState 50 | The current track state. 51 | features : List[ndarray] 52 | A cache of features. On each measurement update, the associated feature 53 | vector is added to this list. 54 | 55 | """*/ 56 | enum TrackState {Tentative = 1, Confirmed, Deleted}; 57 | 58 | public: 59 | Track(KAL_MEAN& mean, KAL_COVA& covariance, int track_id, 60 | int n_init, int max_age, const FEATURE& feature); 61 | Track(KAL_MEAN& mean, KAL_COVA& covariance, int track_id, 62 | int n_init, int max_age, const FEATURE& feature, int cls, float conf); 63 | void predit(MyKalmanFilter* kf); 64 | void update(MyKalmanFilter* const kf, const DETECTION_ROW &detection); 65 | void update(MyKalmanFilter* const kf, const DETECTION_ROW & detection, CLSCONF pair_det); 66 | void mark_missed(); 67 | bool is_confirmed(); 68 | bool is_deleted(); 69 | bool is_tentative(); 70 | DETECTBOX to_tlwh(); 71 | int time_since_update; 72 | int track_id; 73 | FEATURESS features; 74 | KAL_MEAN mean; 75 | KAL_COVA covariance; 76 | 77 | int hits; 78 | int age; 79 | int _n_init; 80 | int _max_age; 81 | TrackState state; 82 | 83 | int cls; 84 | float conf; 85 | private: 86 | void featuresAppendOne(const FEATURE& f); 87 | }; 88 | 89 | #endif // TRACK_H 90 | -------------------------------------------------------------------------------- /deepsort/include/tracker.h: -------------------------------------------------------------------------------- 1 | #ifndef TRACKER_H 2 | #define TRACKER_H 3 | 4 | 5 | #include 6 | 7 | #include "MyKalmanFilter.h" 8 | #include "track.h" 9 | #include "model.hpp" 10 | 11 | using namespace std; 12 | 13 | class NearNeighborDisMetric; 14 | 15 | class tracker 16 | { 17 | public: 18 | NearNeighborDisMetric* metric; 19 | float max_iou_distance; 20 | int max_age; 21 | int n_init; 22 | 23 | MyKalmanFilter* kf; 24 | 25 | int _next_idx; 26 | public: 27 | std::vector tracks; 28 | tracker(/*NearNeighborDisMetric* metric,*/ 29 | float max_cosine_distance, int nn_budget, 30 | float max_iou_distance = 0.7, 31 | int max_age = 30, int n_init=2); 32 | void predict(); 33 | void update(const DETECTIONS& detections); 34 | void update(const DETECTIONSV2& detectionsv2); 35 | typedef DYNAMICM (tracker::* GATED_METRIC_FUNC)( 36 | std::vector& tracks, 37 | const DETECTIONS& dets, 38 | const std::vector& track_indices, 39 | const std::vector& detection_indices); 40 | private: 41 | void _match(const DETECTIONS& detections, TRACHER_MATCHD& res); 42 | void _initiate_track(const DETECTION_ROW& detection); 43 | void _initiate_track(const DETECTION_ROW& detection, CLSCONF clsConf); 44 | public: 45 | DYNAMICM gated_matric( 46 | std::vector& tracks, 47 | const DETECTIONS& dets, 48 | const std::vector& track_indices, 49 | const std::vector& detection_indices); 50 | DYNAMICM iou_cost( 51 | std::vector& tracks, 52 | const DETECTIONS& dets, 53 | const std::vector& track_indices, 54 | const std::vector& detection_indices); 55 | Eigen::VectorXf iou(DETECTBOX& bbox, 56 | DETECTBOXSS &candidates); 57 | }; 58 | 59 | #endif // TRACKER_H 60 | -------------------------------------------------------------------------------- /deepsort/src/deepsort.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | #include "deepsort.h" 8 | #include "common.h" 9 | #include "mytime.h" 10 | using namespace std; 11 | 12 | struct video_property; 13 | extern queue queueDetOut; // output queue 14 | extern queue queueOutput; // output queue 目标追踪输出队列 15 | extern video_property video_probs; // 视频属性类 16 | extern bool bDetecting; // 目标检测进程状态 17 | extern bool bTracking; // 目标追踪进程状态 18 | extern double end_time; // 整个视频追踪结束 19 | 20 | extern mutex mtxQueueOutput; 21 | extern mutex mtxQueueDetOut; 22 | 23 | DeepSort::DeepSort(std::string modelPath, int batchSize, int featureDim, int cpu_id, rknn_core_mask npu_id) { 24 | this->npu_id = npu_id; 25 | this->cpu_id = cpu_id; 26 | this->enginePath = modelPath; 27 | this->batchSize = batchSize; 28 | this->featureDim = featureDim; 29 | this->imgShape = cv::Size(128, 256); 30 | this->maxBudget = 100; 31 | this->maxCosineDist = 0.2; 32 | init(); 33 | } 34 | 35 | void DeepSort::init() { 36 | objTracker = new tracker(maxCosineDist, maxBudget); 37 | 38 | // two Re-ID networks, share same CPU and NPU 39 | featureExtractor1 = new FeatureTensor(enginePath.c_str(), cpu_id, npu_id, 1, 1); 40 | featureExtractor1->init(imgShape, featureDim, NET_INPUTCHANNEL); 41 | 42 | featureExtractor2 = new FeatureTensor(enginePath.c_str(), cpu_id, npu_id, 1, 1); 43 | featureExtractor2->init(imgShape, featureDim, NET_INPUTCHANNEL); 44 | 45 | } 46 | 47 | DeepSort::~DeepSort() { 48 | delete objTracker; 49 | } 50 | 51 | void DeepSort::sort(cv::Mat& frame, vector& dets) { 52 | // preprocess Mat -> DETECTION 53 | DETECTIONS detections; // DETECTIONS: std::vector in model.hpp 54 | vector clsConf; 55 | 56 | // read every detections in current frame and 57 | // store them in detections(bbox) and clsConf(conf scores) 58 | for (DetectBox i : dets) { 59 | DETECTBOX box(i.x1, i.y1, i.x2-i.x1, i.y2-i.y1); 60 | DETECTION_ROW d; 61 | d.tlwh = box; 62 | d.confidence = i.confidence; 63 | detections.push_back(d); 64 | clsConf.push_back(CLSCONF((int)i.classID, i.confidence)); 65 | } 66 | 67 | result.clear(); // result: vector> 68 | results.clear(); // results: vector> 69 | if (detections.size() > 0) { 70 | DETECTIONSV2 detectionsv2 = make_pair(clsConf, detections); 71 | sort(frame, detectionsv2); // sort 72 | } 73 | // postprocess DETECTION -> Mat 74 | dets.clear(); 75 | for (auto r : result) { 76 | DETECTBOX i = r.second; 77 | DetectBox b(i(0), i(1), i(2)+i(0), i(3)+i(1), 1.); 78 | b.trackID = (float)r.first; 79 | dets.push_back(b); 80 | } 81 | for (int i = 0; i < results.size(); ++i) { 82 | CLSCONF c = results[i].first; 83 | dets[i].classID = c.cls; 84 | dets[i].confidence = c.conf; 85 | } 86 | } 87 | 88 | 89 | void DeepSort::sort(cv::Mat& frame, DETECTIONS& detections) { 90 | bool flag = featureExtractor1->getRectsFeature(frame, detections); 91 | if (flag) { 92 | objTracker->predict(); 93 | objTracker->update(detections); 94 | //result.clear(); 95 | for (Track& track : objTracker->tracks) { 96 | if (!track.is_confirmed() || track.time_since_update > 1) 97 | continue; 98 | result.push_back(make_pair(track.track_id, track.to_tlwh())); 99 | } 100 | } 101 | } 102 | 103 | void DeepSort::sort_interval(cv::Mat& frame, vector& dets) { 104 | /* 105 | If frame_id % this->track_interval != 0, there is no new detections 106 | so only predict the tracks using Kalman 107 | */ 108 | if (!dets.empty()) cout << "Error occured! \n"; 109 | 110 | result.clear(); 111 | results.clear(); 112 | objTracker->predict(); // Kalman predict 113 | 114 | // update result and results 115 | // cout << "---------" << objTracker->tracks.size() << "\n"; 116 | for (Track& track : objTracker->tracks) { 117 | // if (!track.is_confirmed() || track.time_since_update > 1) 118 | if (!track.is_confirmed() || track.time_since_update > this->track_interval + 1) 119 | continue; 120 | result.push_back(make_pair(track.track_id, track.to_tlwh())); 121 | results.push_back(make_pair(CLSCONF(track.cls, track.conf) ,track.to_tlwh())); 122 | } 123 | dets.clear(); 124 | for (auto r : result) { 125 | DETECTBOX i = r.second; 126 | DetectBox b(i(0), i(1), i(2)+i(0), i(3)+i(1), 1.); 127 | b.trackID = (float)r.first; 128 | dets.push_back(b); 129 | } 130 | for (int i = 0; i < results.size(); ++i) { 131 | CLSCONF c = results[i].first; 132 | dets[i].classID = c.cls; 133 | dets[i].confidence = c.conf; 134 | } 135 | 136 | } 137 | 138 | void DeepSort::sort(cv::Mat& frame, DETECTIONSV2& detectionsv2) { 139 | std::vector& clsConf = detectionsv2.first; 140 | DETECTIONS& detections = detectionsv2.second; // std::vector 141 | 142 | int numOfDetections = detections.size(); 143 | bool flag1 = true, flag2 = true; 144 | if (numOfDetections < 2){ 145 | // few objects, use single Re-ID 146 | double timeBeforeReID = what_time_is_it_now(); 147 | flag1 = featureExtractor1->getRectsFeature(frame, detections); 148 | double timeAfterReID = what_time_is_it_now(); 149 | 150 | cout << "--------Time cost in ReID: " << timeAfterReID - timeBeforeReID << "\n"; 151 | flag2 = true; 152 | } 153 | else { 154 | DETECTIONS detectionsPart1, detectionsPart2; 155 | int border = numOfDetections >> 1; 156 | auto start = detections.begin(), end = detections.end(); // iterator 157 | 158 | double timeBeforeAssign = what_time_is_it_now(); 159 | detectionsPart1.assign(start, start + border); 160 | detectionsPart2.assign(start + border, end); 161 | double timeAfterAssign = what_time_is_it_now(); 162 | 163 | cout << "--------Time cost in assign: " << timeAfterAssign - timeBeforeAssign << "\n"; 164 | 165 | // NOTE: convert pointer or set global variables 166 | // inference separately 167 | double timeBeforeReID = what_time_is_it_now(); 168 | thread reID1Thread1 (&FeatureTensor::getRectsFeature, featureExtractor1, std::ref(frame), std::ref(detectionsPart1)); 169 | thread reID1Thread2 (&FeatureTensor::getRectsFeature, featureExtractor2, std::ref(frame), std::ref(detectionsPart2)); 170 | 171 | reID1Thread1.join(); reID1Thread2.join(); 172 | 173 | double timeAfterReID = what_time_is_it_now(); 174 | 175 | cout << "--------Time cost in ReID: " << timeAfterReID - timeBeforeReID << "\n"; 176 | 177 | // copy new feature to origin detections 178 | 179 | double timeBeforeUpdateFeatures = what_time_is_it_now(); 180 | for (int idx = 0; flag1 && flag2 && idx < numOfDetections; idx ++) { 181 | if (idx < border) 182 | detections[idx].updateFeature(detectionsPart1[idx].feature); 183 | else 184 | detections[idx].updateFeature(detectionsPart2[idx - border].feature); 185 | } 186 | double timeAfterUpdateFeatures = what_time_is_it_now(); 187 | cout << "--------Time cost in update features: " << timeAfterUpdateFeatures - timeBeforeUpdateFeatures << "\n"; 188 | 189 | } 190 | 191 | // bool flag = featureExtractor->getRectsFeature(frame, detections); 192 | if (flag1 && flag2) { 193 | objTracker->predict(); 194 | // std::cout << "In: \n"; 195 | objTracker->update(detectionsv2); 196 | // std::cout << "Out: \n"; 197 | result.clear(); 198 | results.clear(); 199 | for (Track& track : objTracker->tracks) { 200 | if (!track.is_confirmed() || track.time_since_update > 1) 201 | continue; 202 | result.push_back(make_pair(track.track_id, track.to_tlwh())); 203 | results.push_back(make_pair(CLSCONF(track.cls, track.conf) ,track.to_tlwh())); 204 | } 205 | } 206 | else cout << "Re-ID1 Error? " << flag1 << " Re-ID2 Error? " << flag2 << "\n"; 207 | } 208 | 209 | int DeepSort::track_process(){ 210 | while (1) 211 | { 212 | 213 | if (queueDetOut.empty()) { 214 | if(!bDetecting){ 215 | // queueDetOut为空并且 bDetecting标志已经全部检测完成 说明追踪完毕了 216 | end_time = what_time_is_it_now(); 217 | bTracking = false; 218 | break; 219 | } 220 | usleep(1000); 221 | continue; 222 | } 223 | 224 | // get frame index of queueDetOut.front 225 | int curFrameIdx = queueDetOut.front().dets.id; 226 | // cout << "Is id match with result " << (!queueDetOut.front().dets.results.empty() && !(curFrameIdx % 3)) << "\n"; 227 | 228 | if (curFrameIdx < this->track_interval || !(curFrameIdx % this->track_interval)) // have detections 229 | sort(queueDetOut.front().img , queueDetOut.front().dets.results); // 会更新 dets.results 230 | else 231 | sort_interval(queueDetOut.front().img , queueDetOut.front().dets.results); 232 | mtxQueueOutput.lock(); 233 | // cout << "--------------" << queueDetOut.front().dets.results.size() << "\n"; 234 | queueOutput.push(queueDetOut.front()); 235 | mtxQueueOutput.unlock(); 236 | // showDetection(queueDetOut.front().img, queueDetOut.front().dets.results); 237 | mtxQueueDetOut.lock(); 238 | queueDetOut.pop(); 239 | mtxQueueDetOut.unlock(); 240 | } 241 | cout << "Track is over." << endl; 242 | return 0; 243 | } 244 | 245 | void DeepSort::showDetection(cv::Mat& img, std::vector& boxes) { 246 | cv::Mat temp = img.clone(); 247 | for (auto box : boxes) { 248 | cv::Point lt(box.x1, box.y1); 249 | cv::Point br(box.x2, box.y2); 250 | cv::rectangle(temp, lt, br, cv::Scalar(255, 0, 0), 2); 251 | //std::string lbl = cv::format("ID:%d_C:%d_CONF:%.2f", (int)box.trackID, (int)box.classID, box.confidence); 252 | //std::string lbl = cv::format("ID:%d_C:%d", (int)box.trackID, (int)box.classID); 253 | std::string lbl = cv::format("ID:%d",(int)box.trackID); 254 | cv::putText(temp, lbl, lt, cv::FONT_HERSHEY_COMPLEX, 1.0, cv::Scalar(0,255,0)); 255 | } 256 | cv::imwrite("./display.jpg", temp); 257 | } -------------------------------------------------------------------------------- /deepsort/src/featuretensor.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "featuretensor.h" 5 | #include "mytime.h" 6 | 7 | 8 | void FeatureTensor::init(cv::Size netShape, int featureDim, int channel){ 9 | this->imgShape = netShape; 10 | this->featureDim = featureDim; 11 | this->pre_do = PreResize(netShape.height, netShape.width, channel); 12 | } 13 | 14 | void FeatureTensor::doInference(vector& imgMats, DETECTIONS& det) { 15 | std::queue history_time; 16 | float sum_time = 0; 17 | int cost_time = 0; // rknn接口查询返回 18 | float npu_performance = 0.0; 19 | 20 | for (int i = 0;i < imgMats.size();i++){ 21 | 22 | cost_time = inference(imgMats[i].data); 23 | // std::cout << "in deepsort doInference: " << i << "\n"; 24 | float* output = (float *)_output_buff[0]; 25 | for (int j = 0; j < featureDim; ++j) 26 | det[i].feature[j] = output[j]; 27 | npu_performance = cal_NPU_performance(history_time, sum_time, cost_time / 1.0e3); 28 | // printf("Deepsort: %f NPU(%d) performance : %f\n", what_time_is_it_now()/1000, _cpu_id, npu_performance); 29 | } 30 | } 31 | 32 | bool FeatureTensor::getRectsFeature(const cv::Mat& img, DETECTIONS& det) { 33 | std::vector mats; 34 | 35 | double timeBeforeGetRectsFeature = what_time_is_it_now(); 36 | 37 | for (auto& dbox : det) { 38 | cv::Rect rect = cv::Rect(int(dbox.tlwh(0)), int(dbox.tlwh(1)), 39 | int(dbox.tlwh(2)), int(dbox.tlwh(3))); 40 | // std::cout << img.cols << " " << img.rows << "\n"; 41 | // std::cout << dbox.tlwh(0) << " " << dbox.tlwh(1) << " " << dbox.tlwh(2) << " " << dbox.tlwh(3) << "\n"; 42 | 43 | rect.x -= (rect.height * 0.5 - rect.width) * 0.5; 44 | rect.width = rect.height * 0.5; 45 | rect.x = (rect.x >= 0 ? rect.x : 0); 46 | rect.y = (rect.y >= 0 ? rect.y : 0); 47 | rect.width = (rect.x + rect.width <= img.cols ? rect.width : (img.cols - rect.x)); 48 | rect.height = (rect.y + rect.height <= img.rows ? rect.height : (img.rows - rect.y)); 49 | 50 | if (rect.width < 0 || rect.height < 0) continue; 51 | // std::cout << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << "\n"; 52 | cv::Mat tempMat = img(rect).clone(); 53 | /* 54 | if (tempMat.rows < 128 || tempMat.cols < 256) { 55 | std::cout << tempMat.rows << " " << tempMat.cols << "\n"; 56 | } 57 | */ 58 | if (tempMat.size().empty()) { 59 | std::cout << "tempMat is empty: " << tempMat.cols << " " << tempMat.rows << "\n"; 60 | continue; 61 | } 62 | cv::resize(tempMat, tempMat, imgShape); // opencv 63 | // pre_do.resize(tempMat, tempMat); // rga 64 | mats.push_back(tempMat); 65 | } 66 | 67 | doInference(mats, det); 68 | 69 | double timeAfterGetRectsFeature = what_time_is_it_now(); 70 | std::cout << "--------Time cost in getRectsFeature: " << timeAfterGetRectsFeature- timeBeforeGetRectsFeature << "\n"; 71 | 72 | // std::cout << "in deepsort inference: " << mats.size() << "\n"; 73 | return true; 74 | } -------------------------------------------------------------------------------- /deepsort/src/hungarianoper.cpp: -------------------------------------------------------------------------------- 1 | #include "hungarianoper.h" 2 | 3 | Eigen::Matrix HungarianOper::Solve(const DYNAMICM &cost_matrix) { 4 | int rows = cost_matrix.rows(); 5 | int cols = cost_matrix.cols(); 6 | Matrix matrix(rows, cols); 7 | for (int row = 0; row < rows; row++) { 8 | for (int col = 0; col < cols; col++) { 9 | if (cost_matrix(row, col) != cost_matrix(row, col)) { // nan set to const 10 | // std::cout << "nan occured! \n"; 11 | matrix(row, col) = 0.0; 12 | } 13 | else matrix(row, col) = cost_matrix(row, col); 14 | } 15 | } 16 | //Munkres get matrix; 17 | Munkres m; 18 | m.solve(matrix); 19 | // 20 | std::vector> pairs; 21 | for (int row = 0; row < rows; row++) { 22 | for (int col = 0; col < cols; col++) { 23 | int tmp = (int)matrix(row, col); 24 | if (tmp == 0) pairs.push_back(std::make_pair(row, col)); 25 | } 26 | } 27 | // 28 | int count = pairs.size(); 29 | Eigen::Matrix re(count, 2); 30 | for (int i = 0; i < count; i++) { 31 | re(i, 0) = pairs[i].first; 32 | re(i, 1) = pairs[i].second; 33 | } 34 | return re; 35 | }//end Solve; 36 | -------------------------------------------------------------------------------- /deepsort/src/kalmanfilter.cpp: -------------------------------------------------------------------------------- 1 | #include "MyKalmanFilter.h" 2 | #include 3 | #include 4 | 5 | const double MyKalmanFilter::chi2inv95[10] = { 6 | 0, 7 | 3.8415, 8 | 5.9915, 9 | 7.8147, 10 | 9.4877, 11 | 11.070, 12 | 12.592, 13 | 14.067, 14 | 15.507, 15 | 16.919 16 | }; 17 | MyKalmanFilter::MyKalmanFilter() { 18 | int ndim = 4; 19 | double dt = 1.; 20 | 21 | _motion_mat = Eigen::MatrixXf::Identity(8, 8); 22 | for(int i = 0; i < ndim; i++) { 23 | _motion_mat(i, ndim+i) = dt; 24 | } 25 | _update_mat = Eigen::MatrixXf::Identity(4, 8); 26 | 27 | this->_std_weight_position = 1. / 20; 28 | this->_std_weight_velocity = 1. / 160; 29 | } 30 | 31 | KAL_DATA MyKalmanFilter::initiate(const DETECTBOX& measurement) { 32 | DETECTBOX mean_pos = measurement; 33 | DETECTBOX mean_vel; 34 | for(int i = 0; i < 4; i++) mean_vel(i) = 0; 35 | 36 | KAL_MEAN mean; 37 | for(int i = 0; i < 8; i++){ 38 | if(i < 4) mean(i) = mean_pos(i); 39 | else mean(i) = mean_vel(i - 4); 40 | } 41 | 42 | KAL_MEAN std; 43 | std(0) = 2 * _std_weight_position * measurement[3]; 44 | std(1) = 2 * _std_weight_position * measurement[3]; 45 | std(2) = 1e-2; 46 | std(3) = 2 * _std_weight_position * measurement[3]; 47 | std(4) = 10 * _std_weight_velocity * measurement[3]; 48 | std(5) = 10 * _std_weight_velocity * measurement[3]; 49 | std(6) = 1e-5; 50 | std(7) = 10 * _std_weight_velocity * measurement[3]; 51 | 52 | KAL_MEAN tmp = std.array().square(); 53 | KAL_COVA var = tmp.asDiagonal(); 54 | return std::make_pair(mean, var); 55 | } 56 | 57 | void MyKalmanFilter::predict(KAL_MEAN &mean, KAL_COVA &covariance) { 58 | //revise the data; 59 | DETECTBOX std_pos; 60 | std_pos << _std_weight_position * mean(3), 61 | _std_weight_position * mean(3), 62 | 1e-2, 63 | _std_weight_position * mean(3); 64 | DETECTBOX std_vel; 65 | std_vel << _std_weight_velocity * mean(3), 66 | _std_weight_velocity * mean(3), 67 | 1e-5, 68 | _std_weight_velocity * mean(3); 69 | KAL_MEAN tmp; 70 | tmp.block<1,4>(0,0) = std_pos; 71 | tmp.block<1,4>(0,4) = std_vel; 72 | tmp = tmp.array().square(); 73 | KAL_COVA motion_cov = tmp.asDiagonal(); 74 | KAL_MEAN mean1 = this->_motion_mat * mean.transpose(); 75 | KAL_COVA covariance1 = this->_motion_mat * covariance *(_motion_mat.transpose()); 76 | covariance1 += motion_cov; 77 | 78 | mean = mean1; 79 | covariance = covariance1; 80 | } 81 | 82 | KAL_HDATA MyKalmanFilter::project(const KAL_MEAN &mean, const KAL_COVA &covariance) { 83 | DETECTBOX std; 84 | std << _std_weight_position * mean(3), _std_weight_position * mean(3), 85 | 1e-1, _std_weight_position * mean(3); 86 | KAL_HMEAN mean1 = _update_mat * mean.transpose(); 87 | KAL_HCOVA covariance1 = _update_mat * covariance * (_update_mat.transpose()); 88 | Eigen::Matrix diag = std.asDiagonal(); 89 | diag = diag.array().square().matrix(); 90 | covariance1 += diag; 91 | // covariance1.diagonal() << diag; 92 | return std::make_pair(mean1, covariance1); 93 | } 94 | 95 | KAL_DATA 96 | MyKalmanFilter::update( 97 | const KAL_MEAN &mean, 98 | const KAL_COVA &covariance, 99 | const DETECTBOX &measurement) { 100 | KAL_HDATA pa = project(mean, covariance); 101 | KAL_HMEAN projected_mean = pa.first; 102 | KAL_HCOVA projected_cov = pa.second; 103 | 104 | //chol_factor, lower = 105 | //scipy.linalg.cho_factor(projected_cov, lower=True, check_finite=False) 106 | //kalmain_gain = 107 | //scipy.linalg.cho_solve((cho_factor, lower), 108 | //np.dot(covariance, self._upadte_mat.T).T, 109 | //check_finite=False).T 110 | Eigen::Matrix B = (covariance * (_update_mat.transpose())).transpose(); 111 | Eigen::Matrix kalman_gain = (projected_cov.llt().solve(B)).transpose(); // eg.8x4 112 | Eigen::Matrix innovation = measurement - projected_mean; //eg.1x4 113 | auto tmp = innovation*(kalman_gain.transpose()); 114 | KAL_MEAN new_mean = (mean.array() + tmp.array()).matrix(); 115 | KAL_COVA new_covariance = covariance - kalman_gain*projected_cov*(kalman_gain.transpose()); 116 | return std::make_pair(new_mean, new_covariance); 117 | } 118 | 119 | Eigen::Matrix 120 | MyKalmanFilter::gating_distance( 121 | const KAL_MEAN &mean, 122 | const KAL_COVA &covariance, 123 | const std::vector &measurements, 124 | bool only_position) { 125 | KAL_HDATA pa = this->project(mean, covariance); 126 | if(only_position) { 127 | printf("not implement!"); 128 | exit(0); 129 | } 130 | KAL_HMEAN mean1 = pa.first; 131 | KAL_HCOVA covariance1 = pa.second; 132 | 133 | // Eigen::Matrix d(size, 4); 134 | DETECTBOXSS d(measurements.size(), 4); 135 | int pos = 0; 136 | for(DETECTBOX box:measurements) { 137 | d.row(pos++) = box - mean1; 138 | } 139 | Eigen::Matrix factor = covariance1.llt().matrixL(); 140 | Eigen::Matrix z = factor.triangularView().solve(d).transpose(); 141 | auto zz = ((z.array())*(z.array())).matrix(); 142 | auto square_maha = zz.colwise().sum(); 143 | return square_maha; 144 | } 145 | 146 | -------------------------------------------------------------------------------- /deepsort/src/linear_assignment.cpp: -------------------------------------------------------------------------------- 1 | #include "linear_assignment.h" 2 | #include "hungarianoper.h" 3 | #include 4 | 5 | linear_assignment *linear_assignment::instance = NULL; 6 | linear_assignment::linear_assignment() 7 | { 8 | } 9 | 10 | linear_assignment *linear_assignment::getInstance() 11 | { 12 | if(instance == NULL) instance = new linear_assignment(); 13 | return instance; 14 | } 15 | 16 | TRACHER_MATCHD 17 | linear_assignment::matching_cascade( 18 | tracker *distance_metric, 19 | tracker::GATED_METRIC_FUNC distance_metric_func, 20 | float max_distance, 21 | int cascade_depth, 22 | std::vector &tracks, 23 | const DETECTIONS &detections, 24 | std::vector& track_indices, 25 | std::vector detection_indices) 26 | { 27 | TRACHER_MATCHD res; 28 | // std::cout << "distance_metric" << distance_metric << std::endl; 29 | // std::cout << "max_distance" << max_distance << std::endl; 30 | // std::cout << "cascade_depth" << cascade_depth << std::endl; 31 | // std::cout << "tracks [" << std::endl; 32 | // for (auto i : tracks) 33 | // std::cout << i.hits << ", "; 34 | // std::cout << "]" << endl; 35 | // std::cout << "detections [" << std::endl; 36 | // for (auto i : detections) 37 | // std::cout << i.confidence << ", "; 38 | // std::cout << "]" << endl; 39 | // std::cout << "track_indices [" << std::endl; 40 | // for (auto i : track_indices) 41 | // std::cout << i << ", "; 42 | // std::cout << "]" << endl; 43 | // std::cout << "detection_indices [" << std::endl; 44 | // for (auto i : detection_indices) 45 | // std::cout << i << ", "; 46 | // std::cout << "]" << endl; 47 | // !!!python diff: track_indices will never be None. 48 | // if(track_indices.empty() == true) { 49 | // for(size_t i = 0; i < tracks.size(); i++) { 50 | // track_indices.push_back(i); 51 | // } 52 | // } 53 | 54 | //!!!python diff: detection_indices will always be None. 55 | for(size_t i = 0; i < detections.size(); i++) { 56 | detection_indices.push_back(int(i)); 57 | } 58 | 59 | std::vector unmatched_detections; 60 | unmatched_detections.assign(detection_indices.begin(), detection_indices.end()); 61 | res.matches.clear(); 62 | std::vector track_indices_l; 63 | 64 | std::map matches_trackid; 65 | for(int level = 0; level < cascade_depth; level++) { 66 | if(unmatched_detections.size() == 0) break; //No detections left; 67 | 68 | track_indices_l.clear(); 69 | for(int k:track_indices) { 70 | if(tracks[k].time_since_update == 1+level) 71 | track_indices_l.push_back(k); 72 | } 73 | if(track_indices_l.size() == 0) continue; //Nothing to match at this level. 74 | 75 | TRACHER_MATCHD tmp = min_cost_matching( 76 | distance_metric, distance_metric_func, 77 | max_distance, tracks, detections, track_indices_l, 78 | unmatched_detections); 79 | 80 | unmatched_detections.assign(tmp.unmatched_detections.begin(), tmp.unmatched_detections.end()); 81 | for(size_t i = 0; i < tmp.matches.size(); i++) { 82 | MATCH_DATA pa = tmp.matches[i]; 83 | res.matches.push_back(pa); 84 | matches_trackid.insert(pa); 85 | } 86 | } 87 | res.unmatched_detections.assign(unmatched_detections.begin(), unmatched_detections.end()); 88 | for(size_t i = 0; i < track_indices.size(); i++) { 89 | int tid = track_indices[i]; 90 | if(matches_trackid.find(tid) == matches_trackid.end()) 91 | res.unmatched_tracks.push_back(tid); 92 | } 93 | return res; 94 | } 95 | 96 | TRACHER_MATCHD 97 | linear_assignment::min_cost_matching(tracker *distance_metric, 98 | tracker::GATED_METRIC_FUNC distance_metric_func, 99 | float max_distance, 100 | std::vector &tracks, 101 | const DETECTIONS &detections, 102 | std::vector &track_indices, 103 | std::vector &detection_indices) 104 | { 105 | TRACHER_MATCHD res; 106 | //!!!python diff: track_indices && detection_indices will never be None. 107 | // if(track_indices.empty() == true) { 108 | // for(size_t i = 0; i < tracks.size(); i++) { 109 | // track_indices.push_back(i); 110 | // } 111 | // } 112 | // if(detection_indices.empty() == true) { 113 | // for(size_t i = 0; i < detections.size(); i++) { 114 | // detection_indices.push_back(int(i)); 115 | // } 116 | // } 117 | if((detection_indices.size() == 0) || (track_indices.size() == 0)) { 118 | res.matches.clear(); 119 | res.unmatched_tracks.assign(track_indices.begin(), track_indices.end()); 120 | res.unmatched_detections.assign(detection_indices.begin(), detection_indices.end()); 121 | return res; 122 | } 123 | DYNAMICM cost_matrix = (distance_metric->*(distance_metric_func))( 124 | tracks, detections, track_indices, detection_indices); 125 | for(int i = 0; i < cost_matrix.rows(); i++) { 126 | for(int j = 0; j < cost_matrix.cols(); j++) { 127 | float tmp = cost_matrix(i,j); 128 | if(tmp > max_distance) cost_matrix(i,j) = max_distance + 1e-5; 129 | } 130 | } 131 | 132 | // std::cout << "In: \n"; 133 | // std::cout << cost_matrix << "\n"; 134 | Eigen::Matrix indices = HungarianOper::Solve(cost_matrix); 135 | // std::cout << "Out: \n"; 136 | 137 | res.matches.clear(); 138 | res.unmatched_tracks.clear(); 139 | res.unmatched_detections.clear(); 140 | for(size_t col = 0; col < detection_indices.size(); col++) { 141 | bool flag = false; 142 | for(int i = 0; i < indices.rows(); i++) 143 | if(indices(i, 1) == col) { flag = true; break;} 144 | if(flag == false)res.unmatched_detections.push_back(detection_indices[col]); 145 | } 146 | for(size_t row = 0; row < track_indices.size(); row++) { 147 | bool flag = false; 148 | for(int i = 0; i < indices.rows(); i++) 149 | if(indices(i, 0) == row) { flag = true; break; } 150 | if(flag == false) res.unmatched_tracks.push_back(track_indices[row]); 151 | } 152 | for(int i = 0; i < indices.rows(); i++) { 153 | int row = indices(i, 0); 154 | int col = indices(i, 1); 155 | 156 | int track_idx = track_indices[row]; 157 | int detection_idx = detection_indices[col]; 158 | if(cost_matrix(row, col) > max_distance) { 159 | res.unmatched_tracks.push_back(track_idx); 160 | res.unmatched_detections.push_back(detection_idx); 161 | } else res.matches.push_back(std::make_pair(track_idx, detection_idx)); 162 | } 163 | return res; 164 | } 165 | 166 | DYNAMICM 167 | linear_assignment::gate_cost_matrix( 168 | MyKalmanFilter *kf, 169 | DYNAMICM &cost_matrix, 170 | std::vector &tracks, 171 | const DETECTIONS &detections, 172 | const std::vector &track_indices, 173 | const std::vector &detection_indices, 174 | float gated_cost, bool only_position) 175 | { 176 | // std::cout << "input cost matric" << cost_matrix << std::endl; 177 | int gating_dim = (only_position == true?2:4); 178 | double gating_threshold = MyKalmanFilter::chi2inv95[gating_dim]; 179 | std::vector measurements; 180 | for(int i:detection_indices) { 181 | DETECTION_ROW t = detections[i]; 182 | measurements.push_back(t.to_xyah()); 183 | } 184 | for(size_t i = 0; i < track_indices.size(); i++) { 185 | Track& track = tracks[track_indices[i]]; 186 | Eigen::Matrix gating_distance = kf->gating_distance( 187 | track.mean, track.covariance, measurements, only_position); 188 | for (int j = 0; j < gating_distance.cols(); j++) { 189 | if (gating_distance(0, j) > gating_threshold) cost_matrix(i, j) = gated_cost; 190 | } 191 | } 192 | // std::cout << "out cost matrix" << cost_matrix << std::endl; 193 | return cost_matrix; 194 | } 195 | 196 | -------------------------------------------------------------------------------- /deepsort/src/nn_matching.cpp: -------------------------------------------------------------------------------- 1 | #include "nn_matching.h" 2 | #include 3 | 4 | using namespace Eigen; 5 | 6 | NearNeighborDisMetric::NearNeighborDisMetric( 7 | NearNeighborDisMetric::METRIC_TYPE metric, 8 | float matching_threshold, int budget) 9 | { 10 | if (metric == euclidean) { 11 | _metric = 12 | &NearNeighborDisMetric::_nneuclidean_distance; 13 | } else if (metric == cosine) { 14 | _metric = 15 | &NearNeighborDisMetric::_nncosine_distance; 16 | } 17 | 18 | this->mating_threshold = matching_threshold; 19 | this->budget = budget; 20 | this->samples.clear(); 21 | } 22 | 23 | DYNAMICM 24 | NearNeighborDisMetric::distance( 25 | const FEATURESS & features, 26 | const std::vector < int >&targets) 27 | { 28 | DYNAMICM cost_matrix = Eigen::MatrixXf::Zero(targets.size(), features.rows()); 29 | int idx = 0; 30 | for (int target:targets) { 31 | cost_matrix.row(idx) = (this->*_metric) (this->samples[target], features); 32 | idx++; 33 | } 34 | return cost_matrix; 35 | } 36 | 37 | void 38 | NearNeighborDisMetric::partial_fit( 39 | std::vector &tid_feats, 40 | std::vector < int >&active_targets) 41 | { 42 | /*python code: 43 | * let feature(target_id) append to samples; 44 | * && delete not comfirmed target_id from samples. 45 | * update samples; 46 | */ 47 | for (TRACKER_DATA & data:tid_feats) { 48 | int track_id = data.first; 49 | FEATURESS newFeatOne = data.second; 50 | 51 | if (samples.find(track_id) != samples.end()) { //append 52 | int oldSize = samples[track_id].rows(); 53 | int addSize = newFeatOne.rows(); 54 | int newSize = oldSize + addSize; 55 | 56 | if (newSize <= this->budget) { 57 | FEATURESS newSampleFeatures(newSize, 512); 58 | newSampleFeatures.block(0, 0, oldSize, 512) = samples[track_id]; 59 | newSampleFeatures.block(oldSize, 0, addSize, 512) = newFeatOne; 60 | samples[track_id] = newSampleFeatures; 61 | } else { 62 | if (oldSize < this->budget) { //original space is not enough; 63 | FEATURESS newSampleFeatures(this->budget, 512); 64 | if (addSize >= this->budget) { 65 | newSampleFeatures = newFeatOne.block(0, 0, this->budget, 512); 66 | } else { 67 | newSampleFeatures.block(0, 0, this->budget - addSize, 512) = 68 | samples[track_id].block(addSize - 1, 0, this->budget - addSize, 512).eval(); 69 | newSampleFeatures.block(this->budget - addSize, 0, addSize,512) = newFeatOne; 70 | } 71 | samples[track_id] = newSampleFeatures; 72 | } else { //original space is ok; 73 | if (addSize >= this->budget) { 74 | samples[track_id] = newFeatOne.block(0, 0, this->budget, 512); 75 | } else { 76 | samples[track_id].block(0, 0, this->budget - addSize, 512) = 77 | samples[track_id].block(addSize - 1, 0, this->budget - addSize, 512).eval(); 78 | samples[track_id].block(this->budget - addSize, 0, addSize, 512) = newFeatOne; 79 | } 80 | } 81 | } 82 | } else { //not exit, create new one; 83 | samples[track_id] = newFeatOne; 84 | } 85 | } //add features; 86 | 87 | //erase the samples which not in active_targets; 88 | for (std::map < int, FEATURESS >::iterator i = samples.begin(); i != samples.end();) { 89 | bool flag = false; 90 | for (int j:active_targets) if (j == i->first) { flag = true; break; } 91 | if (flag == false)samples.erase(i++); 92 | else i++; 93 | } 94 | } 95 | 96 | Eigen::VectorXf 97 | NearNeighborDisMetric::_nncosine_distance( 98 | const FEATURESS & x, const FEATURESS & y) 99 | { 100 | MatrixXf distances = _cosine_distance(x, y); 101 | VectorXf res = distances.colwise().minCoeff().transpose(); 102 | return res; 103 | } 104 | 105 | Eigen::VectorXf 106 | NearNeighborDisMetric::_nneuclidean_distance( 107 | const FEATURESS & x, const FEATURESS & y) 108 | { 109 | MatrixXf distances = _pdist(x, y); 110 | VectorXf res = distances.colwise().maxCoeff().transpose(); 111 | res = res.array().max(VectorXf::Zero(res.rows()).array()); 112 | return res; 113 | } 114 | 115 | Eigen::MatrixXf 116 | NearNeighborDisMetric::_pdist(const FEATURESS & x, const FEATURESS & y) 117 | { 118 | int len1 = x.rows(), len2 = y.rows(); 119 | if (len1 == 0 || len2 == 0) { 120 | return Eigen::MatrixXf::Zero(len1, len2); 121 | } 122 | MatrixXf res = -2.0 * x * y.transpose(); 123 | res = res.colwise() + x.rowwise().squaredNorm(); 124 | res = res.rowwise() + y.rowwise().squaredNorm().transpose(); 125 | res = res.array().max(MatrixXf::Zero(res.rows(), res.cols()).array()); 126 | return res; 127 | } 128 | 129 | Eigen::MatrixXf 130 | NearNeighborDisMetric::_cosine_distance( 131 | const FEATURESS & a, const FEATURESS & b, bool data_is_normalized) 132 | { 133 | FEATURESS aa = a; 134 | FEATURESS bb = b; 135 | if (!data_is_normalized) { 136 | //undo: 137 | for (int i = 0; i < a.rows(); ++i) { 138 | aa.row(i) = a.row(i) / sqrt(a.row(i).squaredNorm()); 139 | } 140 | for (int i = 0; i < b.rows(); ++i) { 141 | bb.row(i) = b.row(i) / sqrt(b.row(i).squaredNorm()); 142 | } 143 | } 144 | MatrixXf res = 1. - (aa * bb.transpose()).array(); 145 | return res; 146 | } 147 | -------------------------------------------------------------------------------- /deepsort/src/track.cpp: -------------------------------------------------------------------------------- 1 | #include "track.h" 2 | #include 3 | 4 | Track::Track(KAL_MEAN & mean, KAL_COVA & covariance, int track_id, int n_init, int max_age, const FEATURE & feature) 5 | { 6 | this->mean = mean; 7 | this->covariance = covariance; 8 | this->track_id = track_id; 9 | this->hits = 1; 10 | this->age = 1; 11 | this->time_since_update = 0; 12 | this->state = TrackState::Tentative; 13 | features = FEATURESS(1, 512); 14 | features.row(0) = feature; //features.rows() must = 0; 15 | 16 | this->_n_init = n_init; 17 | this->_max_age = max_age; 18 | } 19 | 20 | Track::Track(KAL_MEAN & mean, KAL_COVA & covariance, int track_id, int n_init, int max_age, const FEATURE & feature, int cls, float conf) 21 | { 22 | this->mean = mean; 23 | this->covariance = covariance; 24 | this->track_id = track_id; 25 | this->hits = 1; 26 | this->age = 1; 27 | this->time_since_update = 0; 28 | this->state = TrackState::Tentative; 29 | features = FEATURESS(1, 512); 30 | features.row(0) = feature; //features.rows() must = 0; 31 | 32 | this->_n_init = n_init; 33 | this->_max_age = max_age; 34 | 35 | this->cls = cls; 36 | this->conf = conf; 37 | } 38 | 39 | void Track::predit(MyKalmanFilter * kf) 40 | { 41 | /*Propagate the state distribution to the current time step using a 42 | Kalman filter prediction step. 43 | 44 | Parameters 45 | ---------- 46 | kf : kalman_filter.MyKalmanFilterd 47 | The Kalman filter. 48 | */ 49 | 50 | kf->predict(this->mean, this->covariance); 51 | 52 | 53 | this->age += 1; 54 | this->time_since_update += 1; 55 | } 56 | 57 | void Track::update(MyKalmanFilter * const kf, const DETECTION_ROW & detection) 58 | { 59 | KAL_DATA pa = kf->update(this->mean, this->covariance, detection.to_xyah()); 60 | this->mean = pa.first; 61 | this->covariance = pa.second; 62 | 63 | featuresAppendOne(detection.feature); 64 | // this->features.row(features.rows()) = detection.feature; 65 | this->hits += 1; 66 | this->time_since_update = 0; 67 | if (this->state == TrackState::Tentative && this->hits >= this->_n_init) { 68 | this->state = TrackState::Confirmed; 69 | } 70 | } 71 | 72 | void Track::update(MyKalmanFilter * const kf, const DETECTION_ROW & detection, CLSCONF pair_det) 73 | { 74 | KAL_DATA pa = kf->update(this->mean, this->covariance, detection.to_xyah()); 75 | this->mean = pa.first; 76 | this->covariance = pa.second; 77 | 78 | featuresAppendOne(detection.feature); 79 | // this->features.row(features.rows()) = detection.feature; 80 | this->hits += 1; 81 | this->time_since_update = 0; 82 | if (this->state == TrackState::Tentative && this->hits >= this->_n_init) { 83 | this->state = TrackState::Confirmed; 84 | } 85 | this->cls = pair_det.cls; 86 | this->conf = pair_det.conf; 87 | } 88 | 89 | void Track::mark_missed() 90 | { 91 | if (this->state == TrackState::Tentative) { 92 | this->state = TrackState::Deleted; 93 | } else if (this->time_since_update > this->_max_age) { 94 | this->state = TrackState::Deleted; 95 | } 96 | } 97 | 98 | bool Track::is_confirmed() 99 | { 100 | return this->state == TrackState::Confirmed; 101 | } 102 | 103 | bool Track::is_deleted() 104 | { 105 | return this->state == TrackState::Deleted; 106 | } 107 | 108 | bool Track::is_tentative() 109 | { 110 | return this->state == TrackState::Tentative; 111 | } 112 | 113 | DETECTBOX Track::to_tlwh() 114 | { 115 | DETECTBOX ret = mean.leftCols(4); 116 | ret(2) *= ret(3); 117 | ret.leftCols(2) -= (ret.rightCols(2) / 2); 118 | return ret; 119 | } 120 | 121 | void Track::featuresAppendOne(const FEATURE & f) 122 | { 123 | int size = this->features.rows(); 124 | FEATURESS newfeatures = FEATURESS(size + 1, 512); 125 | newfeatures.block(0, 0, size, 512) = this->features; 126 | newfeatures.row(size) = f; 127 | features = newfeatures; 128 | } 129 | -------------------------------------------------------------------------------- /deepsort/src/tracker.cpp: -------------------------------------------------------------------------------- 1 | #include "tracker.h" 2 | #include "nn_matching.h" 3 | #include "linear_assignment.h" 4 | using namespace std; 5 | 6 | #define MY_inner_DEBUG 7 | #ifdef MY_inner_DEBUG 8 | #include 9 | #include 10 | #endif 11 | using namespace std; 12 | tracker::tracker( /*NearNeighborDisMetric *metric, */ 13 | float max_cosine_distance, int nn_budget, 14 | float max_iou_distance, int max_age, int n_init) 15 | { 16 | this->metric = new NearNeighborDisMetric( 17 | NearNeighborDisMetric::METRIC_TYPE::cosine, 18 | max_cosine_distance, nn_budget); 19 | this->max_iou_distance = max_iou_distance; 20 | this->max_age = max_age; 21 | this->n_init = n_init; 22 | 23 | this->kf = new MyKalmanFilter(); 24 | this->tracks.clear(); 25 | this->_next_idx = 1; 26 | } 27 | 28 | void tracker::predict() 29 | { 30 | for (Track & track:tracks) { 31 | track.predit(kf); 32 | } 33 | } 34 | 35 | void tracker::update(const DETECTIONS & detections) 36 | { 37 | TRACHER_MATCHD res; 38 | _match(detections, res); 39 | 40 | vector < MATCH_DATA > &matches = res.matches; 41 | for (MATCH_DATA & data:matches) { 42 | int track_idx = data.first; 43 | int detection_idx = data.second; 44 | tracks[track_idx].update(this->kf, detections[detection_idx]); 45 | } 46 | vector < int >&unmatched_tracks = res.unmatched_tracks; 47 | for (int &track_idx:unmatched_tracks) { 48 | this->tracks[track_idx].mark_missed(); 49 | } 50 | vector < int >&unmatched_detections = res.unmatched_detections; 51 | for (int &detection_idx:unmatched_detections) { 52 | this->_initiate_track(detections[detection_idx]); 53 | } 54 | vector < Track >::iterator it; 55 | for (it = tracks.begin(); it != tracks.end();) { 56 | if ((*it).is_deleted()) it = tracks.erase(it); 57 | else ++it; 58 | } 59 | vector < int >active_targets; 60 | vector < TRACKER_DATA > tid_features; 61 | for (Track & track:tracks) { 62 | if (track.is_confirmed() == false) continue; 63 | active_targets.push_back(track.track_id); 64 | tid_features.push_back(std::make_pair(track. track_id, track.features)); 65 | FEATURESS t = FEATURESS(0, 512); 66 | track.features = t; 67 | } 68 | this->metric->partial_fit(tid_features, active_targets); 69 | } 70 | 71 | void tracker::update(const DETECTIONSV2 & detectionsv2) 72 | { 73 | const vector& clsConf = detectionsv2.first; 74 | const DETECTIONS& detections = detectionsv2.second; 75 | TRACHER_MATCHD res; 76 | _match(detections, res); 77 | // std::cout << "checkpoint in overloaded sort\n"; 78 | 79 | vector < MATCH_DATA > &matches = res.matches; 80 | for (MATCH_DATA & data:matches) { 81 | int track_idx = data.first; 82 | int detection_idx = data.second; 83 | tracks[track_idx].update(this->kf, detections[detection_idx], clsConf[detection_idx]); 84 | } 85 | vector < int >&unmatched_tracks = res.unmatched_tracks; 86 | for (int &track_idx:unmatched_tracks) { 87 | this->tracks[track_idx].mark_missed(); 88 | } 89 | vector < int >&unmatched_detections = res.unmatched_detections; 90 | for (int &detection_idx:unmatched_detections) { 91 | this->_initiate_track(detections[detection_idx], clsConf[detection_idx]); 92 | } 93 | vector < Track >::iterator it; 94 | for (it = tracks.begin(); it != tracks.end();) { 95 | if ((*it).is_deleted()) it = tracks.erase(it); 96 | else ++it; 97 | } 98 | vector < int >active_targets; 99 | vector < TRACKER_DATA > tid_features; 100 | for (Track & track:tracks) { 101 | if (track.is_confirmed() == false) continue; 102 | active_targets.push_back(track.track_id); 103 | tid_features.push_back(std::make_pair(track. track_id, track.features)); 104 | FEATURESS t = FEATURESS(0, 512); 105 | track.features = t; 106 | } 107 | this->metric->partial_fit(tid_features, active_targets); 108 | } 109 | 110 | void tracker::_match(const DETECTIONS & detections, TRACHER_MATCHD & res) 111 | { 112 | vector < int >confirmed_tracks; 113 | vector < int >unconfirmed_tracks; 114 | int idx = 0; 115 | for (Track & t:tracks) { 116 | if (t.is_confirmed()) confirmed_tracks.push_back(idx); 117 | else unconfirmed_tracks.push_back(idx); 118 | idx++; 119 | } 120 | 121 | TRACHER_MATCHD matcha = linear_assignment::getInstance()-> matching_cascade( 122 | this, &tracker::gated_matric, 123 | this->metric->mating_threshold, 124 | this->max_age, 125 | this->tracks, 126 | detections, 127 | confirmed_tracks); 128 | 129 | 130 | vector < int >iou_track_candidates; 131 | iou_track_candidates.assign(unconfirmed_tracks.begin(), unconfirmed_tracks.end()); 132 | vector < int >::iterator it; 133 | for (it = matcha.unmatched_tracks.begin(); it != matcha.unmatched_tracks.end();) { 134 | int idx = *it; 135 | if (tracks[idx].time_since_update == 1) { //push into unconfirmed 136 | iou_track_candidates.push_back(idx); 137 | it = matcha.unmatched_tracks.erase(it); 138 | continue; 139 | } 140 | ++it; 141 | } 142 | TRACHER_MATCHD matchb = linear_assignment::getInstance()->min_cost_matching( 143 | this, &tracker::iou_cost, 144 | this->max_iou_distance, 145 | this->tracks, 146 | detections, 147 | iou_track_candidates, 148 | matcha.unmatched_detections); 149 | //get result: 150 | res.matches.assign(matcha.matches.begin(), matcha.matches.end()); 151 | res.matches.insert(res.matches.end(), matchb.matches.begin(), matchb.matches.end()); 152 | //unmatched_tracks; 153 | res.unmatched_tracks.assign( 154 | matcha.unmatched_tracks.begin(), 155 | matcha.unmatched_tracks.end()); 156 | res.unmatched_tracks.insert( 157 | res.unmatched_tracks.end(), 158 | matchb.unmatched_tracks.begin(), 159 | matchb.unmatched_tracks.end()); 160 | res.unmatched_detections.assign( 161 | matchb.unmatched_detections.begin(), 162 | matchb.unmatched_detections.end()); 163 | } 164 | 165 | void tracker::_initiate_track(const DETECTION_ROW & detection) 166 | { 167 | KAL_DATA data = kf->initiate(detection.to_xyah()); 168 | KAL_MEAN mean = data.first; 169 | KAL_COVA covariance = data.second; 170 | 171 | this->tracks.push_back(Track(mean, covariance, this->_next_idx, this->n_init, 172 | this->max_age, detection.feature)); 173 | _next_idx += 1; 174 | } 175 | void tracker::_initiate_track(const DETECTION_ROW& detection, CLSCONF clsConf) 176 | { 177 | KAL_DATA data = kf->initiate(detection.to_xyah()); 178 | KAL_MEAN mean = data.first; 179 | KAL_COVA covariance = data.second; 180 | 181 | this->tracks.push_back(Track(mean, covariance, this->_next_idx, this->n_init, 182 | this->max_age, detection.feature, clsConf.cls, clsConf.conf)); 183 | _next_idx += 1; 184 | } 185 | 186 | DYNAMICM tracker::gated_matric( 187 | std::vector < Track > &tracks, 188 | const DETECTIONS & dets, 189 | const std::vector < int >&track_indices, 190 | const std::vector < int >&detection_indices) 191 | { 192 | FEATURESS features(detection_indices.size(), 512); 193 | int pos = 0; 194 | for (int i:detection_indices) { 195 | features.row(pos++) = dets[i].feature; 196 | } 197 | vector < int >targets; 198 | for (int i:track_indices) { 199 | targets.push_back(tracks[i].track_id); 200 | } 201 | DYNAMICM cost_matrix = this->metric->distance(features, targets); 202 | DYNAMICM res = linear_assignment::getInstance()->gate_cost_matrix( 203 | this->kf, cost_matrix, tracks, dets, track_indices, 204 | detection_indices); 205 | return res; 206 | } 207 | 208 | DYNAMICM 209 | tracker::iou_cost( 210 | std::vector < Track > &tracks, 211 | const DETECTIONS & dets, 212 | const std::vector < int >&track_indices, 213 | const std::vector < int >&detection_indices) 214 | { 215 | //!!!python diff: track_indices && detection_indices will never be None. 216 | // if(track_indices.empty() == true) { 217 | // for(size_t i = 0; i < tracks.size(); i++) { 218 | // track_indices.push_back(i); 219 | // } 220 | // } 221 | // if(detection_indices.empty() == true) { 222 | // for(size_t i = 0; i < dets.size(); i++) { 223 | // detection_indices.push_back(i); 224 | // } 225 | // } 226 | int rows = track_indices.size(); 227 | int cols = detection_indices.size(); 228 | DYNAMICM cost_matrix = Eigen::MatrixXf::Zero(rows, cols); 229 | for (int i = 0; i < rows; i++) { 230 | int track_idx = track_indices[i]; 231 | if (tracks[track_idx].time_since_update > 1) { 232 | cost_matrix.row(i) = Eigen::RowVectorXf::Constant(cols, INFTY_COST); 233 | continue; 234 | } 235 | DETECTBOX bbox = tracks[track_idx].to_tlwh(); 236 | int csize = detection_indices.size(); 237 | DETECTBOXSS candidates(csize, 4); 238 | for (int k = 0; k < csize; k++) candidates.row(k) = dets[detection_indices[k]].tlwh; 239 | Eigen::RowVectorXf rowV = (1. - iou(bbox, candidates).array()).matrix().transpose(); 240 | cost_matrix.row(i) = rowV; 241 | } 242 | return cost_matrix; 243 | } 244 | 245 | Eigen::VectorXf 246 | tracker::iou(DETECTBOX & bbox, DETECTBOXSS & candidates) 247 | { 248 | float bbox_tl_1 = bbox[0]; 249 | float bbox_tl_2 = bbox[1]; 250 | float bbox_br_1 = bbox[0] + bbox[2]; 251 | float bbox_br_2 = bbox[1] + bbox[3]; 252 | float area_bbox = bbox[2] * bbox[3]; 253 | 254 | Eigen::Matrix < float, -1, 2 > candidates_tl; 255 | Eigen::Matrix < float, -1, 2 > candidates_br; 256 | candidates_tl = candidates.leftCols(2); 257 | candidates_br = candidates.rightCols(2) + candidates_tl; 258 | 259 | int size = int (candidates.rows()); 260 | // Eigen::VectorXf area_intersection(size); 261 | // Eigen::VectorXf area_candidates(size); 262 | Eigen::VectorXf res(size); 263 | for (int i = 0; i < size; i++) { 264 | float tl_1 = std::max(bbox_tl_1, candidates_tl(i, 0)); 265 | float tl_2 = std::max(bbox_tl_2, candidates_tl(i, 1)); 266 | float br_1 = std::min(bbox_br_1, candidates_br(i, 0)); 267 | float br_2 = std::min(bbox_br_2, candidates_br(i, 1)); 268 | 269 | float w = br_1 - tl_1; w = (w < 0 ? 0 : w); 270 | float h = br_2 - tl_2; h = (h < 0 ? 0 : h); 271 | float area_intersection = w * h; 272 | float area_candidates = candidates(i, 2) * candidates(i, 3); 273 | res[i] = area_intersection / (area_bbox + area_candidates - area_intersection); 274 | } 275 | return res; 276 | } 277 | -------------------------------------------------------------------------------- /detect.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/detect.gif -------------------------------------------------------------------------------- /include/box.h: -------------------------------------------------------------------------------- 1 | #ifndef BOX_H 2 | #define BOX_H 3 | #include 4 | #include "opencv2/opencv.hpp" 5 | 6 | // 标签 7 | #define OBJ_NAME_MAX_SIZE 16 8 | #define OBJ_NUMB_MAX_SIZE 64 9 | 10 | typedef struct DetectBox { 11 | DetectBox(float x1=0, float y1=0, float x2=0, float y2=0, 12 | float confidence=0, float classID=-1, float trackID=-1) { 13 | this->x1 = x1; 14 | this->y1 = y1; 15 | this->x2 = x2; 16 | this->y2 = y2; 17 | this->confidence = confidence; 18 | this->classID = classID; 19 | this->trackID = trackID; 20 | } 21 | char name[OBJ_NAME_MAX_SIZE]; 22 | float x1, y1, x2, y2; 23 | float confidence; 24 | float classID; 25 | float trackID; 26 | } DetectBox; 27 | #endif -------------------------------------------------------------------------------- /include/common.h: -------------------------------------------------------------------------------- 1 | #pragma once // 防止重定义 2 | #include 3 | #include "opencv2/opencv.hpp" 4 | 5 | #ifndef BOX_H 6 | #include "box.h" 7 | #define BOX_H 8 | #endif // BOX_H 9 | 10 | #define BYTE unsigned char 11 | #define IMG_WIDTH 1024 12 | #define IMG_HEIGHT 540 13 | #define IMG_CHANNEL 3 14 | #define IMG_PAD 640 15 | 16 | // 网络的参数 17 | #define NET_INPUTHEIGHT 640 18 | #define NET_INPUTWIDTH 640 19 | #define NET_INPUTCHANNEL 3 20 | #define GRID0 80 21 | #define GRID1 40 22 | #define GRID2 20 23 | #define OBJ_CLASS_NUM 1 24 | #define nyolo 3 //n yolo layers; 25 | #define nanchor 3 //n anchor per yolo layer 26 | #define PROP_BOX_SIZE (5+OBJ_CLASS_NUM) 27 | 28 | // 阈值 29 | #define NMS_THRESH 0.2 30 | #define BOX_THRESH 0.2 31 | 32 | // 每一层nbox的数量 33 | #define nboxes_0 GRID0*GRID0*nanchor 34 | #define nboxes_1 GRID1*GRID1*nanchor 35 | #define nboxes_2 GRID2*GRID2*nanchor 36 | #define nboxes_total nboxes_0+nboxes_1+nboxes_2 37 | 38 | 39 | struct input_image{ 40 | input_image(){ 41 | 42 | } 43 | input_image(int num, cv::Mat img1, cv::Mat img2){ 44 | index = num; 45 | img_src = img1; 46 | img_pad = img2; 47 | } 48 | int index; 49 | cv::Mat img_src; 50 | cv::Mat img_pad; 51 | }; 52 | 53 | 54 | typedef struct _detect_result_group_t 55 | { 56 | int id; 57 | int count; 58 | std::vector results; 59 | } detect_result_group_t; 60 | 61 | /* 62 | 某一帧的所有合理预测结果 + 图片 63 | img： 背景图 64 | dets： 检测结果结构体数组 65 | */ 66 | struct imageout_idx 67 | { 68 | cv::Mat img; 69 | detect_result_group_t dets; 70 | }; -------------------------------------------------------------------------------- /include/mytime.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | double what_time_is_it_now(); -------------------------------------------------------------------------------- /include/resize.h: -------------------------------------------------------------------------------- 1 | #include "opencv2/opencv.hpp" 2 | #include "im2d.h" 3 | #include "RgaUtils.h" 4 | #include "rga.h" 5 | 6 | /* 7 | 图像预处理 8 | img——原图像 _img——缩放后的图像 9 | resize： 缩放图像 10 | get_max_scale： 获取最大缩放比例 11 | */ 12 | class PreResize 13 | { 14 | public: 15 | PreResize(); 16 | PreResize(int, int, int); 17 | void init(double, double); 18 | int input_height; 19 | int input_width; 20 | int input_channel; 21 | double fx; // scale along x 22 | double fy; // scale along y 23 | // init rga context 24 | rga_buffer_t src; 25 | rga_buffer_t dst; 26 | im_rect src_rect; 27 | im_rect dst_rect; 28 | 29 | void resize(cv::Mat &img, cv::Mat &_img); 30 | }; 31 | 32 | -------------------------------------------------------------------------------- /include/rknn_fp.h: -------------------------------------------------------------------------------- 1 | #ifndef RKNN_FP 2 | #define RKNN_FP 3 | 4 | #include "rknn_api.h" 5 | 6 | class rknn_fp{ 7 | public: 8 | /* 9 | NPU初始化 10 | model_path： 模型路径 11 | thread_id： 线程号 12 | device_id： 设备号 13 | */ 14 | rknn_fp(const char *, int, rknn_core_mask, int, int); 15 | ~rknn_fp(void); 16 | void dump_tensor_attr(rknn_tensor_attr*); 17 | int inference(unsigned char *); 18 | float cal_NPU_performance(std::queue &, float &, float); 19 | public: 20 | int _cpu_id; 21 | int _n_input; 22 | int _n_output; 23 | //Inputs and Output sets 24 | rknn_context ctx; 25 | rknn_tensor_attr _input_attrs[1]; 26 | rknn_tensor_attr _output_attrs[3]; 27 | rknn_tensor_mem* _input_mems[1]; 28 | rknn_tensor_mem* _output_mems[3]; 29 | float* _output_buff[3]; 30 | }; 31 | 32 | #endif -------------------------------------------------------------------------------- /model/best.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/best.rknn -------------------------------------------------------------------------------- /model/best_nofocus_hyx.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/best_nofocus_hyx.rknn -------------------------------------------------------------------------------- /model/best_nofocus_hyx_1920.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/best_nofocus_hyx_1920.rknn -------------------------------------------------------------------------------- /model/best_nofocus_relu.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/best_nofocus_relu.rknn -------------------------------------------------------------------------------- /model/hongwai_2_labels_list.txt: -------------------------------------------------------------------------------- 1 | car 2 | -------------------------------------------------------------------------------- /model/osnet_x0_25_market.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/osnet_x0_25_market.rknn -------------------------------------------------------------------------------- /model/osnet_x1_0_imagenet.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/osnet_x1_0_imagenet.rknn -------------------------------------------------------------------------------- /model/yolov5.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/yolov5.rknn -------------------------------------------------------------------------------- /model/yolov5_1920.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/yolov5_1920.rknn -------------------------------------------------------------------------------- /model/yolov5_nofocus.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/yolov5_nofocus.rknn -------------------------------------------------------------------------------- /model/yolov5_nofocus_1920.rknn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JackWoo0831/yolov5_Deepsort_rknn/beab34f82f679748b03b849b5ab03ce688837351/model/yolov5_nofocus_1920.rknn -------------------------------------------------------------------------------- /src/mytime.cpp: -------------------------------------------------------------------------------- 1 | #include "mytime.h" 2 | double what_time_is_it_now() 3 | { 4 | // 单位: ms 5 | struct timeval time; 6 | if (gettimeofday(&time,NULL)){ 7 | return 0; 8 | } 9 | return (double)time.tv_sec * 1000 + (double)time.tv_usec * .001; 10 | } -------------------------------------------------------------------------------- /src/resize.cpp: -------------------------------------------------------------------------------- 1 | #include "resize.h" 2 | 3 | PreResize::PreResize(){ 4 | 5 | } 6 | 7 | PreResize::PreResize(int height, int width, int channel){ 8 | input_height = height; 9 | input_width = width; 10 | input_channel = channel; 11 | } 12 | 13 | void PreResize::init(double fx, double fy){ 14 | this->fx = fx; 15 | this->fy = fy; 16 | } 17 | 18 | void PreResize::resize(cv::Mat &img, cv::Mat &_img) 19 | { 20 | memset(&src_rect, 0, sizeof(src_rect)); 21 | memset(&dst_rect, 0, sizeof(dst_rect)); 22 | memset(&src, 0, sizeof(src)); 23 | memset(&dst, 0, sizeof(dst)); 24 | int img_width = img.cols; 25 | int img_height = img.rows; 26 | void *resize_buf = malloc(input_height * input_width * input_channel); 27 | 28 | src = wrapbuffer_virtualaddr((void *)img.data, img_width, img_height, RK_FORMAT_RGB_888); 29 | dst = wrapbuffer_virtualaddr((void *)resize_buf, input_width, input_height, RK_FORMAT_RGB_888); 30 | int ret = imcheck(src, dst, src_rect, dst_rect); 31 | 32 | IM_STATUS STATUS = imresize(src, dst); 33 | if (ret != IM_STATUS_NOERROR) 34 | { 35 | printf("%d, check error! %s", __LINE__, imStrError((IM_STATUS)ret)); 36 | exit(-1); 37 | } 38 | _img = cv::Mat(cv::Size(input_width, input_height), CV_8UC3, resize_buf); 39 | free(resize_buf); 40 | // cv::imwrite("resize_input.jpg", _img); 41 | } -------------------------------------------------------------------------------- /src/rknn_fp.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "rknn_fp.h" 7 | 8 | // rknn_fp构造函数 NPU初始化 9 | rknn_fp::rknn_fp(const char *model_path, int cpuid, rknn_core_mask core_mask, 10 | int n_input, int n_output) 11 | { 12 | int ret = 0; 13 | cpu_set_t mask; 14 | 15 | CPU_ZERO(&mask); 16 | CPU_SET(cpuid, &mask); 17 | 18 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 19 | std::cerr << "set thread affinity failed" << std::endl; 20 | 21 | printf("Bind NPU process on CPU %d\n", cpuid); 22 | 23 | _cpu_id = cpuid; 24 | _n_input = n_input; 25 | _n_output = n_output; 26 | 27 | 28 | // Load model 29 | 30 | FILE *fp = fopen(model_path, "rb"); 31 | if(fp == NULL) { 32 | printf("fopen %s fail!\n", model_path); 33 | exit(-1); 34 | } 35 | // 文件的长度(单位字节) 36 | fseek(fp, 0, SEEK_END); 37 | int model_len = ftell(fp); 38 | // 创建一个存储空间model且读入 39 | void *model = malloc(model_len); 40 | fseek(fp, 0, SEEK_SET); 41 | if(model_len != fread(model, 1, model_len, fp)) { 42 | printf("fread %s fail!\n", model_path); 43 | free(model); 44 | exit(-1); 45 | } 46 | 47 | // ret = rknn_init(&ctx, model, m odel_len, RKNN_FLAG_COLLECT_PERF_MASK, NULL); 48 | ret = rknn_init(&ctx, model, model_len, 0, NULL); 49 | if(ret < 0) 50 | { 51 | printf("rknn_init fail! ret=%d\n", ret); 52 | exit(-1); 53 | } 54 | ret = rknn_set_core_mask(ctx, core_mask); 55 | if(ret < 0) 56 | { 57 | printf("set NPU core_mask fail! ret=%d\n", ret); 58 | exit(-1); 59 | } 60 | // rknn_sdk_version 61 | rknn_sdk_version version; 62 | ret = rknn_query(ctx, RKNN_QUERY_SDK_VERSION, &version, 63 | sizeof(rknn_sdk_version)); 64 | printf("api version: %s\n", version.api_version); 65 | printf("driver version: %s\n", version.drv_version); 66 | 67 | // rknn inputs 68 | printf("input tensors:\n"); 69 | memset(_input_attrs, 0, _n_input * sizeof(rknn_tensor_attr)); 70 | for (uint32_t i = 0; i < _n_input; i++) { 71 | _input_attrs[i].index = i; 72 | // query info 73 | ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(_input_attrs[i]), sizeof(rknn_tensor_attr)); 74 | if (ret < 0) { 75 | printf("rknn_init error! ret=%d\n", ret); 76 | exit(-1); 77 | } 78 | dump_tensor_attr(&_input_attrs[i]); 79 | } 80 | 81 | // Create input tensor memory 82 | rknn_tensor_type input_type = RKNN_TENSOR_UINT8; // default input type is int8 (normalize and quantize need compute in outside) 83 | rknn_tensor_format input_layout = RKNN_TENSOR_NHWC; // default fmt is NHWC, npu only support NHWC in zero copy mode 84 | _input_attrs[0].type = input_type; 85 | _input_attrs[0].fmt = input_layout; 86 | _input_mems[0] = rknn_create_mem(ctx, _input_attrs[0].size_with_stride); 87 | 88 | // rknn outputs 89 | printf("output tensors:\n"); 90 | memset(_output_attrs, 0, _n_output * sizeof(rknn_tensor_attr)); 91 | for (uint32_t i = 0; i < _n_output; i++) { 92 | _output_attrs[i].index = i; 93 | // query info 94 | ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(_output_attrs[i]), sizeof(rknn_tensor_attr)); 95 | if (ret != RKNN_SUCC) { 96 | printf("rknn_query fail! ret=%d\n", ret); 97 | exit(-1); 98 | } 99 | dump_tensor_attr(&_output_attrs[i]); 100 | } 101 | 102 | // Create output tensor memory 103 | for (uint32_t i = 0; i < _n_output; ++i) { 104 | // default output type is depend on model, this require float32 to compute top5 105 | // allocate float32 output tensor 106 | int output_size = _output_attrs[i].n_elems * sizeof(float); 107 | _output_mems[i] = rknn_create_mem(ctx, output_size); 108 | } 109 | 110 | // Set input tensor memory 111 | ret = rknn_set_io_mem(ctx, _input_mems[0], &_input_attrs[0]); 112 | if (ret < 0) { 113 | printf("rknn_set_io_mem fail! ret=%d\n", ret); 114 | exit(-1); 115 | } 116 | 117 | // Set output tensor memory 118 | for (uint32_t i = 0; i < _n_output; ++i) { 119 | // default output type is depend on model, this require float32 to compute top5 120 | _output_attrs[i].type = RKNN_TENSOR_FLOAT32; 121 | // set output memory and attribute 122 | ret = rknn_set_io_mem(ctx, _output_mems[i], &_output_attrs[i]); 123 | if (ret < 0) { 124 | printf("rknn_set_io_mem fail! ret=%d\n", ret); 125 | exit(-1); 126 | } 127 | } 128 | } 129 | 130 | rknn_fp::~rknn_fp(){ 131 | rknn_destroy(ctx); 132 | } 133 | 134 | // 读取rknn模型输入/输出属性 135 | void rknn_fp::dump_tensor_attr(rknn_tensor_attr* attr) 136 | { 137 | printf(" index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, " 138 | "zp=%d, scale=%f\n", 139 | attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3], 140 | attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type), 141 | get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale); 142 | return; 143 | } 144 | 145 | int rknn_fp::inference(unsigned char *data){ 146 | // Rknn调用函数 返回推理时间 147 | int ret; 148 | 149 | // inputs[0].buf = img.data; 150 | int width = _input_attrs[0].dims[2]; 151 | // std::cout << "checkpoint in rknn_fp: " << sizeof(data) << " " << width*_input_attrs[0].dims[1]*_input_attrs[0].dims[3] << "\n"; 152 | memcpy(_input_mems[0]->virt_addr, data, width*_input_attrs[0].dims[1]*_input_attrs[0].dims[3]); 153 | // std::cout << "checkpoint in rknn_fp\n"; 154 | // if(img.data) free(img.data); 155 | unsigned char * buff = (unsigned char *)_input_mems[0]->virt_addr; 156 | 157 | // rknn inference 158 | ret = rknn_run(ctx, nullptr); 159 | if(ret < 0) { 160 | printf("rknn_run fail! ret=%d\n", ret); 161 | return -1; 162 | } 163 | // query1: inference time 164 | rknn_perf_run perf_run; 165 | ret = rknn_query(ctx, RKNN_QUERY_PERF_RUN, &perf_run,sizeof(perf_run)); 166 | // printf("RKNN_QUERY_PERF_RUN: inference time %d\n", perf_run.run_duration); 167 | // query2: inference time per layer 168 | // rknn_perf_detail perf_detail; 169 | // ret = rknn_query(ctx, RKNN_QUERY_PERF_DETAIL, &perf_detail, sizeof(perf_detail)); 170 | // printf("%s \n", perf_detail.perf_data); 171 | 172 | // rknn outputs get 173 | for(int i=0;i<_n_output;i++){ 174 | _output_buff[i] = (float*)_output_mems[i]->virt_addr; 175 | } 176 | 177 | return perf_run.run_duration; 178 | } 179 | 180 | float rknn_fp::cal_NPU_performance(std::queue &history_time, float &sum_time, float cost_time){ 181 | // 统计NPU在最近一段时间内的速度 182 | if(history_time.size()<10){ 183 | history_time.push(cost_time); 184 | sum_time += cost_time; 185 | } 186 | else if(history_time.size()==10){ 187 | sum_time -= history_time.front(); 188 | sum_time += cost_time; 189 | history_time.pop(); 190 | history_time.push(cost_time); 191 | } 192 | else{ 193 | printf("cal_NPU_performance Error\n"); 194 | return -1; 195 | } 196 | return sum_time / history_time.size(); 197 | } -------------------------------------------------------------------------------- /yolov5/include/decode.h: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | // output type: uint8 4 | int post_process_u8(uint8_t *input0, uint8_t *input1, uint8_t *input2, int model_in_h, int model_in_w, 5 | int h_offset, int w_offset, float resize_scale, float conf_threshold, float nms_threshold, 6 | std::vector &qnt_zps, std::vector &qnt_scales, 7 | detect_result_group_t *group); 8 | 9 | // output type: fp32 10 | int post_process_fp(float *input0, float *input1, float *input2, int model_in_h, int model_in_w, 11 | int h_offset, int w_offset, float resize_scale, float conf_threshold, float nms_threshold, 12 | detect_result_group_t *group); 13 | 14 | int readLines(const char *fileName, char *lines[], int max_line); -------------------------------------------------------------------------------- /yolov5/include/detect.h: -------------------------------------------------------------------------------- 1 | #include "rknn_fp.h" 2 | 3 | 4 | class Yolo :public rknn_fp{ 5 | public: 6 | using rknn_fp::rknn_fp; //声明使用基类的构造函数 7 | int detect_process(); 8 | private: 9 | const int det_interval = 1; 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /yolov5/include/videoio.h: -------------------------------------------------------------------------------- 1 | #include "opencv2/videoio/videoio_c.h" 2 | 3 | struct _detect_result_group_t; // 前置声明 4 | 5 | // 视频属性类 6 | struct video_property{ 7 | int Frame_cnt; 8 | int Fps; 9 | int Video_width; 10 | int Video_height; 11 | double Video_fourcc; 12 | }; 13 | 14 | 15 | void videoRead(const char* video_name, int cpuid); 16 | void videoResize(int cpuid); 17 | void get_max_scale(int , int , int , int , double &, double &); 18 | void videoWrite(const char* save_path,int cpuid) ; 19 | int draw_image(cv::Mat& , struct _detect_result_group_t); 20 | -------------------------------------------------------------------------------- /yolov5/src/decode.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 by Rockchip Electronics Co., Ltd. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "decode.h" 24 | 25 | #define LABEL_NALE_TXT_PATH "../model/hongwai_2_labels_list.txt" 26 | 27 | static char *labels[OBJ_CLASS_NUM]; 28 | 29 | const int anchor0[6] = {10, 13, 16, 30, 33, 23}; 30 | const int anchor1[6] = {30, 61, 62, 45, 59, 119}; 31 | const int anchor2[6] = {116, 90, 156, 198, 373, 326}; 32 | 33 | inline static int clamp(float val, int min, int max) 34 | { 35 | return val > min ? (val < max ? val : max) : min; 36 | } 37 | 38 | char *readLine(FILE *fp, char *buffer, int *len) 39 | { 40 | int ch; 41 | int i = 0; 42 | size_t buff_len = 0; 43 | 44 | buffer = (char *)malloc(buff_len + 1); 45 | if (!buffer) 46 | return NULL; // Out of memory 47 | 48 | while ((ch = fgetc(fp)) != '\n' && ch != EOF) 49 | { 50 | buff_len++; 51 | void *tmp = realloc(buffer, buff_len + 1); 52 | if (tmp == NULL) 53 | { 54 | free(buffer); 55 | return NULL; // Out of memory 56 | } 57 | buffer = (char *)tmp; 58 | 59 | buffer[i] = (char)ch; 60 | i++; 61 | } 62 | buffer[i] = '\0'; 63 | 64 | *len = buff_len; 65 | 66 | // Detect end 67 | if (ch == EOF && (i == 0 || ferror(fp))) 68 | { 69 | free(buffer); 70 | return NULL; 71 | } 72 | return buffer; 73 | } 74 | 75 | int readLines(const char *fileName, char *lines[], int max_line) 76 | { 77 | FILE *file = fopen(fileName, "r"); 78 | std::cout << "Does the file exists? " << (file == NULL) << "\n"; 79 | char *s; 80 | int i = 0; 81 | int n = 0; 82 | while ((s = readLine(file, s, &n)) != NULL) 83 | { 84 | lines[i++] = s; 85 | if (i >= max_line) 86 | break; 87 | } 88 | return i; 89 | } 90 | 91 | int loadLabelName(const char *locationFilename, char *label[]) 92 | { 93 | printf("loadLabelName %s\n", locationFilename); 94 | readLines(locationFilename, label, OBJ_CLASS_NUM); 95 | std::cout << "load label done!\n"; 96 | return 0; 97 | } 98 | 99 | static float CalculateOverlap(float xmin0, float ymin0, float xmax0, float ymax0, float xmin1, float ymin1, float xmax1, float ymax1) 100 | { 101 | float w = fmax(0.f, fmin(xmax0, xmax1) - fmax(xmin0, xmin1) + 1.0); 102 | float h = fmax(0.f, fmin(ymax0, ymax1) - fmax(ymin0, ymin1) + 1.0); 103 | float i = w * h; 104 | float u = (xmax0 - xmin0 + 1.0) * (ymax0 - ymin0 + 1.0) + (xmax1 - xmin1 + 1.0) * (ymax1 - ymin1 + 1.0) - i; 105 | return u <= 0.f ? 0.f : (i / u); 106 | } 107 | 108 | static int nms(int validCount, std::vector &outputLocations, std::vector &order, float threshold) 109 | { 110 | for (int i = 0; i < validCount; ++i) 111 | { 112 | if (order[i] == -1) 113 | { 114 | continue; 115 | } 116 | int n = order[i]; 117 | for (int j = i + 1; j < validCount; ++j) 118 | { 119 | int m = order[j]; 120 | if (m == -1) 121 | { 122 | continue; 123 | } 124 | float xmin0 = outputLocations[n * 4 + 0]; 125 | float ymin0 = outputLocations[n * 4 + 1]; 126 | float xmax0 = outputLocations[n * 4 + 0] + outputLocations[n * 4 + 2]; 127 | float ymax0 = outputLocations[n * 4 + 1] + outputLocations[n * 4 + 3]; 128 | 129 | float xmin1 = outputLocations[m * 4 + 0]; 130 | float ymin1 = outputLocations[m * 4 + 1]; 131 | float xmax1 = outputLocations[m * 4 + 0] + outputLocations[m * 4 + 2]; 132 | float ymax1 = outputLocations[m * 4 + 1] + outputLocations[m * 4 + 3]; 133 | 134 | float iou = CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1, xmax1, ymax1); 135 | 136 | if (iou > threshold) 137 | { 138 | order[j] = -1; 139 | } 140 | } 141 | } 142 | return 0; 143 | } 144 | 145 | static int quick_sort_indice_inverse( 146 | std::vector &input, 147 | int left, 148 | int right, 149 | std::vector &indices) 150 | { 151 | float key; 152 | int key_index; 153 | int low = left; 154 | int high = right; 155 | if (input.size() < 2) return 0; 156 | if (left < right) 157 | { 158 | 159 | key_index = indices[left]; 160 | key = input[left]; 161 | while (low < high) 162 | { 163 | 164 | while (low < high && input[high] <= key) 165 | { 166 | // std::cout << low << " " << high << "\n"; 167 | high--; 168 | } 169 | input[low] = input[high]; 170 | indices[low] = indices[high]; 171 | while (low < high && input[low] >= key) 172 | { 173 | // std::cout << low << " " << high << "\n"; 174 | low++; 175 | } 176 | input[high] = input[low]; 177 | indices[high] = indices[low]; 178 | } 179 | input[low] = key; 180 | indices[low] = key_index; 181 | quick_sort_indice_inverse(input, left, low - 1, indices); 182 | quick_sort_indice_inverse(input, low + 1, right, indices); 183 | } 184 | return low; 185 | } 186 | 187 | static float sigmoid(float x) 188 | { 189 | return 1.0 / (1.0 + expf(-x)); 190 | } 191 | 192 | static float unsigmoid(float y) 193 | { 194 | return -1.0 * logf((1.0 / y) - 1.0); 195 | } 196 | 197 | inline static int32_t __clip(float val, float min, float max) 198 | { 199 | float f = val <= min ? min : (val >= max ? max : val); 200 | return f; 201 | } 202 | 203 | static uint8_t qnt_f32_to_affine(float f32, uint8_t zp, float scale) 204 | { 205 | float dst_val = (f32 / scale) + zp; 206 | uint8_t res = (uint8_t)__clip(dst_val, 0, 255); 207 | return res; 208 | } 209 | 210 | static float deqnt_affine_to_f32(uint8_t qnt, uint8_t zp, float scale) 211 | { 212 | return ((float)qnt - (float)zp) * scale; 213 | } 214 | 215 | static int process_u8(uint8_t *input, int *anchor, int grid_h, int grid_w, int height, int width, int stride, 216 | std::vector &boxes, std::vector &boxScores, std::vector &classId, 217 | float threshold, uint8_t zp, float scale) 218 | { 219 | 220 | int validCount = 0; 221 | int grid_len = grid_h * grid_w; 222 | float thres = unsigmoid(threshold); 223 | uint8_t thres_u8 = qnt_f32_to_affine(thres, zp, scale); 224 | for (int a = 0; a < 3; a++) 225 | { 226 | for (int i = 0; i < grid_h; i++) 227 | { 228 | for (int j = 0; j < grid_w; j++) 229 | { 230 | uint8_t box_confidence = input[(PROP_BOX_SIZE * a + 4) * grid_len + i * grid_w + j]; 231 | if (box_confidence >= thres_u8) 232 | { 233 | int offset = (PROP_BOX_SIZE * a) * grid_len + i * grid_w + j; 234 | uint8_t *in_ptr = input + offset; 235 | float box_x = sigmoid(deqnt_affine_to_f32(*in_ptr, zp, scale)) * 2.0 - 0.5; 236 | float box_y = sigmoid(deqnt_affine_to_f32(in_ptr[grid_len], zp, scale)) * 2.0 - 0.5; 237 | float box_w = sigmoid(deqnt_affine_to_f32(in_ptr[2 * grid_len], zp, scale)) * 2.0; 238 | float box_h = sigmoid(deqnt_affine_to_f32(in_ptr[3 * grid_len], zp, scale)) * 2.0; 239 | box_x = (box_x + j) * (float)stride; 240 | box_y = (box_y + i) * (float)stride; 241 | box_w = box_w * box_w * (float)anchor[a * 2]; 242 | box_h = box_h * box_h * (float)anchor[a * 2 + 1]; 243 | box_x -= (box_w / 2.0); 244 | box_y -= (box_h / 2.0); 245 | boxes.push_back(box_x); 246 | boxes.push_back(box_y); 247 | boxes.push_back(box_w); 248 | boxes.push_back(box_h); 249 | 250 | uint8_t maxClassProbs = in_ptr[5 * grid_len]; 251 | int maxClassId = 0; 252 | for (int k = 1; k < OBJ_CLASS_NUM; ++k) 253 | { 254 | uint8_t prob = in_ptr[(5 + k) * grid_len]; 255 | if (prob > maxClassProbs) 256 | { 257 | maxClassId = k; 258 | maxClassProbs = prob; 259 | } 260 | } 261 | float box_conf_f32 = sigmoid(deqnt_affine_to_f32(box_confidence, zp, scale)); 262 | float class_prob_f32 = sigmoid(deqnt_affine_to_f32(maxClassProbs, zp, scale)); 263 | boxScores.push_back(box_conf_f32* class_prob_f32); 264 | classId.push_back(maxClassId); 265 | validCount++; 266 | } 267 | } 268 | } 269 | } 270 | return validCount; 271 | } 272 | 273 | static int process_fp(float *input, int *anchor, int grid_h, int grid_w, int height, int width, int stride, 274 | std::vector &boxes, std::vector &boxScores, std::vector &classId, 275 | float threshold) 276 | { 277 | 278 | int validCount = 0; 279 | int grid_len = grid_h * grid_w; 280 | float thres_sigmoid = unsigmoid(threshold); 281 | 282 | for (int a = 0; a < 3; a++) 283 | { 284 | for (int i = 0; i < grid_h; i++) 285 | { 286 | for (int j = 0; j < grid_w; j++) 287 | { 288 | float box_confidence = input[(PROP_BOX_SIZE * a + 4) * grid_len + i * grid_w + j]; 289 | if (box_confidence >= thres_sigmoid) 290 | { 291 | int offset = (PROP_BOX_SIZE * a) * grid_len + i * grid_w + j; 292 | float *in_ptr = input + offset; 293 | float box_x = sigmoid(*in_ptr) * 2.0 - 0.5; 294 | float box_y = sigmoid(in_ptr[grid_len]) * 2.0 - 0.5; 295 | float box_w = sigmoid(in_ptr[2 * grid_len]) * 2.0; 296 | float box_h = sigmoid(in_ptr[3 * grid_len]) * 2.0; 297 | box_x = (box_x + j) * (float)stride; 298 | box_y = (box_y + i) * (float)stride; 299 | box_w = box_w * box_w * (float)anchor[a * 2]; 300 | box_h = box_h * box_h * (float)anchor[a * 2 + 1]; 301 | box_x -= (box_w / 2.0); 302 | box_y -= (box_h / 2.0); 303 | boxes.push_back(box_x); 304 | boxes.push_back(box_y); 305 | boxes.push_back(box_w); 306 | boxes.push_back(box_h); 307 | 308 | float maxClassProbs = in_ptr[5 * grid_len]; 309 | int maxClassId = 0; 310 | for (int k = 1; k < OBJ_CLASS_NUM; ++k) 311 | { 312 | float prob = in_ptr[(5 + k) * grid_len]; 313 | if (prob > maxClassProbs) 314 | { 315 | maxClassId = k; 316 | maxClassProbs = prob; 317 | } 318 | } 319 | float box_conf_f32 = sigmoid(box_confidence); 320 | float class_prob_f32 = sigmoid(maxClassProbs); 321 | boxScores.push_back(box_conf_f32* class_prob_f32); 322 | classId.push_back(maxClassId); 323 | validCount++; 324 | } 325 | } 326 | } 327 | } 328 | return validCount; 329 | } 330 | 331 | int post_process_u8(uint8_t *input0, uint8_t *input1, uint8_t *input2, int model_in_h, int model_in_w, 332 | int h_offset, int w_offset, float resize_scale, float conf_threshold, float nms_threshold, 333 | std::vector &qnt_zps, std::vector &qnt_scales, 334 | detect_result_group_t *group) 335 | { 336 | static int init = -1; 337 | if (init == -1) 338 | { 339 | int ret = 0; 340 | ret = loadLabelName(LABEL_NALE_TXT_PATH, labels); 341 | if (ret < 0) 342 | { 343 | return -1; 344 | } 345 | 346 | init = 0; 347 | } 348 | memset(group, 0, sizeof(detect_result_group_t)); 349 | 350 | std::vector filterBoxes; 351 | std::vector boxesScore; 352 | std::vector classId; 353 | int stride0 = 8; 354 | int grid_h0 = model_in_h / stride0; 355 | int grid_w0 = model_in_w / stride0; 356 | int validCount0 = 0; 357 | validCount0 = process_u8(input0, (int *)anchor0, grid_h0, grid_w0, model_in_h, model_in_w, 358 | stride0, filterBoxes, boxesScore, classId, conf_threshold, qnt_zps[0], qnt_scales[0]); 359 | 360 | int stride1 = 16; 361 | int grid_h1 = model_in_h / stride1; 362 | int grid_w1 = model_in_w / stride1; 363 | int validCount1 = 0; 364 | validCount1 = process_u8(input1, (int *)anchor1, grid_h1, grid_w1, model_in_h, model_in_w, 365 | stride1, filterBoxes, boxesScore, classId, conf_threshold, qnt_zps[1], qnt_scales[1]); 366 | 367 | int stride2 = 32; 368 | int grid_h2 = model_in_h / stride2; 369 | int grid_w2 = model_in_w / stride2; 370 | int validCount2 = 0; 371 | validCount2 = process_u8(input2, (int *)anchor2, grid_h2, grid_w2, model_in_h, model_in_w, 372 | stride2, filterBoxes, boxesScore, classId, conf_threshold, qnt_zps[2], qnt_scales[2]); 373 | 374 | int validCount = validCount0 + validCount1 + validCount2; 375 | // no object detect 376 | if (validCount <= 0) 377 | { 378 | return 0; 379 | } 380 | 381 | std::vector indexArray; 382 | for (int i = 0; i < validCount; ++i) 383 | { 384 | indexArray.push_back(i); 385 | } 386 | 387 | quick_sort_indice_inverse(boxesScore, 0, validCount - 1, indexArray); 388 | 389 | nms(validCount, filterBoxes, indexArray, nms_threshold); 390 | 391 | int last_count = 0; 392 | group->count = 0; 393 | /* box valid detect target */ 394 | for (int i = 0; i < validCount; ++i) 395 | { 396 | 397 | if (indexArray[i] == -1 || boxesScore[i] < conf_threshold || last_count >= OBJ_NUMB_MAX_SIZE) 398 | { 399 | continue; 400 | } 401 | int n = indexArray[i]; 402 | 403 | float x1 = filterBoxes[n * 4 + 0]; 404 | float y1 = filterBoxes[n * 4 + 1]; 405 | float x2 = x1 + filterBoxes[n * 4 + 2]; 406 | float y2 = y1 + filterBoxes[n * 4 + 3]; 407 | int id = classId[n]; 408 | 409 | DetectBox detbox; 410 | detbox.x1 = (int)((clamp(x1, 0, model_in_w) - w_offset) / resize_scale); 411 | detbox.y1 = (int)((clamp(y1, 0, model_in_h) - h_offset) / resize_scale); 412 | detbox.x2 = (int)((clamp(x2, 0, model_in_w) - w_offset) / resize_scale); 413 | detbox.y2 = (int)((clamp(y2, 0, model_in_h) - h_offset) / resize_scale); 414 | detbox.confidence = boxesScore[i]; 415 | detbox.classID = id; 416 | char *label = labels[id]; 417 | strncpy(detbox.name, label, OBJ_NAME_MAX_SIZE); 418 | group->results.push_back(detbox); 419 | 420 | // printf("result %2d: (%4d, %4d, %4d, %4d), %s\n", i, detbox.box.left, detbox.box.top, 421 | // detbox.box.right, detbox.box.bottom, label); 422 | last_count++; 423 | } 424 | group->count = last_count; 425 | 426 | return 0; 427 | } 428 | 429 | 430 | int post_process_fp(float *input0, float *input1, float *input2, int model_in_h, int model_in_w, 431 | int h_offset, int w_offset, float resize_scale, float conf_threshold, float nms_threshold, 432 | detect_result_group_t *group) 433 | { 434 | static int init = -1; 435 | if (init == -1) 436 | { 437 | int ret = 0; 438 | ret = loadLabelName(LABEL_NALE_TXT_PATH, labels); 439 | if (ret < 0) 440 | { 441 | return -1; 442 | } 443 | 444 | init = 0; 445 | } 446 | // printf("1______________________________________\n"); 447 | memset(group, 0, sizeof(detect_result_group_t)); 448 | 449 | std::vector filterBoxes; 450 | std::vector boxesScore; 451 | std::vector classId; 452 | int stride0 = 8; 453 | int grid_h0 = model_in_h / stride0; 454 | int grid_w0 = model_in_w / stride0; 455 | int validCount0 = 0; 456 | validCount0 = process_fp(input0, (int *)anchor0, grid_h0, grid_w0, model_in_h, model_in_w, 457 | stride0, filterBoxes, boxesScore, classId, conf_threshold); 458 | 459 | // printf("2______________________________________\n"); 460 | int stride1 = 16; 461 | int grid_h1 = model_in_h / stride1; 462 | int grid_w1 = model_in_w / stride1; 463 | int validCount1 = 0; 464 | validCount1 = process_fp(input1, (int *)anchor1, grid_h1, grid_w1, model_in_h, model_in_w, 465 | stride1, filterBoxes, boxesScore, classId, conf_threshold); 466 | 467 | int stride2 = 32; 468 | int grid_h2 = model_in_h / stride2; 469 | int grid_w2 = model_in_w / stride2; 470 | int validCount2 = 0; 471 | validCount2 = process_fp(input2, (int *)anchor2, grid_h2, grid_w2, model_in_h, model_in_w, 472 | stride2, filterBoxes, boxesScore, classId, conf_threshold); 473 | 474 | int validCount = validCount0 + validCount1 + validCount2; 475 | // no object detect 476 | if (validCount <= 0) 477 | { 478 | return 0; 479 | } 480 | 481 | std::vector indexArray; 482 | for (int i = 0; i < validCount; ++i) 483 | { 484 | indexArray.push_back(i); 485 | } 486 | // printf("3______________________________________\n"); 487 | // filterBoxes.resize(300); classId.resize(300); boxesScore.resize(300); indexArray.resize(300); validCount = 300; 488 | // for (int idx = 0; idx < 10; ++idx) std::cout << boxesScore[idx]; 489 | quick_sort_indice_inverse(boxesScore, 0, validCount - 1, indexArray); 490 | // printf("4______________________________________\n"); 491 | 492 | nms(validCount, filterBoxes, indexArray, nms_threshold); 493 | // printf("5_____________________________________\n"); 494 | 495 | int last_count = 0; 496 | group->count = 0; 497 | /* box valid detect target */ 498 | for (int i = 0; i < validCount; ++i) 499 | { 500 | // std::cout << "it's " << i << "\n"; 501 | // std::cout << boxesScore.size() << " " << indexArray.size() << "\n"; 502 | 503 | if (indexArray[i] == -1 || boxesScore[i] < conf_threshold || last_count >= OBJ_NUMB_MAX_SIZE) 504 | { 505 | continue; 506 | } 507 | int n = indexArray[i]; 508 | 509 | float x1 = filterBoxes[n * 4 + 0]; 510 | float y1 = filterBoxes[n * 4 + 1]; 511 | float x2 = x1 + filterBoxes[n * 4 + 2]; 512 | float y2 = y1 + filterBoxes[n * 4 + 3]; 513 | int id = classId[n]; 514 | 515 | DetectBox detbox; 516 | detbox.x1 = (int)((clamp(x1, 0, model_in_w) - w_offset) / resize_scale); 517 | detbox.y1 = (int)((clamp(y1, 0, model_in_h) - h_offset) / resize_scale); 518 | detbox.x2 = (int)((clamp(x2, 0, model_in_w) - w_offset) / resize_scale); 519 | detbox.y2 = (int)((clamp(y2, 0, model_in_h) - h_offset) / resize_scale); 520 | 521 | detbox.confidence = boxesScore[i]; 522 | 523 | detbox.classID = id; 524 | char *label = labels[id]; 525 | strncpy(detbox.name, label, OBJ_NAME_MAX_SIZE); 526 | group->results.push_back(detbox); 527 | // printf("result %2d: (%4d, %4d, %4d, %4d), %s\n", i, detbox.box.left, detbox.box.top, 528 | // detbox.box.right, detbox.box.bottom, label); 529 | 530 | last_count++; 531 | } 532 | group->count = last_count; 533 | 534 | return 0; 535 | } -------------------------------------------------------------------------------- /yolov5/src/detect.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | #include "mytime.h" 5 | #include "decode.h" 6 | #include "detect.h" 7 | #include "videoio.h" 8 | 9 | using namespace std; 10 | 11 | extern bool bReading; 12 | extern bool bDetecting; // 目标检测进程状态 13 | // 开始 结束时间 ns 14 | extern double start_time; 15 | extern video_property video_probs; // 视频属性类 16 | // 多线程控制相关 17 | extern int idxOutputImage; // next frame index to be output 保证queueDetOut_server序号正确 18 | extern mutex mtxQueueInput; 19 | extern queue queueInput; // input queue 20 | extern mutex mtxQueueDetOut; 21 | extern queue queueDetOut;// Det output queue 22 | 23 | int Yolo::detect_process(){ 24 | 25 | queue history_time; 26 | float sum_time = 0; 27 | int cost_time = 0; // rknn接口查询返回 28 | float npu_performance = 0.0; 29 | 30 | // Letter box resize 31 | float img_wh_ratio = (float)IMG_WIDTH / (float)IMG_HEIGHT; 32 | float input_wh_ratio = (float)NET_INPUTWIDTH / (float)NET_INPUTHEIGHT; 33 | float resize_scale=0; 34 | int resize_width; 35 | int resize_height; 36 | int h_pad; 37 | int w_pad; 38 | if (img_wh_ratio >= input_wh_ratio){ 39 | //pad height dim 40 | resize_scale = (float)NET_INPUTWIDTH / (float)IMG_WIDTH; 41 | resize_width = NET_INPUTWIDTH; 42 | resize_height = (int)((float)IMG_HEIGHT * resize_scale); 43 | w_pad = 0; 44 | h_pad = (NET_INPUTHEIGHT - resize_height) / 2; 45 | } 46 | else{ 47 | //pad width dim 48 | resize_scale = (float)NET_INPUTHEIGHT / (float)IMG_HEIGHT; 49 | resize_width = (int)((float)IMG_WIDTH * resize_scale); 50 | resize_height = NET_INPUTHEIGHT; 51 | w_pad = (NET_INPUTWIDTH - resize_width) / 2; 52 | h_pad = 0; 53 | } 54 | 55 | while (1) 56 | { 57 | // cout << "Entering detect process" << queueInput.size() << "\n"; 58 | //Load image 59 | input_image input; 60 | mtxQueueInput.lock(); 61 | //queueInput不为空则进入NPU_process 62 | if (queueInput.empty()) { 63 | // printf("waiting queueInput .........\n"); 64 | mtxQueueInput.unlock(); 65 | usleep(1000); 66 | // 如果queue处于空且 bReading不在可读取状态则销毁 跳出 67 | if (bReading){ 68 | continue; 69 | } 70 | else { 71 | break; 72 | } 73 | } 74 | else{ 75 | // Get an image from input queue 76 | // cout << "已缓存的图片数: " << queueInput.size() << endl; 77 | input = queueInput.front(); 78 | queueInput.pop(); 79 | mtxQueueInput.unlock(); 80 | } 81 | 82 | if(input.index == 0){ 83 | start_time = what_time_is_it_now(); 84 | } 85 | 86 | detect_result_group_t detect_result_group; 87 | // detection interval to speed up 88 | if (input.index < this->det_interval || !(input.index % this->det_interval)) { 89 | double timeBeforeDetection = what_time_is_it_now(); 90 | cost_time = inference(input.img_pad.data); 91 | if(cost_time == -1) 92 | printf("NPU inference Error"); 93 | 94 | std::vector out_scales; 95 | std::vector out_zps; 96 | for (int i = 0; i < _n_output; ++i) { 97 | out_scales.push_back(_output_attrs[i].scale); 98 | out_zps.push_back(_output_attrs[i].zp); 99 | } 100 | 101 | post_process_fp((float *)_output_buff[0], (float *)_output_buff[1], (float *)_output_buff[2], 102 | NET_INPUTHEIGHT, NET_INPUTWIDTH, 0, 0, resize_scale, BOX_THRESH, NMS_THRESH, &detect_result_group); 103 | 104 | double timeAfterDetection = what_time_is_it_now(); 105 | 106 | cout << "--------Time cost in Detection: " << timeAfterDetection - timeBeforeDetection << "\n"; 107 | 108 | } 109 | 110 | // cout << "post process done\n"; 111 | detect_result_group.id = input.index; 112 | // double end_time = what_time_is_it_now(); 113 | // cost_time = end_time - start_time; 114 | npu_performance = cal_NPU_performance(history_time, sum_time, cost_time / 1.0e3); 115 | 116 | while(detect_result_group.id != idxOutputImage){ 117 | usleep(1000); 118 | } 119 | imageout_idx res_pair; 120 | res_pair.img = input.img_src; 121 | res_pair.dets = detect_result_group; 122 | mtxQueueDetOut.lock(); 123 | queueDetOut.push(res_pair); 124 | mtxQueueDetOut.unlock(); 125 | // printf("%f NPU(%d) performance : %f (%d)\n", what_time_is_it_now()/1000, _cpu_id, npu_performance, detect_result_group.id); 126 | // draw_image(input.img_src, post_do.scale, nms_res, nboxes_left, 0.3); 127 | idxOutputImage = idxOutputImage + 1; 128 | if(input.index == video_probs.Frame_cnt-1){ 129 | break; // 不加也可 queueInput.empty() + breading可以跳出 130 | } 131 | } 132 | cout << "Detect is over." << endl; 133 | bDetecting = false; 134 | return 0; 135 | } 136 | -------------------------------------------------------------------------------- /yolov5/src/videoio.cpp: -------------------------------------------------------------------------------- 1 | #include "videoio.h" 2 | #include "resize.h" 3 | #include "common.h" 4 | 5 | using namespace std; 6 | 7 | extern video_property video_probs; 8 | extern vector imagePool; 9 | extern mutex mtxQueueInput; 10 | extern queue queueInput; // input queue client 11 | extern mutex mtxQueueDetOut; 12 | extern queue queueDetOut; // Det output queue 13 | extern mutex mtxQueueOutput; 14 | extern queue queueOutput; // 目标追踪输出队列 15 | 16 | extern bool add_head; 17 | extern bool bReading; // flag of input 18 | extern bool bDetecting; // 目标检测进程状态 19 | extern bool bTracking; 20 | extern int idxInputImage; // image index of input video 21 | 22 | 23 | 24 | /*--------------------------------------------------------- 25 | 读视频 缓存在imagePool 26 | video_name: 视频路径 27 | cpuid: 绑定到某核 28 | ----------------------------------------------------------*/ 29 | void videoRead(const char *video_name, int cpuid) 30 | { 31 | // int initialization_finished = 1; 32 | cpu_set_t mask; 33 | 34 | CPU_ZERO(&mask); 35 | CPU_SET(cpuid, &mask); 36 | 37 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 38 | cerr << "set thread affinity failed" << endl; 39 | 40 | printf("Bind videoReadClient process to CPU %d\n", cpuid); 41 | 42 | cv::VideoCapture video; 43 | if (!video.open(video_name)) { 44 | cout << "Fail to open " << video_name << endl; 45 | return; 46 | } 47 | 48 | video_probs.Frame_cnt = video.get(CV_CAP_PROP_FRAME_COUNT); 49 | video_probs.Fps = video.get(CV_CAP_PROP_FPS); 50 | video_probs.Video_width = video.get(CV_CAP_PROP_FRAME_WIDTH); 51 | video_probs.Video_height = video.get(CV_CAP_PROP_FRAME_HEIGHT); 52 | video_probs.Video_fourcc = video.get(CV_CAP_PROP_FOURCC); 53 | 54 | bReading = true;//读写状态标记 55 | while (1) 56 | { 57 | cv::Mat img_src; 58 | // 如果读不到图片 或者 bReading 不在读取状态则跳出 59 | if (!video.read(img_src)) { 60 | cout << "read video stream failed! Maybe to the end!" << endl; 61 | video.release(); 62 | break; 63 | } 64 | imagePool.emplace_back(img_src); 65 | } 66 | cout << "VideoRead is over." << endl; 67 | cout << "Video Total Length: " << imagePool.size() << "\n"; 68 | } 69 | 70 | /*--------------------------------------------------------- 71 | 调整视频尺寸 72 | cpuid: 绑定到某核 73 | ----------------------------------------------------------*/ 74 | void videoResize(int cpuid){ 75 | // int initialization_finished = 1; 76 | rga_buffer_t src; 77 | rga_buffer_t dst; 78 | im_rect src_rect; 79 | im_rect dst_rect; 80 | memset(&src_rect, 0, sizeof(src_rect)); 81 | memset(&dst_rect, 0, sizeof(dst_rect)); 82 | memset(&src, 0, sizeof(src)); 83 | memset(&dst, 0, sizeof(dst)); 84 | 85 | cpu_set_t mask; 86 | 87 | CPU_ZERO(&mask); 88 | CPU_SET(cpuid, &mask); 89 | 90 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 91 | cerr << "set thread affinity failed" << endl; 92 | 93 | printf("Bind videoTransClient process to CPU %d\n", cpuid); 94 | 95 | PreResize pre_do(NET_INPUTHEIGHT, NET_INPUTWIDTH, NET_INPUTCHANNEL); 96 | bReading = true;//读写状态标记 97 | cout << "total length of video: " << video_probs.Frame_cnt << "\n"; 98 | while (1) 99 | { 100 | // 如果读不到图片 或者 bReading 不在读取状态则跳出 101 | if (!bReading || idxInputImage >= video_probs.Frame_cnt) { 102 | break; 103 | } 104 | cv::Mat img_src = imagePool[idxInputImage]; 105 | cv::Mat img = img_src.clone(); 106 | cv::cvtColor(img_src, img, cv::COLOR_BGR2RGB); 107 | 108 | cv::Mat img_pad; 109 | resize(img, img_pad, cv::Size(640,640), 0, 0, 1); 110 | 111 | if (add_head){ 112 | // adaptive head 113 | } 114 | else{ 115 | // rga resize 116 | 117 | void *resize_buf = malloc(NET_INPUTHEIGHT * NET_INPUTWIDTH * NET_INPUTCHANNEL); 118 | src = wrapbuffer_virtualaddr((void *)img.data, img.cols, img.rows, RK_FORMAT_RGB_888); 119 | dst = wrapbuffer_virtualaddr((void *)resize_buf, NET_INPUTWIDTH, NET_INPUTHEIGHT, RK_FORMAT_RGB_888); 120 | 121 | } 122 | 123 | mtxQueueInput.lock(); 124 | queueInput.push(input_image(idxInputImage, img_src, img_pad)); 125 | mtxQueueInput.unlock(); 126 | idxInputImage++; 127 | } 128 | bReading = false; 129 | cout << "VideoResize is over." << endl; 130 | cout << "Resize Video Total Length: " << queueInput.size() << "\n"; 131 | } 132 | 133 | /* 134 | 预处理的缩放比例 135 | 在不丢失原图比例的同时，尽可能的伸缩；同时为了保证检测效果，只允许缩放，不允许放大。 136 | fx = 1 沿x轴缩放 137 | fy = 1 沿y轴缩放 138 | */ 139 | void get_max_scale(int input_width, int input_height, int net_width, int net_height, double &fx, double &fy) 140 | { 141 | double img_wh_ratio = (double)input_width / (double)input_height; 142 | double input_wh_ratio = (double)net_width / (double)net_height; 143 | if (img_wh_ratio >= input_wh_ratio){ 144 | // 缩放相同倍数 w 先到达边界 145 | fx = (double)net_width / input_width; 146 | fy = (double)net_width / input_width; 147 | } 148 | else{ 149 | fx = (double)net_height / input_height; 150 | fy = (double)net_height / input_height; 151 | } 152 | return; 153 | } 154 | 155 | // 写视频 156 | void videoWrite(const char* save_path,int cpuid) 157 | { 158 | cpu_set_t mask; 159 | CPU_ZERO(&mask); 160 | CPU_SET(cpuid, &mask); 161 | 162 | if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) 163 | cerr << "set thread affinity failed" << endl; 164 | 165 | printf("Bind videoWrite process to CPU %d\n", cpuid); 166 | 167 | cv::VideoWriter vid_writer; 168 | while(1) 169 | { 170 | // cout << "checkpoint! " << queueInput.size() << "\n"; 171 | if(queueInput.size() > 0) 172 | { 173 | // cout << video_probs.Video_width << " " << video_probs.Video_height << endl; 174 | vid_writer = cv::VideoWriter(save_path, video_probs.Video_fourcc, video_probs.Fps, 175 | cv::Size(video_probs.Video_width, video_probs.Video_height)); 176 | break; 177 | } 178 | } 179 | 180 | while (1) 181 | { 182 | // if (queueOutput.size()) cout << "checkpoint in VideoWriter: " << queueOutput.size() << "\n"; 183 | // queueOutput 就尝试写 184 | if (queueOutput.size() > 0) { 185 | mtxQueueOutput.lock(); 186 | imageout_idx res_pair = queueOutput.front(); 187 | queueOutput.pop(); 188 | mtxQueueOutput.unlock(); 189 | draw_image(res_pair.img, res_pair.dets); 190 | vid_writer.write(res_pair.img); // Save-video 191 | } 192 | // 最后一帧检测/追踪结束 bWriting置为false 此时如果queueOutput仍存在元素 继续写 193 | else if(!bTracking){ 194 | vid_writer.release(); 195 | break; 196 | } 197 | } 198 | cout << "VideoWrite is over." << endl; 199 | } 200 | 201 | /*--------------------------------------------------------- 202 | 绘制预测框 203 | ----------------------------------------------------------*/ 204 | string labels[2]={"person", "vehicle"}; 205 | cv::Scalar colorArray[2]={ 206 | cv::Scalar(139,0,0,255), 207 | cv::Scalar(139,0,139,255), 208 | }; 209 | int draw_image(cv::Mat &img,detect_result_group_t detect_result_group) 210 | { 211 | char text[256]; 212 | for (auto det_result : detect_result_group.results) 213 | { 214 | // sprintf(text, "%s %.1f%%", det_result.name, det_result.confidence * 100); 215 | sprintf(text, "ID:%d", (int)det_result.trackID); 216 | int x1 = det_result.x1; 217 | int y1 = det_result.y1 / IMG_WIDTH * IMG_HEIGHT; 218 | int x2 = det_result.x2; 219 | int y2 = det_result.y2 / IMG_WIDTH * IMG_HEIGHT; 220 | int class_id = det_result.classID; 221 | rectangle(img, cv::Point(x1, y1), cv::Point(x2, y2), colorArray[class_id%10], 3); 222 | putText(img, text, cv::Point(x1, y1 - 12), 1, 2, cv::Scalar(0, 255, 0, 255)); 223 | } 224 | // imwrite("./display.jpg", img); 225 | return 0; 226 | } -------------------------------------------------------------------------------- /yolov5_deepsort.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "common.h" 9 | #include "detect.h" 10 | #include "deepsort.h" 11 | #include "mytime.h" 12 | #include "videoio.h" 13 | 14 | using namespace std; 15 | 16 | bool add_head = false; 17 | string PROJECT_DIR = "/home/wjp/codes/yolov5_Deepsort_rknn"; 18 | 19 | 20 | string YOLO_MODEL_PATH = PROJECT_DIR + "/model/best.rknn"; 21 | string SORT_MODEL_PATH = PROJECT_DIR + "/model/osnet_x0_25_market.rknn"; 22 | 23 | string VIDEO_PATH = PROJECT_DIR + "/data/M0201.mp4"; 24 | string VIDEO_SAVEPATH = PROJECT_DIR + "/data/M0201_results.mp4"; 25 | 26 | /* 27 | string YOLO_MODEL_PATH = PROJECT_DIR + "/model/best_nofocus_relu.rknn"; 28 | string SORT_MODEL_PATH = PROJECT_DIR + "/model/osnet_x0_25_market.rknn"; 29 | 30 | string VIDEO_PATH = PROJECT_DIR + "/data/DJI_0001_S_cut.mp4"; 31 | string VIDEO_SAVEPATH = PROJECT_DIR + "/data/results.mp4"; 32 | */ 33 | 34 | 35 | // 各任务进行状态序号 36 | video_property video_probs; // 视频属性类 37 | int idxInputImage = 0; // image index of input video 38 | int idxOutputImage = 0; // image index of output video 39 | int idxTrackImage = 0; // 目标追踪下一帧要处理的对象 40 | bool bReading = true; // flag of input 41 | bool bDetecting = true; // Detect是否完成 42 | bool bTracking = true; // Track是否完成 43 | double start_time; // Video Detection开始时间 44 | double end_time; // Video Detection结束时间 45 | 46 | // 多线程控制相关 47 | vector imagePool; // video cache 48 | mutex mtxQueueInput; // mutex of input queue 49 | queue queueInput; // input queue 50 | mutex mtxQueueDetOut; 51 | queue queueDetOut; // output queue 52 | mutex mtxQueueOutput; 53 | queue queueOutput; // output queue 目标追踪输出队列 54 | 55 | 56 | 57 | void videoRead(const char *video_name, int cpuid); 58 | void videoResize(int cpuid); 59 | void videoWrite(const char* save_path,int cpuid); 60 | 61 | int main() { 62 | 63 | class Yolo detect1(YOLO_MODEL_PATH.c_str(), 4, RKNN_NPU_CORE_0, 1, 3); 64 | class Yolo detect2(YOLO_MODEL_PATH.c_str(), 5, RKNN_NPU_CORE_1, 1, 3); 65 | class DeepSort track(SORT_MODEL_PATH, 1, 512, 6, RKNN_NPU_CORE_2); 66 | 67 | const int thread_num = 5; 68 | std::array threads; 69 | videoRead(VIDEO_PATH.c_str(), 7); 70 | // used CPU: 0, 4, 5, 6, 7 71 | threads = { 72 | thread(&Yolo::detect_process, &detect1), // 类成员函数特殊写法 73 | thread(&Yolo::detect_process, &detect2), 74 | thread(&DeepSort::track_process, &track), 75 | thread(videoResize, 7), 76 | thread(videoWrite, VIDEO_SAVEPATH.c_str(), 0), 77 | }; 78 | for (int i = 0; i < thread_num; i++) threads[i].join(); 79 | printf("Video detection mean cost time(ms): %f\n", (end_time-start_time) / video_probs.Frame_cnt); 80 | return 0; 81 | } 82 | --------------------------------------------------------------------------------