pPrev = NULL;
32 | pCurUnit->pNext = m_pFreeMemBlock; //Insert the new unit at head.
33 |
34 | if(NULL != m_pFreeMemBlock)
35 | {
36 | m_pFreeMemBlock->pPrev = pCurUnit;
37 | }
38 | m_pFreeMemBlock = pCurUnit;
39 | }
40 | }
41 | }
42 |
43 | /*===============================================================
44 | ~CMemPool():
45 | Destructor of this class. Its task is to free memory block.
46 | //===============================================================
47 | */
48 | CMemPool::~CMemPool()
49 | {
50 | free(m_pMemBlock);
51 | }
52 |
53 | /*================================================================
54 | Alloc:
55 | To allocate a memory unit. If memory pool can`t provide proper memory unit,
56 | It will call system function.
57 |
58 | Parameters:
59 | [in]ulSize
60 | Memory unit size.
61 |
62 | [in]bUseMemPool
63 | Whether use memory pool.
64 |
65 | Return Values:
66 | Return a pointer to a memory unit.
67 | //=================================================================
68 | */
69 | void* CMemPool::Alloc(unsigned long ulSize, bool bUseMemPool)
70 | {
71 | if( ulSize > m_ulUnitSize || false == bUseMemPool ||
72 | NULL == m_pMemBlock || NULL == m_pFreeMemBlock)
73 | {
74 | return malloc(ulSize);
75 | }
76 |
77 | //Now FreeList isn`t empty
78 | struct _Unit *pCurUnit = m_pFreeMemBlock;
79 | m_pFreeMemBlock = pCurUnit->pNext; //Get a unit from free linkedlist.
80 | if(NULL != m_pFreeMemBlock)
81 | {
82 | m_pFreeMemBlock->pPrev = NULL;
83 | }
84 |
85 | pCurUnit->pNext = m_pAllocatedMemBlock;
86 |
87 | if(NULL != m_pAllocatedMemBlock)
88 | {
89 | m_pAllocatedMemBlock->pPrev = pCurUnit;
90 | }
91 | m_pAllocatedMemBlock = pCurUnit;
92 |
93 | return (void *)((char *)pCurUnit + sizeof(struct _Unit) );
94 | }
95 |
96 | /*================================================================
97 | Free:
98 | To free a memory unit. If the pointer of parameter point to a memory unit,
99 | then insert it to "Free linked list". Otherwise, call system function "free".
100 |
101 | Parameters:
102 | [in]p
103 | It point to a memory unit and prepare to free it.
104 |
105 | Return Values:
106 | none
107 | //================================================================
108 | */
109 | void CMemPool::Free( void* p )
110 | {
111 | if(m_pMemBlockpNext;
116 | if(NULL != m_pAllocatedMemBlock)
117 | {
118 | m_pAllocatedMemBlock->pPrev = NULL;
119 | }
120 |
121 | pCurUnit->pNext = m_pFreeMemBlock;
122 | if(NULL != m_pFreeMemBlock)
123 | {
124 | m_pFreeMemBlock->pPrev = pCurUnit;
125 | }
126 |
127 | m_pFreeMemBlock = pCurUnit;
128 | }
129 | else
130 | {
131 | free(p);
132 | }
133 | }
134 |
--------------------------------------------------------------------------------
/server/src/mem_pool.hpp:
--------------------------------------------------------------------------------
1 | #ifndef __MEMPOOL_H__
2 | #define __MEMPOOL_H__
3 | // https://www.codeproject.com/Articles/27487/Why-to-use-memory-pool-and-how-to-implement-it
4 | class CMemPool
5 | {
6 | private:
7 | //The purpose of the structure`s definition is that we can operate linkedlist conveniently
8 | struct _Unit //The type of the node of linkedlist.
9 | {
10 | struct _Unit *pPrev, *pNext;
11 | };
12 |
13 | void* m_pMemBlock; //The address of memory pool.
14 |
15 | //Manage all unit with two linkedlist.
16 | struct _Unit* m_pAllocatedMemBlock; //Head pointer to Allocated linkedlist.
17 | struct _Unit* m_pFreeMemBlock; //Head pointer to Free linkedlist.
18 |
19 | unsigned long m_ulUnitSize; //Memory unit size. There are much unit in memory pool.
20 | unsigned long m_ulBlockSize;//Memory pool size. Memory pool is make of memory unit.
21 |
22 | public:
23 | CMemPool(unsigned long lUnitNum = 50, unsigned long lUnitSize = 1024);
24 | ~CMemPool();
25 |
26 | void* Alloc(unsigned long ulSize, bool bUseMemPool = true); //Allocate memory unit
27 | void Free( void* p ); //Free memory unit
28 | };
29 |
30 | #endif //__MEMPOOL_H__
31 |
--------------------------------------------------------------------------------
/server/src/people.cpp:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 | #include
4 | #include
5 | #include
6 | #include
7 | #include
8 | #include
9 | #include
10 | #include
11 | #include
12 | #include
13 | #include "people.hpp"
14 |
15 | void Person::set_part(int part, float x, float y) {
16 | history.front().x[part] = x;
17 | history.front().y[part] = y;
18 |
19 | // bounding box update
20 | if (x < min_x)
21 | min_x = x;
22 | else if (x > max_x)
23 | max_x = x;
24 |
25 | if (y < min_y)
26 | min_y = y;
27 | else if (y > max_y)
28 | max_y = y;
29 | }
30 |
31 | void Person::set_action(int type) {
32 | if (type < 0)
33 | type = ACTION_TYPE_NUM;
34 | action = type;
35 |
36 | if (actions.size() == ACTION_HIS_NUM)
37 | actions.pop_front();
38 | actions.push_back(type);
39 | }
40 |
41 | float Person::get_dist(float x1, float y1, float x2, float y2) {
42 | if (x1 == 0 || x2 == 0)
43 | return 0.0;
44 | else {
45 | return sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1));
46 | }
47 | }
48 |
49 | float Person::get_deg(float x1, float y1, float x2, float y2) {
50 | if (x1 == 0 || x2 == 0)
51 | return 0.0;
52 | double dx = x2 - x1;
53 | double dy = y2 - y1;
54 | double rad = atan2(dy, dx);
55 | double degree = (rad * 180) / M_PI;
56 | if (degree < 0)
57 | degree += 360;
58 | return degree;
59 | }
60 |
61 | bool Person::has_output(void) {
62 | if (overlap_count <= 0 && history.size() == HIS_NUM) {
63 | overlap_count = OVERLAP_NUM;
64 | return true;
65 | }
66 | else
67 | return false;
68 | }
69 |
70 | void Person::update(Person* n_p)
71 | {
72 | static const int change_part[] = {
73 | RELBOW, RWRIST, LELBOW, LWRIST, RKNEE, RANKLE, LKNEE, LANKLE
74 | };
75 | static const int change_pair[] = {
76 | RSHOULDER, RELBOW,
77 | RELBOW, RWRIST,
78 | LSHOULDER, LELBOW,
79 | LELBOW, LWRIST,
80 | RHIP, RKNEE,
81 | RKNEE, RANKLE,
82 | LHIP, LKNEE,
83 | LKNEE, LANKLE
84 | };
85 |
86 | Change c;
87 | double deg, n_deg;
88 | int part, pair_1, pair_2;
89 | const Joint& j = history.back();
90 | const Joint& n_j = n_p->history.front();
91 |
92 | // change calc
93 | for (int i = 0; i < CHANGE_NUM; i++) {
94 | part = change_part[i];
95 | c.dist[i] = abs(get_dist(j.x[part], n_j.x[part], j.y[part], n_j.y[part]));
96 |
97 | pair_1 = change_pair[i * 2];
98 | pair_2 = change_pair[i * 2 + 1];
99 |
100 | deg = get_deg(j.x[pair_1], j.y[pair_1], j.x[pair_2], j.y[pair_2]);
101 | n_deg = get_deg(n_j.x[pair_1], n_j.y[pair_1], n_j.x[pair_2], n_j.y[pair_2]);
102 | c.cur_deg[i] = n_deg;
103 | if (deg == 0 || n_deg == 0)
104 | c.deg[i] = 0.0;
105 | else
106 | c.deg[i] = abs(deg - n_deg);
107 | }
108 |
109 | if (history.size() == HIS_NUM) {
110 | history.pop_front();
111 | change_history.pop_front();
112 | }
113 | history.push_back(n_p->history.front());
114 | change_history.push_back(c);
115 | overlap_count--;
116 |
117 | // delete
118 | delete n_p;
119 | }
120 |
121 | /*
122 | std::string get_history(void) const
123 | {
124 | std::stringstream ss;
125 | for (int i = 0; i < history.size(); i++) {
126 | if (i != 0)
127 | ss << '\n';
128 | for (int j = 0; j < JOINT_NUM; j++) {
129 | if (j != 0)
130 | ss << ',';
131 | ss << history[i].x[j] << ',' << history[i].y[j];
132 | }
133 | }
134 | for (int i = history.size(); i < HIS_NUM; i++) {
135 | if (i != 0)
136 | ss << '\n';
137 | for (int j = 0; j < JOINT_NUM; j++) {
138 | if (j != 0)
139 | ss << ',';
140 | ss << 0.0 << ',' << 0.0;
141 | }
142 | }
143 | return ss.str();
144 | }
145 | */
146 |
147 | bool Person::check_crash(const Person& other) const
148 | {
149 | const static int punch_check_joint[] = {RELBOW, RWRIST, LELBOW, LWRIST};
150 | const static int kick_check_joint[] = {RKNEE, RANKLE, LKNEE, LANKLE};
151 |
152 | const int* check_joint = nullptr;
153 | int my_action = this->get_action();
154 |
155 | if (my_action == PUNCH)
156 | check_joint = punch_check_joint;
157 | else if (my_action == KICK)
158 | check_joint = kick_check_joint;
159 | else
160 | return false;
161 |
162 | cv::Rect_ other_rect = other.get_rect();
163 |
164 | const Joint& j = history.back();
165 | float x, y;
166 | for (int i = 0; i < 4; i++) {
167 | x = j.x[check_joint[i]];
168 | y = j.y[check_joint[i]];
169 |
170 | if (other_rect.x <= x && x <= other_rect.x + other_rect.width &&
171 | other_rect.y <= y && y <= other_rect.y + other_rect.height)
172 | return true;
173 | }
174 | return false;
175 | }
176 |
177 | cv::Rect_ Person::get_crash_rect(const Person& p) const
178 | {
179 | cv::Rect_ a_rect = this->get_rect();
180 | cv::Rect_ b_rect = p.get_rect();
181 |
182 | float min_x, min_y, max_x, max_y;
183 |
184 | min_x = a_rect.x < b_rect.x ? a_rect.x : b_rect.x;
185 | min_y = a_rect.y < b_rect.y ? a_rect.y : b_rect.y;
186 | max_x = (a_rect.x + a_rect.width) > (b_rect.x + b_rect.width) ? (a_rect.x + a_rect.width) : (b_rect.x +
187 | b_rect.width);
188 | max_y = (a_rect.y + a_rect.height) > (b_rect.y + b_rect.height) ? (a_rect.y + a_rect.height) : (b_rect.y +
189 | b_rect.height);
190 |
191 | return cv::Rect_(cv::Point_(min_x, min_y),
192 | cv::Point_(max_x, max_y));
193 | }
194 |
195 | std::string Person::get_history(void) const
196 | {
197 | std::stringstream ss;
198 | for (size_t i = 0; i < change_history.size(); i++) {
199 | if (i != 0)
200 | ss << '\n';
201 | for (int j = 0; j < CHANGE_NUM; j++) {
202 | if (j != 0)
203 | ss << ',';
204 | ss << change_history[i].dist[j] << ',' << change_history[i].deg[j] << ',' << change_history[i].cur_deg[j];
205 | }
206 | }
207 | for (int i = change_history.size(); i < HIS_NUM - 1; i++) {
208 | if (i != 0)
209 | ss << '\n';
210 | for (int j = 0; j < CHANGE_NUM; j++) {
211 | if (j != 0)
212 | ss << ',';
213 | ss << 0.0 << ',' << 0.0 << ',' << 0.0;
214 | }
215 | }
216 | return ss.str();
217 | }
218 |
219 | cv::Rect_ Person::get_rect(void) const
220 | {
221 | return cv::Rect_(cv::Point_(min_x, min_y),
222 | cv::Point_(max_x, max_y));
223 | }
224 |
225 | void Person::set_rect(cv::Rect_& rect)
226 | {
227 | min_x = rect.x;
228 | min_y = rect.y;
229 | max_x = rect.x + rect.width;
230 | max_y = rect.y + rect.height;
231 | }
232 |
233 | Person& Person::operator=(const Person& p)
234 | {
235 | track_id = p.track_id;
236 | max_x = p.max_x;
237 | max_y = p.max_y;
238 | min_x = p.min_x;
239 | min_y = p.min_y;
240 | history = p.history;
241 | change_history = p.change_history;
242 | overlap_count = p.overlap_count;
243 | return *this;
244 | }
245 |
246 | std::ostream& operator<<(std::ostream &out, const Person &p)
247 | {
248 | for (size_t i = 0; i < p.change_history.size(); i++) {
249 | for (int j = 0; j < p.CHANGE_NUM; j++) {
250 | if (j != 0)
251 | out << ',';
252 | out << p.change_history[i].dist[j] << ',' << p.change_history[i].deg[j] << ',' << p.change_history[i].cur_deg[j];
253 | }
254 | out << '\n';
255 | }
256 | return out;
257 | }
258 |
259 |
260 | std::vector People::to_person(void) {
261 | std::vector persons;
262 | int person_num = keyshape[0];
263 | int part_num = keyshape[1];
264 | for (int person = 0; person < person_num; person++) {
265 | Person *p = new Person();
266 | for (int part = 0; part < part_num; part++) {
267 | int index = (person * part_num + part) * keyshape[2];
268 | if (keypoints[index + 2] > thresh) {
269 | p->set_part(part, keypoints[index] * scale, keypoints[index + 1] * scale);
270 | }
271 | }
272 | persons.push_back(p);
273 | }
274 | return persons;
275 | }
276 |
277 | std::string People::get_output(void) {
278 | std::string out_str = "\"people\": [\n";
279 | int person_num = keyshape[0];
280 | int part_num = keyshape[1];
281 | for (int person = 0; person < person_num; person++) {
282 | if (person != 0)
283 | out_str += ",\n";
284 | out_str += " {\n";
285 | for (int part = 0; part < part_num; part++) {
286 | if (part != 0)
287 | out_str += ",\n ";
288 | int index = (person * part_num + part) * keyshape[2];
289 | char *buf = (char*)calloc(2048, sizeof(char));
290 |
291 | if (keypoints[index + 2] > thresh) {
292 | sprintf(buf, " \"%d\":[%f, %f]", part, keypoints[index] * scale, keypoints[index + 1] * scale);
293 | }
294 | else {
295 | sprintf(buf, " \"%d\":[%f, %f]", part, 0.0, 0.0);
296 | }
297 | out_str += buf;
298 | free(buf);
299 | }
300 | out_str += "\n }";
301 | }
302 | out_str += "\n ]";
303 | return out_str;
304 | }
305 |
306 | void People::render_pose_keypoints(cv::Mat& frame)
307 | {
308 | const int num_keypoints = keyshape[1];
309 | unsigned int pairs[] =
310 | {
311 | 1, 2, 1, 5, 2, 3, 3, 4, 5, 6, 6, 7, 1, 8, 8, 9, 9, 10,
312 | 1, 11, 11, 12, 12, 13, 1, 0, 0, 14, 14, 16, 0, 15, 15, 17
313 | };
314 | float colors[] =
315 | {
316 | 255.f, 0.f, 85.f, 255.f, 0.f, 0.f, 255.f, 85.f, 0.f, 255.f, 170.f, 0.f,
317 | 255.f, 255.f, 0.f, 170.f, 255.f, 0.f, 85.f, 255.f, 0.f, 0.f, 255.f, 0.f,
318 | 0.f, 255.f, 85.f, 0.f, 255.f, 170.f, 0.f, 255.f, 255.f, 0.f, 170.f, 255.f,
319 | 0.f, 85.f, 255.f, 0.f, 0.f, 255.f, 255.f, 0.f, 170.f, 170.f, 0.f, 255.f,
320 | 255.f, 0.f, 255.f, 85.f, 0.f, 255.f
321 | };
322 | const int pairs_size = sizeof(pairs) / sizeof(unsigned int);
323 | const int number_colors = sizeof(colors) / sizeof(float);
324 |
325 | for (int person = 0; person < keyshape[0]; ++person)
326 | {
327 | // Draw lines
328 | for (int pair = 0u; pair < pairs_size; pair += 2)
329 | {
330 | const int index1 = (person * num_keypoints + pairs[pair]) * keyshape[2];
331 | const int index2 = (person * num_keypoints + pairs[pair + 1]) * keyshape[2];
332 | if (keypoints[index1 + 2] > thresh && keypoints[index2 + 2] > thresh)
333 | {
334 | const int color_index = pairs[pair + 1] * 3;
335 | cv::Scalar color { colors[(color_index + 2) % number_colors],
336 | colors[(color_index + 1) % number_colors],
337 | colors[(color_index + 0) % number_colors]};
338 | cv::Point keypoint1{ intRoundUp(keypoints[index1] * scale), intRoundUp(keypoints[index1 + 1] * scale) };
339 | cv::Point keypoint2{ intRoundUp(keypoints[index2] * scale), intRoundUp(keypoints[index2 + 1] * scale) };
340 | cv::line(frame, keypoint1, keypoint2, color, 2);
341 | }
342 | }
343 | // Draw circles
344 | for (int part = 0; part < num_keypoints; ++part)
345 | {
346 | const int index = (person * num_keypoints + part) * keyshape[2];
347 | if (keypoints[index + 2] > thresh)
348 | {
349 | const int color_index = part * 3;
350 | cv::Scalar color { colors[(color_index + 2) % number_colors],
351 | colors[(color_index + 1) % number_colors],
352 | colors[(color_index + 0) % number_colors]};
353 | cv::Point center{ intRoundUp(keypoints[index] * scale), intRoundUp(keypoints[index + 1] * scale) };
354 | cv::circle(frame, center, 3, color, -1);
355 | }
356 | }
357 | }
358 | }
359 |
--------------------------------------------------------------------------------
/server/src/people.hpp:
--------------------------------------------------------------------------------
1 | #ifndef __PEOPLE
2 | #define __PEOPLE
3 | #include
4 | #include
5 | #include
6 | #include
7 | #include
8 | #include
9 | #include
10 | #include
11 | #include
12 | #include
13 | #include
14 | #include
15 |
16 | template
17 | inline int intRoundUp(const T a)
18 | {
19 | return int(a+0.5f);
20 | }
21 |
22 | class Person
23 | {
24 | private:
25 | enum {
26 | UNTRACK = -1,
27 | OVERLAP_NUM = 6,
28 | HIS_NUM = 33,
29 | CHANGE_NUM = 8,
30 |
31 | // action
32 | ACTION_TYPE_NUM = 4, ACTION_HIS_NUM = 10,
33 | STAND = 0, WALK = 1, PUNCH = 2, KICK = 3, UNKNOWN = 4,
34 |
35 | // coco joint keypoint
36 | JOINT_NUM = 18,
37 | NOSE = 0, NECK = 1, RSHOULDER = 2, RELBOW = 3, RWRIST = 4, LSHOULDER = 5, LELBOW = 6,
38 | LWRIST = 7, RHIP = 8, RKNEE = 9, RANKLE = 10, LHIP = 11, LKNEE = 12, LANKLE = 13,
39 | REYE = 14, LEYE = 15, REAR = 16, LEAR = 17
40 | };
41 | struct Joint {
42 | float x[JOINT_NUM];
43 | float y[JOINT_NUM];
44 | };
45 |
46 | struct Change {
47 | float dist[CHANGE_NUM];
48 | float deg[CHANGE_NUM];
49 | float cur_deg[CHANGE_NUM];
50 | };
51 |
52 | std::deque history;
53 | std::deque change_history;
54 | std::deque actions;
55 |
56 | int overlap_count;
57 | int track_id;
58 | int action;
59 |
60 | // for bounding box
61 | float max_x;
62 | float max_y;
63 | float min_x;
64 | float min_y;
65 |
66 | Person* enemy;
67 |
68 | public:
69 | Person() : enemy(nullptr), overlap_count(0), track_id(UNTRACK), action(ACTION_TYPE_NUM), max_x(FLT_MIN), max_y(FLT_MIN), min_x(FLT_MAX), min_y(FLT_MAX) { history.assign(1, {0}); }
70 |
71 | ~Person() { if (enemy != nullptr) enemy->set_enemy(nullptr); }
72 | // set
73 | void set_part(int part, float x, float y);
74 | void set_id(int id) { track_id = id; }
75 | void set_action(int type);
76 | void set_enemy(Person* p) { enemy = p; }
77 | void set_rect(cv::Rect_& rect);
78 |
79 | // get
80 | inline int get_id(void) const { return track_id; }
81 | inline int get_action(void) const { return action; }
82 | inline const char* get_action_str(void) const {
83 | static const char* action_str[] = {"STAND", "WALK", "PUNCH", "KICK", "UNKNOWN"};
84 | return action_str[action];
85 | }
86 | inline const Person* get_enemy(void) const { return enemy; }
87 | cv::Rect_ get_rect(void) const;
88 | cv::Rect_ get_crash_rect(const Person& p) const;
89 |
90 | // crash
91 | inline bool is_danger(void) const { return (action == PUNCH || action == KICK); }
92 | bool check_crash(const Person& p) const;
93 |
94 | void update(Person* n_p);
95 | bool has_output(void);
96 | std::string get_history(void) const;
97 |
98 | // util
99 | float get_dist(float x1, float y1, float x2, float y2);
100 | float get_deg(float x1, float y1, float x2, float y2);
101 |
102 | Person& operator=(const Person& p);
103 | friend std::ostream& operator<<(std::ostream &out, const Person &p);
104 | };
105 |
106 | class People
107 | {
108 | public:
109 | friend class boost::serialization::access;
110 | const float thresh = 0.05;
111 | std::vector keypoints;
112 | std::vector keyshape;
113 | float scale;
114 |
115 | People() {}
116 |
117 | People(std::vector _keypoints, std::vector _keyshape, float _scale) :
118 | keypoints(_keypoints), keyshape(_keyshape), scale(_scale) {}
119 |
120 | inline int get_person_num(void) const { return keyshape[0]; };
121 |
122 | std::vector to_person(void);
123 | std::string get_output(void);
124 | void render_pose_keypoints(cv::Mat& frame);
125 |
126 | template
127 | void serialize(Archive & ar, const unsigned int version)
128 | {
129 | ar & keypoints;
130 | ar & keyshape;
131 | ar & scale;
132 | }
133 | };
134 |
135 | #endif
136 |
--------------------------------------------------------------------------------
/server/src/pose_detector.cpp:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 | using namespace std;
4 | #include
5 | #include
6 | #include
7 | using namespace cv;
8 | #include "pose_detector.hpp"
9 |
10 | #define POSE_MAX_PEOPLE 96
11 | #define NET_OUT_CHANNELS 57 // 38 for pafs, 19 for parts
12 |
13 | template
14 | inline int intRound(const T a)
15 | {
16 | return int(a+0.5f);
17 | }
18 |
19 | template
20 | inline T fastMin(const T a, const T b)
21 | {
22 | return (a < b ? a : b);
23 | }
24 |
25 | void PoseDetector::connect_bodyparts
26 | (
27 | vector& pose_keypoints,
28 | const float* const map,
29 | const float* const peaks,
30 | int mapw,
31 | int maph,
32 | const int inter_min_above_th,
33 | const float inter_th,
34 | const int min_subset_cnt,
35 | const float min_subset_score,
36 | vector& keypoint_shape
37 | )
38 | {
39 | keypoint_shape.resize(3);
40 | const int body_part_pairs[] =
41 | {
42 | 1, 2, 1, 5, 2, 3, 3, 4, 5, 6, 6, 7, 1, 8, 8, 9, 9, 10, 1, 11, 11,
43 | 12, 12, 13, 1, 0, 0, 14, 14, 16, 0, 15, 15, 17, 2, 16, 5, 17
44 | };
45 | const int limb_idx[] =
46 | {
47 | 31, 32, 39, 40, 33, 34, 35, 36, 41, 42, 43, 44, 19, 20, 21, 22, 23, 24, 25,
48 | 26, 27, 28, 29, 30, 47, 48, 49, 50, 53, 54, 51, 52, 55, 56, 37, 38, 45, 46
49 | };
50 | const int num_body_parts = 18; // COCO part number
51 | const int num_body_part_pairs = num_body_parts + 1;
52 | std::vector, double>> subset;
53 | const int subset_counter_index = num_body_parts;
54 | const int subset_size = num_body_parts + 1;
55 | const int peaks_offset = 3 * (POSE_MAX_PEOPLE + 1);
56 | const int map_offset = mapw * maph;
57 |
58 | for (unsigned int pair_index = 0u; pair_index < num_body_part_pairs; ++pair_index)
59 | {
60 | const int body_partA = body_part_pairs[2 * pair_index];
61 | const int body_partB = body_part_pairs[2 * pair_index + 1];
62 | const float* candidateA = peaks + body_partA*peaks_offset;
63 | const float* candidateB = peaks + body_partB*peaks_offset;
64 | const int nA = (int)(candidateA[0]); // number of part A candidates
65 | const int nB = (int)(candidateB[0]); // number of part B candidates
66 |
67 | // add parts into the subset in special case
68 | if (nA == 0 || nB == 0)
69 | {
70 | // Change w.r.t. other
71 | if (nA == 0) // nB == 0 or not
72 | {
73 | for (int i = 1; i <= nB; ++i)
74 | {
75 | bool num = false;
76 | for (unsigned int j = 0u; j < subset.size(); ++j)
77 | {
78 | const int off = body_partB*peaks_offset + i * 3 + 2;
79 | if (subset[j].first[body_partB] == off)
80 | {
81 | num = true;
82 | break;
83 | }
84 | }
85 | if (!num)
86 | {
87 | std::vector row_vector(subset_size, 0);
88 | // store the index
89 | row_vector[body_partB] = body_partB*peaks_offset + i * 3 + 2;
90 | // the parts number of that person
91 | row_vector[subset_counter_index] = 1;
92 | // total score
93 | const float subsetScore = candidateB[i * 3 + 2];
94 | subset.emplace_back(std::make_pair(row_vector, subsetScore));
95 | }
96 | }
97 | }
98 | else // if (nA != 0 && nB == 0)
99 | {
100 | for (int i = 1; i <= nA; i++)
101 | {
102 | bool num = false;
103 | for (unsigned int j = 0u; j < subset.size(); ++j)
104 | {
105 | const int off = body_partA*peaks_offset + i * 3 + 2;
106 | if (subset[j].first[body_partA] == off)
107 | {
108 | num = true;
109 | break;
110 | }
111 | }
112 | if (!num)
113 | {
114 | std::vector row_vector(subset_size, 0);
115 | // store the index
116 | row_vector[body_partA] = body_partA*peaks_offset + i * 3 + 2;
117 | // parts number of that person
118 | row_vector[subset_counter_index] = 1;
119 | // total score
120 | const float subsetScore = candidateA[i * 3 + 2];
121 | subset.emplace_back(std::make_pair(row_vector, subsetScore));
122 | }
123 | }
124 | }
125 | }
126 | else // if (nA != 0 && nB != 0)
127 | {
128 | std::vector> temp;
129 | const int num_inter = 10;
130 | // limb PAF x-direction heatmap
131 | const float* const mapX = map + limb_idx[2 * pair_index] * map_offset;
132 | // limb PAF y-direction heatmap
133 | const float* const mapY = map + limb_idx[2 * pair_index + 1] * map_offset;
134 | // start greedy algorithm
135 | for (int i = 1; i <= nA; i++)
136 | {
137 | for (int j = 1; j <= nB; j++)
138 | {
139 | const int dX = candidateB[j * 3] - candidateA[i * 3];
140 | const int dY = candidateB[j * 3 + 1] - candidateA[i * 3 + 1];
141 | const float norm_vec = float(std::sqrt(dX*dX + dY*dY));
142 | // If the peaksPtr are coincident. Don't connect them.
143 | if (norm_vec > 1e-6)
144 | {
145 | const float sX = candidateA[i * 3];
146 | const float sY = candidateA[i * 3 + 1];
147 | const float vecX = dX / norm_vec;
148 | const float vecY = dY / norm_vec;
149 | float sum = 0.;
150 | int count = 0;
151 | for (int lm = 0; lm < num_inter; lm++)
152 | {
153 | const int mX = fastMin(mapw - 1, intRound(sX + lm*dX / num_inter));
154 | const int mY = fastMin(maph - 1, intRound(sY + lm*dY / num_inter));
155 | const int idx = mY * mapw + mX;
156 | const float score = (vecX*mapX[idx] + vecY*mapY[idx]);
157 | if (score > inter_th)
158 | {
159 | sum += score;
160 | ++count;
161 | }
162 | }
163 |
164 | // parts score + connection score
165 | if (count > inter_min_above_th)
166 | {
167 | temp.emplace_back(std::make_tuple(sum / count, i, j));
168 | }
169 | }
170 | }
171 | }
172 | // select the top minAB connection, assuming that each part occur only once
173 | // sort rows in descending order based on parts + connection score
174 | if (!temp.empty())
175 | {
176 | std::sort(temp.begin(), temp.end(), std::greater>());
177 | }
178 | std::vector> connectionK;
179 |
180 | const int minAB = fastMin(nA, nB);
181 | // assuming that each part occur only once, filter out same part1 to different part2
182 | std::vector occurA(nA, 0);
183 | std::vector occurB(nB, 0);
184 | int counter = 0;
185 | for (unsigned int row = 0u; row < temp.size(); row++)
186 | {
187 | const float score = std::get<0>(temp[row]);
188 | const int aidx = std::get<1>(temp[row]);
189 | const int bidx = std::get<2>(temp[row]);
190 | if (!occurA[aidx - 1] && !occurB[bidx - 1])
191 | {
192 | // save two part score "position" and limb mean PAF score
193 | connectionK.emplace_back(std::make_tuple(body_partA*peaks_offset + aidx * 3 + 2,
194 | body_partB*peaks_offset + bidx * 3 + 2, score));
195 | ++counter;
196 | if (counter == minAB)
197 | {
198 | break;
199 | }
200 | occurA[aidx - 1] = 1;
201 | occurB[bidx - 1] = 1;
202 | }
203 | }
204 | // Cluster all the body part candidates into subset based on the part connection
205 | // initialize first body part connection
206 | if (pair_index == 0)
207 | {
208 | for (const auto connectionKI : connectionK)
209 | {
210 | std::vector row_vector(num_body_parts + 3, 0);
211 | const int indexA = std::get<0>(connectionKI);
212 | const int indexB = std::get<1>(connectionKI);
213 | const double score = std::get<2>(connectionKI);
214 | row_vector[body_part_pairs[0]] = indexA;
215 | row_vector[body_part_pairs[1]] = indexB;
216 | row_vector[subset_counter_index] = 2;
217 | // add the score of parts and the connection
218 | const double subset_score = peaks[indexA] + peaks[indexB] + score;
219 | subset.emplace_back(std::make_pair(row_vector, subset_score));
220 | }
221 | }
222 | // Add ears connections (in case person is looking to opposite direction to camera)
223 | else if (pair_index == 17 || pair_index == 18)
224 | {
225 | for (const auto& connectionKI : connectionK)
226 | {
227 | const int indexA = std::get<0>(connectionKI);
228 | const int indexB = std::get<1>(connectionKI);
229 | for (auto& subsetJ : subset)
230 | {
231 | auto& subsetJ_first = subsetJ.first[body_partA];
232 | auto& subsetJ_first_plus1 = subsetJ.first[body_partB];
233 | if (subsetJ_first == indexA && subsetJ_first_plus1 == 0)
234 | {
235 | subsetJ_first_plus1 = indexB;
236 | }
237 | else if (subsetJ_first_plus1 == indexB && subsetJ_first == 0)
238 | {
239 | subsetJ_first = indexA;
240 | }
241 | }
242 | }
243 | }
244 | else
245 | {
246 | if (!connectionK.empty())
247 | {
248 | for (unsigned int i = 0u; i < connectionK.size(); ++i)
249 | {
250 | const int indexA = std::get<0>(connectionK[i]);
251 | const int indexB = std::get<1>(connectionK[i]);
252 | const double score = std::get<2>(connectionK[i]);
253 | int num = 0;
254 | // if A is already in the subset, add B
255 | for (unsigned int j = 0u; j < subset.size(); j++)
256 | {
257 | if (subset[j].first[body_partA] == indexA)
258 | {
259 | subset[j].first[body_partB] = indexB;
260 | ++num;
261 | subset[j].first[subset_counter_index] = subset[j].first[subset_counter_index] + 1;
262 | subset[j].second = subset[j].second + peaks[indexB] + score;
263 | }
264 | }
265 | // if A is not found in the subset, create new one and add both
266 | if (num == 0)
267 | {
268 | std::vector row_vector(subset_size, 0);
269 | row_vector[body_partA] = indexA;
270 | row_vector[body_partB] = indexB;
271 | row_vector[subset_counter_index] = 2;
272 | const float subsetScore = peaks[indexA] + peaks[indexB] + score;
273 | subset.emplace_back(std::make_pair(row_vector, subsetScore));
274 | }
275 | }
276 | }
277 | }
278 | }
279 | }
280 |
281 | // Delete people below thresholds, and save to output
282 | int number_people = 0;
283 | std::vector valid_subset_indexes;
284 | valid_subset_indexes.reserve(fastMin((size_t)POSE_MAX_PEOPLE, subset.size()));
285 | for (unsigned int index = 0; index < subset.size(); ++index)
286 | {
287 | const int subset_counter = subset[index].first[subset_counter_index];
288 | const double subset_score = subset[index].second;
289 | if (subset_counter >= min_subset_cnt && (subset_score / subset_counter) > min_subset_score)
290 | {
291 | ++number_people;
292 | valid_subset_indexes.emplace_back(index);
293 | if (number_people == POSE_MAX_PEOPLE)
294 | {
295 | break;
296 | }
297 | }
298 | }
299 |
300 | // Fill and return pose_keypoints
301 | keypoint_shape = { number_people, (int)num_body_parts, 3 };
302 | if (number_people > 0)
303 | {
304 | pose_keypoints.resize(number_people * (int)num_body_parts * 3);
305 | }
306 | else
307 | {
308 | pose_keypoints.clear();
309 | }
310 | for (unsigned int person = 0u; person < valid_subset_indexes.size(); ++person)
311 | {
312 | const auto& subsetI = subset[valid_subset_indexes[person]].first;
313 | for (int bodyPart = 0u; bodyPart < num_body_parts; bodyPart++)
314 | {
315 | const int base_offset = (person*num_body_parts + bodyPart) * 3;
316 | const int body_part_index = subsetI[bodyPart];
317 | if (body_part_index > 0)
318 | {
319 | pose_keypoints[base_offset] = peaks[body_part_index - 2];
320 | pose_keypoints[base_offset + 1] = peaks[body_part_index - 1];
321 | pose_keypoints[base_offset + 2] = peaks[body_part_index];
322 | }
323 | else
324 | {
325 | pose_keypoints[base_offset] = 0.f;
326 | pose_keypoints[base_offset + 1] = 0.f;
327 | pose_keypoints[base_offset + 2] = 0.f;
328 | }
329 | }
330 | }
331 | }
332 |
333 | void PoseDetector::find_heatmap_peaks
334 | (
335 | const float *src,
336 | float *dst,
337 | const int SRCW,
338 | const int SRCH,
339 | const int SRC_CH,
340 | const float TH
341 | )
342 | {
343 | // find peaks (8-connected neighbor), weights with 7 by 7 area to get sub-pixel location and response
344 | const int SRC_PLANE_OFFSET = SRCW * SRCH;
345 | // add 1 for saving total people count, 3 for x, y, score
346 | const int DST_PLANE_OFFSET = (POSE_MAX_PEOPLE + 1) * 3;
347 | float *dstptr = dst;
348 | int c = 0;
349 | int x = 0;
350 | int y = 0;
351 | int i = 0;
352 | int j = 0;
353 | // TODO: reduce multiplication by using pointer
354 | for(c = 0; c < SRC_CH - 1; ++c)
355 | {
356 | int num_people = 0;
357 | for(y = 1; y < SRCH - 1 && num_people != POSE_MAX_PEOPLE; ++y)
358 | {
359 | for(x = 1; x < SRCW - 1 && num_people != POSE_MAX_PEOPLE; ++x)
360 | {
361 | int idx = y * SRCW + x;
362 | float value = src[idx];
363 | if (value > TH)
364 | {
365 | const float TOPLEFT = src[idx - SRCW - 1];
366 | const float TOP = src[idx - SRCW];
367 | const float TOPRIGHT = src[idx - SRCW + 1];
368 | const float LEFT = src[idx - 1];
369 | const float RIGHT = src[idx + 1];
370 | const float BUTTOMLEFT = src[idx + SRCW - 1];
371 | const float BUTTOM = src[idx + SRCW];
372 | const float BUTTOMRIGHT = src[idx + SRCW + 1];
373 | if(value > TOPLEFT && value > TOP && value > TOPRIGHT && value > LEFT &&
374 | value > RIGHT && value > BUTTOMLEFT && value > BUTTOM && value > BUTTOMRIGHT)
375 | {
376 | float x_acc = 0;
377 | float y_acc = 0;
378 | float score_acc = 0;
379 | for (i = -3; i <= 3; ++i)
380 | {
381 | int ux = x + i;
382 | if (ux >= 0 && ux < SRCW)
383 | {
384 | for (j = -3; j <= 3; ++j)
385 | {
386 | int uy = y + j;
387 | if (uy >= 0 && uy < SRCH)
388 | {
389 | float score = src[uy * SRCW + ux];
390 | x_acc += ux * score;
391 | y_acc += uy * score;
392 | score_acc += score;
393 | }
394 | }
395 | }
396 | }
397 | x_acc /= score_acc;
398 | y_acc /= score_acc;
399 | score_acc = value;
400 | dstptr[(num_people + 1) * 3 + 0] = x_acc;
401 | dstptr[(num_people + 1) * 3 + 1] = y_acc;
402 | dstptr[(num_people + 1) * 3 + 2] = score_acc;
403 | ++num_people;
404 | }
405 | }
406 | }
407 | }
408 | dstptr[0] = num_people;
409 | src += SRC_PLANE_OFFSET;
410 | dstptr += DST_PLANE_OFFSET;
411 | }
412 | }
413 |
414 | Mat PoseDetector::create_netsize_im
415 | (
416 | const Mat &im,
417 | const int netw,
418 | const int neth,
419 | float *scale
420 | )
421 | {
422 | // for tall image
423 | int newh = neth;
424 | float s = newh / (float)im.rows;
425 | int neww = im.cols * s;
426 | if (neww > netw)
427 | {
428 | //for fat image
429 | neww = netw;
430 | s = neww / (float)im.cols;
431 | newh = im.rows * s;
432 | }
433 |
434 | *scale = 1 / s;
435 | Rect dst_area(0, 0, neww, newh);
436 | Mat dst = Mat::zeros(neth, netw, CV_8UC3);
437 | resize(im, dst(dst_area), Size(neww, newh));
438 | return dst;
439 | }
440 |
441 | PoseDetector::PoseDetector(const char *cfg_path, const char *weight_path, int gpu_id) : Detector(cfg_path, weight_path,
442 | gpu_id) {
443 | det_people = nullptr;
444 | // initialize net
445 | net_inw = get_net_width();
446 | net_inh = get_net_height();
447 | net_outw = get_net_out_width();
448 | net_outh = get_net_out_height();
449 | }
450 |
451 | void PoseDetector::detect(cv::Mat im, float thresh) {
452 | // 3. resize to net input size, put scaled image on the top left
453 | float scale = 0.0f;
454 | Mat netim = create_netsize_im(im, net_inw, net_inh, &scale);
455 |
456 | // 4. normalized to float type
457 | netim.convertTo(netim, CV_32F, 1 / 256.f, -0.5);
458 |
459 | // 5. split channels
460 | float *netin_data = new float[net_inw * net_inh * 3]();
461 | float *netin_data_ptr = netin_data;
462 | vector input_channels;
463 | for (int i = 0; i < 3; ++i)
464 | {
465 | Mat channel(net_inh, net_inw, CV_32FC1, netin_data_ptr);
466 | input_channels.emplace_back(channel);
467 | netin_data_ptr += (net_inw * net_inh);
468 | }
469 | split(netim, input_channels);
470 |
471 | // 6. feed forward
472 | double time_begin = getTickCount();
473 | float *netoutdata = Detector::predict(netin_data);
474 | double fee_time = (getTickCount() - time_begin) / getTickFrequency() * 1000;
475 | #ifdef DEBUG
476 | cout << "forward fee: " << fee_time << "ms" << endl;
477 | #endif
478 | // 7. resize net output back to input size to get heatmap
479 | float *heatmap = new float[net_inw * net_inh * NET_OUT_CHANNELS];
480 | for (int i = 0; i < NET_OUT_CHANNELS; ++i)
481 | {
482 | Mat netout(net_outh, net_outw, CV_32F, (netoutdata + net_outh*net_outw*i));
483 | Mat nmsin(net_inh, net_inw, CV_32F, heatmap + net_inh*net_inw*i);
484 | resize(netout, nmsin, Size(net_inw, net_inh), 0, 0, CV_INTER_CUBIC);
485 | }
486 |
487 | // 8. get heatmap peaks
488 | float *heatmap_peaks = new float[3 * (POSE_MAX_PEOPLE+1) * (NET_OUT_CHANNELS-1)];
489 | find_heatmap_peaks(heatmap, heatmap_peaks, net_inw, net_inh, NET_OUT_CHANNELS, 0.05);
490 |
491 | // 9. link parts
492 | vector keypoints;
493 | vector shape;
494 | connect_bodyparts(keypoints, heatmap, heatmap_peaks, net_inw, net_inh, 9, 0.05, 6, 0.4, shape);
495 |
496 | delete [] heatmap_peaks;
497 | delete [] heatmap;
498 | delete [] netin_data;
499 |
500 | // people
501 | if (det_people != nullptr)
502 | delete det_people;
503 | det_people = new People(keypoints, shape, scale);
504 | }
505 |
506 | void PoseDetector::draw(cv::Mat mat)
507 | {
508 | det_people->render_pose_keypoints(mat);
509 | }
510 |
511 | std::string PoseDetector::det_to_json(int frame_id)
512 | {
513 | std::string out_str;
514 | char* tmp_buf = (char *)calloc(1024, sizeof(char));
515 | sprintf(tmp_buf, "{\n \"frame_id\":%d, \n ", frame_id);
516 | out_str = tmp_buf;
517 | out_str += det_people->get_output();
518 | out_str += "\n}";
519 | free(tmp_buf);
520 | return out_str;
521 | }
522 |
523 | PoseDetector::~PoseDetector() {
524 | }
525 |
--------------------------------------------------------------------------------
/server/src/pose_detector.hpp:
--------------------------------------------------------------------------------
1 | #ifndef __POSE_DETECTOR
2 | #define __POSE_DETECTOR
3 | #include
4 | #include
5 | #include
6 | #include
7 | #include
8 | #include "yolo_v2_class.hpp"
9 | #include "DetectorInterface.hpp"
10 | #include "people.hpp"
11 |
12 | class PoseDetector : public Detector, public DetectorInterface
13 | {
14 | private:
15 | int net_inw;
16 | int net_inh;
17 | int net_outw;
18 | int net_outh;
19 | People* det_people;
20 | public:
21 | PoseDetector(const char *cfg_path, const char *weight_path, int gpu_id);
22 | ~PoseDetector();
23 | inline People* get_people(void) { return det_people; }
24 | virtual void detect(cv::Mat mat, float thresh);
25 | virtual void draw(cv::Mat mat);
26 | virtual std::string det_to_json(int frame_id);
27 | private:
28 | void connect_bodyparts
29 | (
30 | std::vector& pose_keypoints,
31 | const float* const map,
32 | const float* const peaks,
33 | int mapw,
34 | int maph,
35 | const int inter_min_above_th,
36 | const float inter_th,
37 | const int min_subset_cnt,
38 | const float min_subset_score,
39 | std::vector& keypoint_shape
40 | );
41 |
42 | void find_heatmap_peaks
43 | (
44 | const float *src,
45 | float *dst,
46 | const int SRCW,
47 | const int SRCH,
48 | const int SRC_CH,
49 | const float TH
50 | );
51 |
52 | cv::Mat create_netsize_im
53 | (
54 | const cv::Mat &im,
55 | const int netw,
56 | const int neth,
57 | float *scale
58 | );
59 | };
60 |
61 | #endif
62 |
--------------------------------------------------------------------------------
/server/src/share_queue.h:
--------------------------------------------------------------------------------
1 | #pragma once
2 | // https://stackoverflow.com/questions/36762248/why-is-stdqueue-not-thread-safe
3 | #include
4 | #include
5 | #include
6 |
7 | template
8 | class SharedQueue
9 | {
10 | public:
11 | SharedQueue();
12 | ~SharedQueue();
13 |
14 | T& front();
15 | void pop_front();
16 |
17 | void push_back(const T& item);
18 | void push_back(T&& item);
19 |
20 | int size();
21 | bool empty();
22 |
23 | private:
24 | std::deque queue_;
25 | std::mutex mutex_;
26 | std::condition_variable cond_;
27 | };
28 |
29 | template
30 | SharedQueue::SharedQueue() {}
31 |
32 | template
33 | SharedQueue::~SharedQueue() {}
34 |
35 | template
36 | T& SharedQueue::front()
37 | {
38 | std::unique_lock mlock(mutex_);
39 | while (queue_.empty())
40 | {
41 | cond_.wait(mlock);
42 | }
43 | return queue_.front();
44 | }
45 |
46 | template
47 | void SharedQueue::pop_front()
48 | {
49 | std::unique_lock mlock(mutex_);
50 | while (queue_.empty())
51 | {
52 | cond_.wait(mlock);
53 | }
54 | queue_.pop_front();
55 | }
56 |
57 | template
58 | void SharedQueue::push_back(const T& item)
59 | {
60 | std::unique_lock mlock(mutex_);
61 | queue_.push_back(item);
62 | mlock.unlock(); // unlock before notificiation to minimize mutex con
63 | cond_.notify_one(); // notify one waiting thread
64 |
65 | }
66 |
67 | template
68 | void SharedQueue::push_back(T&& item)
69 | {
70 | std::unique_lock mlock(mutex_);
71 | queue_.push_back(std::move(item));
72 | mlock.unlock(); // unlock before notificiation to minimize mutex con
73 | cond_.notify_one(); // notify one waiting thread
74 |
75 | }
76 |
77 | template
78 | int SharedQueue::size()
79 | {
80 | std::unique_lock mlock(mutex_);
81 | int size = queue_.size();
82 | mlock.unlock();
83 | return size;
84 | }
85 |
--------------------------------------------------------------------------------
/server/src/sink.cpp:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 | #include
4 | #include
5 | #include
6 | #include
7 | #include
8 | #include
|
|
51 | |:---:|:---:|
52 | |
|
|
53 | 
|
|
57 | |:---:|:---:|
58 | |
|
|
59 | 
|
10 |
11 | ### Pose Estimation and Object Tracking
12 | Made pipeline to get 2D pose time series data in video sequence.
13 |
14 | In worker process, Pose Estimation is performed using ***OpenPose***. Input image pass through the Pose_detector, and get the people object which packed up people joint coordinates. People object serialized and send to sink process.
15 |
16 | In Sink process, people object convert to person objects. and every person object are send to Tracker. Tracker receive person object and produces object identities Using ***SORT***(simple online and realtime tracking algorithm).
17 |
18 | Finally, can get the joint time series data per each person. each person's Time series data is managed by queue container. so person object always maintain recent 32 frame.
19 |
20 | | ```Tracking Pipeline``` | ```Time series data``` |
21 | |:---:|:---:|
22 | |
|
|
23 |
24 | * #### Examples of Result (Pose Estimation and Object Tracking)
25 |
26 | |
|
|
27 | |:---:|:---:|
28 |
29 |
30 | ### Collect action video
31 | Collected Four Action type (Standing, Walking, Punching, Kicking) video.
32 |
33 | Punching Action type video is a subset of the ***Berkeley Multimodal Human Action Database (MHAD) dataset***.
34 | This video data is comprised of 12 subjects doing the punching actions for 5 repetitions, filmed from 4 angles. (http://tele-immersion.citris-uc.org/berkeley_mhad)
35 |
36 | Others (Standing, Walking, Kicking) are subsets of the ***CMU Panoptic Dataset***. In Range of Motion videos(171204_pose3, 171204_pose5, 171204_pose6), cut out 3 action type. I recorded timestamp per action type and cut out the video using python script(util/concat.py). This video data is comprised of 13 subjects doing the three actions, filmed from 31 angles. (http://domedb.perception.cs.cmu.edu/index.html)
37 |
38 | ``` jsonc
39 | 0, 1, 10 // 0 : Standing, 1 : begin timestamp, 10: end timestamp
40 | 1, 11, 15 // 1 : Walking, 11 : begin timestamp, 15: end timestamp
41 | 3, 39, 46 // 3 : Kicking, 39 : begin timestamp, 46: end timestamp
42 | ```
43 |
44 | * #### Examples of Dataset (Stand & Walk)
45 | 
|
|
70 | |:---:|:---:|
71 | |
|
|
72 |
73 | 
|
|
77 | |:---:|:---:|
78 | |
|
|
79 |
80 | 
|
|
96 |
97 | * #### Overview of feature vector
98 |
99 | 
|
115 | |:---:|
116 |
117 | 
|
130 | |:---:|
131 |
132 | ### RNN Training and Result
133 | The network used in this experiment is based on that of :
134 | * Guillaume Chevalier, 'LSTMs for Human Activity Recognition, 2016' https://github.com/guillaume-chevalier/LSTM-Human-Activity-Recognition
135 | * stuarteiffert, 'RNN-for-Human-Activity-Recognition-using-2D-Pose-Input' https://github.com/stuarteiffert/RNN-for-Human-Activity-Recognition-using-2D-Pose-Input
136 |
137 | training was run for 300 epochs with a batch size of 1024. (weights/action.h5)
138 |
139 | After training, To get action recognition result in real time, made action recognition pipeline. Sink process send each person's time series feature vector to action process as string. Action process put received data into RNN network and send back results of prediction. (0 : Standing, 1 : Walking, 2 : Punching, 3 : Kicking)
140 |
141 | | ```Action Recognition Pipeline``` | ```RNN Model``` |
142 | |:---:|:---:|
143 | |
|
|
144 |
145 |
146 | * #### Examples of Result (RNN Action Recognition)
147 | | [```standing```](https://www.youtube.com/watch?v=Orc0Eq9bWOs) | [```Walking```](https://www.youtube.com/watch?v=Orc0Eq9bWOs) | [```Punching```](https://www.youtube.com/watch?v=kbgkeTTSau8) | [```Kicking```](https://www.youtube.com/watch?v=R1UWcG9N6tI) |
148 | |:---:|:---:|:---:|:---:|
149 | |
|
|
|
|
150 |
151 | ### Fight Detection
152 |
153 | This stage check that person who kick or punch is hitting someone. if some person has hit other, those people set enemy each other.
154 | System count it as fight and then track them until they exist in frame.
155 |
156 | | ```Fight Detection Pipeline``` |
157 | |:---:|
158 | |
|
159 |
160 | * #### Examples of Result
161 | | [```Fighting Championship```](https://www.youtube.com/watch?v=cIhoK4cPbC4) | [```CCTV Video```](https://www.youtube.com/watch?v=stJPOb6zW7U) |
162 | |:---:|:---:|
163 | |
|
|
164 |
165 | | [```Sparring video A```](https://www.youtube.com/watch?v=x0kJmieuFzI) | [```Sparring video B```](https://www.youtube.com/watch?v=x0kJmieuFzI) |
166 | |:---:|:---:|
167 | |
|
|
168 |
169 | * #### Examples of Result (Failure case)
170 | | [```Fake Person```](https://www.youtube.com/watch?v=kbgkeTTSau8) | [```Small Person```](https://www.youtube.com/watch?v=aUdKzb4LGJI) |
171 | |:---:|:---:|
172 | |
|
|
173 |
174 | ### References
175 | * #### Darknet : https://github.com/AlexeyAB/darknet
176 | * #### OpenCV : https://github.com/opencv/opencv
177 | * #### ZeroMQ : https://github.com/zeromq/libzmq
178 | * #### json-c : https://github.com/json-c/json-c
179 | * #### openpose-darknet : https://github.com/lincolnhard/openpose-darknet
180 | * #### sort-cpp : https://github.com/mcximing/sort-cpp
181 | * #### cpp-base64 : https://github.com/ReneNyffenegger/cpp-base64
182 | * #### mem_pool : https://www.codeproject.com/Articles/27487/Why-to-use-memory-pool-and-how-to-implement-it
183 | * #### share_queue : https://stackoverflow.com/questions/36762248/why-is-stdqueue-not-thread-safe
184 | * #### RNN-for-Human-Activity-Recognition-using-2D-Pose-Input : https://github.com/stuarteiffert/RNN-for-Human-Activity-Recognition-using-2D-Pose-Input
185 | * #### LSTM-Human-Activity-Recognition : https://github.com/guillaume-chevalier/LSTM-Human-Activity-Recognition
186 |
--------------------------------------------------------------------------------
/client/darknet_client.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio 15
4 | VisualStudioVersion = 15.0.28307.852
5 | MinimumVisualStudioVersion = 10.0.40219.1
6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "darknet_client", "darknet_client\darknet_client.vcxproj", "{D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}"
7 | EndProject
8 | Global
9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
10 | Debug|x64 = Debug|x64
11 | Debug|x86 = Debug|x86
12 | Release|x64 = Release|x64
13 | Release|x86 = Release|x86
14 | EndGlobalSection
15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
16 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Debug|x64.ActiveCfg = Debug|x64
17 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Debug|x64.Build.0 = Debug|x64
18 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Debug|x86.ActiveCfg = Debug|Win32
19 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Debug|x86.Build.0 = Debug|Win32
20 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Release|x64.ActiveCfg = Release|x64
21 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Release|x64.Build.0 = Release|x64
22 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Release|x86.ActiveCfg = Release|Win32
23 | {D8EF8C1B-C4C1-4C68-A033-31AD741BE59A}.Release|x86.Build.0 = Release|Win32
24 | EndGlobalSection
25 | GlobalSection(SolutionProperties) = preSolution
26 | HideSolutionNode = FALSE
27 | EndGlobalSection
28 | GlobalSection(ExtensibilityGlobals) = postSolution
29 | SolutionGuid = {FA14131C-5C1C-41BF-AEB9-201B8E8E0B1E}
30 | EndGlobalSection
31 | EndGlobal
32 |
--------------------------------------------------------------------------------
/client/darknet_client/Makefile:
--------------------------------------------------------------------------------
1 | DEBUG = 1
2 |
3 | CPP = g++
4 | COMMON = -DOPENCV
5 | CXXFLAGS = -g -Wall -O2 -std=c++11 -DOPENCV
6 | LDFLAGS = -lstdc++ -lpthread -lzmq -lrt -ltbb
7 |
8 | CXXFLAGS += `pkg-config --cflags json-c`
9 | CXXFLAGS += `pkg-config --cflags opencv`
10 |
11 | LDFLAGS += `pkg-config --libs json-c`
12 | LDFLAGS += `pkg-config --libs opencv`
13 |
14 | ifeq ($(DEBUG), 1)
15 | COMMON += -DDEBUG
16 | endif
17 |
18 | VPATH = ./src/
19 | OBJDIR = ./obj/
20 | DEPS = $(wildcard src/*.h*)
21 |
22 | EXEC1 = darknet_client
23 | EXEC1_OBJ = main.o frame.o mem_pool.o base64.o args.o util.o
24 | EXEC1_OBJS = $(addprefix $(OBJDIR), $(EXEC1_OBJ))
25 |
26 | OBJS = $(EXEC1_OBJS)
27 | EXECS = $(EXEC1)
28 |
29 | all: $(OBJDIR) $(EXECS)
30 |
31 | $(EXEC1): $(EXEC1_OBJS)
32 | $(CPP) $(COMMON) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
33 |
34 | $(OBJDIR)%.o: %.cpp $(DEPS)
35 | $(CPP) $(COMMON) $(CXXFLAGS) -c $< -o $@
36 |
37 | $(OBJDIR):
38 | mkdir -p $(OBJDIR) res
39 |
40 | clean:
41 | rm -rf $(OBJS) $(EXECS)
42 |
--------------------------------------------------------------------------------
/client/darknet_client/darknet_client.vcxproj:
--------------------------------------------------------------------------------
1 |
2 |