├── ActionsEstLoader.py
├── Actionsrecognition
├── Models.py
├── Utils.py
└── train.py
├── App.py
├── CameraLoader.py
├── Data
├── create_dataset_1.py
├── create_dataset_2.py
└── create_dataset_3.py
├── Detection
├── Models.py
└── Utils.py
├── DetectorLoader.py
├── Models
├── TSSTG
│ └── _.txt
├── sppe
│ └── _.txt
└── yolo-tiny-onecls
│ └── _.txt
├── PoseEstimateLoader.py
├── README.md
├── SPPE
├── LICENSE
├── README.md
└── src
│ ├── main_fast_inference.py
│ ├── models
│ ├── FastPose.py
│ ├── __init__.py
│ ├── hg-prm.py
│ ├── hgPRM.py
│ └── layers
│ │ ├── DUC.py
│ │ ├── PRM.py
│ │ ├── Residual.py
│ │ ├── Resnet.py
│ │ ├── SE_Resnet.py
│ │ ├── SE_module.py
│ │ ├── __init__.py
│ │ └── util_models.py
│ ├── opt.py
│ └── utils
│ ├── __init__.py
│ ├── dataset
│ ├── .coco.py.swp
│ ├── __init__.py
│ ├── coco.py
│ ├── fuse.py
│ └── mpii.py
│ ├── eval.py
│ ├── img.py
│ └── pose.py
├── Track
├── Tracker.py
├── iou_matching.py
├── kalman_filter.py
└── linear_assignment.py
├── Visualizer.py
├── fn.py
├── main.py
├── pPose_nms.py
├── pose_utils.py
└── sample1.gif
/ActionsEstLoader.py:
--------------------------------------------------------------------------------
1 | import os
2 | import torch
3 | import numpy as np
4 |
5 | from Actionsrecognition.Models import TwoStreamSpatialTemporalGraph
6 | from pose_utils import normalize_points_with_size, scale_pose
7 |
8 |
9 | class TSSTG(object):
10 | """Two-Stream Spatial Temporal Graph Model Loader.
11 | Args:
12 | weight_file: (str) Path to trained weights file.
13 | device: (str) Device to load the model on 'cpu' or 'cuda'.
14 | """
15 | def __init__(self,
16 | weight_file='./Models/TSSTG/tsstg-model.pth',
17 | device='cuda'):
18 | self.graph_args = {'strategy': 'spatial'}
19 | self.class_names = ['Standing', 'Walking', 'Sitting', 'Lying Down',
20 | 'Stand up', 'Sit down', 'Fall Down']
21 | self.num_class = len(self.class_names)
22 | self.device = device
23 |
24 | self.model = TwoStreamSpatialTemporalGraph(self.graph_args, self.num_class).to(self.device)
25 | self.model.load_state_dict(torch.load(weight_file))
26 | self.model.eval()
27 |
28 | def predict(self, pts, image_size):
29 | """Predict actions from single person skeleton points and score in time sequence.
30 | Args:
31 | pts: (numpy array) points and score in shape `(t, v, c)` where
32 | t : inputs sequence (time steps).,
33 | v : number of graph node (body parts).,
34 | c : channel (x, y, score).,
35 | image_size: (tuple of int) width, height of image frame.
36 | Returns:
37 | (numpy array) Probability of each class actions.
38 | """
39 | pts[:, :, :2] = normalize_points_with_size(pts[:, :, :2], image_size[0], image_size[1])
40 | pts[:, :, :2] = scale_pose(pts[:, :, :2])
41 | pts = np.concatenate((pts, np.expand_dims((pts[:, 1, :] + pts[:, 2, :]) / 2, 1)), axis=1)
42 |
43 | pts = torch.tensor(pts, dtype=torch.float32)
44 | pts = pts.permute(2, 0, 1)[None, :]
45 |
46 | mot = pts[:, :2, 1:, :] - pts[:, :2, :-1, :]
47 | mot = mot.to(self.device)
48 | pts = pts.to(self.device)
49 |
50 | out = self.model((pts, mot))
51 |
52 | return out.detach().cpu().numpy()
53 |
--------------------------------------------------------------------------------
/Actionsrecognition/Models.py:
--------------------------------------------------------------------------------
1 | ### Reference from: https://github.com/yysijie/st-gcn/tree/master/net
2 |
3 | import torch
4 | import torch.nn as nn
5 | import torch.nn.functional as F
6 | import numpy as np
7 |
8 | from Actionsrecognition.Utils import Graph
9 |
10 |
11 | class GraphConvolution(nn.Module):
12 | """The basic module for applying a graph convolution.
13 | Args:
14 | - in_channel: (int) Number of channels in the input sequence data.
15 | - out_channels: (int) Number of channels produced by the convolution.
16 | - kernel_size: (int) Size of the graph convolving kernel.
17 | - t_kernel_size: (int) Size of the temporal convolving kernel.
18 | - t_stride: (int, optional) Stride of the temporal convolution. Default: 1
19 | - t_padding: (int, optional) Temporal zero-padding added to both sides of
20 | the input. Default: 0
21 | - t_dilation: (int, optional) Spacing between temporal kernel elements. Default: 1
22 | - bias: (bool, optional) If `True`, adds a learnable bias to the output.
23 | Default: `True`
24 | Shape:
25 | - Inputs x: Graph sequence in :math:`(N, in_channels, T_{in}, V)`,
26 | A: Graph adjacency matrix in :math:`(K, V, V)`,
27 | - Output: Graph sequence out in :math:`(N, out_channels, T_{out}, V)`
28 |
29 | where
30 | :math:`N` is a batch size,
31 | :math:`K` is the spatial kernel size, as :math:`K == kernel_size[1]`,
32 | :math:`T_{in}/T_{out}` is a length of input/output sequence,
33 | :math:`V` is the number of graph nodes.
34 |
35 | """
36 | def __init__(self, in_channels, out_channels, kernel_size,
37 | t_kernel_size=1,
38 | t_stride=1,
39 | t_padding=0,
40 | t_dilation=1,
41 | bias=True):
42 | super().__init__()
43 |
44 | self.kernel_size = kernel_size
45 | self.conv = nn.Conv2d(in_channels,
46 | out_channels * kernel_size,
47 | kernel_size=(t_kernel_size, 1),
48 | padding=(t_padding, 0),
49 | stride=(t_stride, 1),
50 | dilation=(t_dilation, 1),
51 | bias=bias)
52 |
53 | def forward(self, x, A):
54 | x = self.conv(x)
55 | n, kc, t, v = x.size()
56 | x = x.view(n, self.kernel_size, kc//self.kernel_size, t, v)
57 | x = torch.einsum('nkctv,kvw->nctw', (x, A))
58 |
59 | return x.contiguous()
60 |
61 |
62 | class st_gcn(nn.Module):
63 | """Applies a spatial temporal graph convolution over an input graph sequence.
64 | Args:
65 | - in_channels: (int) Number of channels in the input sequence data.
66 | - out_channels: (int) Number of channels produced by the convolution.
67 | - kernel_size: (tuple) Size of the temporal convolving kernel and
68 | graph convolving kernel.
69 | - stride: (int, optional) Stride of the temporal convolution. Default: 1
70 | - dropout: (int, optional) Dropout rate of the final output. Default: 0
71 | - residual: (bool, optional) If `True`, applies a residual mechanism.
72 | Default: `True`
73 | Shape:
74 | - Inputs x: Graph sequence in :math: `(N, in_channels, T_{in}, V)`,
75 | A: Graph Adjecency matrix in :math: `(K, V, V)`,
76 | - Output: Graph sequence out in :math: `(N, out_channels, T_{out}, V)`
77 | where
78 | :math:`N` is a batch size,
79 | :math:`K` is the spatial kernel size, as :math:`K == kernel_size[1]`,
80 | :math:`T_{in}/T_{out}` is a length of input/output sequence,
81 | :math:`V` is the number of graph nodes.
82 | """
83 | def __init__(self, in_channels, out_channels, kernel_size,
84 | stride=1,
85 | dropout=0,
86 | residual=True):
87 | super().__init__()
88 | assert len(kernel_size) == 2
89 | assert kernel_size[0] % 2 == 1
90 |
91 | padding = ((kernel_size[0] - 1) // 2, 0)
92 |
93 | self.gcn = GraphConvolution(in_channels, out_channels, kernel_size[1])
94 | self.tcn = nn.Sequential(nn.BatchNorm2d(out_channels),
95 | nn.ReLU(inplace=True),
96 | nn.Conv2d(out_channels,
97 | out_channels,
98 | (kernel_size[0], 1),
99 | (stride, 1),
100 | padding),
101 | nn.BatchNorm2d(out_channels),
102 | nn.Dropout(dropout, inplace=True)
103 | )
104 |
105 | if not residual:
106 | self.residual = lambda x: 0
107 | elif (in_channels == out_channels) and (stride == 1):
108 | self.residual = lambda x: x
109 | else:
110 | self.residual = nn.Sequential(nn.Conv2d(in_channels,
111 | out_channels,
112 | kernel_size=1,
113 | stride=(stride, 1)),
114 | nn.BatchNorm2d(out_channels)
115 | )
116 | self.relu = nn.ReLU(inplace=True)
117 |
118 | def forward(self, x, A):
119 | res = self.residual(x)
120 | x = self.gcn(x, A)
121 | x = self.tcn(x) + res
122 |
123 | return self.relu(x)
124 |
125 |
126 | class StreamSpatialTemporalGraph(nn.Module):
127 | """Spatial temporal graph convolutional networks.
128 | Args:
129 | - in_channels: (int) Number of input channels.
130 | - graph_args: (dict) Args map of `Actionsrecognition.Utils.Graph` Class.
131 | - num_class: (int) Number of class outputs. If `None` return pooling features of
132 | the last st-gcn layer instead.
133 | - edge_importance_weighting: (bool) If `True`, adds a learnable importance
134 | weighting to the edges of the graph.
135 | - **kwargs: (optional) Other parameters for graph convolution units.
136 | Shape:
137 | - Input: :math:`(N, in_channels, T_{in}, V_{in})`
138 | - Output: :math:`(N, num_class)` where
139 | :math:`N` is a batch size,
140 | :math:`T_{in}` is a length of input sequence,
141 | :math:`V_{in}` is the number of graph nodes,
142 | or If num_class is `None`: `(N, out_channels)`
143 | :math:`out_channels` is number of out_channels of the last layer.
144 | """
145 | def __init__(self, in_channels, graph_args, num_class=None,
146 | edge_importance_weighting=True, **kwargs):
147 | super().__init__()
148 | # Load graph.
149 | graph = Graph(**graph_args)
150 | A = torch.tensor(graph.A, dtype=torch.float32, requires_grad=False)
151 | self.register_buffer('A', A)
152 |
153 | # Networks.
154 | spatial_kernel_size = A.size(0)
155 | temporal_kernel_size = 9
156 | kernel_size = (temporal_kernel_size, spatial_kernel_size)
157 | kwargs0 = {k: v for k, v in kwargs.items() if k != 'dropout'}
158 |
159 | self.data_bn = nn.BatchNorm1d(in_channels * A.size(1))
160 | self.st_gcn_networks = nn.ModuleList((
161 | st_gcn(in_channels, 64, kernel_size, 1, residual=False, **kwargs0),
162 | st_gcn(64, 64, kernel_size, 1, **kwargs),
163 | st_gcn(64, 64, kernel_size, 1, **kwargs),
164 | st_gcn(64, 64, kernel_size, 1, **kwargs),
165 | st_gcn(64, 128, kernel_size, 2, **kwargs),
166 | st_gcn(128, 128, kernel_size, 1, **kwargs),
167 | st_gcn(128, 128, kernel_size, 1, **kwargs),
168 | st_gcn(128, 256, kernel_size, 2, **kwargs),
169 | st_gcn(256, 256, kernel_size, 1, **kwargs),
170 | st_gcn(256, 256, kernel_size, 1, **kwargs)
171 | ))
172 |
173 | # initialize parameters for edge importance weighting.
174 | if edge_importance_weighting:
175 | self.edge_importance = nn.ParameterList([
176 | nn.Parameter(torch.ones(A.size()))
177 | for i in self.st_gcn_networks
178 | ])
179 | else:
180 | self.edge_importance = [1] * len(self.st_gcn_networks)
181 |
182 | if num_class is not None:
183 | self.cls = nn.Conv2d(256, num_class, kernel_size=1)
184 | else:
185 | self.cls = lambda x: x
186 |
187 | def forward(self, x):
188 | # data normalization.
189 | N, C, T, V = x.size()
190 | x = x.permute(0, 3, 1, 2).contiguous() # (N, V, C, T)
191 | x = x.view(N, V * C, T)
192 | x = self.data_bn(x)
193 | x = x.view(N, V, C, T)
194 | x = x.permute(0, 2, 3, 1).contiguous()
195 | x = x.view(N, C, T, V)
196 |
197 | # forward.
198 | for gcn, importance in zip(self.st_gcn_networks, self.edge_importance):
199 | x = gcn(x, self.A * importance)
200 |
201 | x = F.avg_pool2d(x, x.size()[2:])
202 | x = self.cls(x)
203 | x = x.view(x.size(0), -1)
204 |
205 | return x
206 |
207 |
208 | class TwoStreamSpatialTemporalGraph(nn.Module):
209 | """Two inputs spatial temporal graph convolutional networks.
210 | Args:
211 | - graph_args: (dict) Args map of `Actionsrecognition.Utils.Graph` Class.
212 | - num_class: (int) Number of class outputs.
213 | - edge_importance_weighting: (bool) If `True`, adds a learnable importance
214 | weighting to the edges of the graph.
215 | - **kwargs: (optional) Other parameters for graph convolution units.
216 | Shape:
217 | - Input: :tuple of math:`((N, 3, T, V), (N, 2, T, V))`
218 | for points and motions stream where.
219 | :math:`N` is a batch size,
220 | :math:`in_channels` is data channels (3 is (x, y, score)), (2 is (mot_x, mot_y))
221 | :math:`T` is a length of input sequence,
222 | :math:`V` is the number of graph nodes,
223 | - Output: :math:`(N, num_class)`
224 | """
225 | def __init__(self, graph_args, num_class, edge_importance_weighting=True,
226 | **kwargs):
227 | super().__init__()
228 | self.pts_stream = StreamSpatialTemporalGraph(3, graph_args, None,
229 | edge_importance_weighting,
230 | **kwargs)
231 | self.mot_stream = StreamSpatialTemporalGraph(2, graph_args, None,
232 | edge_importance_weighting,
233 | **kwargs)
234 |
235 | self.fcn = nn.Linear(256 * 2, num_class)
236 |
237 | def forward(self, inputs):
238 | out1 = self.pts_stream(inputs[0])
239 | out2 = self.mot_stream(inputs[1])
240 |
241 | concat = torch.cat([out1, out2], dim=-1)
242 | out = self.fcn(concat)
243 |
244 | return torch.sigmoid(out)
245 |
--------------------------------------------------------------------------------
/Actionsrecognition/Utils.py:
--------------------------------------------------------------------------------
1 | ### Reference from: https://github.com/yysijie/st-gcn/blob/master/net/utils/graph.py
2 |
3 | import os
4 | import torch
5 | import numpy as np
6 |
7 |
8 | class Graph:
9 | """The Graph to model the skeletons extracted by the Alpha-Pose.
10 | Args:
11 | - strategy: (string) must be one of the follow candidates
12 | - uniform: Uniform Labeling,
13 | - distance: Distance Partitioning,
14 | - spatial: Spatial Configuration,
15 | For more information, please refer to the section 'Partition Strategies'
16 | in our paper (https://arxiv.org/abs/1801.07455).
17 | - layout: (string) must be one of the follow candidates
18 | - coco_cut: Is COCO format but cut 4 joints (L-R ears, L-R eyes) out.
19 | - max_hop: (int) the maximal distance between two connected nodes.
20 | - dilation: (int) controls the spacing between the kernel points.
21 | """
22 | def __init__(self,
23 | layout='coco_cut',
24 | strategy='uniform',
25 | max_hop=1,
26 | dilation=1):
27 | self.max_hop = max_hop
28 | self.dilation = dilation
29 |
30 | self.get_edge(layout)
31 | self.hop_dis = get_hop_distance(self.num_node, self.edge, max_hop)
32 | self.get_adjacency(strategy)
33 |
34 | def get_edge(self, layout):
35 | if layout == 'coco_cut':
36 | self.num_node = 14
37 | self_link = [(i, i) for i in range(self.num_node)]
38 | neighbor_link = [(6, 4), (4, 2), (2, 13), (13, 1), (5, 3), (3, 1), (12, 10),
39 | (10, 8), (8, 2), (11, 9), (9, 7), (7, 1), (13, 0)]
40 | self.edge = self_link + neighbor_link
41 | self.center = 13
42 | else:
43 | raise ValueError('This layout is not supported!')
44 |
45 | def get_adjacency(self, strategy):
46 | valid_hop = range(0, self.max_hop + 1, self.dilation)
47 | adjacency = np.zeros((self.num_node, self.num_node))
48 | for hop in valid_hop:
49 | adjacency[self.hop_dis == hop] = 1
50 | normalize_adjacency = normalize_digraph(adjacency)
51 |
52 | if strategy == 'uniform':
53 | A = np.zeros((1, self.num_node, self.num_node))
54 | A[0] = normalize_adjacency
55 | self.A = A
56 | elif strategy == 'distance':
57 | A = np.zeros((len(valid_hop), self.num_node, self.num_node))
58 | for i, hop in enumerate(valid_hop):
59 | A[i][self.hop_dis == hop] = normalize_adjacency[self.hop_dis ==
60 | hop]
61 | self.A = A
62 | elif strategy == 'spatial':
63 | A = []
64 | for hop in valid_hop:
65 | a_root = np.zeros((self.num_node, self.num_node))
66 | a_close = np.zeros((self.num_node, self.num_node))
67 | a_further = np.zeros((self.num_node, self.num_node))
68 | for i in range(self.num_node):
69 | for j in range(self.num_node):
70 | if self.hop_dis[j, i] == hop:
71 | if self.hop_dis[j, self.center] == self.hop_dis[i, self.center]:
72 | a_root[j, i] = normalize_adjacency[j, i]
73 | elif self.hop_dis[j, self.center] > self.hop_dis[i, self.center]:
74 | a_close[j, i] = normalize_adjacency[j, i]
75 | else:
76 | a_further[j, i] = normalize_adjacency[j, i]
77 | if hop == 0:
78 | A.append(a_root)
79 | else:
80 | A.append(a_root + a_close)
81 | A.append(a_further)
82 | A = np.stack(A)
83 | self.A = A
84 | #self.A = np.swapaxes(np.swapaxes(A, 0, 1), 1, 2)
85 | else:
86 | raise ValueError("This strategy is not supported!")
87 |
88 |
89 | def get_hop_distance(num_node, edge, max_hop=1):
90 | A = np.zeros((num_node, num_node))
91 | for i, j in edge:
92 | A[j, i] = 1
93 | A[i, j] = 1
94 |
95 | # compute hop steps
96 | hop_dis = np.zeros((num_node, num_node)) + np.inf
97 | transfer_mat = [np.linalg.matrix_power(A, d) for d in range(max_hop + 1)]
98 | arrive_mat = (np.stack(transfer_mat) > 0)
99 | for d in range(max_hop, -1, -1):
100 | hop_dis[arrive_mat[d]] = d
101 | return hop_dis
102 |
103 |
104 | def normalize_digraph(A):
105 | Dl = np.sum(A, 0)
106 | num_node = A.shape[0]
107 | Dn = np.zeros((num_node, num_node))
108 | for i in range(num_node):
109 | if Dl[i] > 0:
110 | Dn[i, i] = Dl[i]**(-1)
111 | AD = np.dot(A, Dn)
112 | return AD
113 |
114 |
115 | def normalize_undigraph(A):
116 | Dl = np.sum(A, 0)
117 | num_node = A.shape[0]
118 | Dn = np.zeros((num_node, num_node))
119 | for i in range(num_node):
120 | if Dl[i] > 0:
121 | Dn[i, i] = Dl[i]**(-0.5)
122 | DAD = np.dot(np.dot(Dn, A), Dn)
123 | return DAD
124 |
--------------------------------------------------------------------------------
/Actionsrecognition/train.py:
--------------------------------------------------------------------------------
1 | import os
2 | import time
3 | import torch
4 | import pickle
5 | import numpy as np
6 | import torch.nn.functional as F
7 | from shutil import copyfile
8 | from tqdm import tqdm
9 | from torch.utils import data
10 | from torch.optim.adadelta import Adadelta
11 | from sklearn.model_selection import train_test_split
12 |
13 | from Actionsrecognition.Models import *
14 | from Visualizer import plot_graphs, plot_confusion_metrix
15 |
16 |
17 | save_folder = 'saved/TSSTG(pts+mot)-01(cf+hm-hm)'
18 |
19 | device = 'cuda'
20 | epochs = 30
21 | batch_size = 32
22 |
23 | # DATA FILES.
24 | # Should be in format of
25 | # inputs: (N_samples, time_steps, graph_node, channels),
26 | # labels: (N_samples, num_class)
27 | # and do some of normalizations on it. Default data create from:
28 | # Data.create_dataset_(1-3).py
29 | # where
30 | # time_steps: Number of frame input sequence, Default: 30
31 | # graph_node: Number of node in skeleton, Default: 14
32 | # channels: Inputs data (x, y and scores), Default: 3
33 | # num_class: Number of pose class to train, Default: 7
34 |
35 | data_files = ['../Data/Coffee_room_new-set(labelXscrw).pkl',
36 | '../Data/Home_new-set(labelXscrw).pkl']
37 | class_names = ['Standing', 'Walking', 'Sitting', 'Lying Down',
38 | 'Stand up', 'Sit down', 'Fall Down']
39 | num_class = len(class_names)
40 |
41 |
42 | def load_dataset(data_files, batch_size, split_size=0):
43 | """Load data files into torch DataLoader with/without spliting train-test.
44 | """
45 | features, labels = [], []
46 | for fil in data_files:
47 | with open(fil, 'rb') as f:
48 | fts, lbs = pickle.load(f)
49 | features.append(fts)
50 | labels.append(lbs)
51 | del fts, lbs
52 | features = np.concatenate(features, axis=0)
53 | labels = np.concatenate(labels, axis=0)
54 |
55 | if split_size > 0:
56 | x_train, x_valid, y_train, y_valid = train_test_split(features, labels, test_size=split_size,
57 | random_state=9)
58 | train_set = data.TensorDataset(torch.tensor(x_train, dtype=torch.float32).permute(0, 3, 1, 2),
59 | torch.tensor(y_train, dtype=torch.float32))
60 | valid_set = data.TensorDataset(torch.tensor(x_valid, dtype=torch.float32).permute(0, 3, 1, 2),
61 | torch.tensor(y_valid, dtype=torch.float32))
62 | train_loader = data.DataLoader(train_set, batch_size, shuffle=True)
63 | valid_loader = data.DataLoader(valid_set, batch_size)
64 | else:
65 | train_set = data.TensorDataset(torch.tensor(features, dtype=torch.float32).permute(0, 3, 1, 2),
66 | torch.tensor(labels, dtype=torch.float32))
67 | train_loader = data.DataLoader(train_set, batch_size, shuffle=True)
68 | valid_loader = None
69 | return train_loader, valid_loader
70 |
71 |
72 | def accuracy_batch(y_pred, y_true):
73 | return (y_pred.argmax(1) == y_true.argmax(1)).mean()
74 |
75 |
76 | def set_training(model, mode=True):
77 | for p in model.parameters():
78 | p.requires_grad = mode
79 | model.train(mode)
80 | return model
81 |
82 |
83 | if __name__ == '__main__':
84 | save_folder = os.path.join(os.path.dirname(__file__), save_folder)
85 | if not os.path.exists(save_folder):
86 | os.makedirs(save_folder)
87 |
88 | # DATA.
89 | train_loader, _ = load_dataset(data_files[0:1], batch_size)
90 | valid_loader, train_loader_ = load_dataset(data_files[1:2], batch_size, 0.2)
91 |
92 | train_loader = data.DataLoader(data.ConcatDataset([train_loader.dataset, train_loader_.dataset]),
93 | batch_size, shuffle=True)
94 | dataloader = {'train': train_loader, 'valid': valid_loader}
95 | del train_loader_
96 |
97 | # MODEL.
98 | graph_args = {'strategy': 'spatial'}
99 | model = TwoStreamSpatialTemporalGraph(graph_args, num_class).to(device)
100 |
101 | #optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
102 | optimizer = Adadelta(model.parameters())
103 |
104 | losser = torch.nn.BCELoss()
105 |
106 | # TRAINING.
107 | loss_list = {'train': [], 'valid': []}
108 | accu_list = {'train': [], 'valid': []}
109 | for e in range(epochs):
110 | print('Epoch {}/{}'.format(e, epochs - 1))
111 | for phase in ['train', 'valid']:
112 | if phase == 'train':
113 | model = set_training(model, True)
114 | else:
115 | model = set_training(model, False)
116 |
117 | run_loss = 0.0
118 | run_accu = 0.0
119 | with tqdm(dataloader[phase], desc=phase) as iterator:
120 | for pts, lbs in iterator:
121 | # Create motion input by distance of points (x, y) of the same node
122 | # in two frames.
123 | mot = pts[:, :2, 1:, :] - pts[:, :2, :-1, :]
124 |
125 | mot = mot.to(device)
126 | pts = pts.to(device)
127 | lbs = lbs.to(device)
128 |
129 | # Forward.
130 | out = model((pts, mot))
131 | loss = losser(out, lbs)
132 |
133 | if phase == 'train':
134 | # Backward.
135 | model.zero_grad()
136 | loss.backward()
137 | optimizer.step()
138 |
139 | run_loss += loss.item()
140 | accu = accuracy_batch(out.detach().cpu().numpy(),
141 | lbs.detach().cpu().numpy())
142 | run_accu += accu
143 |
144 | iterator.set_postfix_str(' loss: {:.4f}, accu: {:.4f}'.format(
145 | loss.item(), accu))
146 | iterator.update()
147 | #break
148 | loss_list[phase].append(run_loss / len(iterator))
149 | accu_list[phase].append(run_accu / len(iterator))
150 | #break
151 |
152 | print('Summary epoch:\n - Train loss: {:.4f}, accu: {:.4f}\n - Valid loss:'
153 | ' {:.4f}, accu: {:.4f}'.format(loss_list['train'][-1], accu_list['train'][-1],
154 | loss_list['valid'][-1], accu_list['valid'][-1]))
155 |
156 | # SAVE.
157 | torch.save(model.state_dict(), os.path.join(save_folder, 'tsstg-model.pth'))
158 |
159 | plot_graphs(list(loss_list.values()), list(loss_list.keys()),
160 | 'Last Train: {:.2f}, Valid: {:.2f}'.format(
161 | loss_list['train'][-1], loss_list['valid'][-1]
162 | ), 'Loss', xlim=[0, epochs],
163 | save=os.path.join(save_folder, 'loss_graph.png'))
164 | plot_graphs(list(accu_list.values()), list(accu_list.keys()),
165 | 'Last Train: {:.2f}, Valid: {:.2f}'.format(
166 | accu_list['train'][-1], accu_list['valid'][-1]
167 | ), 'Accu', xlim=[0, epochs],
168 | save=os.path.join(save_folder, 'accu_graph.png'))
169 |
170 | #break
171 |
172 | del train_loader, valid_loader
173 |
174 | model.load_state_dict(torch.load(os.path.join(save_folder, 'tsstg-model.pth')))
175 |
176 | # EVALUATION.
177 | model = set_training(model, False)
178 | data_file = data_files[1]
179 | eval_loader, _ = load_dataset([data_file], 32)
180 |
181 | print('Evaluation.')
182 | run_loss = 0.0
183 | run_accu = 0.0
184 | y_preds = []
185 | y_trues = []
186 | with tqdm(eval_loader, desc='eval') as iterator:
187 | for pts, lbs in iterator:
188 | mot = pts[:, :2, 1:, :] - pts[:, :2, :-1, :]
189 | mot = mot.to(device)
190 | pts = pts.to(device)
191 | lbs = lbs.to(device)
192 |
193 | out = model((pts, mot))
194 | loss = losser(out, lbs)
195 |
196 | run_loss += loss.item()
197 | accu = accuracy_batch(out.detach().cpu().numpy(),
198 | lbs.detach().cpu().numpy())
199 | run_accu += accu
200 |
201 | y_preds.extend(out.argmax(1).detach().cpu().numpy())
202 | y_trues.extend(lbs.argmax(1).cpu().numpy())
203 |
204 | iterator.set_postfix_str(' loss: {:.4f}, accu: {:.4f}'.format(
205 | loss.item(), accu))
206 | iterator.update()
207 |
208 | run_loss = run_loss / len(iterator)
209 | run_accu = run_accu / len(iterator)
210 |
211 | plot_confusion_metrix(y_trues, y_preds, class_names, 'Eval on: {}\nLoss: {:.4f}, Accu{:.4f}'.format(
212 | os.path.basename(data_file), run_loss, run_accu
213 | ), 'true', save=os.path.join(save_folder, '{}-confusion_matrix.png'.format(
214 | os.path.basename(data_file).split('.')[0])))
215 |
216 | print('Eval Loss: {:.4f}, Accu: {:.4f}'.format(run_loss, run_accu))
217 |
--------------------------------------------------------------------------------
/App.py:
--------------------------------------------------------------------------------
1 | import os
2 | import cv2
3 | import time
4 | import torch
5 | import screeninfo
6 | import numpy as np
7 | import tkinter as tk
8 | import matplotlib.pyplot as plt
9 | from PIL import Image, ImageTk
10 |
11 | from Detection.Utils import ResizePadding
12 | from CameraLoader import CamLoader, CamLoader_Q
13 | from DetectorLoader import TinyYOLOv3_onecls
14 |
15 | from PoseEstimateLoader import SPPE_FastPose
16 | from fn import draw_single
17 |
18 | from Track.Tracker import Detection, Tracker
19 | from ActionsEstLoader import TSSTG
20 |
21 | import matplotlib
22 | matplotlib.use('TkAgg')
23 | import matplotlib.pyplot as plt
24 | from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk
25 |
26 |
27 | def get_monitor_from_coord(x, y): # multiple monitor dealing.
28 | monitors = screeninfo.get_monitors()
29 | for m in reversed(monitors):
30 | if m.x <= x <= m.width + m.x and m.y <= y <= m.height + m.y:
31 | return m
32 | return monitors[0]
33 |
34 |
35 | class Models:
36 | def __init__(self):
37 | self.inp_dets = 416
38 | self.inp_pose = (256, 192)
39 | self.pose_backbone = 'resnet50'
40 | self.show_detected = True
41 | self.show_skeleton = True
42 | self.device = 'cuda'
43 |
44 | self.load_models()
45 |
46 | def load_models(self):
47 | self.detect_model = TinyYOLOv3_onecls(self.inp_dets, device=self.device)
48 | self.pose_model = SPPE_FastPose(self.pose_backbone, self.inp_pose[0], self.inp_pose[1],
49 | device=self.device)
50 | self.tracker = Tracker(30, n_init=3)
51 | self.action_model = TSSTG(device=self.device)
52 |
53 | def kpt2bbox(self, kpt, ex=20):
54 | return np.array((kpt[:, 0].min() - ex, kpt[:, 1].min() - ex,
55 | kpt[:, 0].max() + ex, kpt[:, 1].max() + ex))
56 |
57 | def process_frame(self, frame):
58 | detected = self.detect_model.detect(frame, need_resize=False, expand_bb=10)
59 |
60 | self.tracker.predict()
61 | for track in self.tracker.tracks:
62 | det = torch.tensor([track.to_tlbr().tolist() + [1.0, 1.0, 0.0]], dtype=torch.float32)
63 | detected = torch.cat([detected, det], dim=0) if detected is not None else det
64 |
65 | detections = []
66 | if detected is not None:
67 | poses = self.pose_model.predict(frame, detected[:, 0:4], detected[:, 4])
68 | detections = [Detection(self.kpt2bbox(ps['keypoints'].numpy()),
69 | np.concatenate((ps['keypoints'].numpy(),
70 | ps['kp_score'].numpy()), axis=1),
71 | ps['kp_score'].mean().numpy()) for ps in poses]
72 | if self.show_detected:
73 | for bb in detected[:, 0:5]:
74 | frame = cv2.rectangle(frame, (bb[0], bb[1]), (bb[2], bb[3]), (0, 0, 255), 1)
75 |
76 | self.tracker.update(detections)
77 | for i, track in enumerate(self.tracker.tracks):
78 | if not track.is_confirmed():
79 | continue
80 | track_id = track.track_id
81 | bbox = track.to_tlbr().astype(int)
82 | center = track.get_center().astype(int)
83 |
84 | action = 'pending..'
85 | clr = (0, 255, 0)
86 | if len(track.keypoints_list) == 30:
87 | pts = np.array(track.keypoints_list, dtype=np.float32)
88 | out = self.action_model.predict(pts, frame.shape[:2])
89 | action_name = self.action_model.class_names[out[0].argmax()]
90 | action = '{}: {:.2f}%'.format(action_name, out[0].max() * 100)
91 | if action_name == 'Fall Down':
92 | clr = (255, 0, 0)
93 | elif action_name == 'Lying Down':
94 | clr = (255, 200, 0)
95 |
96 | track.actions = out
97 |
98 | if track.time_since_update == 0:
99 | if self.show_skeleton:
100 | frame = draw_single(frame, track.keypoints_list[-1])
101 | frame = cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 1)
102 | frame = cv2.putText(frame, str(track_id), (center[0], center[1]), cv2.FONT_HERSHEY_DUPLEX,
103 | 0.4, (255, 0, 0), 2)
104 | frame = cv2.putText(frame, action, (bbox[0] + 5, bbox[1] + 15), cv2.FONT_HERSHEY_COMPLEX,
105 | 0.4, clr, 1)
106 |
107 | return frame
108 |
109 |
110 | class main:
111 | def __init__(self, master: tk.Tk):
112 | self.master = master
113 | self.master.title('Human Falling Detection')
114 | self.master.protocol('WM_DELETE_WINDOW', self._on_closing)
115 | self.main_screen = get_monitor_from_coord(master.winfo_x(), master.winfo_y())
116 |
117 | self.width = int(self.main_screen.width * .85)
118 | self.height = int(self.main_screen.height * .85)
119 | self.master.geometry('{}x{}'.format(self.width, self.height + 15))
120 |
121 | self.cam = None
122 | self.canvas = tk.Canvas(master, width=int(self.width * .65), height=self.height)
123 | self.canvas.grid(row=0, column=0, padx=5, pady=5, sticky=tk.NSEW)
124 |
125 | fig = plt.Figure(figsize=(6, 8), dpi=100)
126 | fig.suptitle('Actions')
127 | self.ax = fig.add_subplot(111)
128 | self.fig_canvas = FigureCanvasTkAgg(fig, self.master)
129 | self.fig_canvas.get_tk_widget().grid(row=0, column=1, padx=5, pady=5, sticky=tk.NSEW)
130 |
131 | # Load Models
132 | self.resize_fn = ResizePadding(416, 416)
133 | self.models = Models()
134 |
135 | self.actions_graph()
136 |
137 | self.delay = 15
138 | self.load_cam('../Data/falldata/Home/Videos/video (1).avi')
139 | self.update()
140 |
141 | def preproc(self, image):
142 | image = self.resize_fn(image)
143 | image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
144 | return image
145 |
146 | def load_cam(self, source):
147 | if self.cam:
148 | self.cam.__del__()
149 |
150 | if type(source) is str and os.path.isfile(source):
151 | self.cam = CamLoader_Q(source, queue_size=1000, preprocess=self.preproc).start()
152 | else:
153 | self.cam = CamLoader(source, preprocess=self.preproc).start()
154 |
155 | def actions_graph(self):
156 | if len(self.models.tracker.tracks) == 0:
157 | return
158 | track = self.models.tracker.tracks[0]
159 | if hasattr(track, 'actions'):
160 | y_labels = self.models.action_model.class_names
161 | self.ax.barh(np.arange(len(y_labels)), track.actions)
162 | self.fig_canvas.draw()
163 |
164 | def update(self):
165 | if self.cam is None:
166 | return
167 | if self.cam.grabbed():
168 | frame = self.cam.getitem()
169 |
170 | frame = self.models.process_frame(frame)
171 |
172 | frame = cv2.resize(frame, (self.canvas.winfo_width(), self.canvas.winfo_height()),
173 | interpolation=cv2.INTER_CUBIC)
174 | self.photo = ImageTk.PhotoImage(image=Image.fromarray(frame))
175 | self.canvas.create_image(0, 0, image=self.photo, anchor=tk.NW)
176 | else:
177 | self.cam.stop()
178 |
179 | self._cam = self.master.after(self.delay, self.update)
180 |
181 | def _on_closing(self):
182 | self.master.after_cancel(self._cam)
183 | if self.cam:
184 | self.cam.stop()
185 | self.cam.__del__()
186 | self.master.destroy()
187 |
188 |
189 | root = tk.Tk()
190 | app = main(root)
191 | root.mainloop()
192 |
--------------------------------------------------------------------------------
/CameraLoader.py:
--------------------------------------------------------------------------------
1 | import os
2 | import cv2
3 | import time
4 | import torch
5 | import numpy as np
6 |
7 | from queue import Queue
8 | from threading import Thread, Lock
9 |
10 |
11 | class CamLoader:
12 | """Use threading to capture a frame from camera for faster frame load.
13 | Recommend for camera or webcam.
14 |
15 | Args:
16 | camera: (int, str) Source of camera or video.,
17 | preprocess: (Callable function) to process the frame before return.
18 | """
19 | def __init__(self, camera, preprocess=None, ori_return=False):
20 | self.stream = cv2.VideoCapture(camera)
21 | assert self.stream.isOpened(), 'Cannot read camera source!'
22 | self.fps = self.stream.get(cv2.CAP_PROP_FPS)
23 | self.frame_size = (int(self.stream.get(cv2.CAP_PROP_FRAME_WIDTH)),
24 | int(self.stream.get(cv2.CAP_PROP_FRAME_HEIGHT)))
25 |
26 | self.stopped = False
27 | self.ret = False
28 | self.frame = None
29 | self.ori_frame = None
30 | self.read_lock = Lock()
31 | self.ori = ori_return
32 |
33 | self.preprocess_fn = preprocess
34 |
35 | def start(self):
36 | self.t = Thread(target=self.update, args=()) # , daemon=True)
37 | self.t.start()
38 | c = 0
39 | while not self.ret:
40 | time.sleep(0.1)
41 | c += 1
42 | if c > 20:
43 | self.stop()
44 | raise TimeoutError('Can not get a frame from camera!!!')
45 | return self
46 |
47 | def update(self):
48 | while not self.stopped:
49 | ret, frame = self.stream.read()
50 | self.read_lock.acquire()
51 | self.ori_frame = frame.copy()
52 | if ret and self.preprocess_fn is not None:
53 | frame = self.preprocess_fn(frame)
54 |
55 | self.ret, self.frame = ret, frame
56 | self.read_lock.release()
57 |
58 | def grabbed(self):
59 | """Return `True` if can read a frame."""
60 | return self.ret
61 |
62 | def getitem(self):
63 | self.read_lock.acquire()
64 | frame = self.frame.copy()
65 | ori_frame = self.ori_frame.copy()
66 | self.read_lock.release()
67 | if self.ori:
68 | return frame, ori_frame
69 | else:
70 | return frame
71 |
72 | def stop(self):
73 | if self.stopped:
74 | return
75 | self.stopped = True
76 | if self.t.is_alive():
77 | self.t.join()
78 | self.stream.release()
79 |
80 | def __del__(self):
81 | if self.stream.isOpened():
82 | self.stream.release()
83 |
84 | def __exit__(self, exc_type, exc_val, exc_tb):
85 | if self.stream.isOpened():
86 | self.stream.release()
87 |
88 |
89 | class CamLoader_Q:
90 | """Use threading and queue to capture a frame and store to queue for pickup in sequence.
91 | Recommend for video file.
92 |
93 | Args:
94 | camera: (int, str) Source of camera or video.,
95 | batch_size: (int) Number of batch frame to store in queue. Default: 1,
96 | queue_size: (int) Maximum queue size. Default: 256,
97 | preprocess: (Callable function) to process the frame before return.
98 | """
99 | def __init__(self, camera, batch_size=1, queue_size=256, preprocess=None):
100 | self.stream = cv2.VideoCapture(camera)
101 | assert self.stream.isOpened(), 'Cannot read camera source!'
102 | self.fps = self.stream.get(cv2.CAP_PROP_FPS)
103 | self.frame_size = (int(self.stream.get(cv2.CAP_PROP_FRAME_WIDTH)),
104 | int(self.stream.get(cv2.CAP_PROP_FRAME_HEIGHT)))
105 |
106 | # Queue for storing each frames.
107 |
108 | self.stopped = False
109 | self.batch_size = batch_size
110 | self.Q = Queue(maxsize=queue_size)
111 |
112 | self.preprocess_fn = preprocess
113 |
114 | def start(self):
115 | t = Thread(target=self.update, args=(), daemon=True).start()
116 | c = 0
117 | while not self.grabbed():
118 | time.sleep(0.1)
119 | c += 1
120 | if c > 20:
121 | self.stop()
122 | raise TimeoutError('Can not get a frame from camera!!!')
123 | return self
124 |
125 | def update(self):
126 | while not self.stopped:
127 | if not self.Q.full():
128 | frames = []
129 | for k in range(self.batch_size):
130 | ret, frame = self.stream.read()
131 | if not ret:
132 | self.stop()
133 | return
134 |
135 | if self.preprocess_fn is not None:
136 | frame = self.preprocess_fn(frame)
137 |
138 | frames.append(frame)
139 | frames = np.stack(frames)
140 | self.Q.put(frames)
141 | else:
142 | with self.Q.mutex:
143 | self.Q.queue.clear()
144 | # time.sleep(0.05)
145 |
146 | def grabbed(self):
147 | """Return `True` if can read a frame."""
148 | return self.Q.qsize() > 0
149 |
150 | def getitem(self):
151 | return self.Q.get().squeeze()
152 |
153 | def stop(self):
154 | if self.stopped:
155 | return
156 | self.stopped = True
157 | self.stream.release()
158 |
159 | def __len__(self):
160 | return self.Q.qsize()
161 |
162 | def __del__(self):
163 | if self.stream.isOpened():
164 | self.stream.release()
165 |
166 | def __exit__(self, exc_type, exc_val, exc_tb):
167 | if self.stream.isOpened():
168 | self.stream.release()
169 |
170 |
171 | if __name__ == '__main__':
172 | fps_time = 0
173 |
174 | # Using threading.
175 | cam = CamLoader(0).start()
176 | while cam.grabbed():
177 | frames = cam.getitem()
178 |
179 | frames = cv2.putText(frames, 'FPS: %f' % (1.0 / (time.time() - fps_time)),
180 | (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
181 | fps_time = time.time()
182 | cv2.imshow('frame', frames)
183 |
184 | if cv2.waitKey(1) & 0xFF == ord('q'):
185 | break
186 | cam.stop()
187 | cv2.destroyAllWindows()
188 |
189 | # Normal video capture.
190 | """cam = cv2.VideoCapture(0)
191 | while True:
192 | ret, frame = cam.read()
193 | if ret:
194 | #time.sleep(0.05)
195 | #frame = (cv2.flip(frame, 1) / 255.).astype(np.float)
196 |
197 | frame = cv2.putText(frame, 'FPS: %f' % (1.0 / (time.time() - fps_time)),
198 | (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
199 | fps_time = time.time()
200 | cv2.imshow('frame', frame)
201 | if cv2.waitKey(1) & 0xFF == ord('q'):
202 | break
203 | cam.release()
204 | cv2.destroyAllWindows()"""
205 |
--------------------------------------------------------------------------------
/Data/create_dataset_1.py:
--------------------------------------------------------------------------------
1 | """
2 | This script to create .csv videos frames action annotation file.
3 |
4 | - It will play a video frame by frame control the flow by [a] and [d]
5 | to play previos or next frame.
6 | - Open the annot_file (.csv) and label each frame of video with number
7 | of action class.
8 | """
9 |
10 | import os
11 | import cv2
12 | import time
13 | import numpy as np
14 | import pandas as pd
15 | import matplotlib.pyplot as plt
16 |
17 | class_names = ['Standing', 'Walking', 'Sitting', 'Lying Down',
18 | 'Stand up', 'Sit down', 'Fall Down'] # label.
19 |
20 | video_folder = '../Data/falldata/Home/Videos'
21 | annot_file = '../Data/Home_new.csv'
22 |
23 | index_video_to_play = 0 # Choose video to play.
24 |
25 |
26 | def create_csv(folder):
27 | list_file = sorted(os.listdir(folder))
28 | cols = ['video', 'frame', 'label']
29 | df = pd.DataFrame(columns=cols)
30 | for fil in list_file:
31 | cap = cv2.VideoCapture(os.path.join(folder, fil))
32 | frames_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
33 | video = np.array([fil] * frames_count)
34 | frame = np.arange(1, frames_count + 1)
35 | label = np.array([0] * frames_count)
36 | rows = np.stack([video, frame, label], axis=1)
37 | df = df.append(pd.DataFrame(rows, columns=cols),
38 | ignore_index=True)
39 | cap.release()
40 | df.to_csv(annot_file, index=False)
41 |
42 |
43 | if not os.path.exists(annot_file):
44 | create_csv(video_folder)
45 |
46 | annot = pd.read_csv(annot_file)
47 | video_list = annot.iloc[:, 0].unique()
48 | video_file = os.path.join(video_folder, video_list[index_video_to_play])
49 | print(os.path.basename(video_file))
50 |
51 | annot = annot[annot['video'] == video_list[index_video_to_play]].reset_index(drop=True)
52 | frames_idx = annot.iloc[:, 1].tolist()
53 |
54 | cap = cv2.VideoCapture(video_file)
55 | frames_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
56 |
57 | assert frames_count == len(frames_idx), 'frame count not equal! {} and {}'.format(
58 | len(frames_idx), frames_count
59 | )
60 |
61 | i = 0
62 | while True:
63 | cap.set(cv2.CAP_PROP_POS_FRAMES, i)
64 | ret, frame = cap.read()
65 | if ret:
66 | cls_name = class_names[int(annot.iloc[i, -1]) - 1]
67 | frame = cv2.resize(frame, (0, 0), fx=1.5, fy=1.5)
68 | frame = cv2.putText(frame, 'Frame: {} Pose: {}'.format(i+1, cls_name),
69 | (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
70 | cv2.imshow('frame', frame)
71 |
72 | key = cv2.waitKey(0) & 0xFF
73 | if key == ord('q'):
74 | break
75 | elif key == ord('d'):
76 | i += 1
77 | continue
78 | elif key == ord('a'):
79 | i -= 1
80 | continue
81 | else:
82 | break
83 |
84 | cap.release()
85 | cv2.destroyAllWindows()
86 |
--------------------------------------------------------------------------------
/Data/create_dataset_2.py:
--------------------------------------------------------------------------------
1 | """
2 | This script to extract skeleton joints position and score.
3 |
4 | - This 'annot_folder' is a action class and bounding box for each frames that came with dataset.
5 | Should be in format of [frame_idx, action_cls, xmin, ymin, xmax, ymax]
6 | Use for crop a person to use in pose estimation model.
7 | - If have no annotation file you can leave annot_folder = '' for use Detector model to get the
8 | bounding box.
9 | """
10 |
11 | import os
12 | import cv2
13 | import time
14 | import torch
15 | import pandas as pd
16 | import numpy as np
17 | import torchvision.transforms as transforms
18 |
19 | from DetectorLoader import TinyYOLOv3_onecls
20 | from PoseEstimateLoader import SPPE_FastPose
21 | from fn import vis_frame_fast
22 |
23 | save_path = '../../Data/Home_new-pose+score.csv'
24 |
25 | annot_file = '../../Data/Home_new.csv' # from create_dataset_1.py
26 | video_folder = '../Data/falldata/Home/Videos'
27 | annot_folder = '../Data/falldata/Home/Annotation_files' # bounding box annotation for each frame.
28 |
29 | # DETECTION MODEL.
30 | detector = TinyYOLOv3_onecls()
31 |
32 | # POSE MODEL.
33 | inp_h = 320
34 | inp_w = 256
35 | pose_estimator = SPPE_FastPose(inp_h, inp_w)
36 |
37 | # with score.
38 | columns = ['video', 'frame', 'Nose_x', 'Nose_y', 'Nose_s', 'LShoulder_x', 'LShoulder_y', 'LShoulder_s',
39 | 'RShoulder_x', 'RShoulder_y', 'RShoulder_s', 'LElbow_x', 'LElbow_y', 'LElbow_s', 'RElbow_x',
40 | 'RElbow_y', 'RElbow_s', 'LWrist_x', 'LWrist_y', 'LWrist_s', 'RWrist_x', 'RWrist_y', 'RWrist_s',
41 | 'LHip_x', 'LHip_y', 'LHip_s', 'RHip_x', 'RHip_y', 'RHip_s', 'LKnee_x', 'LKnee_y', 'LKnee_s',
42 | 'RKnee_x', 'RKnee_y', 'RKnee_s', 'LAnkle_x', 'LAnkle_y', 'LAnkle_s', 'RAnkle_x', 'RAnkle_y',
43 | 'RAnkle_s', 'label']
44 |
45 |
46 | def normalize_points_with_size(points_xy, width, height, flip=False):
47 | points_xy[:, 0] /= width
48 | points_xy[:, 1] /= height
49 | if flip:
50 | points_xy[:, 0] = 1 - points_xy[:, 0]
51 | return points_xy
52 |
53 |
54 | annot = pd.read_csv(annot_file)
55 | vid_list = annot['video'].unique()
56 | for vid in vid_list:
57 | print(f'Process on: {vid}')
58 | df = pd.DataFrame(columns=columns)
59 | cur_row = 0
60 |
61 | # Pose Labels.
62 | frames_label = annot[annot['video'] == vid].reset_index(drop=True)
63 |
64 | cap = cv2.VideoCapture(os.path.join(video_folder, vid))
65 | frames_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
66 | frame_size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
67 | int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)))
68 |
69 | # Bounding Boxs Labels.
70 | annot_file = os.path.join(annot_folder, vid.split('.')[0], '.txt')
71 | annot = None
72 | if os.path.exists(annot_file):
73 | annot = pd.read_csv(annot_file, header=None,
74 | names=['frame_idx', 'class', 'xmin', 'ymin', 'xmax', 'ymax'])
75 | annot = annot.dropna().reset_index(drop=True)
76 |
77 | assert frames_count == len(annot), 'frame count not equal! {} and {}'.format(frames_count, len(annot))
78 |
79 | fps_time = 0
80 | i = 1
81 | while True:
82 | ret, frame = cap.read()
83 | if ret:
84 | frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
85 | cls_idx = int(frames_label[frames_label['frame'] == i]['label'])
86 |
87 | if annot:
88 | bb = np.array(annot.iloc[i-1, 2:].astype(int))
89 | else:
90 | bb = detector.detect(frame)[0, :4].numpy().astype(int)
91 | bb[:2] = np.maximum(0, bb[:2] - 5)
92 | bb[2:] = np.minimum(frame_size, bb[2:] + 5) if bb[2:].any() != 0 else bb[2:]
93 |
94 | result = []
95 | if bb.any() != 0:
96 | result = pose_estimator.predict(frame, torch.tensor(bb[None, ...]),
97 | torch.tensor([[1.0]]))
98 |
99 | if len(result) > 0:
100 | pt_norm = normalize_points_with_size(result[0]['keypoints'].numpy().copy(),
101 | frame_size[0], frame_size[1])
102 | pt_norm = np.concatenate((pt_norm, result[0]['kp_score']), axis=1)
103 |
104 | #idx = result[0]['kp_score'] <= 0.05
105 | #pt_norm[idx.squeeze()] = np.nan
106 | row = [vid, i, *pt_norm.flatten().tolist(), cls_idx]
107 | scr = result[0]['kp_score'].mean()
108 | else:
109 | row = [vid, i, *[np.nan] * (13 * 3), cls_idx]
110 | scr = 0.0
111 |
112 | df.loc[cur_row] = row
113 | cur_row += 1
114 |
115 | # VISUALIZE.
116 | frame = vis_frame_fast(frame, result)
117 | frame = cv2.rectangle(frame, (bb[0], bb[1]), (bb[2], bb[3]), (0, 255, 0), 2)
118 | frame = cv2.putText(frame, 'Frame: {}, Pose: {}, Score: {:.4f}'.format(i, cls_idx, scr),
119 | (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
120 | frame = frame[:, :, ::-1]
121 | fps_time = time.time()
122 | i += 1
123 |
124 | cv2.imshow('frame', frame)
125 | if cv2.waitKey(1) & 0xFF == ord('q'):
126 | break
127 | else:
128 | break
129 |
130 | cap.release()
131 | cv2.destroyAllWindows()
132 |
133 | if os.path.exists(save_path):
134 | df.to_csv(save_path, mode='a', header=False, index=False)
135 | else:
136 | df.to_csv(save_path, mode='w', index=False)
137 |
138 |
--------------------------------------------------------------------------------
/Data/create_dataset_3.py:
--------------------------------------------------------------------------------
1 | """
2 | This script to create dataset and labels by clean off some NaN, do a normalization,
3 | label smoothing and label weights by scores.
4 |
5 | """
6 | import os
7 | import pickle
8 | import numpy as np
9 | import pandas as pd
10 |
11 |
12 | class_names = ['Standing', 'Walking', 'Sitting', 'Lying Down',
13 | 'Stand up', 'Sit down', 'Fall Down']
14 | main_parts = ['LShoulder_x', 'LShoulder_y', 'RShoulder_x', 'RShoulder_y', 'LHip_x', 'LHip_y',
15 | 'RHip_x', 'RHip_y']
16 | main_idx_parts = [1, 2, 7, 8, -1] # 1.5
17 |
18 | csv_pose_file = '../Data/Coffee_room_new-pose+score.csv'
19 | save_path = '../../Data/Coffee_room_new-set(labelXscrw).pkl'
20 |
21 | # Params.
22 | smooth_labels_step = 8
23 | n_frames = 30
24 | skip_frame = 1
25 |
26 | annot = pd.read_csv(csv_pose_file)
27 |
28 | # Remove NaN.
29 | idx = annot.iloc[:, 2:-1][main_parts].isna().sum(1) > 0
30 | idx = np.where(idx)[0]
31 | annot = annot.drop(idx)
32 | # One-Hot Labels.
33 | label_onehot = pd.get_dummies(annot['label'])
34 | annot = annot.drop('label', axis=1).join(label_onehot)
35 | cols = label_onehot.columns.values
36 |
37 |
38 | def scale_pose(xy):
39 | """
40 | Normalize pose points by scale with max/min value of each pose.
41 | xy : (frames, parts, xy) or (parts, xy)
42 | """
43 | if xy.ndim == 2:
44 | xy = np.expand_dims(xy, 0)
45 | xy_min = np.nanmin(xy, axis=1)
46 | xy_max = np.nanmax(xy, axis=1)
47 | for i in range(xy.shape[0]):
48 | xy[i] = ((xy[i] - xy_min[i]) / (xy_max[i] - xy_min[i])) * 2 - 1
49 | return xy.squeeze()
50 |
51 |
52 | def seq_label_smoothing(labels, max_step=10):
53 | steps = 0
54 | remain_step = 0
55 | target_label = 0
56 | active_label = 0
57 | start_change = 0
58 | max_val = np.max(labels)
59 | min_val = np.min(labels)
60 | for i in range(labels.shape[0]):
61 | if remain_step > 0:
62 | if i >= start_change:
63 | labels[i][active_label] = max_val * remain_step / steps
64 | labels[i][target_label] = max_val * (steps - remain_step) / steps \
65 | if max_val * (steps - remain_step) / steps else min_val
66 | remain_step -= 1
67 | continue
68 |
69 | diff_index = np.where(np.argmax(labels[i:i+max_step], axis=1) - np.argmax(labels[i]) != 0)[0]
70 | if len(diff_index) > 0:
71 | start_change = i + remain_step // 2
72 | steps = diff_index[0]
73 | remain_step = steps
74 | target_label = np.argmax(labels[i + remain_step])
75 | active_label = np.argmax(labels[i])
76 | return labels
77 |
78 |
79 | feature_set = np.empty((0, n_frames, 14, 3))
80 | labels_set = np.empty((0, len(cols)))
81 | vid_list = annot['video'].unique()
82 | for vid in vid_list:
83 | print(f'Process on: {vid}')
84 | data = annot[annot['video'] == vid].reset_index(drop=True).drop(columns='video')
85 |
86 | # Label Smoothing.
87 | esp = 0.1
88 | data[cols] = data[cols] * (1 - esp) + (1 - data[cols]) * esp / (len(cols) - 1)
89 | data[cols] = seq_label_smoothing(data[cols].values, smooth_labels_step)
90 |
91 | # Separate continuous frames.
92 | frames = data['frame'].values
93 | frames_set = []
94 | fs = [0]
95 | for i in range(1, len(frames)):
96 | if frames[i] < frames[i-1] + 10:
97 | fs.append(i)
98 | else:
99 | frames_set.append(fs)
100 | fs = [i]
101 | frames_set.append(fs)
102 |
103 | for fs in frames_set:
104 | xys = data.iloc[fs, 1:-len(cols)].values.reshape(-1, 13, 3)
105 | # Scale pose normalize.
106 | xys[:, :, :2] = scale_pose(xys[:, :, :2])
107 | # Add center point.
108 | xys = np.concatenate((xys, np.expand_dims((xys[:, 1, :] + xys[:, 2, :]) / 2, 1)), axis=1)
109 |
110 | # Weighting main parts score.
111 | scr = xys[:, :, -1].copy()
112 | scr[:, main_idx_parts] = np.minimum(scr[:, main_idx_parts] * 1.5, 1.0)
113 | # Mean score.
114 | scr = scr.mean(1)
115 |
116 | # Targets.
117 | lb = data.iloc[fs, -len(cols):].values
118 | # Apply points score mean to all labels.
119 | lb = lb * scr[:, None]
120 |
121 | for i in range(xys.shape[0] - n_frames):
122 | feature_set = np.append(feature_set, xys[i:i+n_frames][None, ...], axis=0)
123 | labels_set = np.append(labels_set, lb[i:i+n_frames].mean(0)[None, ...], axis=0)
124 |
125 |
126 | """with open(save_path, 'wb') as f:
127 | pickle.dump((feature_set, labels_set), f)"""
128 |
--------------------------------------------------------------------------------
/Detection/Utils.py:
--------------------------------------------------------------------------------
1 | import cv2
2 | import math
3 | import time
4 | import tqdm
5 | import torch
6 | import torch.nn as nn
7 | import torch.nn.functional as F
8 | import numpy as np
9 | from torch.utils.data import DataLoader
10 |
11 |
12 | def to_cpu(tensor):
13 | return tensor.detach().cpu()
14 |
15 |
16 | def load_classes(path):
17 | """
18 | Loads class labels at 'path'
19 | """
20 | fp = open(path, "r")
21 | names = fp.read().split("\n")[:-1]
22 | return names
23 |
24 |
25 | def weights_init_normal(m):
26 | classname = m.__class__.__name__
27 | if classname.find("Conv") != -1:
28 | torch.nn.init.normal_(m.weight.data, 0.0, 0.02)
29 | elif classname.find("BatchNorm2d") != -1:
30 | torch.nn.init.normal_(m.weight.data, 1.0, 0.02)
31 | torch.nn.init.constant_(m.bias.data, 0.0)
32 |
33 |
34 | def rescale_boxes(boxes, current_dim, original_shape):
35 | """ Rescales bounding boxes to the original shape """
36 | orig_h, orig_w = original_shape
37 | # The amount of padding that was added
38 | pad_x = max(orig_h - orig_w, 0) * (current_dim / max(original_shape))
39 | pad_y = max(orig_w - orig_h, 0) * (current_dim / max(original_shape))
40 | # Image height and width after padding is removed
41 | unpad_h = current_dim - pad_y
42 | unpad_w = current_dim - pad_x
43 | # Rescale bounding boxes to dimension of original image
44 | boxes[:, 0] = ((boxes[:, 0] - pad_x // 2) / unpad_w) * orig_w
45 | boxes[:, 1] = ((boxes[:, 1] - pad_y // 2) / unpad_h) * orig_h
46 | boxes[:, 2] = ((boxes[:, 2] - pad_x // 2) / unpad_w) * orig_w
47 | boxes[:, 3] = ((boxes[:, 3] - pad_y // 2) / unpad_h) * orig_h
48 | return boxes
49 |
50 |
51 | def xywh2xyxy(x):
52 | y = x.new(x.shape)
53 | y[..., 0] = x[..., 0] - x[..., 2] / 2
54 | y[..., 1] = x[..., 1] - x[..., 3] / 2
55 | y[..., 2] = x[..., 0] + x[..., 2] / 2
56 | y[..., 3] = x[..., 1] + x[..., 3] / 2
57 | return y
58 |
59 |
60 | def ap_per_class(tp, conf, pred_cls, target_cls):
61 | """ Compute the average precision, given the recall and precision curves.
62 | Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
63 | # Arguments
64 | tp: True positives (list).
65 | conf: Objectness value from 0-1 (list).
66 | pred_cls: Predicted object classes (list).
67 | target_cls: True object classes (list).
68 | # Returns
69 | The average precision as computed in py-faster-rcnn.
70 | """
71 | # Sort by objectness
72 | i = np.argsort(-conf)
73 | tp, conf, pred_cls = tp[i], conf[i], pred_cls[i]
74 |
75 | # Find unique classes
76 | unique_classes = np.unique(target_cls)
77 |
78 | # Create Precision-Recall curve and compute AP for each class
79 | ap, p, r = [], [], []
80 | for c in tqdm.tqdm(unique_classes, desc="Computing AP"):
81 | i = pred_cls == c
82 | n_gt = (target_cls == c).sum() # Number of ground truth objects
83 | n_p = i.sum() # Number of predicted objects
84 |
85 | if n_p == 0 and n_gt == 0:
86 | continue
87 | elif n_p == 0 or n_gt == 0:
88 | ap.append(0)
89 | r.append(0)
90 | p.append(0)
91 | else:
92 | # Accumulate FPs and TPs
93 | fpc = (1 - tp[i]).cumsum()
94 | tpc = (tp[i]).cumsum()
95 |
96 | # Recall
97 | recall_curve = tpc / (n_gt + 1e-16)
98 | r.append(recall_curve[-1])
99 |
100 | # Precision
101 | precision_curve = tpc / (tpc + fpc)
102 | p.append(precision_curve[-1])
103 |
104 | # AP from recall-precision curve
105 | ap.append(compute_ap(recall_curve, precision_curve))
106 |
107 | # Compute F1 score (harmonic mean of precision and recall)
108 | p, r, ap = np.array(p), np.array(r), np.array(ap)
109 | f1 = 2 * p * r / (p + r + 1e-16)
110 |
111 | return p, r, ap, f1, unique_classes.astype("int32")
112 |
113 |
114 | def compute_ap(recall, precision):
115 | """ Compute the average precision, given the recall and precision curves.
116 | Code originally from https://github.com/rbgirshick/py-faster-rcnn.
117 | # Arguments
118 | recall: The recall curve (list).
119 | precision: The precision curve (list).
120 | # Returns
121 | The average precision as computed in py-faster-rcnn.
122 | """
123 | # correct AP calculation
124 | # first append sentinel values at the end
125 | mrec = np.concatenate(([0.0], recall, [1.0]))
126 | mpre = np.concatenate(([0.0], precision, [0.0]))
127 |
128 | # compute the precision envelope
129 | for i in range(mpre.size - 1, 0, -1):
130 | mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
131 |
132 | # to calculate area under PR curve, look for points
133 | # where X axis (recall) changes value
134 | i = np.where(mrec[1:] != mrec[:-1])[0]
135 |
136 | # and sum (\Delta recall) * prec
137 | ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
138 | return ap
139 |
140 |
141 | def get_batch_statistics(outputs, targets, iou_threshold):
142 | """ Compute true positives, predicted scores and predicted labels per sample """
143 | batch_metrics = []
144 | for sample_i in range(len(outputs)):
145 |
146 | if outputs[sample_i] is None:
147 | continue
148 |
149 | output = outputs[sample_i]
150 | pred_boxes = output[:, :4]
151 | pred_scores = output[:, 4]
152 | pred_labels = output[:, -1]
153 |
154 | true_positives = np.zeros(pred_boxes.shape[0])
155 |
156 | annotations = targets[targets[:, 0] == sample_i][:, 1:]
157 | target_labels = annotations[:, 0] if len(annotations) else []
158 | if len(annotations):
159 | detected_boxes = []
160 | target_boxes = annotations[:, 1:]
161 |
162 | for pred_i, (pred_box, pred_label) in enumerate(zip(pred_boxes, pred_labels)):
163 |
164 | # If targets are found break
165 | if len(detected_boxes) == len(annotations):
166 | break
167 |
168 | # Ignore if label is not one of the target labels
169 | if pred_label not in target_labels:
170 | continue
171 |
172 | iou, box_index = bbox_iou(pred_box.unsqueeze(0), target_boxes).max(0)
173 | if iou >= iou_threshold and box_index not in detected_boxes:
174 | true_positives[pred_i] = 1
175 | detected_boxes += [box_index]
176 | batch_metrics.append([true_positives, pred_scores, pred_labels])
177 | return batch_metrics
178 |
179 |
180 | def bbox_wh_iou(wh1, wh2):
181 | wh2 = wh2.t()
182 | w1, h1 = wh1[0], wh1[1]
183 | w2, h2 = wh2[0], wh2[1]
184 | inter_area = torch.min(w1, w2) * torch.min(h1, h2)
185 | union_area = (w1 * h1 + 1e-16) + w2 * h2 - inter_area
186 | return inter_area / union_area
187 |
188 |
189 | def bbox_iou(box1, box2, x1y1x2y2=True):
190 | """
191 | Returns the IoU of two bounding boxes
192 | """
193 | if not x1y1x2y2:
194 | # Transform from center and width to exact coordinates
195 | b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2
196 | b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2
197 | b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2
198 | b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2
199 | else:
200 | # Get the coordinates of bounding boxes
201 | b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3]
202 | b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3]
203 |
204 | # get the corrdinates of the intersection rectangle
205 | inter_rect_x1 = torch.max(b1_x1, b2_x1)
206 | inter_rect_y1 = torch.max(b1_y1, b2_y1)
207 | inter_rect_x2 = torch.min(b1_x2, b2_x2)
208 | inter_rect_y2 = torch.min(b1_y2, b2_y2)
209 | # Intersection area
210 | inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp(
211 | inter_rect_y2 - inter_rect_y1 + 1, min=0
212 | )
213 | # Union Area
214 | b1_area = (b1_x2 - b1_x1 + 1) * (b1_y2 - b1_y1 + 1)
215 | b2_area = (b2_x2 - b2_x1 + 1) * (b2_y2 - b2_y1 + 1)
216 |
217 | iou = inter_area / (b1_area + b2_area - inter_area + 1e-16)
218 |
219 | return iou
220 |
221 |
222 | def non_max_suppression(prediction, conf_thres=0.5, nms_thres=0.4):
223 | """
224 | Removes detections with lower object confidence score than 'conf_thres' and performs
225 | Non-Maximum Suppression to further filter detections.
226 | Returns detections with shape:
227 | (x1, y1, x2, y2, object_conf, class_score, class_pred)
228 | """
229 | # From (center x, center y, width, height) to (x1, y1, x2, y2)
230 | prediction[..., :4] = xywh2xyxy(prediction[..., :4])
231 | output = [None for _ in range(len(prediction))]
232 | for image_i, image_pred in enumerate(prediction):
233 | # Filter out confidence scores below threshold
234 | image_pred = image_pred[image_pred[:, 4] >= conf_thres]
235 | # If none are remaining => process next image
236 | if not image_pred.size(0):
237 | continue
238 | # Object confidence times class confidence
239 | score = image_pred[:, 4] * image_pred[:, 5:].max(1)[0]
240 | # Sort by it
241 | image_pred = image_pred[(-score).argsort()]
242 | class_confs, class_preds = image_pred[:, 5:].max(1, keepdim=True)
243 | detections = torch.cat((image_pred[:, :5], class_confs.float(), class_preds.float()), 1)
244 | # Perform non-maximum suppression
245 | keep_boxes = []
246 | while detections.size(0):
247 | large_overlap = bbox_iou(detections[0, :4].unsqueeze(0), detections[:, :4]) > nms_thres
248 | label_match = detections[0, -1] == detections[:, -1]
249 | # Indices of boxes with lower confidence scores, large IOUs and matching labels
250 | invalid = large_overlap & label_match
251 | weights = detections[invalid, 4:5]
252 | # Merge overlapping bboxes by order of confidence
253 | detections[0, :4] = (weights * detections[invalid, :4]).sum(0) / weights.sum()
254 | keep_boxes += [detections[0]]
255 | detections = detections[~invalid]
256 | if keep_boxes:
257 | output[image_i] = torch.stack(keep_boxes)
258 |
259 | return output
260 |
261 |
262 | def build_targets(pred_boxes, pred_cls, target, anchors, ignore_thres):
263 | ByteTensor = torch.cuda.ByteTensor if pred_boxes.is_cuda else torch.ByteTensor
264 | FloatTensor = torch.cuda.FloatTensor if pred_boxes.is_cuda else torch.FloatTensor
265 |
266 | nB = pred_boxes.size(0)
267 | nA = pred_boxes.size(1)
268 | nC = pred_cls.size(-1)
269 | nG = pred_boxes.size(2)
270 |
271 | # Output tensors
272 | obj_mask = ByteTensor(nB, nA, nG, nG).fill_(0)
273 | noobj_mask = ByteTensor(nB, nA, nG, nG).fill_(1)
274 | class_mask = FloatTensor(nB, nA, nG, nG).fill_(0)
275 | iou_scores = FloatTensor(nB, nA, nG, nG).fill_(0)
276 | tx = FloatTensor(nB, nA, nG, nG).fill_(0)
277 | ty = FloatTensor(nB, nA, nG, nG).fill_(0)
278 | tw = FloatTensor(nB, nA, nG, nG).fill_(0)
279 | th = FloatTensor(nB, nA, nG, nG).fill_(0)
280 | tcls = FloatTensor(nB, nA, nG, nG, nC).fill_(0)
281 |
282 | # Convert to position relative to box
283 | target_boxes = target[:, 2:6] * nG
284 | gxy = target_boxes[:, :2]
285 | gwh = target_boxes[:, 2:]
286 | # Get anchors with best iou
287 | ious = torch.stack([bbox_wh_iou(anchor, gwh) for anchor in anchors])
288 | best_ious, best_n = ious.max(0)
289 | # Separate target values
290 | b, target_labels = target[:, :2].long().t()
291 | gx, gy = gxy.t()
292 | gw, gh = gwh.t()
293 | gi, gj = gxy.long().t()
294 | # Set masks
295 | obj_mask[b, best_n, gj, gi] = 1
296 | noobj_mask[b, best_n, gj, gi] = 0
297 |
298 | # Set noobj mask to zero where iou exceeds ignore threshold
299 | for i, anchor_ious in enumerate(ious.t()):
300 | noobj_mask[b[i], anchor_ious > ignore_thres, gj[i], gi[i]] = 0
301 |
302 | # Coordinates
303 | tx[b, best_n, gj, gi] = gx - gx.floor()
304 | ty[b, best_n, gj, gi] = gy - gy.floor()
305 | # Width and height
306 | tw[b, best_n, gj, gi] = torch.log(gw / anchors[best_n][:, 0] + 1e-16)
307 | th[b, best_n, gj, gi] = torch.log(gh / anchors[best_n][:, 1] + 1e-16)
308 | # One-hot encoding of label
309 | tcls[b, best_n, gj, gi, target_labels] = 1
310 | # Compute label correctness and iou at best anchor
311 | class_mask[b, best_n, gj, gi] = (pred_cls[b, best_n, gj, gi].argmax(-1) == target_labels).float()
312 | iou_scores[b, best_n, gj, gi] = bbox_iou(pred_boxes[b, best_n, gj, gi], target_boxes, x1y1x2y2=False)
313 |
314 | tconf = obj_mask.float()
315 | return iou_scores, class_mask, obj_mask, noobj_mask, tx, ty, tw, th, tcls, tconf
316 |
317 |
318 | def parse_model_config(path):
319 | """Parses the yolo-v3 layer configuration file and returns module definitions"""
320 | file = open(path, 'r')
321 | lines = file.read().split('\n')
322 | lines = [x for x in lines if x and not x.startswith('#')]
323 | lines = [x.rstrip().lstrip() for x in lines] # get rid of fringe whitespaces
324 | module_defs = []
325 | for line in lines:
326 | if line.startswith('['): # This marks the start of a new block
327 | module_defs.append({})
328 | module_defs[-1]['type'] = line[1:-1].rstrip()
329 | if module_defs[-1]['type'] == 'convolutional':
330 | module_defs[-1]['batch_normalize'] = 0
331 | else:
332 | key, value = line.split("=")
333 | value = value.strip()
334 | module_defs[-1][key.rstrip()] = value.strip()
335 |
336 | return module_defs
337 |
338 |
339 | def parse_data_config(path):
340 | """Parses the data configuration file"""
341 | options = dict()
342 | options['gpus'] = '0,1,2,3'
343 | options['num_workers'] = '10'
344 | with open(path, 'r') as fp:
345 | lines = fp.readlines()
346 | for line in lines:
347 | line = line.strip()
348 | if line == '' or line.startswith('#'):
349 | continue
350 | key, value = line.split('=')
351 | options[key.strip()] = value.strip()
352 | return options
353 |
354 |
355 | def ResizePadding(height, width):
356 | desized_size = (height, width)
357 |
358 | def resizePadding(image, **kwargs):
359 | old_size = image.shape[:2]
360 | max_size_idx = old_size.index(max(old_size))
361 | ratio = float(desized_size[max_size_idx]) / max(old_size)
362 | new_size = tuple([int(x * ratio) for x in old_size])
363 |
364 | if new_size > desized_size:
365 | min_size_idx = old_size.index(min(old_size))
366 | ratio = float(desized_size[min_size_idx]) / min(old_size)
367 | new_size = tuple([int(x * ratio) for x in old_size])
368 |
369 | image = cv2.resize(image, (new_size[1], new_size[0]))
370 | delta_w = desized_size[1] - new_size[1]
371 | delta_h = desized_size[0] - new_size[0]
372 | top, bottom = delta_h // 2, delta_h - (delta_h // 2)
373 | left, right = delta_w // 2, delta_w - (delta_w // 2)
374 |
375 | image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT)
376 | return image
377 | return resizePadding
378 |
379 |
380 | class AverageValueMeter(object):
381 | def __init__(self):
382 | self.reset()
383 | self.val = 0
384 |
385 | def add(self, value, n=1):
386 | self.val = value
387 | self.sum += value
388 | self.var += value * value
389 | self.n += n
390 |
391 | if self.n == 0:
392 | self.mean, self.std = np.nan, np.nan
393 | elif self.n == 1:
394 | self.mean = 0.0 + self.sum # This is to force a copy in torch/numpy
395 | self.std = np.inf
396 | self.mean_old = self.mean
397 | self.m_s = 0.0
398 | else:
399 | self.mean = self.mean_old + (value - n * self.mean_old) / float(self.n)
400 | self.m_s += (value - self.mean_old) * (value - self.mean)
401 | self.mean_old = self.mean
402 | self.std = np.sqrt(self.m_s / (self.n - 1.0))
403 |
404 | def value(self):
405 | return self.mean, self.std
406 |
407 | def reset(self):
408 | self.n = 0
409 | self.sum = 0.0
410 | self.var = 0.0
411 | self.val = 0.0
412 | self.mean = np.nan
413 | self.mean_old = 0.0
414 | self.m_s = 0.0
415 | self.std = np.nan
416 |
--------------------------------------------------------------------------------
/DetectorLoader.py:
--------------------------------------------------------------------------------
1 | import time
2 | import torch
3 | import numpy as np
4 | import torchvision.transforms as transforms
5 |
6 | from queue import Queue
7 | from threading import Thread
8 |
9 | from Detection.Models import Darknet
10 | from Detection.Utils import non_max_suppression, ResizePadding
11 |
12 |
13 | class TinyYOLOv3_onecls(object):
14 | """Load trained Tiny-YOLOv3 one class (person) detection model.
15 | Args:
16 | input_size: (int) Size of input image must be divisible by 32. Default: 416,
17 | config_file: (str) Path to Yolo model structure config file.,
18 | weight_file: (str) Path to trained weights file.,
19 | nms: (float) Non-Maximum Suppression overlap threshold.,
20 | conf_thres: (float) Minimum Confidence threshold of predicted bboxs to cut off.,
21 | device: (str) Device to load the model on 'cpu' or 'cuda'.
22 | """
23 | def __init__(self,
24 | input_size=416,
25 | config_file='Models/yolo-tiny-onecls/yolov3-tiny-onecls.cfg',
26 | weight_file='Models/yolo-tiny-onecls/best-model.pth',
27 | nms=0.2,
28 | conf_thres=0.45,
29 | device='cuda'):
30 | self.input_size = input_size
31 | self.model = Darknet(config_file).to(device)
32 | self.model.load_state_dict(torch.load(weight_file))
33 | self.model.eval()
34 | self.device = device
35 |
36 | self.nms = nms
37 | self.conf_thres = conf_thres
38 |
39 | self.resize_fn = ResizePadding(input_size, input_size)
40 | self.transf_fn = transforms.ToTensor()
41 |
42 | def detect(self, image, need_resize=True, expand_bb=5):
43 | """Feed forward to the model.
44 | Args:
45 | image: (numpy array) Single RGB image to detect.,
46 | need_resize: (bool) Resize to input_size before feed and will return bboxs
47 | with scale to image original size.,
48 | expand_bb: (int) Expand boundary of the boxs.
49 | Returns:
50 | (torch.float32) Of each detected object contain a
51 | [top, left, bottom, right, bbox_score, class_score, class]
52 | return `None` if no detected.
53 | """
54 | image_size = (self.input_size, self.input_size)
55 | if need_resize:
56 | image_size = image.shape[:2]
57 | image = self.resize_fn(image)
58 |
59 | image = self.transf_fn(image)[None, ...]
60 | scf = torch.min(self.input_size / torch.FloatTensor([image_size]), 1)[0]
61 |
62 | detected = self.model(image.to(self.device))
63 | detected = non_max_suppression(detected, self.conf_thres, self.nms)[0]
64 | if detected is not None:
65 | detected[:, [0, 2]] -= (self.input_size - scf * image_size[1]) / 2
66 | detected[:, [1, 3]] -= (self.input_size - scf * image_size[0]) / 2
67 | detected[:, 0:4] /= scf
68 |
69 | detected[:, 0:2] = np.maximum(0, detected[:, 0:2] - expand_bb)
70 | detected[:, 2:4] = np.minimum(image_size[::-1], detected[:, 2:4] + expand_bb)
71 |
72 | return detected
73 |
74 |
75 | class ThreadDetection(object):
76 | def __init__(self,
77 | dataloader,
78 | model,
79 | queue_size=256):
80 | self.model = model
81 |
82 | self.dataloader = dataloader
83 | self.stopped = False
84 | self.Q = Queue(maxsize=queue_size)
85 |
86 | def start(self):
87 | t = Thread(target=self.update, args=(), daemon=True).start()
88 | return self
89 |
90 | def update(self):
91 | while True:
92 | if self.stopped:
93 | return
94 |
95 | images = self.dataloader.getitem()
96 |
97 | outputs = self.model.detect(images)
98 |
99 | if self.Q.full():
100 | time.sleep(2)
101 | self.Q.put((images, outputs))
102 |
103 | def getitem(self):
104 | return self.Q.get()
105 |
106 | def stop(self):
107 | self.stopped = True
108 |
109 | def __len__(self):
110 | return self.Q.qsize()
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
--------------------------------------------------------------------------------
/Models/TSSTG/_.txt:
--------------------------------------------------------------------------------
1 | tsstg-model.pth
--------------------------------------------------------------------------------
/Models/sppe/_.txt:
--------------------------------------------------------------------------------
1 | fast_res50_256x192.pth
2 | fast_res101_320x256.pth
--------------------------------------------------------------------------------
/Models/yolo-tiny-onecls/_.txt:
--------------------------------------------------------------------------------
1 | best-model.pth
2 | yolov3-tiny-onecls.cfg
--------------------------------------------------------------------------------
/PoseEstimateLoader.py:
--------------------------------------------------------------------------------
1 | import os
2 | import cv2
3 | import torch
4 |
5 | from SPPE.src.main_fast_inference import InferenNet_fast, InferenNet_fastRes50
6 | from SPPE.src.utils.img import crop_dets
7 | from pPose_nms import pose_nms
8 | from SPPE.src.utils.eval import getPrediction
9 |
10 |
11 | class SPPE_FastPose(object):
12 | def __init__(self,
13 | backbone,
14 | input_height=320,
15 | input_width=256,
16 | device='cuda'):
17 | assert backbone in ['resnet50', 'resnet101'], '{} backbone is not support yet!'.format(backbone)
18 |
19 | self.inp_h = input_height
20 | self.inp_w = input_width
21 | self.device = device
22 |
23 | if backbone == 'resnet101':
24 | self.model = InferenNet_fast().to(device)
25 | else:
26 | self.model = InferenNet_fastRes50().to(device)
27 | self.model.eval()
28 |
29 | def predict(self, image, bboxs, bboxs_scores):
30 | inps, pt1, pt2 = crop_dets(image, bboxs, self.inp_h, self.inp_w)
31 | pose_hm = self.model(inps.to(self.device)).cpu().data
32 |
33 | # Cut eyes and ears.
34 | pose_hm = torch.cat([pose_hm[:, :1, ...], pose_hm[:, 5:, ...]], dim=1)
35 |
36 | xy_hm, xy_img, scores = getPrediction(pose_hm, pt1, pt2, self.inp_h, self.inp_w,
37 | pose_hm.shape[-2], pose_hm.shape[-1])
38 | result = pose_nms(bboxs, bboxs_scores, xy_img, scores)
39 | return result
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Human Falling Detection and Tracking
2 |
3 | Using Tiny-YOLO oneclass to detect each person in the frame and use
4 | [AlphaPose](https://github.com/MVIG-SJTU/AlphaPose) to get skeleton-pose and then use
5 | [ST-GCN](https://github.com/yysijie/st-gcn) model to predict action from every 30 frames
6 | of each person tracks.
7 |
8 | Which now support 7 actions: Standing, Walking, Sitting, Lying Down, Stand up, Sit down, Fall Down.
9 |
10 |
11 |
12 |