├── model └── keras │ └── readme.md ├── sample_images ├── ski.jpg ├── ski2.jpg └── ski副本.jpg ├── README.md ├── config ├── config_reader.py ├── util.py ├── model.py └── demo_camera.py /model/keras/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sanfooh/camera-openpose-keras/HEAD/model/keras/readme.md -------------------------------------------------------------------------------- /sample_images/ski.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sanfooh/camera-openpose-keras/HEAD/sample_images/ski.jpg -------------------------------------------------------------------------------- /sample_images/ski2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sanfooh/camera-openpose-keras/HEAD/sample_images/ski2.jpg -------------------------------------------------------------------------------- /sample_images/ski副本.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sanfooh/camera-openpose-keras/HEAD/sample_images/ski副本.jpg -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # camera-openpose-keras 2 | keras使用openpose模型的一个小例子 3 | 4 | 此项目基于 5 | https://github.com/michalfaber/keras_Realtime_Multi-Person_Pose_Estimation 6 | 7 | 修改了部分代码以支持摄像头 8 | 9 | 运行前确保模型文件存在 /model/keras/model.h5 10 | 此文件可在 https://www.dropbox.com/s/llpxd14is7gyj0z/model.h5 下载 11 | 12 | 在主分支目录下运行 python demo_camera.py 即可 -------------------------------------------------------------------------------- /config: -------------------------------------------------------------------------------- 1 | [param] 2 | 3 | # CPU mode or GPU mode 4 | use_gpu = 1 5 | 6 | # GPU device number (doesn't matter for CPU mode) 7 | GPUdeviceNumber = 0 8 | 9 | # Select model (default: 1) 10 | modelID = 1 11 | 12 | # Look in matlab counterpart for explanation 13 | octave = 3 14 | starting_range = 0.8 15 | ending_range = 2 16 | scale_search = 0.5, 1, 1.5, 2 17 | thre1 = 0.1 18 | thre2 = 0.05 19 | thre3 = 0.5 20 | min_num = 4 21 | mid_num = 10 22 | crop_ratio = 2.5 23 | bbox_ratio = 0.25 24 | 25 | [models] 26 | ## don't edit this part 27 | 28 | [[1]] 29 | caffemodel = './model/_trained_COCO/pose_iter_440000.caffemodel' 30 | deployFile = './model/_trained_COCO/pose_deploy.prototxt' 31 | description = 'COCO Pose56 Two-level Linevec' 32 | boxsize = 368 33 | padValue = 128 34 | np = 12 35 | stride = 8 36 | part_str = [nose, neck, Rsho, Relb, Rwri, Lsho, Lelb, Lwri, Rhip, Rkne, Rank, Lhip, Lkne, Lank, Leye, Reye, Lear, Rear, pt19] 37 | -------------------------------------------------------------------------------- /config_reader.py: -------------------------------------------------------------------------------- 1 | from configobj import ConfigObj 2 | import numpy as np 3 | 4 | 5 | def config_reader(): 6 | config = ConfigObj('config') 7 | 8 | param = config['param'] 9 | model_id = param['modelID'] 10 | model = config['models'][model_id] 11 | model['boxsize'] = int(model['boxsize']) 12 | model['stride'] = int(model['stride']) 13 | model['padValue'] = int(model['padValue']) 14 | #param['starting_range'] = float(param['starting_range']) 15 | #param['ending_range'] = float(param['ending_range']) 16 | param['octave'] = int(param['octave']) 17 | param['use_gpu'] = int(param['use_gpu']) 18 | param['starting_range'] = float(param['starting_range']) 19 | param['ending_range'] = float(param['ending_range']) 20 | param['scale_search'] = map(float, param['scale_search']) 21 | param['thre1'] = float(param['thre1']) 22 | param['thre2'] = float(param['thre2']) 23 | param['thre3'] = float(param['thre3']) 24 | param['mid_num'] = int(param['mid_num']) 25 | param['min_num'] = int(param['min_num']) 26 | param['crop_ratio'] = float(param['crop_ratio']) 27 | param['bbox_ratio'] = float(param['bbox_ratio']) 28 | param['GPUdeviceNumber'] = int(param['GPUdeviceNumber']) 29 | 30 | return param, model 31 | 32 | if __name__ == "__main__": 33 | config_reader() 34 | -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from io import StringIO 3 | import PIL.Image 4 | from IPython.display import Image, display 5 | 6 | def showBGRimage(a, fmt='jpeg'): 7 | a = np.uint8(np.clip(a, 0, 255)) 8 | a[:,:,[0,2]] = a[:,:,[2,0]] # for B,G,R order 9 | f = StringIO() 10 | PIL.Image.fromarray(a).save(f, fmt) 11 | display(Image(data=f.getvalue())) 12 | 13 | def showmap(a, fmt='png'): 14 | a = np.uint8(np.clip(a, 0, 255)) 15 | f = StringIO() 16 | PIL.Image.fromarray(a).save(f, fmt) 17 | display(Image(data=f.getvalue())) 18 | 19 | #def checkparam(param): 20 | # octave = param['octave'] 21 | # starting_range = param['starting_range'] 22 | # ending_range = param['ending_range'] 23 | # assert starting_range <= ending_range, 'starting ratio should <= ending ratio' 24 | # assert octave >= 1, 'octave should >= 1' 25 | # return starting_range, ending_range, octave 26 | 27 | def getJetColor(v, vmin, vmax): 28 | c = np.zeros((3)) 29 | if (v < vmin): 30 | v = vmin 31 | if (v > vmax): 32 | v = vmax 33 | dv = vmax - vmin 34 | if (v < (vmin + 0.125 * dv)): 35 | c[0] = 256 * (0.5 + (v * 4)) #B: 0.5 ~ 1 36 | elif (v < (vmin + 0.375 * dv)): 37 | c[0] = 255 38 | c[1] = 256 * (v - 0.125) * 4 #G: 0 ~ 1 39 | elif (v < (vmin + 0.625 * dv)): 40 | c[0] = 256 * (-4 * v + 2.5) #B: 1 ~ 0 41 | c[1] = 255 42 | c[2] = 256 * (4 * (v - 0.375)) #R: 0 ~ 1 43 | elif (v < (vmin + 0.875 * dv)): 44 | c[1] = 256 * (-4 * v + 3.5) #G: 1 ~ 0 45 | c[2] = 255 46 | else: 47 | c[2] = 256 * (-4 * v + 4.5) #R: 1 ~ 0.5 48 | return c 49 | 50 | def colorize(gray_img): 51 | out = np.zeros(gray_img.shape + (3,)) 52 | for y in range(out.shape[0]): 53 | for x in range(out.shape[1]): 54 | out[y,x,:] = getJetColor(gray_img[y,x], 0, 1) 55 | return out 56 | 57 | def padRightDownCorner(img, stride, padValue): 58 | h = img.shape[0] 59 | w = img.shape[1] 60 | 61 | pad = 4 * [None] 62 | pad[0] = 0 # up 63 | pad[1] = 0 # left 64 | pad[2] = 0 if (h%stride==0) else stride - (h % stride) # down 65 | pad[3] = 0 if (w%stride==0) else stride - (w % stride) # right 66 | 67 | img_padded = img 68 | pad_up = np.tile(img_padded[0:1,:,:]*0 + padValue, (pad[0], 1, 1)) 69 | img_padded = np.concatenate((pad_up, img_padded), axis=0) 70 | pad_left = np.tile(img_padded[:,0:1,:]*0 + padValue, (1, pad[1], 1)) 71 | img_padded = np.concatenate((pad_left, img_padded), axis=1) 72 | pad_down = np.tile(img_padded[-2:-1,:,:]*0 + padValue, (pad[2], 1, 1)) 73 | img_padded = np.concatenate((img_padded, pad_down), axis=0) 74 | pad_right = np.tile(img_padded[:,-2:-1,:]*0 + padValue, (1, pad[3], 1)) 75 | img_padded = np.concatenate((img_padded, pad_right), axis=1) 76 | 77 | return img_padded, pad 78 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | from keras.models import Model 2 | from keras.layers.merge import Concatenate 3 | from keras.layers import Activation, Input, Lambda 4 | from keras.layers.convolutional import Conv2D 5 | from keras.layers.pooling import MaxPooling2D 6 | from keras.layers.merge import Multiply 7 | from keras.regularizers import l2 8 | from keras.initializers import random_normal,constant 9 | 10 | def relu(x): return Activation('relu')(x) 11 | 12 | def conv(x, nf, ks, name, weight_decay): 13 | kernel_reg = l2(weight_decay[0]) if weight_decay else None 14 | bias_reg = l2(weight_decay[1]) if weight_decay else None 15 | 16 | x = Conv2D(nf, (ks, ks), padding='same', name=name, 17 | kernel_regularizer=kernel_reg, 18 | bias_regularizer=bias_reg, 19 | kernel_initializer=random_normal(stddev=0.01), 20 | bias_initializer=constant(0.0))(x) 21 | return x 22 | 23 | def pooling(x, ks, st, name): 24 | x = MaxPooling2D((ks, ks), strides=(st, st), name=name)(x) 25 | return x 26 | 27 | def vgg_block(x, weight_decay): 28 | # Block 1 29 | x = conv(x, 64, 3, "conv1_1", (weight_decay, 0)) 30 | x = relu(x) 31 | x = conv(x, 64, 3, "conv1_2", (weight_decay, 0)) 32 | x = relu(x) 33 | x = pooling(x, 2, 2, "pool1_1") 34 | 35 | # Block 2 36 | x = conv(x, 128, 3, "conv2_1", (weight_decay, 0)) 37 | x = relu(x) 38 | x = conv(x, 128, 3, "conv2_2", (weight_decay, 0)) 39 | x = relu(x) 40 | x = pooling(x, 2, 2, "pool2_1") 41 | 42 | # Block 3 43 | x = conv(x, 256, 3, "conv3_1", (weight_decay, 0)) 44 | x = relu(x) 45 | x = conv(x, 256, 3, "conv3_2", (weight_decay, 0)) 46 | x = relu(x) 47 | x = conv(x, 256, 3, "conv3_3", (weight_decay, 0)) 48 | x = relu(x) 49 | x = conv(x, 256, 3, "conv3_4", (weight_decay, 0)) 50 | x = relu(x) 51 | x = pooling(x, 2, 2, "pool3_1") 52 | 53 | # Block 4 54 | x = conv(x, 512, 3, "conv4_1", (weight_decay, 0)) 55 | x = relu(x) 56 | x = conv(x, 512, 3, "conv4_2", (weight_decay, 0)) 57 | x = relu(x) 58 | 59 | # Additional non vgg layers 60 | x = conv(x, 256, 3, "conv4_3_CPM", (weight_decay, 0)) 61 | x = relu(x) 62 | x = conv(x, 128, 3, "conv4_4_CPM", (weight_decay, 0)) 63 | x = relu(x) 64 | 65 | return x 66 | 67 | 68 | def stage1_block(x, num_p, branch, weight_decay): 69 | # Block 1 70 | x = conv(x, 128, 3, "Mconv1_stage1_L%d" % branch, (weight_decay, 0)) 71 | x = relu(x) 72 | x = conv(x, 128, 3, "Mconv2_stage1_L%d" % branch, (weight_decay, 0)) 73 | x = relu(x) 74 | x = conv(x, 128, 3, "Mconv3_stage1_L%d" % branch, (weight_decay, 0)) 75 | x = relu(x) 76 | x = conv(x, 512, 1, "Mconv4_stage1_L%d" % branch, (weight_decay, 0)) 77 | x = relu(x) 78 | x = conv(x, num_p, 1, "Mconv5_stage1_L%d" % branch, (weight_decay, 0)) 79 | 80 | return x 81 | 82 | 83 | def stageT_block(x, num_p, stage, branch, weight_decay): 84 | # Block 1 85 | x = conv(x, 128, 7, "Mconv1_stage%d_L%d" % (stage, branch), (weight_decay, 0)) 86 | x = relu(x) 87 | x = conv(x, 128, 7, "Mconv2_stage%d_L%d" % (stage, branch), (weight_decay, 0)) 88 | x = relu(x) 89 | x = conv(x, 128, 7, "Mconv3_stage%d_L%d" % (stage, branch), (weight_decay, 0)) 90 | x = relu(x) 91 | x = conv(x, 128, 7, "Mconv4_stage%d_L%d" % (stage, branch), (weight_decay, 0)) 92 | x = relu(x) 93 | x = conv(x, 128, 7, "Mconv5_stage%d_L%d" % (stage, branch), (weight_decay, 0)) 94 | x = relu(x) 95 | x = conv(x, 128, 1, "Mconv6_stage%d_L%d" % (stage, branch), (weight_decay, 0)) 96 | x = relu(x) 97 | x = conv(x, num_p, 1, "Mconv7_stage%d_L%d" % (stage, branch), (weight_decay, 0)) 98 | 99 | return x 100 | 101 | 102 | def apply_mask(x, mask1, mask2, num_p, stage, branch): 103 | w_name = "weight_stage%d_L%d" % (stage, branch) 104 | if num_p == 38: 105 | w = Multiply(name=w_name)([x, mask1]) # vec_weight 106 | 107 | else: 108 | w = Multiply(name=w_name)([x, mask2]) # vec_heat 109 | return w 110 | 111 | 112 | def get_training_model(weight_decay): 113 | 114 | stages = 6 115 | np_branch1 = 38 116 | np_branch2 = 19 117 | 118 | img_input_shape = (None, None, 3) 119 | vec_input_shape = (None, None, 38) 120 | heat_input_shape = (None, None, 19) 121 | 122 | inputs = [] 123 | outputs = [] 124 | 125 | img_input = Input(shape=img_input_shape) 126 | vec_weight_input = Input(shape=vec_input_shape) 127 | heat_weight_input = Input(shape=heat_input_shape) 128 | 129 | inputs.append(img_input) 130 | inputs.append(vec_weight_input) 131 | inputs.append(heat_weight_input) 132 | 133 | img_normalized = Lambda(lambda x: x / 256 - 0.5)(img_input) # [-0.5, 0.5] 134 | 135 | # VGG 136 | stage0_out = vgg_block(img_normalized, weight_decay) 137 | 138 | # stage 1 - branch 1 (PAF) 139 | stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1, weight_decay) 140 | w1 = apply_mask(stage1_branch1_out, vec_weight_input, heat_weight_input, np_branch1, 1, 1) 141 | 142 | # stage 1 - branch 2 (confidence maps) 143 | stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2, weight_decay) 144 | w2 = apply_mask(stage1_branch2_out, vec_weight_input, heat_weight_input, np_branch2, 1, 2) 145 | 146 | x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out]) 147 | 148 | outputs.append(w1) 149 | outputs.append(w2) 150 | 151 | # stage sn >= 2 152 | for sn in range(2, stages + 1): 153 | # stage SN - branch 1 (PAF) 154 | stageT_branch1_out = stageT_block(x, np_branch1, sn, 1, weight_decay) 155 | w1 = apply_mask(stageT_branch1_out, vec_weight_input, heat_weight_input, np_branch1, sn, 1) 156 | 157 | # stage SN - branch 2 (confidence maps) 158 | stageT_branch2_out = stageT_block(x, np_branch2, sn, 2, weight_decay) 159 | w2 = apply_mask(stageT_branch2_out, vec_weight_input, heat_weight_input, np_branch2, sn, 2) 160 | 161 | outputs.append(w1) 162 | outputs.append(w2) 163 | 164 | if (sn < stages): 165 | x = Concatenate()([stageT_branch1_out, stageT_branch2_out, stage0_out]) 166 | 167 | model = Model(inputs=inputs, outputs=outputs) 168 | 169 | return model 170 | 171 | 172 | def get_testing_model(): 173 | stages = 6 174 | np_branch1 = 38 175 | np_branch2 = 19 176 | 177 | img_input_shape = (None, None, 3) 178 | 179 | img_input = Input(shape=img_input_shape) 180 | 181 | img_normalized = Lambda(lambda x: x / 256 - 0.5)(img_input) # [-0.5, 0.5] 182 | 183 | # VGG 184 | stage0_out = vgg_block(img_normalized, None) 185 | 186 | # stage 1 - branch 1 (PAF) 187 | stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1, None) 188 | 189 | # stage 1 - branch 2 (confidence maps) 190 | stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2, None) 191 | 192 | x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out]) 193 | 194 | # stage t >= 2 195 | stageT_branch1_out = None 196 | stageT_branch2_out = None 197 | for sn in range(2, stages + 1): 198 | stageT_branch1_out = stageT_block(x, np_branch1, sn, 1, None) 199 | stageT_branch2_out = stageT_block(x, np_branch2, sn, 2, None) 200 | 201 | if (sn < stages): 202 | x = Concatenate()([stageT_branch1_out, stageT_branch2_out, stage0_out]) 203 | 204 | model = Model(inputs=[img_input], outputs=[stageT_branch1_out, stageT_branch2_out]) 205 | 206 | return model -------------------------------------------------------------------------------- /demo_camera.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import cv2 3 | import math 4 | import time 5 | import numpy as np 6 | import util 7 | from config_reader import config_reader 8 | from scipy.ndimage.filters import gaussian_filter 9 | from model import get_testing_model 10 | 11 | tic=0 12 | # find connection in the specified sequence, center 29 is in the position 15 13 | limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], \ 14 | [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], \ 15 | [1, 16], [16, 18], [3, 17], [6, 18]] 16 | 17 | # the middle joints heatmap correpondence 18 | mapIdx = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44], [19, 20], [21, 22], \ 19 | [23, 24], [25, 26], [27, 28], [29, 30], [47, 48], [49, 50], [53, 54], [51, 52], \ 20 | [55, 56], [37, 38], [45, 46]] 21 | 22 | # visualize 23 | colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], 24 | [0, 255, 0], \ 25 | [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], 26 | [85, 0, 255], \ 27 | [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]] 28 | 29 | 30 | def process (input_image, params, model_params): 31 | 32 | oriImg = cv2.imread(input_image) # B,G,R order 33 | multiplier = [x * model_params['boxsize'] / oriImg.shape[0] for x in params['scale_search']] 34 | 35 | heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) 36 | paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) 37 | 38 | #for m in range(len(multiplier)): 39 | for m in range(1): 40 | scale = multiplier[m] 41 | 42 | imageToTest = cv2.resize(oriImg, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) 43 | imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model_params['stride'], 44 | model_params['padValue']) 45 | 46 | input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2)) # required shape (1, width, height, channels) 47 | 48 | 49 | output_blobs = model.predict(input_img) 50 | 51 | 52 | 53 | # extract outputs, resize, and remove padding 54 | heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps 55 | heatmap = cv2.resize(heatmap, (0, 0), fx=model_params['stride'], fy=model_params['stride'], 56 | interpolation=cv2.INTER_CUBIC) 57 | heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], 58 | :] 59 | heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) 60 | 61 | paf = np.squeeze(output_blobs[0]) # output 0 is PAFs 62 | paf = cv2.resize(paf, (0, 0), fx=model_params['stride'], fy=model_params['stride'], 63 | interpolation=cv2.INTER_CUBIC) 64 | paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] 65 | paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC) 66 | 67 | heatmap_avg = heatmap_avg + heatmap / len(multiplier) 68 | paf_avg = paf_avg + paf / len(multiplier) 69 | 70 | all_peaks = [] 71 | peak_counter = 0 72 | prinfTick(1) 73 | for part in range(18): 74 | map_ori = heatmap_avg[:, :, part] 75 | map = gaussian_filter(map_ori, sigma=3) 76 | 77 | map_left = np.zeros(map.shape) 78 | map_left[1:, :] = map[:-1, :] 79 | map_right = np.zeros(map.shape) 80 | map_right[:-1, :] = map[1:, :] 81 | map_up = np.zeros(map.shape) 82 | map_up[:, 1:] = map[:, :-1] 83 | map_down = np.zeros(map.shape) 84 | map_down[:, :-1] = map[:, 1:] 85 | 86 | peaks_binary = np.logical_and.reduce( 87 | (map >= map_left, map >= map_right, map >= map_up, map >= map_down, map > params['thre1'])) 88 | peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse 89 | peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks] 90 | id = range(peak_counter, peak_counter + len(peaks)) 91 | peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))] 92 | 93 | all_peaks.append(peaks_with_score_and_id) 94 | peak_counter += len(peaks) 95 | 96 | connection_all = [] 97 | special_k = [] 98 | mid_num = 10 99 | prinfTick(2) 100 | for k in range(len(mapIdx)): 101 | score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]] 102 | candA = all_peaks[limbSeq[k][0] - 1] 103 | candB = all_peaks[limbSeq[k][1] - 1] 104 | nA = len(candA) 105 | nB = len(candB) 106 | indexA, indexB = limbSeq[k] 107 | if (nA != 0 and nB != 0): 108 | connection_candidate = [] 109 | for i in range(nA): 110 | for j in range(nB): 111 | vec = np.subtract(candB[j][:2], candA[i][:2]) 112 | norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) 113 | # failure case when 2 body parts overlaps 114 | if norm == 0: 115 | continue 116 | vec = np.divide(vec, norm) 117 | 118 | startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \ 119 | np.linspace(candA[i][1], candB[j][1], num=mid_num))) 120 | 121 | vec_x = np.array( 122 | [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \ 123 | for I in range(len(startend))]) 124 | vec_y = np.array( 125 | [score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \ 126 | for I in range(len(startend))]) 127 | 128 | score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1]) 129 | score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min( 130 | 0.5 * oriImg.shape[0] / norm - 1, 0) 131 | criterion1 = len(np.nonzero(score_midpts > params['thre2'])[0]) > 0.8 * len( 132 | score_midpts) 133 | criterion2 = score_with_dist_prior > 0 134 | if criterion1 and criterion2: 135 | connection_candidate.append([i, j, score_with_dist_prior, 136 | score_with_dist_prior + candA[i][2] + candB[j][2]]) 137 | 138 | connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) 139 | connection = np.zeros((0, 5)) 140 | for c in range(len(connection_candidate)): 141 | i, j, s = connection_candidate[c][0:3] 142 | if (i not in connection[:, 3] and j not in connection[:, 4]): 143 | connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]]) 144 | if (len(connection) >= min(nA, nB)): 145 | break 146 | 147 | connection_all.append(connection) 148 | else: 149 | special_k.append(k) 150 | connection_all.append([]) 151 | 152 | # last number in each row is the total parts number of that person 153 | # the second last number in each row is the score of the overall configuration 154 | subset = -1 * np.ones((0, 20)) 155 | candidate = np.array([item for sublist in all_peaks for item in sublist]) 156 | prinfTick(3) 157 | for k in range(len(mapIdx)): 158 | if k not in special_k: 159 | partAs = connection_all[k][:, 0] 160 | partBs = connection_all[k][:, 1] 161 | indexA, indexB = np.array(limbSeq[k]) - 1 162 | 163 | for i in range(len(connection_all[k])): # = 1:size(temp,1) 164 | found = 0 165 | subset_idx = [-1, -1] 166 | for j in range(len(subset)): # 1:size(subset,1): 167 | if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]: 168 | subset_idx[found] = j 169 | found += 1 170 | 171 | if found == 1: 172 | j = subset_idx[0] 173 | if (subset[j][indexB] != partBs[i]): 174 | subset[j][indexB] = partBs[i] 175 | subset[j][-1] += 1 176 | subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] 177 | elif found == 2: # if found 2 and disjoint, merge them 178 | j1, j2 = subset_idx 179 | membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] 180 | if len(np.nonzero(membership == 2)[0]) == 0: # merge 181 | subset[j1][:-2] += (subset[j2][:-2] + 1) 182 | subset[j1][-2:] += subset[j2][-2:] 183 | subset[j1][-2] += connection_all[k][i][2] 184 | subset = np.delete(subset, j2, 0) 185 | else: # as like found == 1 186 | subset[j1][indexB] = partBs[i] 187 | subset[j1][-1] += 1 188 | subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] 189 | 190 | # if find no partA in the subset, create a new subset 191 | elif not found and k < 17: 192 | row = -1 * np.ones(20) 193 | row[indexA] = partAs[i] 194 | row[indexB] = partBs[i] 195 | row[-1] = 2 196 | row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + \ 197 | connection_all[k][i][2] 198 | subset = np.vstack([subset, row]) 199 | 200 | # delete some rows of subset which has few parts occur 201 | deleteIdx = []; 202 | for i in range(len(subset)): 203 | if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: 204 | deleteIdx.append(i) 205 | subset = np.delete(subset, deleteIdx, axis=0) 206 | 207 | canvas = cv2.imread(input_image) # B,G,R order 208 | for i in range(18): 209 | for j in range(len(all_peaks[i])): 210 | cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1) 211 | 212 | stickwidth = 4 213 | for i in range(17): 214 | for n in range(len(subset)): 215 | index = subset[n][np.array(limbSeq[i]) - 1] 216 | if -1 in index: 217 | continue 218 | cur_canvas = canvas.copy() 219 | Y = candidate[index.astype(int), 0] 220 | X = candidate[index.astype(int), 1] 221 | mX = np.mean(X) 222 | mY = np.mean(Y) 223 | length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 224 | angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) 225 | polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 226 | 360, 1) 227 | cv2.fillConvexPoly(cur_canvas, polygon, colors[i]) 228 | canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0) 229 | 230 | return canvas 231 | 232 | def prinfTick(i): 233 | toc = time.time() 234 | print ('processing time%d is %.5f' % (i,toc - tic)) 235 | 236 | if __name__ == '__main__': 237 | parser = argparse.ArgumentParser() 238 | parser.add_argument('--image', type=str, default='sample_images/ski1.jpg', help='input image') 239 | parser.add_argument('--output', type=str, default='result.png', help='output image') 240 | parser.add_argument('--model', type=str, default='model/keras/model.h5', help='path to the weights file') 241 | 242 | args = parser.parse_args() 243 | input_image = args.image 244 | output = args.output 245 | keras_weights_file = args.model 246 | 247 | tic = time.time() 248 | print('start processing...') 249 | 250 | # load model 251 | 252 | # authors of original model don't use 253 | # vgg normalization (subtracting mean) on input images 254 | model = get_testing_model() 255 | model.load_weights(keras_weights_file) 256 | 257 | cap=cv2.VideoCapture(0) 258 | vi=cap.isOpened() 259 | if(vi == False): 260 | time.sleep(2) #必须要此步骤，否则失败 261 | fr = cv2.imread('./sample_images/ski2.jpg',1) 262 | tic = time.time() 263 | cv2.imwrite(input_image, fr) 264 | params, model_params = config_reader() 265 | canvas = process(input_image, params, model_params) 266 | cv2.imshow("capture",canvas) 267 | cv2.waitKey(0) 268 | 269 | 270 | if(vi == True): 271 | cap.set(3,160) 272 | cap.set(4,120) 273 | time.sleep(2) #必须要此步骤，否则失败 274 | 275 | while(1): 276 | tic = time.time() 277 | 278 | ret,frame=cap.read() 279 | cv2.imwrite(input_image, frame) 280 | params, model_params = config_reader() 281 | 282 | # generate image with body parts 283 | canvas = process(input_image, params, model_params) 284 | cv2.imshow("capture",canvas) 285 | if cv2.waitKey(1) & 0xFF==ord('q'): 286 | break 287 | cap.release() 288 | cv2.destroyAllWindows() 289 | 290 | 291 | 292 | 293 | 294 | --------------------------------------------------------------------------------