实时人体姿态评估算法学习笔记
2018-02-05 本文已影响796人
gaoshine
0. 实时人体姿态评估算法学习笔记
本文是以 2017 CVPR的一篇论文为基础的,采用深度学习算法,实时评估人体姿态的代码重构的学习笔记.
叠加生成人体姿态
本文采用keras,以TensorFlow为backend.
import keras
from keras.models import Sequential
from keras.models import Model
from keras.layers import Input, Dense, Activation
from keras.layers.convolutional import Conv2D
from keras.layers.pooling import MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.layers.merge import Concatenate
from config_reader import config_reader
import scipy
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
Using TensorFlow backend.
1.定义模型
模型基于VGG,1-4层采用VGG的模型,在VGG后面增加了非VGG的层.
def relu(x):
return Activation('relu')(x)
def conv(x, nf, ks, name):
x1 = Conv2D(nf, (ks, ks), padding='same', name=name)(x)
return x1
def pooling(x, ks, st, name):
x = MaxPooling2D((ks, ks), strides=(st, st), name=name)(x)
return x
def vgg_block(x):
# Block 1
x = conv(x, 64, 3, "conv1_1")
x = relu(x)
x = conv(x, 64, 3, "conv1_2")
x = relu(x)
x = pooling(x, 2, 2, "pool1_1")
# Block 2
x = conv(x, 128, 3, "conv2_1")
x = relu(x)
x = conv(x, 128, 3, "conv2_2")
x = relu(x)
x = pooling(x, 2, 2, "pool2_1")
# Block 3
x = conv(x, 256, 3, "conv3_1")
x = relu(x)
x = conv(x, 256, 3, "conv3_2")
x = relu(x)
x = conv(x, 256, 3, "conv3_3")
x = relu(x)
x = conv(x, 256, 3, "conv3_4")
x = relu(x)
x = pooling(x, 2, 2, "pool3_1")
# Block 4
x = conv(x, 512, 3, "conv4_1")
x = relu(x)
x = conv(x, 512, 3, "conv4_2")
x = relu(x)
# Additional non vgg layers
x = conv(x, 256, 3, "conv4_3_CPM")
x = relu(x)
x = conv(x, 128, 3, "conv4_4_CPM")
x = relu(x)
return x
def stage1_block(x, num_p, branch):
# Block 1
x = conv(x, 128, 3, "conv5_1_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 128, 3, "conv5_2_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 128, 3, "conv5_3_CPM_L%d" % branch)
x = relu(x)
x = conv(x, 512, 1, "conv5_4_CPM_L%d" % branch)
x = relu(x)
x = conv(x, num_p, 1, "conv5_5_CPM_L%d" % branch)
return x
def stageT_block(x, num_p, stage, branch):
# Block 1
x = conv(x, 128, 7, "Mconv1_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv2_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv3_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv4_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 7, "Mconv5_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, 128, 1, "Mconv6_stage%d_L%d" % (stage, branch))
x = relu(x)
x = conv(x, num_p, 1, "Mconv7_stage%d_L%d" % (stage, branch))
return x
weights_path = "model/keras/model.h5"
input_shape = (None,None,3)
img_input = Input(shape=input_shape)
stages = 6
np_branch1 = 38
np_branch2 = 19
# VGG
stage0_out = vgg_block(img_input)
# stage 1
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1)
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2)
x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out])
# stage t >= 2
for sn in range(2, stages + 1):
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1)
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2)
if (sn < stages):
x = Concatenate()([stageT_branch1_out, stageT_branch2_out, stage0_out])
model = Model(img_input, [stageT_branch1_out, stageT_branch2_out])
model.load_weights(weights_path)
model.summary()
SVG(model_to_dot(model,show_shapes=True).create(prog='dot', format='svg'))
#模型的结构如下列表和图形
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) (None, None, None, 3 0
__________________________________________________________________________________________________
conv1_1 (Conv2D) (None, None, None, 6 1792 input_1[0][0]
__________________________________________________________________________________________________
activation_1 (Activation) (None, None, None, 6 0 conv1_1[0][0]
__________________________________________________________________________________________________
conv1_2 (Conv2D) (None, None, None, 6 36928 activation_1[0][0]
__________________________________________________________________________________________________
activation_2 (Activation) (None, None, None, 6 0 conv1_2[0][0]
__________________________________________________________________________________________________
pool1_1 (MaxPooling2D) (None, None, None, 6 0 activation_2[0][0]
__________________________________________________________________________________________________
conv2_1 (Conv2D) (None, None, None, 1 73856 pool1_1[0][0]
__________________________________________________________________________________________________
activation_3 (Activation) (None, None, None, 1 0 conv2_1[0][0]
__________________________________________________________________________________________________
conv2_2 (Conv2D) (None, None, None, 1 147584 activation_3[0][0]
__________________________________________________________________________________________________
activation_4 (Activation) (None, None, None, 1 0 conv2_2[0][0]
__________________________________________________________________________________________________
pool2_1 (MaxPooling2D) (None, None, None, 1 0 activation_4[0][0]
__________________________________________________________________________________________________
conv3_1 (Conv2D) (None, None, None, 2 295168 pool2_1[0][0]
__________________________________________________________________________________________________
activation_5 (Activation) (None, None, None, 2 0 conv3_1[0][0]
__________________________________________________________________________________________________
conv3_2 (Conv2D) (None, None, None, 2 590080 activation_5[0][0]
__________________________________________________________________________________________________
activation_6 (Activation) (None, None, None, 2 0 conv3_2[0][0]
__________________________________________________________________________________________________
conv3_3 (Conv2D) (None, None, None, 2 590080 activation_6[0][0]
__________________________________________________________________________________________________
activation_7 (Activation) (None, None, None, 2 0 conv3_3[0][0]
__________________________________________________________________________________________________
conv3_4 (Conv2D) (None, None, None, 2 590080 activation_7[0][0]
__________________________________________________________________________________________________
activation_8 (Activation) (None, None, None, 2 0 conv3_4[0][0]
__________________________________________________________________________________________________
pool3_1 (MaxPooling2D) (None, None, None, 2 0 activation_8[0][0]
__________________________________________________________________________________________________
conv4_1 (Conv2D) (None, None, None, 5 1180160 pool3_1[0][0]
__________________________________________________________________________________________________
activation_9 (Activation) (None, None, None, 5 0 conv4_1[0][0]
__________________________________________________________________________________________________
conv4_2 (Conv2D) (None, None, None, 5 2359808 activation_9[0][0]
__________________________________________________________________________________________________
activation_10 (Activation) (None, None, None, 5 0 conv4_2[0][0]
__________________________________________________________________________________________________
conv4_3_CPM (Conv2D) (None, None, None, 2 1179904 activation_10[0][0]
__________________________________________________________________________________________________
activation_11 (Activation) (None, None, None, 2 0 conv4_3_CPM[0][0]
__________________________________________________________________________________________________
conv4_4_CPM (Conv2D) (None, None, None, 1 295040 activation_11[0][0]
__________________________________________________________________________________________________
activation_12 (Activation) (None, None, None, 1 0 conv4_4_CPM[0][0]
__________________________________________________________________________________________________
conv5_1_CPM_L1 (Conv2D) (None, None, None, 1 147584 activation_12[0][0]
__________________________________________________________________________________________________
conv5_1_CPM_L2 (Conv2D) (None, None, None, 1 147584 activation_12[0][0]
__________________________________________________________________________________________________
activation_13 (Activation) (None, None, None, 1 0 conv5_1_CPM_L1[0][0]
__________________________________________________________________________________________________
activation_17 (Activation) (None, None, None, 1 0 conv5_1_CPM_L2[0][0]
__________________________________________________________________________________________________
conv5_2_CPM_L1 (Conv2D) (None, None, None, 1 147584 activation_13[0][0]
__________________________________________________________________________________________________
conv5_2_CPM_L2 (Conv2D) (None, None, None, 1 147584 activation_17[0][0]
__________________________________________________________________________________________________
activation_14 (Activation) (None, None, None, 1 0 conv5_2_CPM_L1[0][0]
__________________________________________________________________________________________________
activation_18 (Activation) (None, None, None, 1 0 conv5_2_CPM_L2[0][0]
__________________________________________________________________________________________________
conv5_3_CPM_L1 (Conv2D) (None, None, None, 1 147584 activation_14[0][0]
__________________________________________________________________________________________________
conv5_3_CPM_L2 (Conv2D) (None, None, None, 1 147584 activation_18[0][0]
__________________________________________________________________________________________________
activation_15 (Activation) (None, None, None, 1 0 conv5_3_CPM_L1[0][0]
__________________________________________________________________________________________________
activation_19 (Activation) (None, None, None, 1 0 conv5_3_CPM_L2[0][0]
__________________________________________________________________________________________________
conv5_4_CPM_L1 (Conv2D) (None, None, None, 5 66048 activation_15[0][0]
__________________________________________________________________________________________________
conv5_4_CPM_L2 (Conv2D) (None, None, None, 5 66048 activation_19[0][0]
__________________________________________________________________________________________________
activation_16 (Activation) (None, None, None, 5 0 conv5_4_CPM_L1[0][0]
__________________________________________________________________________________________________
activation_20 (Activation) (None, None, None, 5 0 conv5_4_CPM_L2[0][0]
__________________________________________________________________________________________________
conv5_5_CPM_L1 (Conv2D) (None, None, None, 3 19494 activation_16[0][0]
__________________________________________________________________________________________________
conv5_5_CPM_L2 (Conv2D) (None, None, None, 1 9747 activation_20[0][0]
__________________________________________________________________________________________________
concatenate_1 (Concatenate) (None, None, None, 1 0 conv5_5_CPM_L1[0][0]
conv5_5_CPM_L2[0][0]
activation_12[0][0]
__________________________________________________________________________________________________
Mconv1_stage2_L1 (Conv2D) (None, None, None, 1 1160448 concatenate_1[0][0]
__________________________________________________________________________________________________
Mconv1_stage2_L2 (Conv2D) (None, None, None, 1 1160448 concatenate_1[0][0]
__________________________________________________________________________________________________
activation_21 (Activation) (None, None, None, 1 0 Mconv1_stage2_L1[0][0]
__________________________________________________________________________________________________
activation_27 (Activation) (None, None, None, 1 0 Mconv1_stage2_L2[0][0]
__________________________________________________________________________________________________
Mconv2_stage2_L1 (Conv2D) (None, None, None, 1 802944 activation_21[0][0]
__________________________________________________________________________________________________
Mconv2_stage2_L2 (Conv2D) (None, None, None, 1 802944 activation_27[0][0]
__________________________________________________________________________________________________
activation_22 (Activation) (None, None, None, 1 0 Mconv2_stage2_L1[0][0]
__________________________________________________________________________________________________
activation_28 (Activation) (None, None, None, 1 0 Mconv2_stage2_L2[0][0]
__________________________________________________________________________________________________
Mconv3_stage2_L1 (Conv2D) (None, None, None, 1 802944 activation_22[0][0]
__________________________________________________________________________________________________
Mconv3_stage2_L2 (Conv2D) (None, None, None, 1 802944 activation_28[0][0]
__________________________________________________________________________________________________
activation_23 (Activation) (None, None, None, 1 0 Mconv3_stage2_L1[0][0]
__________________________________________________________________________________________________
activation_29 (Activation) (None, None, None, 1 0 Mconv3_stage2_L2[0][0]
__________________________________________________________________________________________________
Mconv4_stage2_L1 (Conv2D) (None, None, None, 1 802944 activation_23[0][0]
__________________________________________________________________________________________________
Mconv4_stage2_L2 (Conv2D) (None, None, None, 1 802944 activation_29[0][0]
__________________________________________________________________________________________________
activation_24 (Activation) (None, None, None, 1 0 Mconv4_stage2_L1[0][0]
__________________________________________________________________________________________________
activation_30 (Activation) (None, None, None, 1 0 Mconv4_stage2_L2[0][0]
__________________________________________________________________________________________________
Mconv5_stage2_L1 (Conv2D) (None, None, None, 1 802944 activation_24[0][0]
__________________________________________________________________________________________________
Mconv5_stage2_L2 (Conv2D) (None, None, None, 1 802944 activation_30[0][0]
__________________________________________________________________________________________________
activation_25 (Activation) (None, None, None, 1 0 Mconv5_stage2_L1[0][0]
__________________________________________________________________________________________________
activation_31 (Activation) (None, None, None, 1 0 Mconv5_stage2_L2[0][0]
__________________________________________________________________________________________________
Mconv6_stage2_L1 (Conv2D) (None, None, None, 1 16512 activation_25[0][0]
__________________________________________________________________________________________________
Mconv6_stage2_L2 (Conv2D) (None, None, None, 1 16512 activation_31[0][0]
__________________________________________________________________________________________________
activation_26 (Activation) (None, None, None, 1 0 Mconv6_stage2_L1[0][0]
__________________________________________________________________________________________________
activation_32 (Activation) (None, None, None, 1 0 Mconv6_stage2_L2[0][0]
__________________________________________________________________________________________________
Mconv7_stage2_L1 (Conv2D) (None, None, None, 3 4902 activation_26[0][0]
__________________________________________________________________________________________________
Mconv7_stage2_L2 (Conv2D) (None, None, None, 1 2451 activation_32[0][0]
__________________________________________________________________________________________________
concatenate_2 (Concatenate) (None, None, None, 1 0 Mconv7_stage2_L1[0][0]
Mconv7_stage2_L2[0][0]
activation_12[0][0]
__________________________________________________________________________________________________
Mconv1_stage3_L1 (Conv2D) (None, None, None, 1 1160448 concatenate_2[0][0]
__________________________________________________________________________________________________
Mconv1_stage3_L2 (Conv2D) (None, None, None, 1 1160448 concatenate_2[0][0]
__________________________________________________________________________________________________
activation_33 (Activation) (None, None, None, 1 0 Mconv1_stage3_L1[0][0]
__________________________________________________________________________________________________
activation_39 (Activation) (None, None, None, 1 0 Mconv1_stage3_L2[0][0]
__________________________________________________________________________________________________
Mconv2_stage3_L1 (Conv2D) (None, None, None, 1 802944 activation_33[0][0]
__________________________________________________________________________________________________
Mconv2_stage3_L2 (Conv2D) (None, None, None, 1 802944 activation_39[0][0]
__________________________________________________________________________________________________
activation_34 (Activation) (None, None, None, 1 0 Mconv2_stage3_L1[0][0]
__________________________________________________________________________________________________
activation_40 (Activation) (None, None, None, 1 0 Mconv2_stage3_L2[0][0]
__________________________________________________________________________________________________
Mconv3_stage3_L1 (Conv2D) (None, None, None, 1 802944 activation_34[0][0]
__________________________________________________________________________________________________
Mconv3_stage3_L2 (Conv2D) (None, None, None, 1 802944 activation_40[0][0]
__________________________________________________________________________________________________
activation_35 (Activation) (None, None, None, 1 0 Mconv3_stage3_L1[0][0]
__________________________________________________________________________________________________
activation_41 (Activation) (None, None, None, 1 0 Mconv3_stage3_L2[0][0]
__________________________________________________________________________________________________
Mconv4_stage3_L1 (Conv2D) (None, None, None, 1 802944 activation_35[0][0]
__________________________________________________________________________________________________
Mconv4_stage3_L2 (Conv2D) (None, None, None, 1 802944 activation_41[0][0]
__________________________________________________________________________________________________
activation_36 (Activation) (None, None, None, 1 0 Mconv4_stage3_L1[0][0]
__________________________________________________________________________________________________
activation_42 (Activation) (None, None, None, 1 0 Mconv4_stage3_L2[0][0]
__________________________________________________________________________________________________
Mconv5_stage3_L1 (Conv2D) (None, None, None, 1 802944 activation_36[0][0]
__________________________________________________________________________________________________
Mconv5_stage3_L2 (Conv2D) (None, None, None, 1 802944 activation_42[0][0]
__________________________________________________________________________________________________
activation_37 (Activation) (None, None, None, 1 0 Mconv5_stage3_L1[0][0]
__________________________________________________________________________________________________
activation_43 (Activation) (None, None, None, 1 0 Mconv5_stage3_L2[0][0]
__________________________________________________________________________________________________
Mconv6_stage3_L1 (Conv2D) (None, None, None, 1 16512 activation_37[0][0]
__________________________________________________________________________________________________
Mconv6_stage3_L2 (Conv2D) (None, None, None, 1 16512 activation_43[0][0]
__________________________________________________________________________________________________
activation_38 (Activation) (None, None, None, 1 0 Mconv6_stage3_L1[0][0]
__________________________________________________________________________________________________
activation_44 (Activation) (None, None, None, 1 0 Mconv6_stage3_L2[0][0]
__________________________________________________________________________________________________
Mconv7_stage3_L1 (Conv2D) (None, None, None, 3 4902 activation_38[0][0]
__________________________________________________________________________________________________
Mconv7_stage3_L2 (Conv2D) (None, None, None, 1 2451 activation_44[0][0]
__________________________________________________________________________________________________
concatenate_3 (Concatenate) (None, None, None, 1 0 Mconv7_stage3_L1[0][0]
Mconv7_stage3_L2[0][0]
activation_12[0][0]
__________________________________________________________________________________________________
Mconv1_stage4_L1 (Conv2D) (None, None, None, 1 1160448 concatenate_3[0][0]
__________________________________________________________________________________________________
Mconv1_stage4_L2 (Conv2D) (None, None, None, 1 1160448 concatenate_3[0][0]
__________________________________________________________________________________________________
activation_45 (Activation) (None, None, None, 1 0 Mconv1_stage4_L1[0][0]
__________________________________________________________________________________________________
activation_51 (Activation) (None, None, None, 1 0 Mconv1_stage4_L2[0][0]
__________________________________________________________________________________________________
Mconv2_stage4_L1 (Conv2D) (None, None, None, 1 802944 activation_45[0][0]
__________________________________________________________________________________________________
Mconv2_stage4_L2 (Conv2D) (None, None, None, 1 802944 activation_51[0][0]
__________________________________________________________________________________________________
activation_46 (Activation) (None, None, None, 1 0 Mconv2_stage4_L1[0][0]
__________________________________________________________________________________________________
activation_52 (Activation) (None, None, None, 1 0 Mconv2_stage4_L2[0][0]
__________________________________________________________________________________________________
Mconv3_stage4_L1 (Conv2D) (None, None, None, 1 802944 activation_46[0][0]
__________________________________________________________________________________________________
Mconv3_stage4_L2 (Conv2D) (None, None, None, 1 802944 activation_52[0][0]
__________________________________________________________________________________________________
activation_47 (Activation) (None, None, None, 1 0 Mconv3_stage4_L1[0][0]
__________________________________________________________________________________________________
activation_53 (Activation) (None, None, None, 1 0 Mconv3_stage4_L2[0][0]
__________________________________________________________________________________________________
Mconv4_stage4_L1 (Conv2D) (None, None, None, 1 802944 activation_47[0][0]
__________________________________________________________________________________________________
Mconv4_stage4_L2 (Conv2D) (None, None, None, 1 802944 activation_53[0][0]
__________________________________________________________________________________________________
activation_48 (Activation) (None, None, None, 1 0 Mconv4_stage4_L1[0][0]
__________________________________________________________________________________________________
activation_54 (Activation) (None, None, None, 1 0 Mconv4_stage4_L2[0][0]
__________________________________________________________________________________________________
Mconv5_stage4_L1 (Conv2D) (None, None, None, 1 802944 activation_48[0][0]
__________________________________________________________________________________________________
Mconv5_stage4_L2 (Conv2D) (None, None, None, 1 802944 activation_54[0][0]
__________________________________________________________________________________________________
activation_49 (Activation) (None, None, None, 1 0 Mconv5_stage4_L1[0][0]
__________________________________________________________________________________________________
activation_55 (Activation) (None, None, None, 1 0 Mconv5_stage4_L2[0][0]
__________________________________________________________________________________________________
Mconv6_stage4_L1 (Conv2D) (None, None, None, 1 16512 activation_49[0][0]
__________________________________________________________________________________________________
Mconv6_stage4_L2 (Conv2D) (None, None, None, 1 16512 activation_55[0][0]
__________________________________________________________________________________________________
activation_50 (Activation) (None, None, None, 1 0 Mconv6_stage4_L1[0][0]
__________________________________________________________________________________________________
activation_56 (Activation) (None, None, None, 1 0 Mconv6_stage4_L2[0][0]
__________________________________________________________________________________________________
Mconv7_stage4_L1 (Conv2D) (None, None, None, 3 4902 activation_50[0][0]
__________________________________________________________________________________________________
Mconv7_stage4_L2 (Conv2D) (None, None, None, 1 2451 activation_56[0][0]
__________________________________________________________________________________________________
concatenate_4 (Concatenate) (None, None, None, 1 0 Mconv7_stage4_L1[0][0]
Mconv7_stage4_L2[0][0]
activation_12[0][0]
__________________________________________________________________________________________________
Mconv1_stage5_L1 (Conv2D) (None, None, None, 1 1160448 concatenate_4[0][0]
__________________________________________________________________________________________________
Mconv1_stage5_L2 (Conv2D) (None, None, None, 1 1160448 concatenate_4[0][0]
__________________________________________________________________________________________________
activation_57 (Activation) (None, None, None, 1 0 Mconv1_stage5_L1[0][0]
__________________________________________________________________________________________________
activation_63 (Activation) (None, None, None, 1 0 Mconv1_stage5_L2[0][0]
__________________________________________________________________________________________________
Mconv2_stage5_L1 (Conv2D) (None, None, None, 1 802944 activation_57[0][0]
__________________________________________________________________________________________________
Mconv2_stage5_L2 (Conv2D) (None, None, None, 1 802944 activation_63[0][0]
__________________________________________________________________________________________________
activation_58 (Activation) (None, None, None, 1 0 Mconv2_stage5_L1[0][0]
__________________________________________________________________________________________________
activation_64 (Activation) (None, None, None, 1 0 Mconv2_stage5_L2[0][0]
__________________________________________________________________________________________________
Mconv3_stage5_L1 (Conv2D) (None, None, None, 1 802944 activation_58[0][0]
__________________________________________________________________________________________________
Mconv3_stage5_L2 (Conv2D) (None, None, None, 1 802944 activation_64[0][0]
__________________________________________________________________________________________________
activation_59 (Activation) (None, None, None, 1 0 Mconv3_stage5_L1[0][0]
__________________________________________________________________________________________________
activation_65 (Activation) (None, None, None, 1 0 Mconv3_stage5_L2[0][0]
__________________________________________________________________________________________________
Mconv4_stage5_L1 (Conv2D) (None, None, None, 1 802944 activation_59[0][0]
__________________________________________________________________________________________________
Mconv4_stage5_L2 (Conv2D) (None, None, None, 1 802944 activation_65[0][0]
__________________________________________________________________________________________________
activation_60 (Activation) (None, None, None, 1 0 Mconv4_stage5_L1[0][0]
__________________________________________________________________________________________________
activation_66 (Activation) (None, None, None, 1 0 Mconv4_stage5_L2[0][0]
__________________________________________________________________________________________________
Mconv5_stage5_L1 (Conv2D) (None, None, None, 1 802944 activation_60[0][0]
__________________________________________________________________________________________________
Mconv5_stage5_L2 (Conv2D) (None, None, None, 1 802944 activation_66[0][0]
__________________________________________________________________________________________________
activation_61 (Activation) (None, None, None, 1 0 Mconv5_stage5_L1[0][0]
__________________________________________________________________________________________________
activation_67 (Activation) (None, None, None, 1 0 Mconv5_stage5_L2[0][0]
__________________________________________________________________________________________________
Mconv6_stage5_L1 (Conv2D) (None, None, None, 1 16512 activation_61[0][0]
__________________________________________________________________________________________________
Mconv6_stage5_L2 (Conv2D) (None, None, None, 1 16512 activation_67[0][0]
__________________________________________________________________________________________________
activation_62 (Activation) (None, None, None, 1 0 Mconv6_stage5_L1[0][0]
__________________________________________________________________________________________________
activation_68 (Activation) (None, None, None, 1 0 Mconv6_stage5_L2[0][0]
__________________________________________________________________________________________________
Mconv7_stage5_L1 (Conv2D) (None, None, None, 3 4902 activation_62[0][0]
__________________________________________________________________________________________________
Mconv7_stage5_L2 (Conv2D) (None, None, None, 1 2451 activation_68[0][0]
__________________________________________________________________________________________________
concatenate_5 (Concatenate) (None, None, None, 1 0 Mconv7_stage5_L1[0][0]
Mconv7_stage5_L2[0][0]
activation_12[0][0]
__________________________________________________________________________________________________
Mconv1_stage6_L1 (Conv2D) (None, None, None, 1 1160448 concatenate_5[0][0]
__________________________________________________________________________________________________
Mconv1_stage6_L2 (Conv2D) (None, None, None, 1 1160448 concatenate_5[0][0]
__________________________________________________________________________________________________
activation_69 (Activation) (None, None, None, 1 0 Mconv1_stage6_L1[0][0]
__________________________________________________________________________________________________
activation_75 (Activation) (None, None, None, 1 0 Mconv1_stage6_L2[0][0]
__________________________________________________________________________________________________
Mconv2_stage6_L1 (Conv2D) (None, None, None, 1 802944 activation_69[0][0]
__________________________________________________________________________________________________
Mconv2_stage6_L2 (Conv2D) (None, None, None, 1 802944 activation_75[0][0]
__________________________________________________________________________________________________
activation_70 (Activation) (None, None, None, 1 0 Mconv2_stage6_L1[0][0]
__________________________________________________________________________________________________
activation_76 (Activation) (None, None, None, 1 0 Mconv2_stage6_L2[0][0]
__________________________________________________________________________________________________
Mconv3_stage6_L1 (Conv2D) (None, None, None, 1 802944 activation_70[0][0]
__________________________________________________________________________________________________
Mconv3_stage6_L2 (Conv2D) (None, None, None, 1 802944 activation_76[0][0]
__________________________________________________________________________________________________
activation_71 (Activation) (None, None, None, 1 0 Mconv3_stage6_L1[0][0]
__________________________________________________________________________________________________
activation_77 (Activation) (None, None, None, 1 0 Mconv3_stage6_L2[0][0]
__________________________________________________________________________________________________
Mconv4_stage6_L1 (Conv2D) (None, None, None, 1 802944 activation_71[0][0]
__________________________________________________________________________________________________
Mconv4_stage6_L2 (Conv2D) (None, None, None, 1 802944 activation_77[0][0]
__________________________________________________________________________________________________
activation_72 (Activation) (None, None, None, 1 0 Mconv4_stage6_L1[0][0]
__________________________________________________________________________________________________
activation_78 (Activation) (None, None, None, 1 0 Mconv4_stage6_L2[0][0]
__________________________________________________________________________________________________
Mconv5_stage6_L1 (Conv2D) (None, None, None, 1 802944 activation_72[0][0]
__________________________________________________________________________________________________
Mconv5_stage6_L2 (Conv2D) (None, None, None, 1 802944 activation_78[0][0]
__________________________________________________________________________________________________
activation_73 (Activation) (None, None, None, 1 0 Mconv5_stage6_L1[0][0]
__________________________________________________________________________________________________
activation_79 (Activation) (None, None, None, 1 0 Mconv5_stage6_L2[0][0]
__________________________________________________________________________________________________
Mconv6_stage6_L1 (Conv2D) (None, None, None, 1 16512 activation_73[0][0]
__________________________________________________________________________________________________
Mconv6_stage6_L2 (Conv2D) (None, None, None, 1 16512 activation_79[0][0]
__________________________________________________________________________________________________
activation_74 (Activation) (None, None, None, 1 0 Mconv6_stage6_L1[0][0]
__________________________________________________________________________________________________
activation_80 (Activation) (None, None, None, 1 0 Mconv6_stage6_L2[0][0]
__________________________________________________________________________________________________
Mconv7_stage6_L1 (Conv2D) (None, None, None, 3 4902 activation_74[0][0]
__________________________________________________________________________________________________
Mconv7_stage6_L2 (Conv2D) (None, None, None, 1 2451 activation_80[0][0]
==================================================================================================
Total params: 52,311,446
Trainable params: 52,311,446
Non-trainable params: 0
__________________________________________________________________________________________________
svg
开始加载图片,并做测试
%matplotlib inline
import cv2
import matplotlib
import pylab as plt
import numpy as np
import util
test_image = './sample_images/ski.jpg'
oriImg = cv2.imread(test_image) # B,G,R order
plt.imshow(oriImg[:,:,[2,1,0]])
<matplotlib.image.AxesImage at 0x116303358>
output_7_1.png
output_11_0.png
加载配置文件,准备运算.
对关键点开始做热力图分析,找到关键点.
肘和手腕和髋,颈部等等关键节点的热力图.
param, model_params = config_reader()
multiplier = [x * model_params['boxsize'] / oriImg.shape[0] for x in param['scale_search']]
heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19))
paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38))
# first figure shows padded images
f, axarr = plt.subplots(1, len(multiplier))
f.set_size_inches((20, 5))
# second figure shows heatmaps
f2, axarr2 = plt.subplots(1, len(multiplier))
f2.set_size_inches((20, 5))
# third figure shows PAFs
f3, axarr3 = plt.subplots(2, len(multiplier))
f3.set_size_inches((20, 10))
for m in range(len(multiplier)):
scale = multiplier[m]
imageToTest = cv2.resize(oriImg, (0,0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
imageToTest_padded, pad = util.padRightDownCorner(imageToTest, model_params['stride'], model_params['padValue'])
axarr[m].imshow(imageToTest_padded[:,:,[2,1,0]])
axarr[m].set_title('Input image: scale %d' % m)
input_img = np.transpose(np.float32(imageToTest_padded[:,:,:,np.newaxis]), (3,0,1,2))/256 - 0.5; # required shape (1, width, height, channels)
print("Input shape: " + str(input_img.shape))
output_blobs = model.predict(input_img)
print("Output shape (heatmap): " + str(output_blobs[1].shape))
# extract outputs, resize, and remove padding
heatmap = np.squeeze(output_blobs[1]) # output 1 is heatmaps
heatmap = cv2.resize(heatmap, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
heatmap = heatmap[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
heatmap = cv2.resize(heatmap, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
paf = np.squeeze(output_blobs[0]) # output 0 is PAFs
paf = cv2.resize(paf, (0,0), fx=model_params['stride'], fy=model_params['stride'], interpolation=cv2.INTER_CUBIC)
paf = paf[:imageToTest_padded.shape[0]-pad[2], :imageToTest_padded.shape[1]-pad[3], :]
paf = cv2.resize(paf, (oriImg.shape[1], oriImg.shape[0]), interpolation=cv2.INTER_CUBIC)
# visualization
axarr2[m].imshow(oriImg[:,:,[2,1,0]])
ax2 = axarr2[m].imshow(heatmap[:,:,3], alpha=.5) # right elbow
axarr2[m].set_title('Heatmaps (Relb): scale %d' % m)
axarr3.flat[m].imshow(oriImg[:,:,[2,1,0]])
ax3x = axarr3.flat[m].imshow(paf[:,:,16], alpha=.5) # right elbow
axarr3.flat[m].set_title('PAFs (x comp. of Rwri to Relb): scale %d' % m)
axarr3.flat[len(multiplier) + m].imshow(oriImg[:,:,[2,1,0]])
ax3y = axarr3.flat[len(multiplier) + m].imshow(paf[:,:,17], alpha=.5) # right wrist
axarr3.flat[len(multiplier) + m].set_title('PAFs (y comp. of Relb to Rwri): scale %d' % m)
heatmap_avg = heatmap_avg + heatmap / len(multiplier)
paf_avg = paf_avg + paf / len(multiplier)
f2.subplots_adjust(right=0.93)
cbar_ax = f2.add_axes([0.95, 0.15, 0.01, 0.7])
_ = f2.colorbar(ax2, cax=cbar_ax)
f3.subplots_adjust(right=0.93)
cbar_axx = f3.add_axes([0.95, 0.57, 0.01, 0.3])
_ = f3.colorbar(ax3x, cax=cbar_axx)
cbar_axy = f3.add_axes([0.95, 0.15, 0.01, 0.3])
_ = f3.colorbar(ax3y, cax=cbar_axy)
Input shape: (1, 184, 200, 3)
Output shape (heatmap): (1, 23, 25, 19)
Input shape: (1, 368, 392, 3)
Output shape (heatmap): (1, 46, 49, 19)
Input shape: (1, 552, 584, 3)
Output shape (heatmap): (1, 69, 73, 19)
Input shape: (1, 736, 784, 3)
Output shape (heatmap): (1, 92, 98, 19)
output_9_1.png
output_9_2.png
output_9_3.png
我们看上图输出的热点图,能过看到,经过运算把关键点(肘和手腕和髋,颈部等等)表现出来.
plt.imshow(oriImg[:,:,[2,1,0]])
plt.imshow(heatmap_avg[:,:,9], alpha=.5)
fig = matplotlib.pyplot.gcf()
cax = matplotlib.pyplot.gca()
fig.set_size_inches(20, 20)
fig.subplots_adjust(right=0.93)
cbar_ax = fig.add_axes([0.95, 0.15, 0.01, 0.7])
_ = fig.colorbar(ax2, cax=cbar_ax)
output_11_0.png
右膝盖的热力图
from numpy import ma
U = paf_avg[:,:,16] * -1
V = paf_avg[:,:,17]
X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0]))
M = np.zeros(U.shape, dtype='bool')
M[U**2 + V**2 < 0.5 * 0.5] = True
U = ma.masked_array(U, mask=M)
V = ma.masked_array(V, mask=M)
# 1
plt.figure()
plt.imshow(oriImg[:,:,[2,1,0]], alpha = .5)
s = 5
Q = plt.quiver(X[::s,::s], Y[::s,::s], U[::s,::s], V[::s,::s],
scale=50, headaxislength=4, alpha=.5, width=0.001, color='r')
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(20, 20)
output_13_0.png
右臂的热力图
from scipy.ndimage.filters import gaussian_filter
all_peaks = []
peak_counter = 0
for part in range(19-1):
map_ori = heatmap_avg[:,:,part]
map = gaussian_filter(map_ori, sigma=3)
map_left = np.zeros(map.shape)
map_left[1:,:] = map[:-1,:]
map_right = np.zeros(map.shape)
map_right[:-1,:] = map[1:,:]
map_up = np.zeros(map.shape)
map_up[:,1:] = map[:,:-1]
map_down = np.zeros(map.shape)
map_down[:,:-1] = map[:,1:]
peaks_binary = np.logical_and.reduce((map>=map_left, map>=map_right, map>=map_up, map>=map_down, map > param['thre1']))
peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse
peaks_with_score = [x + (map_ori[x[1],x[0]],) for x in peaks]
id = range(peak_counter, peak_counter + len(peaks))
peaks_with_score_and_id = [peaks_with_score[i] + (id[i],) for i in range(len(id))]
all_peaks.append(peaks_with_score_and_id)
peak_counter += len(peaks)
# find connection in the specified sequence, center 29 is in the position 15
limbSeq = [[2,3], [2,6], [3,4], [4,5], [6,7], [7,8], [2,9], [9,10], \
[10,11], [2,12], [12,13], [13,14], [2,1], [1,15], [15,17], \
[1,16], [16,18], [3,17], [6,18]]
# the middle joints heatmap correpondence
mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], \
[23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], \
[55,56], [37,38], [45,46]]
import math
connection_all = []
special_k = []
mid_num = 10
for k in range(len(mapIdx)):
score_mid = paf_avg[:,:,[x-19 for x in mapIdx[k]]]
candA = all_peaks[limbSeq[k][0]-1]
candB = all_peaks[limbSeq[k][1]-1]
nA = len(candA)
nB = len(candB)
indexA, indexB = limbSeq[k]
if(nA != 0 and nB != 0):
connection_candidate = []
for i in range(nA):
for j in range(nB):
vec = np.subtract(candB[j][:2], candA[i][:2])
norm = math.sqrt(vec[0]*vec[0] + vec[1]*vec[1])
vec = np.divide(vec, norm)
startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \
np.linspace(candA[i][1], candB[j][1], num=mid_num)))
vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \
for I in range(len(startend))])
vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \
for I in range(len(startend))])
score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
score_with_dist_prior = sum(score_midpts)/len(score_midpts) + min(0.5*oriImg.shape[0]/norm-1, 0)
criterion1 = len(np.nonzero(score_midpts > param['thre2'])[0]) > 0.8 * len(score_midpts)
criterion2 = score_with_dist_prior > 0
if criterion1 and criterion2:
connection_candidate.append([i, j, score_with_dist_prior, score_with_dist_prior+candA[i][2]+candB[j][2]])
connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
connection = np.zeros((0,5))
for c in range(len(connection_candidate)):
i,j,s = connection_candidate[c][0:3]
if(i not in connection[:,3] and j not in connection[:,4]):
connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
if(len(connection) >= min(nA, nB)):
break
connection_all.append(connection)
else:
special_k.append(k)
connection_all.append([])
# last number in each row is the total parts number of that person
# the second last number in each row is the score of the overall configuration
subset = -1 * np.ones((0, 20))
candidate = np.array([item for sublist in all_peaks for item in sublist])
for k in range(len(mapIdx)):
if k not in special_k:
partAs = connection_all[k][:,0]
partBs = connection_all[k][:,1]
indexA, indexB = np.array(limbSeq[k]) - 1
for i in range(len(connection_all[k])): #= 1:size(temp,1)
found = 0
subset_idx = [-1, -1]
for j in range(len(subset)): #1:size(subset,1):
if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
subset_idx[found] = j
found += 1
if found == 1:
j = subset_idx[0]
if(subset[j][indexB] != partBs[i]):
subset[j][indexB] = partBs[i]
subset[j][-1] += 1
subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
elif found == 2: # if found 2 and disjoint, merge them
j1, j2 = subset_idx
print ("found = 2")
membership = ((subset[j1]>=0).astype(int) + (subset[j2]>=0).astype(int))[:-2]
if len(np.nonzero(membership == 2)[0]) == 0: #merge
subset[j1][:-2] += (subset[j2][:-2] + 1)
subset[j1][-2:] += subset[j2][-2:]
subset[j1][-2] += connection_all[k][i][2]
subset = np.delete(subset, j2, 0)
else: # as like found == 1
subset[j1][indexB] = partBs[i]
subset[j1][-1] += 1
subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
# if find no partA in the subset, create a new subset
elif not found and k < 17:
row = -1 * np.ones(20)
row[indexA] = partAs[i]
row[indexB] = partBs[i]
row[-1] = 2
row[-2] = sum(candidate[connection_all[k][i,:2].astype(int), 2]) + connection_all[k][i][2]
subset = np.vstack([subset, row])
# delete some rows of subset which has few parts occur
deleteIdx = [];
for i in range(len(subset)):
if subset[i][-1] < 4 or subset[i][-2]/subset[i][-1] < 0.4:
deleteIdx.append(i)
subset = np.delete(subset, deleteIdx, axis=0)
我们把关键点全部可视化一下:
colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \
[0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \
[170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
cmap = matplotlib.cm.get_cmap('hsv')
canvas = cv2.imread(test_image) # B,G,R order
for i in range(18):
rgba = np.array(cmap(1 - i/18. - 1./36))
rgba[0:3] *= 255
for j in range(len(all_peaks[i])):
cv2.circle(canvas, all_peaks[i][j][0:2], 4, colors[i], thickness=-1)
to_plot = cv2.addWeighted(oriImg, 0.3, canvas, 0.7, 0)
plt.imshow(to_plot[:,:,[2,1,0]])
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(12, 12)
output_18_0.png
下一步我们把关键节点连起来就可以了
stickwidth = 4
for i in range(17):
for n in range(len(subset)):
index = subset[n][np.array(limbSeq[i])-1]
if -1 in index:
continue
cur_canvas = canvas.copy()
Y = candidate[index.astype(int), 0]
X = candidate[index.astype(int), 1]
mX = np.mean(X)
mY = np.mean(Y)
length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
polygon = cv2.ellipse2Poly((int(mY),int(mX)), (int(length/2), stickwidth), int(angle), 0, 360, 1)
cv2.fillConvexPoly(cur_canvas, polygon, colors[i])
canvas = cv2.addWeighted(canvas, 0.4, cur_canvas, 0.6, 0)
plt.imshow(canvas[:,:,[2,1,0]])
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(12, 12)
output_20_0.png
以上过程可以学习到深度学习对人体姿态的算法过程,通过关键节点的识别和链接,完整形成人体姿态的描述.
下一步,我们可以通过人体姿态的识别可以实现如下功能:
- 人体姿态控制计算机或设备
- 通过姿态预测人的行为
- 实时姿态生成动画人物
等等吧
