├── dev_requirements.txt ├── requirements.txt ├── classification_probability.png ├── data └── train01_16_128_171_mean.npy.bz2 ├── download_test_video.sh ├── sports1m └── get_labels.sh ├── models └── get_weights_and_mean.sh ├── Dockerfile ├── do_everything.sh ├── LICENSE.md ├── README.md ├── convert_caffe_model.py ├── c3d_model.py └── test_model.py /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | pydot==1.1.0 2 | youtube-dl 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | h5py==2.6.0 2 | keras==2.0.0 3 | numpy==1.12.0 4 | -------------------------------------------------------------------------------- /classification_probability.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/axon-research/c3d-keras/HEAD/classification_probability.png -------------------------------------------------------------------------------- /data/train01_16_128_171_mean.npy.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/axon-research/c3d-keras/HEAD/data/train01_16_128_171_mean.npy.bz2 -------------------------------------------------------------------------------- /download_test_video.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | youtube-dl \ 4 | -f 18 \ 5 | -o '%(id)s.%(ext)s' \ 6 | 'https://www.youtube.com/watch?v=dM06AMFLsrc' 7 | -------------------------------------------------------------------------------- /sports1m/get_labels.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | echo --------------------------------------------- 6 | echo Downloading sports-1m-dataset labels... 7 | wget \ 8 | -N \ 9 | https://raw.githubusercontent.com/gtoderici/sports-1m-dataset/master/labels.txt \ 10 | --directory-prefix=${DIR} 11 | 12 | echo --------------------------------------------- 13 | echo Done! 14 | -------------------------------------------------------------------------------- /models/get_weights_and_mean.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | echo --------------------------------------------- 6 | echo Downloading Sports1mil pre-trained model... 7 | wget \ 8 | -N \ 9 | --content-disposition \ 10 | http://vlg.cs.dartmouth.edu/c3d/conv3d_deepnetA_sport1m_iter_1900000 \ 11 | --directory-prefix=${DIR} 12 | 13 | echo --------------------------------------------- 14 | echo Unpacking mean cube... 15 | cp data/train01_16_128_171_mean.npy.bz2 ${DIR} 16 | bunzip2 ${DIR}/train01_16_128_171_mean.npy.bz2 17 | 18 | echo --------------------------------------------- 19 | echo Done! 20 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/tensorflow:0.12.0 2 | 3 | RUN \ 4 | apt-get update; \ 5 | apt-get install -y protobuf-compiler wget libcv-dev python-opencv; \ 6 | apt-get install -y python-tk graphviz; \ 7 | mkdir c3d-keras 8 | 9 | ADD requirements.txt c3d-keras/requirements.txt 10 | RUN pip install -r c3d-keras/requirements.txt 11 | 12 | ADD models c3d-keras/models 13 | ADD sports1m c3d-keras/sports1m 14 | ADD data c3d-keras/data 15 | 16 | RUN \ 17 | cd c3d-keras; \ 18 | bash models/get_weights_and_mean.sh; \ 19 | bash sports1m/get_labels.sh; \ 20 | wget -N https://raw.githubusercontent.com/facebook/C3D/master/C3D-v1.0/src/caffe/proto/caffe.proto; \ 21 | protoc --python_out=. caffe.proto 22 | 23 | ADD convert_caffe_model.py c3d-keras/convert_caffe_model.py 24 | ADD c3d_model.py c3d-keras/c3d_model.py 25 | 26 | RUN \ 27 | cd c3d-keras; \ 28 | echo import keras | python; \ 29 | python convert_caffe_model.py 30 | 31 | ADD download_test_video.sh c3d-keras/download_test_video.sh 32 | ADD test_model.py c3d-keras/test_model.py 33 | ADD dev_requirements.txt c3d-keras/dev_requirements.txt 34 | RUN pip install -r c3d-keras/dev_requirements.txt 35 | 36 | RUN \ 37 | cd c3d-keras; \ 38 | bash download_test_video.sh; \ 39 | python test_model.py 40 | -------------------------------------------------------------------------------- /do_everything.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # suppress some logs 4 | export TF_CPP_MIN_LOG_LEVEL=1 5 | 6 | # get weights / mean cube 7 | bash models/get_weights_and_mean.sh 8 | 9 | # get sports1mil labels 10 | bash sports1m/get_labels.sh 11 | 12 | # get caffe.proto from facebook/C3D repo (thanks to Du Tran) 13 | wget -N https://raw.githubusercontent.com/facebook/C3D/master/C3D-v1.0/src/caffe/proto/caffe.proto 14 | 15 | # protobuf combile caffe.proto 16 | if [ "$(which protoc 2> /dev/null)" ]; then 17 | protoc --python_out=. caffe.proto 18 | else 19 | echo 'Please install protobuf-compiler and rerun this script. e.g. "sudo apt-get install protobuf-compiler"' 20 | exit -1 21 | fi 22 | 23 | # make sure the default keras config (in `~/.keras/keras.json`) has: `tf` image_dim_ordering, and `tensorflow` backend. 24 | KERASCONF=~/.keras/keras.json 25 | if [ -z "$(grep image_dim_ordering.*tf ${KERASCONF})" ]; then 26 | echo 'Please set "image_dim_ordering" in ${KERASCONF} to be "tf"' 27 | exit -1 28 | fi 29 | if [ -z "$(grep backend.*tensorflow ${KERASCONF})" ]; then 30 | echo 'Please set "backend" in ${KERASCONF} to be "tensorflow"' 31 | exit -1 32 | fi 33 | 34 | # finally do the conversion! 35 | python convert_caffe_model.py 36 | 37 | # download test video (basketball clip) 38 | bash download_test_video.sh 39 | 40 | # run classification on this video 41 | python test_model.py 42 | 43 | echo "---------------------------------------------------" 44 | echo "You should have something close to the following:" 45 | echo "---------------------------------------------------" 46 | echo "Position of maximum probability: 367" 47 | echo "Maximum probability: 0.57953" 48 | echo "Corresponding label: basketball" 49 | echo "" 50 | echo "Top 5 probabilities and labels:" 51 | echo "basketball: 0.57953" 52 | echo "volleyball: 0.14435" 53 | echo "streetball: 0.06718" 54 | echo "freestyle wrestling: 0.03323" 55 | echo "greco-roman wrestling: 0.03293" 56 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Source Code License 2 | =================== 3 | 4 | Copyright 2017 TASER International, Inc. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | 1. Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | 13 | 2. Redistributions in binary form must reproduce the above copyright 14 | notice, this list of conditions and the following disclaimer in the 15 | documentation and/or other materials provided with the distribution. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | C3D Caffe Model 30 | =============== 31 | 32 | The FaceBook C3D Caffe Model is provided under a Creative Commons 33 | Attribution-NonCommercial 3.0 license: 34 | 35 | https://github.com/facebook/C3D/blob/master/LICENSE 36 | https://creativecommons.org/licenses/by-nc/3.0/ 37 | 38 | Sports-1M Labels 39 | ================ 40 | 41 | Labels of the YouTube Sports-1M Dataset are provided under a 42 | Creative Commons License: 43 | https://github.com/gtoderici/sports-1m-dataset/blob/master/LICENSE.md 44 | http://creativecommons.org/licenses/by/3.0/ 45 | 46 | Test Video 47 | ========== 48 | 49 | A test video is downloaded from youtube. No ownership or license 50 | is asserted for this video. 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | C3D Model for Keras + TensorFlow 2 | ================================ 3 | 4 | The scripts here are inspired by [`C3D Model for Keras`](https://gist.github.com/albertomontesg/d8b21a179c1e6cca0480ebdf292c34d2) gist, but specifically for Keras + TensorFlow (not Theano-backend). 5 | 6 | To reproduce results: 7 | 8 | - Run a script that does everything: `bash do_everything.sh` 9 | 10 | OR, build a docker image, which will do all the steps of replication 11 | during the build: 12 | 13 | ``` 14 | docker build -t c3d-keras . 15 | ``` 16 | 17 | OR, run each of these steps: 18 | 19 | 1. Download pretrained model: `bash models/get_weights_and_mean.sh` 20 | 2. Download sport1mil labels: `bash sports1m/get_labels.sh` 21 | 3. Download facebook/C3D `caffe.proto` file for conversion from caffe to Keras: `wget https://raw.githubusercontent.com/facebook/C3D/master/C3D-v1.0/src/caffe/proto/caffe.proto` 22 | 4. Install protobuf per instruction in https://github.com/google/protobuf. In Ubuntu, `sudo apt-get install protobuf-compiler` will do. 23 | 5. Compile the caffe.proto file for python: `protoc --python_out=. caffe.proto` 24 | 6. Make sure the default keras config (in `~/.keras/keras.json`) has: `tf` image_dim_ordering, and `tensorflow` backend. 25 | 7. Convert the pre-trained model from Caffe format to Keras: `python convert_caffe_model.py` 26 | 8. Download test video: `bash download_test_video.sh` 27 | 9. Run test: `python test_model.py` 28 | 29 | Prerequisites 30 | ============= 31 | Known to work with the following python packages: 32 | - Keras==2.0.0 33 | - tensorflow==0.12.1 34 | - h5py==2.6.0 35 | - numpy==1.12.0 36 | - cv2==3.1.0 37 | - pydot==1.1.0 38 | - graphviz 39 | 40 | Some basic command-line tools: 41 | - [protobuf compiler](https://developers.google.com/protocol-buffers/docs/downloads) 42 | - wget 43 | - [youtube-dl](https://rg3.github.io/youtube-dl/) 44 | 45 | Results 46 | ======= 47 | A following classification probability plot is expected (saved as `probabilities.png`). A peak at 367th class (probability = 71%) corresponds to basketball label. 48 | 49 | Classification Probability Plot 50 | 51 | The top 5 labels will also be reported, and should look something like: 52 | 53 | ``` 54 | Position of maximum probability: 367 55 | Maximum probability: 0.57953 56 | Corresponding label: basketball 57 | 58 | Top 5 probabilities and labels: 59 | basketball: 0.57953 60 | volleyball: 0.14435 61 | streetball: 0.06718 62 | freestyle wrestling: 0.03323 63 | greco-roman wrestling: 0.03293 64 | ``` 65 | 66 | References 67 | ========== 68 | 69 | 1. [C3D Model for Keras](https://gist.github.com/albertomontesg/d8b21a179c1e6cca0480ebdf292c34d2) 70 | 2. [Original C3D implementation in Caffe](https://github.com/facebook/C3D) 71 | 3. [C3D paper](https://arxiv.org/abs/1412.0767) 72 | 73 | Comment / Feedback 74 | =================== 75 | Feel free to contact Chuck Cho (cycho at axon.com) for any comment or feedback. 76 | 77 | License 78 | ======= 79 | 80 | * Source code: 2-clause BSD. 81 | * Data: various Creative Commons licenses. See [LICENSE.md](LICENSE.md) for details. 82 | -------------------------------------------------------------------------------- /convert_caffe_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import c3d_model 4 | import caffe_pb2 as caffe 5 | import numpy as np 6 | import h5py 7 | import os 8 | 9 | def reindex(x): 10 | # https://github.com/fchollet/keras/blob/master/keras/utils/np_utils.py#L90-L115 11 | # invert the last three axes 12 | if x.ndim != 5: 13 | print "[Error] Input to reindex must be 5D nparray." 14 | return None 15 | 16 | N = x.shape[0] 17 | C = x.shape[1] 18 | L = x.shape[2] 19 | H = x.shape[3] 20 | W = x.shape[4] 21 | y = np.zeros_like(x) 22 | for n in range(N): 23 | for c in range(C): 24 | for l in range(L): 25 | for h in range(H): 26 | for w in range(W): 27 | y[n, c, l, h, w] = x[n, c, 28 | L - l - 1, 29 | H - h - 1, 30 | W - w - 1] 31 | return y 32 | 33 | def convert_dense(w): 34 | # kernel: (8192, 4096): (512x1x4x4, 4096) -> (1x4x4x512, 4096) 35 | wo = np.zeros_like(w) 36 | for i in range(w.shape[1]): 37 | wi = np.squeeze(w[:,i]) 38 | wo[:,i] = np.transpose(np.reshape(wi, (512,4,4)), (1, 2, 0)).flatten() 39 | return wo 40 | 41 | def main(): 42 | 43 | #dim_ordering = 'th' 44 | #dim_ordering = 'th' 45 | import keras.backend as K 46 | dim_ordering = K.image_dim_ordering() 47 | print "[Info] image_dim_order (from default ~/.keras/keras.json)={}".format( 48 | dim_ordering) 49 | 50 | # get C3D model placeholder 51 | model = c3d_model.get_model(summary=True, backend=dim_ordering) 52 | 53 | # input caffe model 54 | caffe_model_filename = './models/conv3d_deepnetA_sport1m_iter_1900000' 55 | 56 | # output dir/files 57 | model_dir = './models' 58 | if not os.path.exists(model_dir): 59 | os.makedirs(model_dir) 60 | output_model_filename = os.path.join(model_dir, 'sports1M_weights_{}.h5'.format(dim_ordering)) 61 | output_json_filename = os.path.join(model_dir, 'sports1M_weights_{}.json'.format(dim_ordering)) 62 | 63 | # read caffe model 64 | print "-" * 19 65 | print "Reading model file={}...".format(caffe_model_filename) 66 | p = caffe.NetParameter() 67 | p.ParseFromString(open(caffe_model_filename, 'rb').read()) 68 | 69 | params = [] 70 | print "-" * 19 71 | print "Converting model..." 72 | 73 | # read every conv/fc layer and append to "params" list 74 | for i in range(len(p.layers)): 75 | layer = p.layers[i] 76 | # skip non-conv/fc layers 77 | if 'conv' not in layer.name and 'fc' not in layer.name: 78 | continue 79 | print "[Info] Massaging \"{}\" layer...".format(layer.name) 80 | weights_b = np.array(layer.blobs[1].data, dtype=np.float32) 81 | weights_p = np.array(layer.blobs[0].data, dtype=np.float32).reshape( 82 | layer.blobs[0].num, 83 | layer.blobs[0].channels, 84 | layer.blobs[0].length, 85 | layer.blobs[0].height, 86 | layer.blobs[0].width, 87 | ) 88 | if 'conv' in layer.name: 89 | # theano vs tensorflow: https://github.com/fchollet/keras/blob/master/keras/utils/np_utils.py#L90-L115 90 | if dim_ordering == 'th': 91 | weights_p = reindex(weights_p) 92 | else: 93 | weights_p = np.transpose(weights_p, (2, 3, 4, 1, 0)) 94 | elif 'fc' in layer.name: 95 | weights_p = weights_p[0, 0, 0, :, :].T 96 | if 'fc6' in layer.name: 97 | print("[Info] First FC layer after flattening layer needs " 98 | "special care...") 99 | weights_p = convert_dense(weights_p) 100 | params.append([weights_p, weights_b]) 101 | 102 | valid_layer_count = 0 103 | for layer_indx in range(len(model.layers)): 104 | layer_name = model.layers[layer_indx].name 105 | if 'conv' in layer_name or 'fc' in layer_name: 106 | print "[Info] Transplanting \"{}\" layer...".format(layer_name) 107 | model.layers[layer_indx].set_weights(params[valid_layer_count]) 108 | valid_layer_count += 1 109 | 110 | print "-" * 19 111 | print "Saving pre-trained model weights as {}...".format(output_model_filename) 112 | model.save_weights(output_model_filename, overwrite=True) 113 | json_string = model.to_json() 114 | with open(output_json_filename, 'w') as f: 115 | f.write(json_string) 116 | print "-" * 39 117 | print "Conversion done!" 118 | print "-" * 39 119 | 120 | if __name__ == '__main__': 121 | main() 122 | -------------------------------------------------------------------------------- /c3d_model.py: -------------------------------------------------------------------------------- 1 | from keras.models import Sequential 2 | from keras.layers.core import Dense, Dropout, Flatten 3 | from keras.layers.convolutional import Convolution3D, MaxPooling3D, ZeroPadding3D 4 | from keras.optimizers import SGD 5 | 6 | ''' 7 | dim_ordering issue: 8 | - 'th'-style dim_ordering: [batch, channels, depth, height, width] 9 | - 'tf'-style dim_ordering: [batch, depth, height, width, channels] 10 | ''' 11 | 12 | def get_model(summary=False, backend='tf'): 13 | """ Return the Keras model of the network 14 | """ 15 | model = Sequential() 16 | if backend == 'tf': 17 | input_shape=(16, 112, 112, 3) # l, h, w, c 18 | else: 19 | input_shape=(3, 16, 112, 112) # c, l, h, w 20 | model.add(Convolution3D(64, 3, 3, 3, activation='relu', 21 | border_mode='same', name='conv1', 22 | input_shape=input_shape)) 23 | model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), 24 | border_mode='valid', name='pool1')) 25 | # 2nd layer group 26 | model.add(Convolution3D(128, 3, 3, 3, activation='relu', 27 | border_mode='same', name='conv2')) 28 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 29 | border_mode='valid', name='pool2')) 30 | # 3rd layer group 31 | model.add(Convolution3D(256, 3, 3, 3, activation='relu', 32 | border_mode='same', name='conv3a')) 33 | model.add(Convolution3D(256, 3, 3, 3, activation='relu', 34 | border_mode='same', name='conv3b')) 35 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 36 | border_mode='valid', name='pool3')) 37 | # 4th layer group 38 | model.add(Convolution3D(512, 3, 3, 3, activation='relu', 39 | border_mode='same', name='conv4a')) 40 | model.add(Convolution3D(512, 3, 3, 3, activation='relu', 41 | border_mode='same', name='conv4b')) 42 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 43 | border_mode='valid', name='pool4')) 44 | # 5th layer group 45 | model.add(Convolution3D(512, 3, 3, 3, activation='relu', 46 | border_mode='same', name='conv5a')) 47 | model.add(Convolution3D(512, 3, 3, 3, activation='relu', 48 | border_mode='same', name='conv5b')) 49 | model.add(ZeroPadding3D(padding=((0, 0), (0, 1), (0, 1)), name='zeropad5')) 50 | model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 51 | border_mode='valid', name='pool5')) 52 | model.add(Flatten()) 53 | # FC layers group 54 | model.add(Dense(4096, activation='relu', name='fc6')) 55 | model.add(Dropout(.5)) 56 | model.add(Dense(4096, activation='relu', name='fc7')) 57 | model.add(Dropout(.5)) 58 | model.add(Dense(487, activation='softmax', name='fc8')) 59 | 60 | if summary: 61 | print(model.summary()) 62 | 63 | return model 64 | 65 | def get_int_model(model, layer, backend='tf'): 66 | 67 | if backend == 'tf': 68 | input_shape=(16, 112, 112, 3) # l, h, w, c 69 | else: 70 | input_shape=(3, 16, 112, 112) # c, l, h, w 71 | 72 | int_model = Sequential() 73 | 74 | int_model.add(Convolution3D(64, 3, 3, 3, activation='relu', 75 | border_mode='same', name='conv1', 76 | input_shape=input_shape, 77 | weights=model.layers[0].get_weights())) 78 | if layer == 'conv1': 79 | return int_model 80 | int_model.add(MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), 81 | border_mode='valid', name='pool1')) 82 | if layer == 'pool1': 83 | return int_model 84 | 85 | # 2nd layer group 86 | int_model.add(Convolution3D(128, 3, 3, 3, activation='relu', 87 | border_mode='same', name='conv2', 88 | weights=model.layers[2].get_weights())) 89 | if layer == 'conv2': 90 | return int_model 91 | int_model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 92 | border_mode='valid', name='pool2')) 93 | if layer == 'pool2': 94 | return int_model 95 | 96 | # 3rd layer group 97 | int_model.add(Convolution3D(256, 3, 3, 3, activation='relu', 98 | border_mode='same', name='conv3a', 99 | weights=model.layers[4].get_weights())) 100 | if layer == 'conv3a': 101 | return int_model 102 | int_model.add(Convolution3D(256, 3, 3, 3, activation='relu', 103 | border_mode='same', name='conv3b', 104 | weights=model.layers[5].get_weights())) 105 | if layer == 'conv3b': 106 | return int_model 107 | int_model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 108 | border_mode='valid', name='pool3')) 109 | if layer == 'pool3': 110 | return int_model 111 | 112 | # 4th layer group 113 | int_model.add(Convolution3D(512, 3, 3, 3, activation='relu', 114 | border_mode='same', name='conv4a', 115 | weights=model.layers[7].get_weights())) 116 | if layer == 'conv4a': 117 | return int_model 118 | int_model.add(Convolution3D(512, 3, 3, 3, activation='relu', 119 | border_mode='same', name='conv4b', 120 | weights=model.layers[8].get_weights())) 121 | if layer == 'conv4b': 122 | return int_model 123 | int_model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 124 | border_mode='valid', name='pool4')) 125 | if layer == 'pool4': 126 | return int_model 127 | 128 | # 5th layer group 129 | int_model.add(Convolution3D(512, 3, 3, 3, activation='relu', 130 | border_mode='same', name='conv5a', 131 | weights=model.layers[10].get_weights())) 132 | if layer == 'conv5a': 133 | return int_model 134 | int_model.add(Convolution3D(512, 3, 3, 3, activation='relu', 135 | border_mode='same', name='conv5b', 136 | weights=model.layers[11].get_weights())) 137 | if layer == 'conv5b': 138 | return int_model 139 | int_model.add(ZeroPadding3D(padding=(0, 1, 1), name='zeropad')) 140 | int_model.add(MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), 141 | border_mode='valid', name='pool5')) 142 | if layer == 'pool5': 143 | return int_model 144 | 145 | int_model.add(Flatten()) 146 | # FC layers group 147 | int_model.add(Dense(4096, activation='relu', name='fc6', 148 | weights=model.layers[15].get_weights())) 149 | if layer == 'fc6': 150 | return int_model 151 | int_model.add(Dropout(.5)) 152 | int_model.add(Dense(4096, activation='relu', name='fc7', 153 | weights=model.layers[17].get_weights())) 154 | if layer == 'fc7': 155 | return int_model 156 | int_model.add(Dropout(.5)) 157 | int_model.add(Dense(487, activation='softmax', name='fc8', 158 | weights=model.layers[19].get_weights())) 159 | if layer == 'fc8': 160 | return int_model 161 | 162 | return None 163 | 164 | if __name__ == '__main__': 165 | model = get_model(summary=True) 166 | -------------------------------------------------------------------------------- /test_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import matplotlib 4 | matplotlib.use('Agg') 5 | from keras.models import model_from_json 6 | import os 7 | import cv2 8 | import numpy as np 9 | import matplotlib.pyplot as plt 10 | import c3d_model 11 | import sys 12 | import keras.backend as K 13 | dim_ordering = K.image_dim_ordering() 14 | print "[Info] image_dim_order (from default ~/.keras/keras.json)={}".format( 15 | dim_ordering) 16 | backend = dim_ordering 17 | 18 | def diagnose(data, verbose=True, label='input', plots=False, backend='tf'): 19 | # Convolution3D? 20 | if data.ndim > 2: 21 | if backend == 'th': 22 | data = np.transpose(data, (1, 2, 3, 0)) 23 | #else: 24 | # data = np.transpose(data, (0, 2, 1, 3)) 25 | min_num_spatial_axes = 10 26 | max_outputs_to_show = 3 27 | ndim = data.ndim 28 | print "[Info] {}.ndim={}".format(label, ndim) 29 | print "[Info] {}.shape={}".format(label, data.shape) 30 | for d in range(ndim): 31 | num_this_dim = data.shape[d] 32 | if num_this_dim >= min_num_spatial_axes: # check for spatial axes 33 | # just first, center, last indices 34 | range_this_dim = [0, num_this_dim/2, num_this_dim - 1] 35 | else: 36 | # sweep all indices for non-spatial axes 37 | range_this_dim = range(num_this_dim) 38 | for i in range_this_dim: 39 | new_dim = tuple([d] + range(d) + range(d + 1, ndim)) 40 | sliced = np.transpose(data, new_dim)[i, ...] 41 | print("[Info] {}, dim:{} {}-th slice: " 42 | "(min, max, mean, std)=({}, {}, {}, {})".format( 43 | label, 44 | d, i, 45 | np.min(sliced), 46 | np.max(sliced), 47 | np.mean(sliced), 48 | np.std(sliced))) 49 | if plots: 50 | # assume (l, h, w, c)-shaped input 51 | if data.ndim != 4: 52 | print("[Error] data (shape={}) is not 4-dim. Check data".format( 53 | data.shape)) 54 | return 55 | l, h, w, c = data.shape 56 | if l >= min_num_spatial_axes or \ 57 | h < min_num_spatial_axes or \ 58 | w < min_num_spatial_axes: 59 | print("[Error] data (shape={}) does not look like in (l,h,w,c) " 60 | "format. Do reshape/transpose.".format(data.shape)) 61 | return 62 | nrows = int(np.ceil(np.sqrt(data.shape[0]))) 63 | # BGR 64 | if c == 3: 65 | for i in range(l): 66 | mng = plt.get_current_fig_manager() 67 | mng.resize(*mng.window.maxsize()) 68 | plt.subplot(nrows, nrows, i + 1) # doh, one-based! 69 | im = np.squeeze(data[i, ...]).astype(np.float32) 70 | im = im[:, :, ::-1] # BGR to RGB 71 | # force it to range [0,1] 72 | im_min, im_max = im.min(), im.max() 73 | if im_max > im_min: 74 | im_std = (im - im_min) / (im_max - im_min) 75 | else: 76 | print "[Warning] image is constant!" 77 | im_std = np.zeros_like(im) 78 | plt.imshow(im_std) 79 | plt.axis('off') 80 | plt.title("{}: t={}".format(label, i)) 81 | plt.show() 82 | #plt.waitforbuttonpress() 83 | else: 84 | for j in range(min(c, max_outputs_to_show)): 85 | for i in range(l): 86 | mng = plt.get_current_fig_manager() 87 | mng.resize(*mng.window.maxsize()) 88 | plt.subplot(nrows, nrows, i + 1) # doh, one-based! 89 | im = np.squeeze(data[i, ...]).astype(np.float32) 90 | im = im[:, :, j] 91 | # force it to range [0,1] 92 | im_min, im_max = im.min(), im.max() 93 | if im_max > im_min: 94 | im_std = (im - im_min) / (im_max - im_min) 95 | else: 96 | print "[Warning] image is constant!" 97 | im_std = np.zeros_like(im) 98 | plt.imshow(im_std) 99 | plt.axis('off') 100 | plt.title("{}: o={}, t={}".format(label, j, i)) 101 | plt.show() 102 | #plt.waitforbuttonpress() 103 | elif data.ndim == 1: 104 | print("[Info] {} (min, max, mean, std)=({}, {}, {}, {})".format( 105 | label, 106 | np.min(data), 107 | np.max(data), 108 | np.mean(data), 109 | np.std(data))) 110 | print("[Info] data[:10]={}".format(data[:10])) 111 | 112 | return 113 | 114 | def main(): 115 | show_images = False 116 | diagnose_plots = False 117 | model_dir = './models' 118 | global backend 119 | 120 | # override backend if provided as an input arg 121 | if len(sys.argv) > 1: 122 | if 'tf' in sys.argv[1].lower(): 123 | backend = 'tf' 124 | else: 125 | backend = 'th' 126 | print "[Info] Using backend={}".format(backend) 127 | 128 | if backend == 'th': 129 | model_weight_filename = os.path.join(model_dir, 'sports1M_weights_th.h5') 130 | model_json_filename = os.path.join(model_dir, 'sports1M_weights_th.json') 131 | else: 132 | model_weight_filename = os.path.join(model_dir, 'sports1M_weights_tf.h5') 133 | model_json_filename = os.path.join(model_dir, 'sports1M_weights_tf.json') 134 | 135 | print("[Info] Reading model architecture...") 136 | model = model_from_json(open(model_json_filename, 'r').read()) 137 | #model = c3d_model.get_model(backend=backend) 138 | 139 | # visualize model 140 | model_img_filename = os.path.join(model_dir, 'c3d_model.png') 141 | if not os.path.exists(model_img_filename): 142 | from keras.utils import plot_model 143 | plot_model(model, to_file=model_img_filename) 144 | 145 | print("[Info] Loading model weights...") 146 | model.load_weights(model_weight_filename) 147 | print("[Info] Loading model weights -- DONE!") 148 | model.compile(loss='mean_squared_error', optimizer='sgd') 149 | 150 | print("[Info] Loading labels...") 151 | with open('sports1m/labels.txt', 'r') as f: 152 | labels = [line.strip() for line in f.readlines()] 153 | print('Total labels: {}'.format(len(labels))) 154 | 155 | print("[Info] Loading a sample video...") 156 | cap = cv2.VideoCapture('dM06AMFLsrc.mp4') 157 | 158 | vid = [] 159 | while True: 160 | ret, img = cap.read() 161 | if not ret: 162 | break 163 | vid.append(cv2.resize(img, (171, 128))) 164 | vid = np.array(vid, dtype=np.float32) 165 | 166 | #plt.imshow(vid[2000]/256) 167 | #plt.show() 168 | 169 | # sample 16-frame clip 170 | #start_frame = 100 171 | start_frame = 2000 172 | X = vid[start_frame:(start_frame + 16), :, :, :] 173 | #diagnose(X, verbose=True, label='X (16-frame clip)', plots=show_images) 174 | 175 | # subtract mean 176 | mean_cube = np.load('models/train01_16_128_171_mean.npy') 177 | mean_cube = np.transpose(mean_cube, (1, 2, 3, 0)) 178 | #diagnose(mean_cube, verbose=True, label='Mean cube', plots=show_images) 179 | X -= mean_cube 180 | #diagnose(X, verbose=True, label='Mean-subtracted X', plots=show_images) 181 | 182 | # center crop 183 | X = X[:, 8:120, 30:142, :] # (l, h, w, c) 184 | #diagnose(X, verbose=True, label='Center-cropped X', plots=show_images) 185 | 186 | if backend == 'th': 187 | X = np.transpose(X, (3, 0, 1, 2)) # input_shape = (3,16,112,112) 188 | else: 189 | pass # input_shape = (16,112,112,3) 190 | 191 | # get activations for intermediate layers if needed 192 | inspect_layers = [ 193 | # 'fc6', 194 | # 'fc7', 195 | ] 196 | for layer in inspect_layers: 197 | int_model = c3d_model.get_int_model(model=model, layer=layer, backend=backend) 198 | int_output = int_model.predict_on_batch(np.array([X])) 199 | int_output = int_output[0, ...] 200 | print "[Debug] at layer={}: output.shape={}".format(layer, int_output.shape) 201 | diagnose(int_output, 202 | verbose=True, 203 | label='{} activation'.format(layer), 204 | plots=diagnose_plots, 205 | backend=backend) 206 | 207 | # inference 208 | output = model.predict_on_batch(np.array([X])) 209 | 210 | # show results 211 | print('Saving class probabilitities in probabilities.png') 212 | plt.plot(output[0]) 213 | plt.title('Probability') 214 | plt.savefig("probabilities.png") 215 | print('Position of maximum probability: {}'.format(output[0].argmax())) 216 | print('Maximum probability: {:.5f}'.format(max(output[0]))) 217 | print('Corresponding label: {}'.format(labels[output[0].argmax()])) 218 | 219 | # sort top five predictions from softmax output 220 | top_inds = output[0].argsort()[::-1][:5] # reverse sort and take five largest items 221 | print('\nTop 5 probabilities and labels:') 222 | for i in top_inds: 223 | print('{1}: {0:.5f}'.format(output[0][i], labels[i])) 224 | 225 | if __name__ == '__main__': 226 | main() 227 | --------------------------------------------------------------------------------