├── C3D_model.py ├── LICENSE ├── README.md ├── data └── roger │ ├── frame_00030.png │ ├── frame_00031.png │ ├── frame_00032.png │ ├── frame_00033.png │ ├── frame_00034.png │ ├── frame_00035.png │ ├── frame_00036.png │ ├── frame_00037.png │ ├── frame_00038.png │ ├── frame_00039.png │ ├── frame_00040.png │ ├── frame_00041.png │ ├── frame_00042.png │ ├── frame_00043.png │ ├── frame_00044.png │ └── frame_00045.png ├── labels.txt └── predict.py /C3D_model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class C3D(nn.Module): 5 | """ 6 | The C3D network as described in [1]. 7 | """ 8 | 9 | def __init__(self): 10 | super(C3D, self).__init__() 11 | 12 | self.conv1 = nn.Conv3d(3, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 13 | self.pool1 = nn.MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2)) 14 | 15 | self.conv2 = nn.Conv3d(64, 128, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 16 | self.pool2 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) 17 | 18 | self.conv3a = nn.Conv3d(128, 256, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 19 | self.conv3b = nn.Conv3d(256, 256, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 20 | self.pool3 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) 21 | 22 | self.conv4a = nn.Conv3d(256, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 23 | self.conv4b = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 24 | self.pool4 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) 25 | 26 | self.conv5a = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 27 | self.conv5b = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) 28 | self.pool5 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=(0, 1, 1)) 29 | 30 | self.fc6 = nn.Linear(8192, 4096) 31 | self.fc7 = nn.Linear(4096, 4096) 32 | self.fc8 = nn.Linear(4096, 487) 33 | 34 | self.dropout = nn.Dropout(p=0.5) 35 | 36 | self.relu = nn.ReLU() 37 | self.softmax = nn.Softmax() 38 | 39 | def forward(self, x): 40 | 41 | h = self.relu(self.conv1(x)) 42 | h = self.pool1(h) 43 | 44 | h = self.relu(self.conv2(h)) 45 | h = self.pool2(h) 46 | 47 | h = self.relu(self.conv3a(h)) 48 | h = self.relu(self.conv3b(h)) 49 | h = self.pool3(h) 50 | 51 | h = self.relu(self.conv4a(h)) 52 | h = self.relu(self.conv4b(h)) 53 | h = self.pool4(h) 54 | 55 | h = self.relu(self.conv5a(h)) 56 | h = self.relu(self.conv5b(h)) 57 | h = self.pool5(h) 58 | 59 | h = h.view(-1, 8192) 60 | h = self.relu(self.fc6(h)) 61 | h = self.dropout(h) 62 | h = self.relu(self.fc7(h)) 63 | h = self.dropout(h) 64 | 65 | logits = self.fc8(h) 66 | probs = self.softmax(logits) 67 | 68 | return probs 69 | 70 | """ 71 | References 72 | ---------- 73 | [1] Tran, Du, et al. "Learning spatiotemporal features with 3d convolutional networks." 74 | Proceedings of the IEEE international conference on computer vision. 2015. 75 | """ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Davide Abati 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # C3D for pytorch 2 | 3 | This is a pytorch porting of the network presented in the paper [Learning Spatiotemporal Features with 3D Convolutional Networks](http://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Tran_Learning_Spatiotemporal_Features_ICCV_2015_paper.pdf) 4 | 5 | --- 6 | ### How to use: 7 | * Download the pretrained weights (Sports1M) from [here](http://imagelab.ing.unimore.it/files/c3d_pytorch/c3d.pickle). 8 | * Run the predict script. 9 | 10 | **disclaimer**: Performances tested only qualitatively! No warranty! 11 | -------------------------------------------------------------------------------- /data/roger/frame_00030.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00030.png -------------------------------------------------------------------------------- /data/roger/frame_00031.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00031.png -------------------------------------------------------------------------------- /data/roger/frame_00032.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00032.png -------------------------------------------------------------------------------- /data/roger/frame_00033.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00033.png -------------------------------------------------------------------------------- /data/roger/frame_00034.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00034.png -------------------------------------------------------------------------------- /data/roger/frame_00035.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00035.png -------------------------------------------------------------------------------- /data/roger/frame_00036.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00036.png -------------------------------------------------------------------------------- /data/roger/frame_00037.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00037.png -------------------------------------------------------------------------------- /data/roger/frame_00038.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00038.png -------------------------------------------------------------------------------- /data/roger/frame_00039.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00039.png -------------------------------------------------------------------------------- /data/roger/frame_00040.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00040.png -------------------------------------------------------------------------------- /data/roger/frame_00041.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00041.png -------------------------------------------------------------------------------- /data/roger/frame_00042.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00042.png -------------------------------------------------------------------------------- /data/roger/frame_00043.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00043.png -------------------------------------------------------------------------------- /data/roger/frame_00044.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00044.png -------------------------------------------------------------------------------- /data/roger/frame_00045.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavideA/c3d-pytorch/1111ccf0de77cf1ba9f1027f7f065e889fc97169/data/roger/frame_00045.png -------------------------------------------------------------------------------- /labels.txt: -------------------------------------------------------------------------------- 1 | boomerang 2 | boxing 3 | bowling 4 | candlepin bowling 5 | bowls 6 | skittles (sport) 7 | ten-pin bowling 8 | cycling 9 | unicycle 10 | mountain unicycling 11 | bicycle 12 | bmx 13 | freestyle bmx 14 | cyclo-cross 15 | cross-country cycling 16 | road bicycle racing 17 | track cycling 18 | downhill mountain biking 19 | freeride 20 | dirt jumping 21 | slopestyle 22 | equestrianism 23 | fencing 24 | figure skating 25 | speed skating 26 | gymnastics 27 | tumbling (gymnastics) 28 | baton twirling 29 | artistic gymnastics 30 | balance beam 31 | floor (gymnastics) 32 | horizontal bar 33 | parallel bars 34 | pommel horse 35 | rings (gymnastics) 36 | sport aerobics 37 | uneven bars 38 | vault (gymnastics) 39 | majorette (dancer) 40 | rhythmic gymnastics 41 | hoop (rhythmic gymnastics) 42 | ribbon (rhythmic gymnastics) 43 | rope (rhythmic gymnastics) 44 | ball (rhythmic gymnastics) 45 | juggling club 46 | tricking 47 | skipping rope 48 | acrobatics 49 | slacklining 50 | trampolining 51 | trapeze 52 | flying trapeze 53 | judo 54 | brazilian jiu-jitsu 55 | tennis 56 | mixed martial arts 57 | krav maga 58 | yoga 59 | sanshou 60 | baguazhang 61 | bujinkan 62 | hapkido 63 | jeet kune do 64 | kajukenbo 65 | kalaripayattu 66 | kuk sool won 67 | northern praying mantis (martial art) 68 | ninjutsu 69 | pankration 70 | pencak silat 71 | shidokan 72 | shoot boxing 73 | shorinji kempo 74 | systema 75 | t'ai chi ch'uan 76 | vovinam 77 | xing yi quan 78 | skiing 79 | skijoring 80 | alpine skiing 81 | cross-country skiing 82 | freestyle skiing 83 | nordic skiing 84 | ski jumping 85 | ski touring 86 | telemark skiing 87 | squash (sport) 88 | swimming (sport) 89 | backstroke 90 | breaststroke 91 | butterfly stroke 92 | freestyle swimming 93 | medley swimming 94 | synchronized swimming 95 | finswimming 96 | water polo 97 | table tennis 98 | wrestling 99 | pehlwani 100 | shuai jiao 101 | yağlı güreş 102 | amateur wrestling 103 | greco-roman wrestling 104 | freestyle wrestling 105 | sport kite 106 | kite 107 | kite landboarding 108 | parasailing 109 | snowkiting 110 | kite buggy 111 | kitesurfing 112 | sledding 113 | skeleton (sport) 114 | toboggan 115 | skibobbing 116 | sled dog racing 117 | bobsleigh 118 | luge 119 | archery 120 | kyūdō 121 | pitch and putt 122 | croquet 123 | cue sports 124 | eight-ball 125 | blackball (pool) 126 | nine-ball 127 | straight pool 128 | ten-ball 129 | trick shot 130 | russian pyramid 131 | snooker 132 | carom billiards 133 | darts 134 | golf 135 | match play 136 | shooting sport 137 | practical shooting 138 | cowboy action shooting 139 | clay pigeon shooting 140 | skeet shooting 141 | trap shooting 142 | sporting clays 143 | skee ball 144 | knife throwing 145 | boules 146 | bocce 147 | boccia 148 | curling 149 | shuffleboard 150 | pétanque 151 | climbing 152 | canyoning 153 | mountaineering 154 | rope climbing 155 | ice climbing 156 | pole climbing (gymnastic) 157 | hiking 158 | rock climbing 159 | sport climbing 160 | abseiling 161 | bouldering 162 | walking 163 | backpacking (wilderness) 164 | race walking 165 | triathlon 166 | adventure racing 167 | biathlon 168 | duathlon 169 | decathlon 170 | heptathlon 171 | modern pentathlon 172 | pentathlon 173 | crossfit 174 | board sports 175 | freestyle scootering 176 | freeboard (skateboard) 177 | longboarding 178 | streetluge 179 | skimboarding 180 | wakesurfing 181 | bodyboarding 182 | riverboarding 183 | wakeboarding 184 | snowboarding 185 | sandboarding 186 | mountainboarding 187 | surfing 188 | skateboarding 189 | free running 190 | freestyle football 191 | powerbocking 192 | calisthenics 193 | footbag 194 | physical fitness 195 | pilates 196 | weight training 197 | highland games 198 | olympic weightlifting 199 | strength athletics 200 | bodybuilding 201 | powerlifting 202 | wallball 203 | running 204 | orienteering 205 | geocaching 206 | sprint (running) 207 | hurdles 208 | endurance 209 | ultramarathon 210 | cross country running 211 | half marathon 212 | marathon 213 | aggressive inline skating 214 | artistic roller skating 215 | ice skating 216 | inline speed skating 217 | rink bandy 218 | freestyle slalom skating 219 | roller derby 220 | roller skating 221 | short track speed skating 222 | synchronized skating 223 | snowmobile 224 | hunting 225 | deer hunting 226 | fox hunting 227 | disc dog 228 | dog agility 229 | bikejoring 230 | carting 231 | obedience trial 232 | conformation show 233 | dock jumping 234 | flyball 235 | french ring sport 236 | greyhound racing 237 | lure coursing 238 | mushing 239 | obedience training 240 | rally obedience 241 | schutzhund 242 | sheepdog trial 243 | sighthound 244 | weight pulling 245 | dachshund racing 246 | barrel racing 247 | charreada 248 | chilean rodeo 249 | cross-country equestrianism 250 | dressage 251 | endurance riding 252 | english pleasure 253 | equitation 254 | eventing 255 | equestrian vaulting 256 | gymkhana 257 | harness racing 258 | horse racing 259 | horseball 260 | hunt seat 261 | reining 262 | rodeo 263 | bull riding 264 | bullfighting 265 | show jumping 266 | steeplechase 267 | team penning 268 | western pleasure 269 | motorcycle racing 270 | auto race 271 | road racing 272 | endurance racing (motorsport) 273 | enduro 274 | freestyle motocross 275 | grand prix motorcycle racing 276 | rally raid 277 | motocross 278 | ice racing 279 | motorcycle speedway 280 | superbike racing 281 | supermoto 282 | track racing 283 | trial 284 | isle of man tt 285 | motorcycle drag racing 286 | all-terrain vehicle 287 | remote control 288 | radio-controlled car 289 | robot combat 290 | auto racing 291 | autocross 292 | autograss 293 | banger racing 294 | demolition derby 295 | desert racing 296 | dirt track racing 297 | drag racing 298 | drifting (motorsport) 299 | folkrace 300 | formula racing 301 | kart racing 302 | monster truck 303 | mud bogging 304 | off-road racing 305 | race of champions 306 | rallycross 307 | rallying 308 | short track motor racing 309 | sports car racing 310 | sprint car racing 311 | street racing 312 | stock car racing 313 | time attack 314 | tractor pulling 315 | touring car racing 316 | truck racing 317 | motorboat racing 318 | f1 powerboat racing 319 | hydroplane racing 320 | jetsprint 321 | personal water craft 322 | drag boat racing 323 | badminton 324 | basque pelota 325 | frontenis 326 | jai alai 327 | beach tennis 328 | lacrosse 329 | women's lacrosse 330 | padel tennis 331 | pickleball 332 | racquetball 333 | soft tennis 334 | team handball 335 | goalball 336 | tchoukball 337 | beach handball 338 | hockey 339 | underwater hockey 340 | ball hockey 341 | bandy 342 | broomball 343 | field hockey 344 | indoor field hockey 345 | floorball 346 | ice hockey 347 | ringette 348 | pond hockey 349 | roller hockey 350 | inline hockey 351 | roller hockey (quad) 352 | sledge hockey 353 | street hockey 354 | polo 355 | canoe polo 356 | sepak takraw 357 | association football 358 | ulama 359 | paintball 360 | volleyball 361 | beach volleyball 362 | color guard (flag spinning) 363 | capture the flag 364 | hide-and-seek 365 | kabaddi 366 | ultimate (sport) 367 | hurling 368 | basketball 369 | netball 370 | 3x3 (basketball) 371 | streetball 372 | wheelchair basketball 373 | korfball 374 | slamball 375 | kickball 376 | baseball 377 | cricket 378 | limited overs cricket 379 | one day international 380 | test cricket 381 | twenty20 382 | rounders 383 | softball 384 | tee ball 385 | wiffle ball 386 | beach soccer 387 | futsal 388 | indoor soccer 389 | street football 390 | australian rules football 391 | gaelic football 392 | gridiron football 393 | american football 394 | flag football 395 | indoor american football 396 | canadian football 397 | arena football 398 | rugby 399 | rugby sevens 400 | football tennis 401 | footvolley 402 | dodgeball 403 | ga-ga 404 | rundown 405 | fistball 406 | jianzi 407 | peteca 408 | valencian frontó 409 | fives 410 | valencian pilota 411 | disc golf 412 | flying disc freestyle 413 | capoeira 414 | fujian white crane 415 | karate 416 | kenpō 417 | kickboxing 418 | muay thai 419 | pradal serey 420 | savate 421 | shaolin kung fu 422 | silat 423 | taekwondo 424 | taido 425 | tang soo do 426 | wing chun 427 | wing tsun 428 | zui quan 429 | airsoft 430 | laser tag 431 | grappling 432 | jujutsu 433 | sambo (martial art) 434 | sumo 435 | daitō-ryū aiki-jūjutsu 436 | aikido 437 | kenjutsu 438 | kung fu (term) 439 | modern arnis 440 | okinawan kobudō 441 | wushu (sport) 442 | battōjutsu 443 | eskrima 444 | gatka 445 | haidong gumdo 446 | iaidō 447 | jōdō 448 | kendo 449 | sailing 450 | land sailing 451 | windsurfing 452 | dinghy sailing 453 | rowing (sport) 454 | dragon boat 455 | fishing 456 | angling 457 | big-game fishing 458 | casting (fishing) 459 | noodling 460 | spearfishing 461 | surf fishing 462 | rock fishing 463 | fly fishing 464 | diving 465 | free-diving 466 | scuba diving 467 | snorkeling 468 | canoeing 469 | kayaking 470 | creeking 471 | sea kayak 472 | surf kayaking 473 | whitewater kayaking 474 | rafting 475 | gliding 476 | hang gliding 477 | powered hang glider 478 | paragliding 479 | powered paragliding 480 | parachuting 481 | base jumping 482 | wingsuit flying 483 | ultralight aviation 484 | aerobatics 485 | air racing 486 | hot air ballooning 487 | model aircraft -------------------------------------------------------------------------------- /predict.py: -------------------------------------------------------------------------------- 1 | """ How to use C3D network. """ 2 | import numpy as np 3 | 4 | import torch 5 | from torch.autograd import Variable 6 | 7 | from os.path import join 8 | from glob import glob 9 | 10 | import skimage.io as io 11 | from skimage.transform import resize 12 | 13 | from C3D_model import C3D 14 | 15 | 16 | def get_sport_clip(clip_name, verbose=True): 17 | """ 18 | Loads a clip to be fed to C3D for classification. 19 | TODO: should I remove mean here? 20 | 21 | Parameters 22 | ---------- 23 | clip_name: str 24 | the name of the clip (subfolder in 'data'). 25 | verbose: bool 26 | if True, shows the unrolled clip (default is True). 27 | 28 | Returns 29 | ------- 30 | Tensor 31 | a pytorch batch (n, ch, fr, h, w). 32 | """ 33 | 34 | clip = sorted(glob(join('data', clip_name, '*.png'))) 35 | clip = np.array([resize(io.imread(frame), output_shape=(112, 200), preserve_range=True) for frame in clip]) 36 | clip = clip[:, :, 44:44+112, :] # crop centrally 37 | 38 | if verbose: 39 | clip_img = np.reshape(clip.transpose(1, 0, 2, 3), (112, 16 * 112, 3)) 40 | io.imshow(clip_img.astype(np.uint8)) 41 | io.show() 42 | 43 | clip = clip.transpose(3, 0, 1, 2) # ch, fr, h, w 44 | clip = np.expand_dims(clip, axis=0) # batch axis 45 | clip = np.float32(clip) 46 | 47 | return torch.from_numpy(clip) 48 | 49 | 50 | def read_labels_from_file(filepath): 51 | """ 52 | Reads Sport1M labels from file 53 | 54 | Parameters 55 | ---------- 56 | filepath: str 57 | the file. 58 | 59 | Returns 60 | ------- 61 | list 62 | list of sport names. 63 | """ 64 | with open(filepath, 'r') as f: 65 | labels = [line.strip() for line in f.readlines()] 66 | return labels 67 | 68 | 69 | def main(): 70 | """ 71 | Main function. 72 | """ 73 | 74 | # load a clip to be predicted 75 | X = get_sport_clip('roger') 76 | X = Variable(X) 77 | X = X.cuda() 78 | 79 | # get network pretrained model 80 | net = C3D() 81 | net.load_state_dict(torch.load('c3d.pickle')) 82 | net.cuda() 83 | net.eval() 84 | 85 | # perform prediction 86 | prediction = net(X) 87 | prediction = prediction.data.cpu().numpy() 88 | 89 | # read labels 90 | labels = read_labels_from_file('labels.txt') 91 | 92 | # print top predictions 93 | top_inds = prediction[0].argsort()[::-1][:5] # reverse sort and take five largest items 94 | print('\nTop 5:') 95 | for i in top_inds: 96 | print('{:.5f} {}'.format(prediction[0][i], labels[i])) 97 | 98 | 99 | # entry point 100 | if __name__ == '__main__': 101 | main() 102 | --------------------------------------------------------------------------------