├── test.py ├── layer ├── spatiallppooling.py ├── depthconcat.py ├── lambdas.py ├── normalize.py └── spatialcrossmaplrn.py ├── README.md └── net.py /test.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | import net 4 | 5 | 6 | model = net.model 7 | model.load_state_dict(torch.load('net.pth')) 8 | model.eval() 9 | 10 | input = torch.rand(6, 3, 96, 96) # input: RGB image of 96*96 11 | 12 | # CPU 13 | output = model(Variable(input)) 14 | print('CPU done') 15 | 16 | # GPU 17 | model.cuda() 18 | input = input.cuda() 19 | output = model(Variable(input)) 20 | print('GPU done') -------------------------------------------------------------------------------- /layer/spatiallppooling.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class SpatialLPPooling(nn.Module): 5 | def __init__(self, pnorm, kW, kH, dW, dH): 6 | super(SpatialLPPooling, self).__init__() 7 | self.pnorm = pnorm 8 | self.scale = kW * kH 9 | 10 | self.pool = nn.AvgPool2d((kH, kW), (dH, dW)) 11 | 12 | def forward(self, x): 13 | x = x ** self.pnorm 14 | x = self.pool(x) 15 | x *= self.scale 16 | x = x ** (1.0/self.pnorm) 17 | return x -------------------------------------------------------------------------------- /layer/depthconcat.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def calculate_pad(hei, wid, hei_pad, wid_pad): 4 | left = (wid_pad - wid) // 2 5 | right = wid_pad - wid - left 6 | top = (hei_pad - hei) // 2 7 | bottom = hei_pad - hei - top 8 | return [left, right, top, bottom] 9 | 10 | # concat 3D/4D variable 11 | # dim = 0: 3D; = 1: 4D 12 | def concat_with_pad(seq, dim): 13 | # get maximum size 14 | hei_pad = 0 15 | wid_pad = 0 16 | for input in seq: 17 | hei_pad = max(hei_pad, input.size(dim + 1)) 18 | wid_pad = max(hei_pad, input.size(dim + 2)) 19 | 20 | # pad each input 21 | output = [] 22 | for input in seq: 23 | pad = calculate_pad(input.size(dim + 1), input.size(dim + 2), hei_pad, wid_pad) 24 | input_pad = torch.nn.functional.pad(input, pad) 25 | output.append(input_pad) 26 | return torch.cat(output, dim) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenFace-PyTorch 2 | 3 | # Introduction 4 | Here we provide [PyTorch](http://pytorch.org/) version for [OpenFace](https://github.com/cmusatyalab/openface)'s facial feature extraction model. 5 | 6 | The original model is written in Torch. 7 | 8 | The conversion is mostly done by clarwin's [convert_torch_to_pytorch](https://github.com/clcarwin/convert_torch_to_pytorch), with some added layers, e.g. Inception. 9 | 10 | # Usage 11 | First, download binary model files(.pth) from [BaiduPan](http://pan.baidu.com/s/1o84qACE) or [DropBox](https://www.dropbox.com/sh/sani3kabk11hfxe/AAD6p4CE28PhmW76C1PnXxnDa?dl=0). 12 | 13 | Then, refer to test.py for how to use the model. 14 | 15 | In short, net.py contains model structure, and net.pth contains model parameters. 16 | 17 | They are converted from 'nn4.small2.v1.t7' which is the default model of OpenFace project. 18 | 19 | More models from OpenFace can be found under \models folder. -------------------------------------------------------------------------------- /layer/lambdas.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | from functools import reduce 3 | 4 | 5 | class LambdaBase(nn.Sequential): 6 | def __init__(self, fn, *args): 7 | super(LambdaBase, self).__init__(*args) 8 | self.lambda_func = fn 9 | 10 | def forward_prepare(self, input): 11 | output = [] 12 | for module in self._modules.values(): 13 | output.append(module(input)) 14 | return output if output else input 15 | 16 | class Lambda(LambdaBase): 17 | def forward(self, input): 18 | return self.lambda_func(self.forward_prepare(input)) 19 | 20 | class LambdaMap(LambdaBase): 21 | def forward(self, input): 22 | # result is Variables list [Variable1, Variable2, ...] 23 | return map(self.lambda_func,self.forward_prepare(input)) 24 | 25 | class LambdaReduce(LambdaBase): 26 | def forward(self, input): 27 | # result is a Variable 28 | return reduce(self.lambda_func,self.forward_prepare(input)) -------------------------------------------------------------------------------- /layer/normalize.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | 5 | class Normalize(nn.Module): 6 | def __init__(self, p): 7 | super(Normalize, self).__init__() 8 | self.p = p 9 | 10 | def forward(self, x): 11 | if x.dim()==1: 12 | if self.p == float('inf'): 13 | norm = torch.max(x.abs()) 14 | else: 15 | norm = torch.norm(x, self.p, 0) 16 | x.div_(norm.data[0]) 17 | return x 18 | elif x.dim()==2: 19 | if self.p == float('inf'): 20 | x = x.abs() 21 | norm = Variable(torch.Tensor(x.size(0), 1)) 22 | for i in range(norm.size(0)): 23 | norm[i, 0] = torch.max(x[i]) 24 | else: 25 | norm = torch.norm(x, self.p, 1) # batch_size * 1 26 | 27 | for i in range(x.size(0)): 28 | x[i].div_(norm.data[i][0]) 29 | return x 30 | else: 31 | raise RuntimeError("expected dim=1 or 2, got {}".format(x.dim())) -------------------------------------------------------------------------------- /layer/spatialcrossmaplrn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.autograd import Variable 4 | 5 | 6 | class SpatialCrossMapLRN(nn.Module): 7 | def __init__(self, size, alpha=0.0001, beta=0.75, k=1): 8 | super(SpatialCrossMapLRN, self).__init__() 9 | self.size = size 10 | self.alpha = alpha 11 | self.beta = beta 12 | self.k = k 13 | 14 | def forward(self, x): 15 | # transfer to 4D 16 | assert x.dim() == 3 or x.dim() == 4, 'support only 3D or 4D input' 17 | is_batch = True 18 | if x.dim() == 3: 19 | x = torch.unsqueeze(x, 0) 20 | is_batch = False 21 | 22 | layer_square = x * x 23 | 24 | pad = int((self.size - 1) / 2 + 1) 25 | channels = x.size(1) 26 | if pad > channels: # pad_crop: input range of first layer 27 | pad_crop = channels 28 | else: 29 | pad_crop = pad 30 | 31 | # sum for first layer 32 | list_sum = [] 33 | current_sum = Variable(x.data.new(x.size(0), 1, x.size(2), x.size(3))) 34 | current_sum.data.zero_() 35 | for c in range(pad_crop): 36 | current_sum = current_sum + layer_square.select(1, c) 37 | list_sum.append(current_sum) 38 | 39 | # sum other layers by 'add + remove' 40 | for c in range(1, channels): 41 | current_sum = list_sum[c-1] # copy previous sum 42 | # add 43 | if c < channels - pad + 1: 44 | index_next = c + pad - 1 45 | current_sum = current_sum + layer_square.select(1, index_next) 46 | # remove 47 | if c > pad: 48 | index_prev = c - pad 49 | current_sum = current_sum - layer_square.select(1, index_prev) 50 | list_sum.append(current_sum) 51 | layer_square_sum = torch.cat(list_sum, 1) 52 | 53 | # y = x / { (k + alpha / size * sum)^beta } 54 | layer_square_sum = layer_square_sum * self.alpha / self.size + self.k 55 | output = x * torch.pow(layer_square_sum, -self.beta) 56 | 57 | # recover 3D 58 | if not is_batch: 59 | output = torch.squeeze(output, 1) 60 | return output -------------------------------------------------------------------------------- /net.py: -------------------------------------------------------------------------------- 1 | 2 | import torch 3 | import torch.nn as nn 4 | from torch.autograd import Variable 5 | 6 | from layer.lambdas import LambdaBase, Lambda, LambdaMap, LambdaReduce 7 | from layer.normalize import Normalize 8 | from layer.spatiallppooling import SpatialLPPooling 9 | from layer.depthconcat import concat_with_pad 10 | from layer.spatialcrossmaplrn import SpatialCrossMapLRN 11 | 12 | 13 | model = nn.Sequential( # Sequential, 14 | nn.Conv2d(3, 64, (7, 7), (2, 2), (3, 3), 1, 1,bias=True), 15 | nn.BatchNorm2d(64), 16 | nn.ReLU(), 17 | nn.MaxPool2d((3, 3), (2, 2), (1, 1)), 18 | SpatialCrossMapLRN(5, 0.0001, 0.75, 1), #SpatialCrossMapLRN, 19 | nn.Conv2d(64, 64, (1, 1), (1, 1), (0, 0), 1, 1,bias=True), 20 | nn.BatchNorm2d(64), 21 | nn.ReLU(), 22 | nn.Conv2d(64, 192, (3, 3), (1, 1), (1, 1), 1, 1,bias=True), 23 | nn.BatchNorm2d(192), 24 | nn.ReLU(), 25 | SpatialCrossMapLRN(5, 0.0001, 0.75, 1), #SpatialCrossMapLRN, 26 | nn.MaxPool2d((3, 3), (2, 2), (1, 1)), 27 | nn.Sequential( # Sequential, 28 | LambdaReduce(lambda x, y, dim=1: concat_with_pad((x, y), dim), # DepthConcat, 29 | nn.Sequential( # Sequential, 30 | nn.Conv2d(192, 96, (1, 1), (1, 1)), 31 | nn.BatchNorm2d(96), 32 | nn.ReLU(), 33 | nn.Conv2d(96, 128, (3, 3), (1, 1), (1, 1)), 34 | nn.BatchNorm2d(128), 35 | nn.ReLU(), 36 | ), 37 | nn.Sequential( # Sequential, 38 | nn.Conv2d(192, 16, (1, 1), (1, 1)), 39 | nn.BatchNorm2d(16), 40 | nn.ReLU(), 41 | nn.Conv2d(16, 32, (5, 5), (1, 1), (2, 2)), 42 | nn.BatchNorm2d(32), 43 | nn.ReLU(), 44 | ), 45 | nn.Sequential( # Sequential, 46 | nn.MaxPool2d((3, 3), (2, 2)), 47 | nn.Conv2d(192, 32, (1, 1), (1, 1)), 48 | nn.BatchNorm2d(32), 49 | nn.ReLU(), 50 | ), 51 | nn.Sequential( # Sequential, 52 | nn.Conv2d(192, 64, (1, 1), (1, 1)), 53 | nn.BatchNorm2d(64), 54 | nn.ReLU(), 55 | ), 56 | ), 57 | ), 58 | nn.Sequential( # Sequential, 59 | LambdaReduce(lambda x, y, dim=1: concat_with_pad((x, y), dim), # DepthConcat, 60 | nn.Sequential( # Sequential, 61 | nn.Conv2d(256, 96, (1, 1), (1, 1)), 62 | nn.BatchNorm2d(96), 63 | nn.ReLU(), 64 | nn.Conv2d(96, 128, (3, 3), (1, 1), (1, 1)), 65 | nn.BatchNorm2d(128), 66 | nn.ReLU(), 67 | ), 68 | nn.Sequential( # Sequential, 69 | nn.Conv2d(256, 32, (1, 1), (1, 1)), 70 | nn.BatchNorm2d(32), 71 | nn.ReLU(), 72 | nn.Conv2d(32, 64, (5, 5), (1, 1), (2, 2)), 73 | nn.BatchNorm2d(64), 74 | nn.ReLU(), 75 | ), 76 | nn.Sequential( # Sequential, 77 | SpatialLPPooling(2, 3, 3, 3, 3), #SpatialLPPooling, 78 | nn.Conv2d(256, 64, (1, 1), (1, 1)), 79 | nn.BatchNorm2d(64), 80 | nn.ReLU(), 81 | ), 82 | nn.Sequential( # Sequential, 83 | nn.Conv2d(256, 64, (1, 1), (1, 1)), 84 | nn.BatchNorm2d(64), 85 | nn.ReLU(), 86 | ), 87 | ), 88 | ), 89 | nn.Sequential( # Sequential, 90 | LambdaReduce(lambda x, y, dim=1: concat_with_pad((x, y), dim), # DepthConcat, 91 | nn.Sequential( # Sequential, 92 | nn.Conv2d(320, 128, (1, 1), (1, 1)), 93 | nn.BatchNorm2d(128), 94 | nn.ReLU(), 95 | nn.Conv2d(128, 256, (3, 3), (2, 2), (1, 1)), 96 | nn.BatchNorm2d(256), 97 | nn.ReLU(), 98 | ), 99 | nn.Sequential( # Sequential, 100 | nn.Conv2d(320, 32, (1, 1), (1, 1)), 101 | nn.BatchNorm2d(32), 102 | nn.ReLU(), 103 | nn.Conv2d(32, 64, (5, 5), (2, 2), (2, 2)), 104 | nn.BatchNorm2d(64), 105 | nn.ReLU(), 106 | ), 107 | nn.Sequential( # Sequential, 108 | nn.MaxPool2d((3, 3), (2, 2)), 109 | ), 110 | ), 111 | ), 112 | nn.Sequential( # Sequential, 113 | LambdaReduce(lambda x, y, dim=1: concat_with_pad((x, y), dim), # DepthConcat, 114 | nn.Sequential( # Sequential, 115 | nn.Conv2d(640, 96, (1, 1), (1, 1)), 116 | nn.BatchNorm2d(96), 117 | nn.ReLU(), 118 | nn.Conv2d(96, 192, (3, 3), (1, 1), (1, 1)), 119 | nn.BatchNorm2d(192), 120 | nn.ReLU(), 121 | ), 122 | nn.Sequential( # Sequential, 123 | nn.Conv2d(640, 32, (1, 1), (1, 1)), 124 | nn.BatchNorm2d(32), 125 | nn.ReLU(), 126 | nn.Conv2d(32, 64, (5, 5), (1, 1), (2, 2)), 127 | nn.BatchNorm2d(64), 128 | nn.ReLU(), 129 | ), 130 | nn.Sequential( # Sequential, 131 | SpatialLPPooling(2, 3, 3, 3, 3), #SpatialLPPooling, 132 | nn.Conv2d(640, 128, (1, 1), (1, 1)), 133 | nn.BatchNorm2d(128), 134 | nn.ReLU(), 135 | ), 136 | nn.Sequential( # Sequential, 137 | nn.Conv2d(640, 256, (1, 1), (1, 1)), 138 | nn.BatchNorm2d(256), 139 | nn.ReLU(), 140 | ), 141 | ), 142 | ), 143 | nn.Sequential( # Sequential, 144 | LambdaReduce(lambda x, y, dim=1: concat_with_pad((x, y), dim), # DepthConcat, 145 | nn.Sequential( # Sequential, 146 | nn.Conv2d(640, 160, (1, 1), (1, 1)), 147 | nn.BatchNorm2d(160), 148 | nn.ReLU(), 149 | nn.Conv2d(160, 256, (3, 3), (2, 2), (1, 1)), 150 | nn.BatchNorm2d(256), 151 | nn.ReLU(), 152 | ), 153 | nn.Sequential( # Sequential, 154 | nn.Conv2d(640, 64, (1, 1), (1, 1)), 155 | nn.BatchNorm2d(64), 156 | nn.ReLU(), 157 | nn.Conv2d(64, 128, (5, 5), (2, 2), (2, 2)), 158 | nn.BatchNorm2d(128), 159 | nn.ReLU(), 160 | ), 161 | nn.Sequential( # Sequential, 162 | nn.MaxPool2d((3, 3), (2, 2)), 163 | ), 164 | ), 165 | ), 166 | nn.Sequential( # Sequential, 167 | LambdaReduce(lambda x, y, dim=1: concat_with_pad((x, y), dim), # DepthConcat, 168 | nn.Sequential( # Sequential, 169 | nn.Conv2d(1024, 96, (1, 1), (1, 1)), 170 | nn.BatchNorm2d(96), 171 | nn.ReLU(), 172 | nn.Conv2d(96, 384, (3, 3), (1, 1), (1, 1)), 173 | nn.BatchNorm2d(384), 174 | nn.ReLU(), 175 | ), 176 | nn.Sequential( # Sequential, 177 | SpatialLPPooling(2, 3, 3, 3, 3), #SpatialLPPooling, 178 | nn.Conv2d(1024, 96, (1, 1), (1, 1)), 179 | nn.BatchNorm2d(96), 180 | nn.ReLU(), 181 | ), 182 | nn.Sequential( # Sequential, 183 | nn.Conv2d(1024, 256, (1, 1), (1, 1)), 184 | nn.BatchNorm2d(256), 185 | nn.ReLU(), 186 | ), 187 | ), 188 | ), 189 | Lambda(lambda x: x.view(-1, 736, 3, 3)), # View, 190 | nn.Sequential( # Sequential, 191 | LambdaReduce(lambda x, y, dim=1: concat_with_pad((x, y), dim), # DepthConcat, 192 | nn.Sequential( # Sequential, 193 | nn.Conv2d(736, 96, (1, 1), (1, 1)), 194 | nn.BatchNorm2d(96), 195 | nn.ReLU(), 196 | nn.Conv2d(96, 384, (3, 3), (1, 1), (1, 1)), 197 | nn.BatchNorm2d(384), 198 | nn.ReLU(), 199 | ), 200 | nn.Sequential( # Sequential, 201 | nn.MaxPool2d((3, 3), (2, 2)), 202 | nn.Conv2d(736, 96, (1, 1), (1, 1)), 203 | nn.BatchNorm2d(96), 204 | nn.ReLU(), 205 | ), 206 | nn.Sequential( # Sequential, 207 | nn.Conv2d(736, 256, (1, 1), (1, 1)), 208 | nn.BatchNorm2d(256), 209 | nn.ReLU(), 210 | ), 211 | ), 212 | ), 213 | nn.AvgPool2d((3, 3), (1, 1)), 214 | Lambda(lambda x: x.view(-1, 736)), # View, 215 | Lambda(lambda x: x.view(x.size(0), -1)), # View, 216 | nn.Sequential(Lambda(lambda x: x.view(1, -1) if 1==len(x.size()) else x ), nn.Linear(736, 128)), # Linear, 217 | Normalize(2), #Normalize, 218 | ) --------------------------------------------------------------------------------