├── README.md
└── selu.py


/README.md:
--------------------------------------------------------------------------------
1 | A pytorch implementation of selu and dropout_selu in "Self-Normalizing Neural Networks" by Günter Klambauer, Thomas Unterthiner, Andreas Mayr, Sepp Hochreiter. (borrowed heavily from the original tf implementation)
2 | 


--------------------------------------------------------------------------------
/selu.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*- 
 2 | """
 3 | Created on Sun Jun 11 14:46:31 2017
 4 | 
 5 | @author: danny
 6 | """
 7 | import torch
 8 | import numpy as np
 9 | import torch.nn as nn
10 | from torch.autograd import Variable
11 | from torch.nn import functional as F
12 | 
13 | class selu(nn.Module):
14 |     def __init__(self):
15 |         super(selu, self).__init__()
16 |         self.alpha = 1.6732632423543772848170429916717
17 |         self.scale = 1.0507009873554804934193349852946
18 |     def forward(self, x):
19 |         temp1 = self.scale * F.relu(x)
20 |         temp2 = self.scale * self.alpha * (F.elu(-1*F.relu(-1*x)))
21 |         return temp1 + temp2
22 | 
23 | class alpha_drop(nn.Module):
24 |     def __init__(self, p = 0.05, alpha=-1.7580993408473766, fixedPointMean=0, fixedPointVar=1):
25 |         super(alpha_drop, self).__init__()
26 |         keep_prob = 1 - p
27 |         self.a = np.sqrt(fixedPointVar / (keep_prob *((1-keep_prob) * pow(alpha-fixedPointMean,2) + fixedPointVar)))
28 |         self.b = fixedPointMean - self.a * (keep_prob * fixedPointMean + (1 - keep_prob) * alpha)
29 |         self.alpha = alpha
30 |         self.keep_prob = 1 - p
31 |         self.drop_prob = p
32 |     def forward(self, x):
33 |         if self.keep_prob == 1 or not self.training:
34 |             # print("testing mode, direct return")
35 |             return x
36 |         else:
37 |             random_tensor  = self.keep_prob + torch.rand(x.size())
38 |             
39 |             binary_tensor = Variable(torch.floor(random_tensor))
40 | 
41 |             if torch.cuda.is_available():
42 |                 binary_tensor = binary_tensor.cuda()
43 |             
44 |             x = x.mul(binary_tensor)
45 |             ret = x + self.alpha * (1-binary_tensor)
46 |             ret.mul_(self.a).add_(self.b)
47 |             return ret
48 | 
49 | #Selu = selu()
50 | #dropout_selu = alpha_drop(0.05)
51 | #x = torch.normal(torch.rand(1000, 3, 3, 224), 1)
52 | #w = Selu(Variable(x))
53 | #y = dropout_selu(w)
54 | 
55 | 
56 | 


--------------------------------------------------------------------------------