├── .gitignore ├── LICENSE ├── PytorchModels ├── __init__.py ├── base_model.py └── xnor_model.py ├── README.md ├── datasets ├── __init__.py └── pytorch_provider.py └── pytorch_train.py /.gitignore: -------------------------------------------------------------------------------- 1 | # project files 2 | *.sublime-* 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # dotenv 86 | .env 87 | 88 | # virtualenv 89 | .venv 90 | venv/ 91 | ENV/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Illarion 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PytorchModels/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ikhlestov/XNOR-Net/b95a57911858b9d750366c0d9d5e46ab3c7bf220/PytorchModels/__init__.py -------------------------------------------------------------------------------- /PytorchModels/base_model.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | class UsualConv2d(nn.Module): 5 | def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, 6 | padding=1, avg_kernel_size=2, avg_stride=2, bn_size=0): 7 | super().__init__() 8 | self.conv = nn.Conv2d( 9 | in_channels=in_channels, 10 | out_channels=out_channels, 11 | kernel_size=kernel_size, 12 | stride=stride, 13 | padding=padding) 14 | self.bn = nn.BatchNorm2d(out_channels) 15 | self.activ = nn.ReLU() 16 | self.max_pool = nn.MaxPool2d(kernel_size=2, stride=2) 17 | 18 | def forward(self, x): 19 | x = self.conv(x) 20 | x = self.bn(x) 21 | x = self.activ(x) 22 | x = self.max_pool(x) 23 | return x 24 | 25 | 26 | class Model(nn.Module): 27 | def __init__(self): 28 | super().__init__() 29 | self.features = nn.Sequential( 30 | UsualConv2d(3, 12, kernel_size=3, padding=1, stride=1), 31 | UsualConv2d(12, 24, kernel_size=3, padding=1, stride=1), 32 | UsualConv2d(24, 32, kernel_size=3, padding=1, stride=1), 33 | UsualConv2d(32, 48, kernel_size=3, padding=1, stride=1), 34 | UsualConv2d(48, 64, kernel_size=3, padding=1, stride=1), 35 | nn.BatchNorm2d(64), 36 | nn.Conv2d(64, 10, kernel_size=1) 37 | ) 38 | 39 | def forward(self, x): 40 | x = self.features(x) 41 | x = x.view(x.size()[0], -1) 42 | return x 43 | -------------------------------------------------------------------------------- /PytorchModels/xnor_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from .base_model import UsualConv2d 5 | 6 | 7 | class _BinActive(torch.autograd.Function): 8 | 9 | def forward(self, input): 10 | self.save_for_backward(input) 11 | output = torch.sign(input) 12 | return output 13 | 14 | def backward(self, grad_output): 15 | # saved tensors - tuple of tensors with one element 16 | input, = self.saved_tensors 17 | grad_output[input.ge(1)] = 0 18 | grad_output[input.le(-1)] = 0 19 | return grad_output 20 | 21 | 22 | class BinActive(nn.Module): 23 | def forward(self, x): 24 | return _BinActive()(x) 25 | 26 | 27 | class BinConv2D(nn.Module): 28 | def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, 29 | padding=1, avg_kernel_size=2, avg_stride=2): 30 | super().__init__() 31 | self.conv = nn.Conv2d( 32 | in_channels=in_channels, 33 | out_channels=out_channels, 34 | kernel_size=kernel_size, 35 | stride=stride, 36 | padding=padding) 37 | self.bn = nn.BatchNorm2d(in_channels) 38 | self.activ = BinActive() 39 | self.max_pool = nn.MaxPool2d(kernel_size=2, stride=2) 40 | 41 | def forward(self, x): 42 | x = self.bn(x) 43 | x = self.activ(x) 44 | x = self.conv(x) 45 | x = self.max_pool(x) 46 | return x 47 | 48 | 49 | class XNORModel(nn.Module): 50 | def __init__(self): 51 | super().__init__() 52 | self.features = nn.Sequential( 53 | BinConv2D(3, 12, kernel_size=3, padding=1, stride=1), 54 | BinConv2D(12, 24, kernel_size=3, padding=1, stride=1), 55 | BinConv2D(24, 32, kernel_size=3, padding=1, stride=1), 56 | BinConv2D(32, 48, kernel_size=3, padding=1, stride=1), 57 | BinConv2D(48, 64, kernel_size=3, padding=1, stride=1), 58 | nn.BatchNorm2d(64), 59 | nn.ReLU(inplace=True), 60 | nn.Conv2d(64, 10, kernel_size=1) 61 | ) 62 | 63 | def forward(self, x): 64 | x = self.features(x) 65 | x = x.view(x.size()[0], -1) 66 | return x 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # XNOR-Net 2 | Implementations of the XNOR networks. 3 | 4 | Initially binary networks were proposed at [Binarized Neural Networks](https://arxiv.org/abs/1602.02830) paper. After a while idea was updated at [XNOR-Net](https://arxiv.org/abs/1603.05279) paper. In this repo additional layers for such networks exists. Now layers implemented only naive way, without real binary operations. 5 | 6 | # TODO 7 | 8 | - [x] Implement pytorch layers 9 | - [x] Implement trainer 10 | - [ ] Add keras implementation 11 | - [ ] Update trainer 12 | - [ ] Networks with real binary operations 13 | -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ikhlestov/XNOR-Net/b95a57911858b9d750366c0d9d5e46ab3c7bf220/datasets/__init__.py -------------------------------------------------------------------------------- /datasets/pytorch_provider.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torchvision.transforms as transforms 4 | 5 | 6 | transform = transforms.Compose( 7 | [transforms.ToTensor(), 8 | transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) 9 | 10 | 11 | def get_loaders(batch_size): 12 | trainset = torchvision.datasets.CIFAR10( 13 | root='/tmp/cifar10', train=True, 14 | download=True, transform=transform) 15 | train_loader = torch.utils.data.DataLoader( 16 | trainset, batch_size=4, 17 | shuffle=True, num_workers=2) 18 | 19 | testset = torchvision.datasets.CIFAR10( 20 | root='/tmp/cifar10', train=False, 21 | download=True, transform=transform) 22 | test_loader = torch.utils.data.DataLoader( 23 | testset, batch_size=4, 24 | shuffle=False, num_workers=2) 25 | 26 | return train_loader, test_loader 27 | -------------------------------------------------------------------------------- /pytorch_train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.autograd import Variable 3 | 4 | from PytorchModels.base_model import Model 5 | from PytorchModels.xnor_model import XNORModel 6 | from datasets.pytorch_provider import get_loaders 7 | 8 | 9 | # Initialize model 10 | cfg = { 11 | 'epochs': 10, 12 | 'lr': 0.001, 13 | 'momentum': 0.9, 14 | 'batch_size': 4, 15 | 'report_step': 2000, 16 | 'model_type': 'xnor' # choices from ['base', 'xnor'] 17 | } 18 | 19 | train_loader, test_loader = get_loaders(batch_size=cfg['batch_size']) 20 | if cfg['model_type'] == 'base': 21 | net = Model() 22 | elif cfg['model_type'] == 'xnor': 23 | net = XNORModel() 24 | else: 25 | raise NotImplementedError() 26 | criterion = torch.nn.CrossEntropyLoss() 27 | optimizer = torch.optim.SGD( 28 | net.parameters(), 29 | lr=cfg['lr'], 30 | momentum=cfg['momentum']) 31 | 32 | 33 | # Train the model 34 | for epoch in range(cfg['epochs']): 35 | 36 | running_loss = 0.0 37 | for batch_n, (inputs, labels) in enumerate(train_loader): 38 | inputs, labels = Variable(inputs), Variable(labels) 39 | 40 | optimizer.zero_grad() 41 | outputs = net(inputs) 42 | loss = criterion(outputs, labels) 43 | loss.backward() 44 | optimizer.step() 45 | 46 | running_loss += loss.data[0] 47 | if batch_n % cfg['report_step'] == 1999: 48 | print('[%d, %5d] loss: %.3f' % 49 | (epoch + 1, batch_n + 1, running_loss / cfg['report_step'])) 50 | running_loss = 0.0 51 | 52 | # Test the model 53 | correct = 0 54 | total = 0 55 | for (images, labels) in test_loader: 56 | outputs = net(Variable(images)) 57 | _, predicted = torch.max(outputs.data, 1) 58 | total += labels.size(0) 59 | correct += (predicted == labels).sum() 60 | 61 | print('Accuracy of the network on the 10000 test images: %d %%' % ( 62 | 100 * correct / total)) 63 | --------------------------------------------------------------------------------