├── log ├── fc2_b.summary ├── fc2_w.summary ├── fc2_b_grad.summary ├── fc2_w_grad.summary ├── fc5_b_grad.summary ├── fc3_w.summary ├── fc5_w.summary ├── fc4_w.summary ├── fc5_b.summary ├── fc5_w_grad.summary ├── fc3_b_grad.summary ├── fc4_b_grad.summary ├── fc4_w_grad.summary ├── fc3_b.summary ├── fc3_w_grad.summary ├── fc4_b.summary ├── conv1_b.summary ├── conv1_w.summary ├── pred_w.summary ├── conv1_b_grad.summary ├── conv2_w.summary ├── conv2_b.summary ├── pred_w_grad.summary ├── pred_b_grad.summary ├── pred_b.summary ├── conv1_w_grad.summary ├── conv2_b_grad.summary └── conv2_w_grad.summary ├── src ├── test_binary_tool.py ├── binary_tool.py ├── utils.py ├── MNIST_main.py └── models.py ├── tutorial ├── binary_net.py ├── mnist_example.py ├── MNIST-light.ipynb └── MNIST_BNN.ipynb ├── README.md ├── .gitignore └── test └── run_test.py /log/fc2_b.summary: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /log/fc2_w.summary: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /log/fc2_b_grad.summary: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /log/fc2_w_grad.summary: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /log/fc5_b_grad.summary: -------------------------------------------------------------------------------- 1 | -0.0496944 0.0301778 -1.11759e-09 0.0274461 2 | -0.0985398 0.0754847 -7.45058e-10 0.054761 3 | -0.0904916 0.178293 0 0.0801946 4 | -------------------------------------------------------------------------------- /log/fc3_w.summary: -------------------------------------------------------------------------------- 1 | -0.0273468 0.0272871 1.24917e-06 0.0154879 2 | -0.0274982 0.0274768 9.93357e-07 0.015488 3 | -0.0274833 0.0275279 -2.50228e-06 0.0154878 4 | -------------------------------------------------------------------------------- /log/fc5_w.summary: -------------------------------------------------------------------------------- 1 | -0.0293146 0.0296865 -4.22294e-05 0.0156151 2 | -0.029896 0.0342668 -4.2229e-05 0.0156321 3 | -0.0324405 0.03153 -4.22298e-05 0.0156296 4 | -------------------------------------------------------------------------------- /log/fc4_w.summary: -------------------------------------------------------------------------------- 1 | -0.0272668 0.0272893 -2.171e-06 0.0154897 2 | -0.0274164 0.0275138 -2.41159e-06 0.0154897 3 | -0.0276005 0.0275451 -5.47345e-06 0.0154896 4 | -------------------------------------------------------------------------------- /log/fc5_b.summary: -------------------------------------------------------------------------------- 1 | -0.00301477 0.00496448 1.39698e-10 0.00274186 2 | -0.00511823 0.011939 -4.65661e-11 0.0051441 3 | -0.015514 0.00645313 2.32831e-10 0.00600414 4 | -------------------------------------------------------------------------------- /log/fc5_w_grad.summary: -------------------------------------------------------------------------------- 1 | -0.0488585 0.02913 -1.74072e-11 0.00557307 2 | -0.0458957 0.0501027 2.45552e-09 0.00723293 3 | -0.0573505 0.108083 5.45409e-09 0.0107047 4 | -------------------------------------------------------------------------------- /log/fc3_b_grad.summary: -------------------------------------------------------------------------------- 1 | -0.00389626 0.00414804 -2.43943e-07 0.000862834 2 | -0.00490212 0.005584 2.83714e-05 0.00118354 3 | -0.0066908 0.00806987 0.000191591 0.00160764 4 | -------------------------------------------------------------------------------- /log/fc4_b_grad.summary: -------------------------------------------------------------------------------- 1 | -0.00483944 0.00433116 1.10527e-05 0.00108238 2 | -0.00837028 0.00822624 3.25825e-05 0.00173079 3 | -0.00787189 0.0104088 0.000239843 0.00237037 4 | -------------------------------------------------------------------------------- /log/fc4_w_grad.summary: -------------------------------------------------------------------------------- 1 | -0.00459439 0.00409596 1.94438e-06 0.000294283 2 | -0.00694306 0.00600496 2.4153e-06 0.000351194 3 | -0.00723067 0.0102613 3.07003e-05 0.000465008 4 | -------------------------------------------------------------------------------- /log/fc3_b.summary: -------------------------------------------------------------------------------- 1 | -0.000414389 0.000389237 2.43704e-08 8.61971e-05 2 | -0.00055187 0.000521022 -2.8071e-06 0.000145056 3 | -0.000735133 0.000685688 -2.19088e-05 0.000184821 4 | -------------------------------------------------------------------------------- /log/fc3_w_grad.summary: -------------------------------------------------------------------------------- 1 | -0.00525978 0.0051553 8.79661e-07 0.000311377 2 | -0.00446473 0.00531214 2.56017e-06 0.000344449 3 | -0.00683981 0.00733824 3.50635e-05 0.000420189 4 | -------------------------------------------------------------------------------- /log/fc4_b.summary: -------------------------------------------------------------------------------- 1 | -0.000432682 0.00048346 -1.10417e-06 0.00010813 2 | -0.00081886 0.00072858 -4.35592e-06 0.000191605 3 | -0.000832858 0.000962636 -2.82683e-05 0.00023295 4 | -------------------------------------------------------------------------------- /src/test_binary_tool.py: -------------------------------------------------------------------------------- 1 | from caffe2.python import core, workspace 2 | import numpy as np 3 | import binary_tool as bt 4 | 5 | ''' 6 | for implement binary_net on caffe2 7 | original theano version would be in the upper comment 8 | ''' 9 | 10 | #Generate an array with random number elements 11 | X = np.random.randn(2, 3).astype(np.float32) 12 | Y = 2 13 | print("Generated X from numpy:\n{}".format(X)) 14 | print("Generated Y :\n{}".format(Y)) 15 | workspace.FeedBlob("X", X) 16 | 17 | 18 | #X = bt.tt(X) 19 | #print("Generated X from tt(X):\n{}".format(X)) 20 | 21 | X = bt.hard_sigmoid(X) 22 | print("Generated X from hard_sigmoid(X):\n{}".format(X)) 23 | 24 | #X = bt.binary_tanh_unit(X) 25 | #print("Generated X from hard_sigmoid(X):\n{}".format(X)) 26 | -------------------------------------------------------------------------------- /tutorial/binary_net.py: -------------------------------------------------------------------------------- 1 | from caffe2.python import core, workspace 2 | 3 | ''' 4 | for implement binary_net on caffe2 5 | original theano version would be in the upper comment 6 | ''' 7 | 8 | ''' 9 | def hard_sigmoid(x): 10 | return clip((x+1.)/2.,0,1) 11 | ''' 12 | def hard_sigmoid(x): 13 | x = (x + 1.) / 2. 14 | workspace.FeedBlob("x", x) 15 | Clip = core.CreateOperator( 16 | "Clip", 17 | ["x"], 18 | ["x"], 19 | min=0., 20 | max=1. 21 | ) 22 | workspace.RunOperatorOnce(Clip) 23 | return workspace.FetchBlob("x") 24 | 25 | 26 | ''' 27 | def binary_tanh_unit(x): 28 | return 2.*round(hard_sigmoid(x))-1. 29 | ''' 30 | #TODO: implement binary_tanh_unit 31 | 32 | ''' 33 | def binary_sigmoid_unit(x): 34 | return round(hard_sigmoid(x)) 35 | ''' 36 | #TODO: implement binary_sigmoid_unit 37 | 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Note 2 | This repository is not maintian any more. 3 | 4 | # BNN_Caffe2 5 | Quantized Neural Network on Caffe2 6 | 7 | Goal: 8 | Porting QNN(BNN) from Theano version to Caffe2 version. 9 | 10 | ## TODO: 11 | - [x] MNIST with LeNet 12 | - [x] MNIST with layers from QNN paper, without Batch Normalization, without Binarization. 13 | - [x] MNIST with layers from QNN paper, with Batch Normalization, without Binarization. 14 | - [ ] Binarization functions (in progress) 15 | - [ ] Layers with Binarization 16 | - [ ] MNIST with layers from QNN paper, with Batch Normalization, with Binarization. 17 | - [ ] Testing and debugging 18 | - [ ] Training / evaluation on other dataset 19 | 20 | ## Acknowledgment 21 | 22 | Refs. 23 | 24 | QNN Theano version paper 25 | https://arxiv.org/abs/1609.07061 26 | 27 | Theano version github 28 | https://github.com/MatthieuCourbariaux/BinaryNet 29 | 30 | 31 | ## License 32 | -------------------------------------------------------------------------------- /src/binary_tool.py: -------------------------------------------------------------------------------- 1 | from caffe2.python import core, workspace 2 | 3 | ''' 4 | for implement binary_net on caffe2 5 | original theano version would be in the upper comment 6 | ''' 7 | 8 | ''' 9 | def hard_sigmoid(x): 10 | return clip((x+1.)/2.,0,1) 11 | ''' 12 | 13 | def tt(x): 14 | workspace.FeedBlob("x", x) 15 | Clip = core.CreateOperator( 16 | "Clip", 17 | ["x"], 18 | ["x"], 19 | min=0., 20 | max=1. 21 | ) 22 | workspace.RunOperatorOnce(Clip) 23 | return workspace.FetchBlob("x") 24 | 25 | 26 | def hard_sigmoid(x): 27 | x = (x + 1.) / 2. 28 | workspace.FeedBlob("x", x) 29 | Clip = core.CreateOperator( 30 | "Clip", 31 | ["x"], 32 | ["x"], 33 | min=0., 34 | max=1. 35 | ) 36 | workspace.RunOperatorOnce(Clip) 37 | return workspace.FetchBlob("x") 38 | 39 | 40 | ''' 41 | def binary_tanh_unit(x): 42 | return 2.*round(hard_sigmoid(x))-1. 43 | ''' 44 | #TODO: implement binary_tanh_unit 45 | 46 | def binary_tanh_unit(x): 47 | return 2.*round(hard_sigmoid(x))-1. 48 | 49 | 50 | ''' 51 | def binary_sigmoid_unit(x): 52 | return round(hard_sigmoid(x)) 53 | ''' 54 | #TODO: implement binary_sigmoid_unit 55 | 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | current_folder = os.path.join(os.path.expanduser('~'), 'caffe2_notebooks') 5 | 6 | data_folder = os.path.join(current_folder, 'tutorial_data', 'mnist') 7 | root_folder = os.path.join(current_folder, 'tutorial_files', 'tutorial_mnist') 8 | image_file_train = os.path.join(data_folder, "train-images-idx3-ubyte") 9 | label_file_train = os.path.join(data_folder, "train-labels-idx1-ubyte") 10 | image_file_test = os.path.join(data_folder, "t10k-images-idx3-ubyte") 11 | label_file_test = os.path.join(data_folder, "t10k-labels-idx1-ubyte") 12 | 13 | # Get the dataset if it is missing 14 | def DownloadDataset(url, path): 15 | import requests, zipfile, StringIO 16 | print "Downloading... ", url, " to ", path 17 | r = requests.get(url, stream=True) 18 | z = zipfile.ZipFile(StringIO.StringIO(r.content)) 19 | z.extractall(path) 20 | 21 | def GenerateDB(image, label, name): 22 | name = os.path.join(data_folder, name) 23 | print 'DB: ', name 24 | if not os.path.exists(name): 25 | syscall = "/usr/local/bin/make_mnist_db --channel_first --db leveldb --image_file " + image + " --label_file " + label + " --output_file " + name 26 | # print "Creating database with: ", syscall 27 | os.system(syscall) 28 | else: 29 | print "Database exists already. Delete the folder if you have issues/corrupted DB, then rerun this." 30 | if os.path.exists(os.path.join(name, "LOCK")): 31 | # print "Deleting the pre-existing lock file" 32 | os.remove(os.path.join(name, "LOCK")) 33 | 34 | def PrepareDataset(): 35 | if not os.path.exists(data_folder): 36 | os.makedirs(data_folder) 37 | if not os.path.exists(label_file_train): 38 | DownloadDataset("https://download.caffe2.ai/datasets/mnist/mnist.zip", data_folder) 39 | 40 | if os.path.exists(root_folder): 41 | print("Looks like you ran this before, so we need to cleanup those old files...") 42 | shutil.rmtree(root_folder) 43 | 44 | os.makedirs(root_folder) 45 | #workspace.ResetWorkspace(root_folder) 46 | 47 | # (Re)generate the leveldb database (known to get corrupted...) 48 | GenerateDB(image_file_train, label_file_train, "mnist-train-nchw-leveldb") 49 | GenerateDB(image_file_test, label_file_test, "mnist-test-nchw-leveldb") 50 | 51 | print("training data folder:" + data_folder) 52 | #print("workspace root folder:" + root_folder) 53 | 54 | -------------------------------------------------------------------------------- /test/run_test.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 3 | GOAL : verify the functionality/correctness of operation/layer 4 | Testing methodology : compare the output of operation/layer to Theano's 5 | 6 | Preliminary, use the static generated data for comparison. 7 | Note: Theano adopts symbolic computing, intermediate values in a computation cannot be printed 8 | in the normal python way with the print statement, because Theano has no statements. 9 | Instead there is the 'Print' Op. 10 | http://deeplearning.net/software/theano/library/printing.html 11 | 12 | input = 13 | [[8.53484750e-01 1.00000000e+00 0.00000000e+00 9.02203321e-01 14 | 6.46270394e-01 9.61881950e-02 7.77260303e-01 3.87970775e-01 ... 15 | 16 | expected = 17 | [[1. 1. 0. 1. 1. 0. 1. 0. 0. 0. 1. 1. 0. 1. 0. 1. 1. 1. ... 18 | 19 | ''' 20 | import sys 21 | sys.path.append("../src") 22 | 23 | import binary_tool 24 | import numpy as np 25 | import re 26 | 27 | def flatten_from_file(fd): 28 | array = [] 29 | for line in fd: 30 | for x in re.split(r'\[|\]|\s*', line): 31 | if not x == '': 32 | array.append(float(x)) 33 | return array 34 | 35 | # fn : testing function 36 | # fracton : there is a little bit precision different between Theano and Caffe 37 | def comparison(fn, input_fd_name, expected_fd_name, fraction): 38 | # Reading the data from file and flat to 1d list 39 | f_input = open(input_fd_name, "r") 40 | inputs = flatten_from_file(f_input) 41 | 42 | inputs = np.array(inputs).astype(np.float32) 43 | outputs = fn(inputs) 44 | 45 | f_expected = open(expected_fd_name, "r") 46 | expecteds = flatten_from_file(f_expected) 47 | 48 | # Perform the element-wise comparison one by one 49 | assert len(outputs) == len(expecteds), "len(input) = %r, len(expecteds) = %r" \ 50 | % (len(outputs), len(expecteds)) 51 | 52 | for i in range(0, len(outputs)): 53 | out = round(outputs[i], fraction) 54 | exp = round(expecteds[i], fraction) 55 | assert out == exp, "input = %r, expected = %r" % (out, exp) 56 | 57 | # def test_round3(): 58 | # comparison(binary_tool.round3, 'input.txt', 'expected.txt', 5) 59 | 60 | # test_round3() 61 | 62 | def test_hard_sigmoid(): 63 | # test failed against fraction above 5 64 | comparison(binary_tool.hard_sigmoid, 'clip_input.txt', 'clip_output.txt', 5) 65 | 66 | test_hard_sigmoid() 67 | 68 | # TODO if we has a round3 implementation 69 | # static test : by file format 70 | # def test_round3(): 71 | # generated_from_caffe2 = round3(input_from_theano) // invoke round3() 72 | # comparison(generated_from_caffe2, generated_from_theano) 73 | # 74 | # def test_xxx(): ... 75 | 76 | ''' 77 | output = [] 78 | for x in range(0, 100): 79 | temp = [] 80 | for y in range(0, 4096): 81 | temp.append(array.pop(0)) 82 | output.append(temp) 83 | 84 | # output_file = open("test.txt", "a") 85 | output_file = open("test2.txt", "a") 86 | output_file.write(str(output)) 87 | output_file.close() 88 | ''' 89 | -------------------------------------------------------------------------------- /src/MNIST_main.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import shutil 4 | import utils 5 | import models 6 | 7 | from caffe2.python import core, model_helper, net_drawer, workspace, visualize, brew 8 | 9 | core.GlobalInit(['caffe2', '--caffe2_log_level=0']) 10 | print("Necessities imported!") 11 | 12 | ################# 13 | # Prepare data 14 | 15 | utils.PrepareDataset() 16 | 17 | 18 | ############################ 19 | # Start constructing models 20 | 21 | arg_scope = {"order": "NCHW"} 22 | train_model = model_helper.ModelHelper(name="mnist_train", arg_scope=arg_scope) 23 | 24 | data, label = models.AddInput( 25 | train_model, batch_size=64, 26 | db=os.path.join(utils.data_folder, 'mnist-train-nchw-leveldb'), 27 | db_type='leveldb') 28 | #softmax = models.AddLeNetModel(train_model, data) 29 | softmax = models.AddMLP(train_model, data, batch_size=64) 30 | models.AddTrainingOperators(train_model, softmax, label) 31 | models.AddBookkeepingOperators(train_model) 32 | 33 | # Testing model. We will set the batch size to 100, so that the testing 34 | # pass is 100 iterations (10,000 images in total). 35 | # For the testing model, we need the data input part, the main LeNetModel 36 | # part, and an accuracy part. Note that init_params is set False because 37 | # we will be using the parameters obtained from the train model. 38 | test_model = model_helper.ModelHelper( 39 | name="mnist_test", arg_scope=arg_scope, init_params=False) 40 | data, label = models.AddInput( 41 | test_model, batch_size=100, 42 | db=os.path.join(utils.data_folder, 'mnist-test-nchw-leveldb'), 43 | db_type='leveldb') 44 | #softmax = models.AddLeNetModel(test_model, data) 45 | softmax = models.AddMLP(test_model, data, batch_size=100) 46 | models.AddAccuracy(test_model, softmax, label) 47 | 48 | #print(str(train_model.param_init_net.Proto()) + '\n...') 49 | #print(str(test_model.param_init_net.Proto()) + '\n...') 50 | 51 | 52 | print('Training...') 53 | 54 | #train_model.param_init_net.RunAllOnGPU() 55 | #train_model.net.RunAllOnGPU() 56 | 57 | 58 | # The parameter initialization network only needs to be run once. 59 | workspace.RunNetOnce(train_model.param_init_net) 60 | # creating the network 61 | workspace.CreateNet(train_model.net, overwrite=True) 62 | 63 | 64 | 65 | 66 | # set the number of iterations and track the accuracy & loss 67 | total_iters = 1000 68 | accuracy = np.zeros(total_iters) 69 | loss = np.zeros(total_iters) 70 | # Now, we will manually run the network for 200 iterations. 71 | 72 | data_array = [] 73 | drop1_array = [] 74 | fc2_array = [] 75 | 76 | for i in range(total_iters): 77 | #for i in range(1): 78 | workspace.RunNet(train_model.net) 79 | accuracy[i] = workspace.FetchBlob('accuracy') 80 | loss[i] = workspace.FetchBlob('loss') 81 | print('iter {0} loss = {1} '.format(i, loss[i])) 82 | print(' accuracy = {0} '.format(accuracy[i])) 83 | 84 | print("Current blobs in the workspace: {}".format(workspace.Blobs())) 85 | 86 | print("Workspace has blob 'data'? {}".format(workspace.HasBlob("data"))) 87 | #print("Fetched data:\n{}".format(workspace.FetchBlob("data"))) 88 | data_array.append(workspace.FetchBlob("data")) 89 | print('data_array',np.shape(data_array)) 90 | 91 | 92 | print("Workspace has blob 'drop1'? {}".format(workspace.HasBlob("drop1"))) 93 | #print("Fetched drop1:\n{}".format(workspace.FetchBlob("drop1"))) 94 | drop1_array.append(workspace.FetchBlob("drop1")) 95 | print('drop1_array',np.shape(drop1_array)) 96 | 97 | print("Workspace has blob 'fc2'? {}".format(workspace.HasBlob("fc2"))) 98 | #print("Fetched fc2:\n{}".format(workspace.FetchBlob("fc2"))) 99 | fc2_array.append(workspace.FetchBlob("fc2")) 100 | print('fc2_array',np.shape(fc2_array)) 101 | 102 | # store params of train_model 103 | train_params = {p: workspace.FetchBlob(p) for p in train_model.GetParams()} 104 | 105 | # run a test pass on the test net 106 | print('Testing...') 107 | workspace.RunNetOnce(test_model.param_init_net) 108 | workspace.CreateNet(test_model.net, overwrite=True) 109 | test_accuracy = np.zeros(100) 110 | 111 | # set params of test_model according to the stored train_params 112 | for p in train_params: 113 | print(p) 114 | workspace.FeedBlob(p, train_params[p]) 115 | 116 | test_iter = 100 117 | for i in range(test_iter): 118 | workspace.RunNet(test_model.net.Proto().name) 119 | test_accuracy[i] = workspace.FetchBlob('accuracy') 120 | 121 | print('average test accuracy = {0}'.format(np.mean(test_accuracy))) 122 | 123 | 124 | -------------------------------------------------------------------------------- /src/models.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from caffe2.python import core, model_helper, net_drawer, workspace, visualize, brew 4 | 5 | def AddLeNetModel(model, data): 6 | # Image size: 28 x 28 -> 24 x 24 7 | conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5) 8 | # Image size: 24 x 24 -> 12 x 12 9 | pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) 10 | # Image size: 12 x 12 -> 8 x 8 11 | conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) 12 | # Image size: 8 x 8 -> 4 x 4 13 | pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) 14 | # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size 15 | fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500) 16 | fc3 = brew.relu(model, fc3, fc3) 17 | pred = brew.fc(model, fc3, 'pred', 500, 10) 18 | softmax = brew.softmax(model, pred, 'softmax') 19 | return softmax 20 | 21 | def AddMLP(model, data, batch_size): 22 | ''' 23 | Implement MLP model on MNIST 24 | ''' 25 | num_units = 4096 # number of nuerons in fc layer 26 | num_labels = 10 # for 10 classes in mnist 27 | drop1 = brew.dropout(model, data, 'drop1', ratio=0.5, is_test=0) 28 | 29 | fc2 = brew.fc(model, drop1, 'fc2', dim_in=1 * 28 * 28, dim_out=num_units) 30 | model.Reshape([fc2], [fc2, 'fc2_old_shape'], shape=(batch_size, num_units, 1, 1)) 31 | bn2 = brew.spatial_bn(model, fc2, 'bn2', num_units, epsilon=1e-4, momentum=0.9) 32 | relu2 = brew.relu(model, bn2, 'relu2') 33 | 34 | fc3 = brew.fc(model, relu2, 'fc3', dim_in=num_units, dim_out=num_units) 35 | model.Reshape([fc3], [fc3, 'fc3_old_shape'], shape=(batch_size, num_units, 1, 1)) 36 | bn3 = brew.spatial_bn(model, fc3, 'bn3', num_units, epsilon=1e-4, momentum=0.9) 37 | relu3 = brew.relu(model, bn3, 'relu3') 38 | 39 | fc4 = brew.fc(model, relu3, 'fc4', dim_in=num_units, dim_out=num_units) 40 | model.Reshape([fc4], [fc4, 'fc4_old_shape'], shape=(batch_size, num_units, 1, 1)) 41 | bn4 = brew.spatial_bn(model, fc4, 'bn4', num_units, epsilon=1e-4, momentum=0.9) 42 | relu4 = brew.relu(model, bn4, 'relu4') 43 | 44 | fc5 = brew.fc(model, relu4, 'fc5', dim_in=num_units, dim_out=num_labels) 45 | model.Reshape([fc5], [fc5, 'fc5_old_shape'], shape=(batch_size, num_labels, 1, 1)) 46 | bn5 = brew.spatial_bn(model, fc5, 'bn5', num_labels, epsilon=1e-4, momentum=0.9) 47 | softmax = brew.softmax(model, bn5, 'softmax') 48 | return softmax 49 | 50 | 51 | def AddMLP_BN(model, data): 52 | ''' 53 | Implement MLP model on MNIST 54 | ''' 55 | # number of nuerons in fc layer 56 | num_units = 4096 57 | 58 | # NCHW: 64 x 1 x 28 x 28 -> 64 x 1 x 28 x 28 59 | drop1 = brew.dropout(model, data, 'drop1', ratio=0.5, is_test=0) 60 | 61 | # NCHW: 64 x 1 x 28 x 28 -> 64 x 4096 62 | fc2 = brew.fc(model, drop1, 'fc2', dim_in=1 * 28 * 28, dim_out=num_units) 63 | 64 | 65 | fc2_reshaped = model.Reshape( 66 | [fc2], 67 | ['fc2_reshaped'], 68 | shape=(4,2) 69 | ) 70 | 71 | 72 | # bn2 = brew.spatial_bn(model, fc2, 'bn2', num_units, epsilon=1e-3, momentum=0.9, is_test=is_test) 73 | #relu2 = brew.relu(model, fc2, 'relu2') 74 | 75 | #fc3 = brew.fc(model, relu2, 'fc3', dim_in=num_units, dim_out=num_units) 76 | # bn3 = brew.spatial_bn(model, fc3, 'bn3', bn_units, epsilon=1e-3, momentum=0.9, is_test=is_test) 77 | #relu3 = brew.relu(model, fc3, 'relu3') 78 | 79 | #fc4 = brew.fc(model, relu3, 'fc4', dim_in=num_units, dim_out=num_units) 80 | # bn4 = brew.spatial_bn(model, fc4, 'bn4', bn_units, epsilon=1e-3, momentum=0.9, is_test=is_test) 81 | #relu4 = brew.relu(model, fc4, 'relu4') 82 | 83 | #fc5 = brew.fc(model, relu4, 'fc5', dim_in=num_units, dim_out=10) # 10 for 10-classes 84 | # bn5 = brew.spatial_bn(model, fc5, 'bn5', 10, epsilon=1e-3, momentum=0.9, is_test=is_test) 85 | softmax = brew.softmax(model, fc2, 'softmax') 86 | return softmax 87 | 88 | def AddAccuracy(model, softmax, label): 89 | accuracy = brew.accuracy(model, [softmax, label], "accuracy") 90 | return accuracy 91 | 92 | def AddTrainingOperators(model, softmax, label): 93 | xent = model.LabelCrossEntropy([softmax, label], 'xent') 94 | # compute the expected loss 95 | loss = model.AveragedLoss(xent, "loss") 96 | # track the accuracy of the model 97 | AddAccuracy(model, softmax, label) 98 | # use the average loss we just computed to add gradient operators to the model 99 | model.AddGradientOperators([loss]) 100 | # do a simple stochastic gradient descent 101 | ITER = brew.iter(model, "iter") 102 | # set the learning rate schedule 103 | LR = model.LearningRate( 104 | ITER, "LR", base_lr=-0.1, policy="step", stepsize=1, gamma=0.999 ) 105 | # ONE is a constant value that is used in the gradient update. We only need 106 | # to create it once, so it is explicitly placed in param_init_net. 107 | ONE = model.param_init_net.ConstantFill([], "ONE", shape=[1], value=1.0) 108 | # Now, for each parameter, we do the gradient updates. 109 | for param in model.params: 110 | # Note how we get the gradient of each parameter - ModelHelper keeps 111 | # track of that. 112 | param_grad = model.param_to_grad[param] 113 | # The update is a simple weighted sum: param = param + param_grad * LR 114 | model.WeightedSum([param, ONE, param_grad, LR], param) 115 | 116 | def AddInput(model, batch_size, db, db_type): 117 | # load the data 118 | data_uint8, label = model.TensorProtosDBInput( 119 | [], ["data_uint8", "label"], batch_size=batch_size, 120 | db=db, db_type=db_type) 121 | # cast the data to float 122 | data = model.Cast(data_uint8, "data", to=core.DataType.FLOAT) 123 | # scale data from [0,255] down to [0,1] 124 | data = model.Scale(data, data, scale=float(1./256)) 125 | # don't need the gradient for the backward pass 126 | data = model.StopGradient(data, data) 127 | return data, label 128 | 129 | def AddBookkeepingOperators(model): 130 | # Print basically prints out the content of the blob. to_file=1 routes the 131 | # printed output to a file. The file is going to be stored under 132 | # root_folder/[blob name] 133 | model.Print('accuracy', [], to_file=1) 134 | model.Print('loss', [], to_file=1) 135 | # Summarizes the parameters. Different from Print, Summarize gives some 136 | # statistics of the parameter, such as mean, std, min and max. 137 | for param in model.params: 138 | model.Summarize(param, [], to_file=1) 139 | model.Summarize(model.param_to_grad[param], [], to_file=1) 140 | 141 | 142 | -------------------------------------------------------------------------------- /log/conv1_b.summary: -------------------------------------------------------------------------------- 1 | -0.00951836 0.0123174 0.000904883 0.00516232 2 | -0.0117628 0.0217478 0.00160772 0.00877819 3 | -0.0214147 0.0242367 -0.00104836 0.012727 4 | -0.0314187 0.032033 -0.000817228 0.0166336 5 | -0.0390059 0.0347795 -0.00230794 0.020291 6 | -0.0520021 0.0431842 -0.00335313 0.0247015 7 | -0.0560472 0.0521624 -0.0052308 0.0277256 8 | -0.0689494 0.0625729 -0.00622592 0.0320247 9 | -0.070136 0.0616864 -0.00715054 0.0340137 10 | -0.0865023 0.07289 -0.00927038 0.0414469 11 | -0.0855674 0.0620793 -0.00928442 0.0388569 12 | -0.112782 0.0895257 -0.0153411 0.0540324 13 | -0.120839 0.0964905 -0.0170945 0.0572004 14 | -0.115197 0.0950688 -0.015517 0.0571466 15 | -0.125921 0.0935235 -0.0190263 0.057119 16 | -0.130018 0.0978612 -0.0183898 0.0603068 17 | -0.127421 0.101676 -0.0191148 0.0609522 18 | -0.136053 0.108592 -0.0203293 0.0644832 19 | -0.137407 0.111452 -0.0212455 0.0669917 20 | -0.13313 0.117294 -0.0202568 0.0669592 21 | -0.157308 0.119554 -0.0247822 0.0705502 22 | -0.151428 0.114187 -0.0229811 0.0704706 23 | -0.151961 0.131289 -0.0225298 0.0729418 24 | -0.154977 0.136202 -0.0233539 0.0753862 25 | -0.158035 0.132086 -0.0240035 0.0773611 26 | -0.16286 0.131899 -0.024956 0.0791795 27 | -0.16536 0.133226 -0.0246356 0.0801818 28 | -0.165865 0.130897 -0.0247836 0.0802064 29 | -0.176718 0.149994 -0.0258766 0.0871517 30 | -0.171366 0.13532 -0.0271156 0.0821092 31 | -0.176341 0.142107 -0.0272593 0.0851083 32 | -0.181865 0.145818 -0.0282149 0.0878703 33 | -0.183242 0.146069 -0.0279035 0.0882408 34 | -0.186834 0.148043 -0.028272 0.0889812 35 | -0.187691 0.152727 -0.0275035 0.0908112 36 | -0.186577 0.15744 -0.0287445 0.090909 37 | -0.189887 0.159821 -0.0291923 0.0939839 38 | -0.195282 0.156938 -0.0297362 0.094986 39 | -0.196213 0.160994 -0.0298542 0.096746 40 | -0.192704 0.152421 -0.0306135 0.0914991 41 | -0.199097 0.156456 -0.0313129 0.0953705 42 | -0.201113 0.149839 -0.0313566 0.0958269 43 | -0.20135 0.158193 -0.0305246 0.096218 44 | -0.201289 0.157759 -0.030811 0.0957962 45 | -0.204428 0.160735 -0.0321945 0.09851 46 | -0.20243 0.16248 -0.0315695 0.0978818 47 | -0.203437 0.161062 -0.0322934 0.0976896 48 | -0.211972 0.165856 -0.0336793 0.101462 49 | -0.210293 0.165548 -0.0338178 0.101494 50 | -0.212648 0.169385 -0.0341414 0.103038 51 | -0.220356 0.173 -0.0347084 0.104639 52 | -0.22522 0.174794 -0.0352864 0.106544 53 | -0.221844 0.172786 -0.0357573 0.108031 54 | -0.225602 0.178916 -0.0354078 0.108787 55 | -0.230748 0.17664 -0.0367575 0.109869 56 | -0.232751 0.178589 -0.0363768 0.110251 57 | -0.233287 0.179953 -0.0371405 0.111198 58 | -0.230189 0.176991 -0.0370504 0.109768 59 | -0.232819 0.175674 -0.0370371 0.10997 60 | -0.238108 0.182649 -0.0371036 0.111762 61 | -0.240646 0.184401 -0.0376865 0.112678 62 | -0.244203 0.183771 -0.038285 0.114017 63 | -0.246046 0.186546 -0.0376498 0.114647 64 | -0.243416 0.18696 -0.0369764 0.114115 65 | -0.246244 0.182828 -0.0376041 0.11321 66 | -0.243251 0.181182 -0.0377448 0.111318 67 | -0.238858 0.178619 -0.037205 0.110622 68 | -0.248456 0.186962 -0.0385865 0.11528 69 | -0.236635 0.180407 -0.0373032 0.113607 70 | -0.242431 0.183001 -0.0379386 0.114692 71 | -0.242351 0.187226 -0.0378995 0.117016 72 | -0.240941 0.185318 -0.0381343 0.11579 73 | -0.239368 0.18174 -0.037674 0.11404 74 | -0.239412 0.180927 -0.0377326 0.115674 75 | -0.247544 0.190779 -0.0381579 0.118265 76 | -0.244918 0.186785 -0.0390198 0.119652 77 | -0.24261 0.187554 -0.0387493 0.117253 78 | -0.256529 0.188251 -0.0409801 0.122315 79 | -0.25362 0.181459 -0.0410623 0.120709 80 | -0.263207 0.187764 -0.0416558 0.123924 81 | -0.257896 0.184076 -0.0405793 0.12068 82 | -0.265185 0.191378 -0.0418166 0.124311 83 | -0.264124 0.190315 -0.0424229 0.122614 84 | -0.257937 0.188733 -0.0411796 0.12085 85 | -0.260339 0.189783 -0.041688 0.122266 86 | -0.262041 0.192318 -0.0421506 0.124335 87 | -0.260834 0.187877 -0.0423247 0.12331 88 | -0.269049 0.19554 -0.0438608 0.125439 89 | -0.265788 0.197156 -0.04357 0.125463 90 | -0.274339 0.195011 -0.0454636 0.125618 91 | -0.277886 0.195245 -0.0457329 0.126395 92 | -0.278949 0.195637 -0.0456086 0.127427 93 | -0.270262 0.189467 -0.0449844 0.125501 94 | -0.281586 0.197245 -0.0466672 0.130332 95 | -0.277831 0.191217 -0.0462169 0.128136 96 | -0.28206 0.196326 -0.0467933 0.13045 97 | -0.285042 0.198011 -0.0475304 0.130455 98 | -0.293329 0.199893 -0.0483781 0.131945 99 | -0.2877 0.19422 -0.0477423 0.130579 100 | -0.289236 0.20318 -0.0477973 0.131979 101 | -0.285298 0.202016 -0.0468095 0.130049 102 | -0.275947 0.194173 -0.0466251 0.128434 103 | -0.279158 0.19949 -0.0470173 0.131503 104 | -0.28913 0.205781 -0.0481486 0.133294 105 | -0.286178 0.202941 -0.0481481 0.131971 106 | -0.283103 0.202819 -0.0479051 0.131754 107 | -0.275397 0.195886 -0.0471981 0.13041 108 | -0.284023 0.206536 -0.0477434 0.135582 109 | -0.272884 0.193028 -0.0467151 0.12935 110 | -0.288116 0.208223 -0.0491504 0.136747 111 | -0.279609 0.201338 -0.047229 0.13309 112 | -0.27654 0.19802 -0.0461868 0.131363 113 | -0.283392 0.200636 -0.0468975 0.132888 114 | -0.28822 0.211682 -0.0479703 0.136356 115 | -0.278282 0.205344 -0.0469834 0.133249 116 | -0.280798 0.206017 -0.0475958 0.134802 117 | -0.284996 0.202805 -0.0487152 0.136859 118 | -0.286222 0.203907 -0.0487892 0.137316 119 | -0.285873 0.199706 -0.0481502 0.136283 120 | -0.292985 0.201173 -0.0497965 0.139531 121 | -0.284494 0.200428 -0.04854 0.135946 122 | -0.286057 0.196528 -0.047882 0.135636 123 | -0.295648 0.203675 -0.0497619 0.138821 124 | -0.290022 0.197357 -0.0495984 0.13843 125 | -0.293504 0.20264 -0.0501151 0.139154 126 | -0.287566 0.20471 -0.0480994 0.136898 127 | -0.286925 0.207322 -0.0481469 0.137847 128 | -0.283286 0.208032 -0.0475636 0.135854 129 | -0.283878 0.205916 -0.0473377 0.135085 130 | -0.293483 0.212502 -0.0497038 0.140998 131 | -0.285802 0.209123 -0.0486183 0.139766 132 | -0.282397 0.210003 -0.0480646 0.137671 133 | -0.287646 0.211536 -0.0490439 0.139762 134 | -0.283913 0.208269 -0.0485532 0.138548 135 | -0.283601 0.210724 -0.0482555 0.138505 136 | -0.278326 0.209528 -0.046535 0.137168 137 | -0.273123 0.208219 -0.0444559 0.133283 138 | -0.286828 0.21025 -0.047284 0.139965 139 | -0.281982 0.203111 -0.0464241 0.139478 140 | -0.275787 0.192104 -0.0449419 0.135423 141 | -0.267217 0.186951 -0.0431828 0.132916 142 | -0.275052 0.19506 -0.0445732 0.135639 143 | -0.27532 0.192857 -0.0450857 0.136212 144 | -0.269471 0.192101 -0.0439079 0.133163 145 | -0.271845 0.193794 -0.0450044 0.133839 146 | -0.272573 0.199771 -0.0448729 0.134254 147 | -0.272561 0.195785 -0.0452051 0.135092 148 | -0.27655 0.193734 -0.046241 0.136031 149 | -0.269087 0.191805 -0.0450609 0.133953 150 | -0.272437 0.190528 -0.0451574 0.133885 151 | -0.272896 0.190136 -0.0448317 0.133874 152 | -0.277035 0.189354 -0.0454779 0.135256 153 | -0.284515 0.18876 -0.0470925 0.137073 154 | -0.285152 0.18665 -0.0476184 0.137295 155 | -0.284963 0.188053 -0.04793 0.138032 156 | -0.279759 0.185235 -0.0475672 0.137054 157 | -0.282433 0.18887 -0.0476367 0.137756 158 | -0.283972 0.191199 -0.0481512 0.13776 159 | -0.285562 0.193391 -0.04837 0.139008 160 | -0.294095 0.199826 -0.0496817 0.141954 161 | -0.288831 0.195681 -0.0490789 0.139866 162 | -0.28626 0.193771 -0.0486475 0.139657 163 | -0.28568 0.194072 -0.0485532 0.139151 164 | -0.281923 0.189311 -0.0479587 0.139569 165 | -0.283309 0.190994 -0.0482602 0.140312 166 | -0.285536 0.192132 -0.0487371 0.139744 167 | -0.285923 0.194271 -0.0484359 0.139833 168 | -0.294291 0.192076 -0.0511227 0.142709 169 | -0.291946 0.192231 -0.0497775 0.140452 170 | -0.289513 0.18927 -0.0496182 0.140168 171 | -0.293164 0.191779 -0.0502999 0.141089 172 | -0.294285 0.189147 -0.0504612 0.141166 173 | -0.293414 0.186524 -0.0505435 0.140868 174 | -0.293636 0.183275 -0.0505374 0.141623 175 | -0.292985 0.184534 -0.0506188 0.141647 176 | -0.2879 0.181828 -0.049853 0.139684 177 | -0.288711 0.181456 -0.0500304 0.140154 178 | -0.285126 0.183256 -0.0494824 0.139332 179 | -0.287291 0.184138 -0.0495591 0.139317 180 | -0.288911 0.185083 -0.0498002 0.139684 181 | -0.288455 0.192566 -0.0490802 0.138681 182 | -0.285054 0.182996 -0.0489039 0.139191 183 | -0.285998 0.185592 -0.0493474 0.136936 184 | -0.28336 0.183405 -0.0493563 0.137006 185 | -0.291916 0.188884 -0.0507448 0.138878 186 | -0.279636 0.183043 -0.0494456 0.136809 187 | -0.288268 0.189386 -0.0507423 0.138755 188 | -0.28916 0.180765 -0.0519021 0.141057 189 | -0.291236 0.181066 -0.0521248 0.14285 190 | -0.292121 0.182451 -0.052023 0.143131 191 | -0.291046 0.183572 -0.0519347 0.143034 192 | -0.292821 0.182777 -0.0530139 0.14384 193 | -0.292543 0.183452 -0.0531998 0.143941 194 | -0.288885 0.182196 -0.0526353 0.142195 195 | -0.291062 0.183183 -0.0530139 0.142419 196 | -0.287589 0.181626 -0.0527746 0.142004 197 | -0.291003 0.184398 -0.0533423 0.141757 198 | -0.283596 0.183479 -0.0504905 0.141921 199 | -0.289407 0.192234 -0.0519372 0.142816 200 | -0.28628 0.18702 -0.0521219 0.142434 201 | -------------------------------------------------------------------------------- /log/conv1_w.summary: -------------------------------------------------------------------------------- 1 | -0.347528 0.346677 -0.00742127 0.197561 2 | -0.348026 0.349915 -0.00742447 0.197601 3 | -0.3469 0.350647 -0.0075125 0.19744 4 | -0.348683 0.351261 -0.00713013 0.197752 5 | -0.348124 0.35311 -0.00655613 0.198186 6 | -0.34692 0.353606 -0.00594319 0.198751 7 | -0.34738 0.35472 -0.00526063 0.199379 8 | -0.347697 0.357144 -0.00474518 0.199985 9 | -0.347174 0.35938 -0.00440644 0.200403 10 | -0.347063 0.360955 -0.00417737 0.20069 11 | -0.347913 0.360663 -0.00471399 0.200824 12 | -0.346903 0.36219 -0.00476615 0.200526 13 | -0.346913 0.365234 -0.00450943 0.200889 14 | -0.347806 0.367876 -0.00422372 0.201207 15 | -0.34727 0.369988 -0.00425166 0.20159 16 | -0.347793 0.371896 -0.00379429 0.202171 17 | -0.348666 0.372014 -0.00380065 0.202416 18 | -0.348382 0.378389 -0.00319545 0.202813 19 | -0.349558 0.379532 -0.00286143 0.203218 20 | -0.349324 0.379599 -0.00247458 0.203499 21 | -0.351437 0.378961 -0.00316756 0.20334 22 | -0.350073 0.376943 -0.00344755 0.203172 23 | -0.352744 0.379686 -0.00301424 0.203531 24 | -0.352998 0.3828 -0.00269524 0.203929 25 | -0.352757 0.384944 -0.00225864 0.204323 26 | -0.35451 0.385953 -0.00205287 0.20462 27 | -0.355403 0.386947 -0.00177096 0.204869 28 | -0.355137 0.389556 -0.00155877 0.20513 29 | -0.35573 0.3902 -0.00176688 0.205218 30 | -0.355496 0.390316 -0.00181819 0.205446 31 | -0.355387 0.392296 -0.00161743 0.205585 32 | -0.356155 0.39216 -0.0017275 0.205595 33 | -0.356298 0.391579 -0.0017885 0.205641 34 | -0.357103 0.391859 -0.00151132 0.205865 35 | -0.357082 0.392626 -0.00160687 0.205991 36 | -0.358395 0.393929 -0.00152012 0.206168 37 | -0.357535 0.393526 -0.00127988 0.206368 38 | -0.358589 0.394679 -0.00121215 0.20661 39 | -0.358567 0.398669 -0.0010139 0.206735 40 | -0.360504 0.399896 -0.00135172 0.206726 41 | -0.360184 0.399524 -0.00118955 0.206797 42 | -0.358769 0.399762 -0.00120719 0.206785 43 | -0.359388 0.398551 -0.00127359 0.206797 44 | -0.361261 0.400058 -0.00124022 0.206967 45 | -0.359938 0.399392 -0.0010705 0.207134 46 | -0.359604 0.399892 -0.00112709 0.207205 47 | -0.360512 0.402401 -0.000909475 0.207288 48 | -0.360515 0.403451 -0.00100628 0.207373 49 | -0.360704 0.404413 -0.000827771 0.207583 50 | -0.360996 0.40646 -0.000492772 0.207809 51 | -0.36194 0.408942 -0.0003272 0.20798 52 | -0.362529 0.408865 -0.000297541 0.208093 53 | -0.362286 0.408229 -9.98991e-05 0.208225 54 | -0.363895 0.409029 -0.000115397 0.208345 55 | -0.363816 0.41056 -4.67398e-05 0.20845 56 | -0.364549 0.408689 -1.71158e-05 0.20853 57 | -0.364315 0.408729 4.88777e-05 0.208632 58 | -0.363962 0.40877 -0.000137126 0.208552 59 | -0.364225 0.408545 1.09643e-07 0.208534 60 | -0.365094 0.408584 -8.14138e-05 0.208615 61 | -0.365743 0.409083 -3.14724e-05 0.208747 62 | -0.365494 0.410892 0.0002228 0.20893 63 | -0.366587 0.41002 -2.86821e-05 0.208997 64 | -0.366616 0.410696 0.000158936 0.209009 65 | -0.366541 0.410967 -6.43262e-05 0.208933 66 | -0.365654 0.413242 -0.00011618 0.209038 67 | -0.365444 0.413189 8.88783e-06 0.209173 68 | -0.367189 0.413569 -0.000147197 0.209223 69 | -0.367077 0.413831 0.000195745 0.209354 70 | -0.367684 0.4152 0.000108579 0.209463 71 | -0.367483 0.41499 0.000253264 0.209503 72 | -0.368167 0.415768 0.000346383 0.209624 73 | -0.368264 0.417387 0.000256105 0.209578 74 | -0.365969 0.415919 0.000408316 0.209652 75 | -0.365933 0.417382 0.000306066 0.20968 76 | -0.365477 0.416923 0.000792466 0.209714 77 | -0.367174 0.418692 0.000723563 0.209819 78 | -0.367342 0.415739 0.000337451 0.209633 79 | -0.367719 0.415729 0.00043729 0.209817 80 | -0.367988 0.416141 0.000336962 0.209919 81 | -0.369272 0.417425 0.00044624 0.209869 82 | -0.368405 0.416502 0.000271646 0.209904 83 | -0.369233 0.418296 0.000436264 0.210024 84 | -0.369568 0.419531 0.000676172 0.210157 85 | -0.369518 0.419574 0.000743489 0.210238 86 | -0.368543 0.417816 0.000901541 0.210312 87 | -0.367992 0.417803 0.000954895 0.210348 88 | -0.368353 0.420678 0.000955847 0.210419 89 | -0.368704 0.420085 0.0011049 0.210539 90 | -0.369554 0.421804 0.000829744 0.210552 91 | -0.370461 0.421261 0.00078936 0.210673 92 | -0.371102 0.420688 0.000808534 0.210668 93 | -0.37085 0.420569 0.00102484 0.21073 94 | -0.369595 0.42003 0.000971435 0.210718 95 | -0.369467 0.42031 0.00111556 0.210783 96 | -0.369025 0.420168 0.00107666 0.210719 97 | -0.369867 0.421986 0.00116005 0.210848 98 | -0.37 0.423887 0.00101631 0.210901 99 | -0.370392 0.424446 0.00129735 0.211068 100 | -0.370515 0.425272 0.00141992 0.211149 101 | -0.370112 0.426076 0.00146245 0.211223 102 | -0.370225 0.422684 0.00147553 0.211197 103 | -0.369795 0.422381 0.00147274 0.211228 104 | -0.370811 0.424239 0.00121392 0.211206 105 | -0.371555 0.424453 0.00111855 0.211288 106 | -0.3717 0.424575 0.00120398 0.211413 107 | -0.372346 0.424748 0.00141475 0.211568 108 | -0.370604 0.422323 0.00077863 0.211301 109 | -0.371514 0.425196 0.00110362 0.211395 110 | -0.372087 0.4231 0.000493671 0.211162 111 | -0.373039 0.424567 0.00064474 0.2113 112 | -0.372937 0.424538 0.000574683 0.211308 113 | -0.373193 0.425716 0.000509683 0.211424 114 | -0.373852 0.427776 0.000384776 0.211505 115 | -0.374748 0.428662 0.000730309 0.211641 116 | -0.374349 0.428339 0.000708564 0.211738 117 | -0.373811 0.428059 0.000764496 0.211827 118 | -0.373681 0.4286 0.000892844 0.211893 119 | -0.373668 0.427328 0.000900809 0.211944 120 | -0.373004 0.426709 0.000933643 0.211993 121 | -0.373851 0.428589 0.00117576 0.212107 122 | -0.373352 0.430434 0.00143017 0.212288 123 | -0.374266 0.429945 0.00118365 0.212121 124 | -0.372599 0.428177 0.00119842 0.212124 125 | -0.373261 0.428636 0.000944594 0.212058 126 | -0.3746 0.429795 0.00111864 0.212101 127 | -0.374479 0.428269 0.000814578 0.212058 128 | -0.37538 0.429832 0.000930599 0.212144 129 | -0.375551 0.431065 0.000949554 0.21213 130 | -0.375041 0.430831 0.00078094 0.212059 131 | -0.375792 0.431277 0.00121597 0.21223 132 | -0.377349 0.434365 0.00141407 0.212354 133 | -0.376208 0.430239 0.000714331 0.212317 134 | -0.376717 0.430995 0.000898963 0.212372 135 | -0.377185 0.430179 0.000930901 0.212427 136 | -0.377658 0.430176 0.00105389 0.212456 137 | -0.378739 0.432164 0.00110253 0.212399 138 | -0.376181 0.428122 0.000434369 0.212354 139 | -0.375321 0.426923 0.00075585 0.212446 140 | -0.374863 0.425852 0.000837038 0.212454 141 | -0.374751 0.42634 0.000975366 0.212503 142 | -0.37546 0.427376 0.000582768 0.212513 143 | -0.374955 0.424974 0.00057171 0.212467 144 | -0.37607 0.426885 0.000809851 0.212609 145 | -0.375901 0.426642 0.000623134 0.212644 146 | -0.376439 0.428266 0.000676278 0.212614 147 | -0.376472 0.426552 0.000709389 0.21268 148 | -0.376236 0.425845 0.000512479 0.212667 149 | -0.375713 0.426953 0.000850197 0.212764 150 | -0.376086 0.427715 0.000944783 0.212829 151 | -0.376324 0.428631 0.0011628 0.212979 152 | -0.376492 0.428561 0.00100364 0.213025 153 | -0.375973 0.429617 0.00105068 0.213002 154 | -0.376387 0.429514 0.00110716 0.213054 155 | -0.375599 0.429062 0.00119337 0.213049 156 | -0.375202 0.429396 0.00143011 0.213151 157 | -0.375826 0.429637 0.00143018 0.213151 158 | -0.376628 0.430573 0.00137723 0.213207 159 | -0.376331 0.430348 0.00125934 0.213226 160 | -0.377248 0.43141 0.000449479 0.213235 161 | -0.376949 0.429786 0.000620018 0.213413 162 | -0.376946 0.429604 0.000762311 0.213448 163 | -0.377509 0.431106 0.000897137 0.213554 164 | -0.375736 0.42963 0.00135902 0.213617 165 | -0.375912 0.430148 0.00144561 0.213678 166 | -0.377124 0.430798 0.00143794 0.213725 167 | -0.37741 0.432138 0.00149406 0.213782 168 | -0.377056 0.432372 0.00144531 0.213724 169 | -0.377696 0.433489 0.00153189 0.213792 170 | -0.377311 0.433121 0.00157218 0.213795 171 | -0.377679 0.433977 0.00163468 0.213865 172 | -0.377959 0.434411 0.00147771 0.213896 173 | -0.377541 0.434533 0.00132927 0.213882 174 | -0.376693 0.43359 0.0014928 0.213892 175 | -0.377603 0.434537 0.00170519 0.21397 176 | -0.377393 0.434145 0.00183345 0.21398 177 | -0.377288 0.434125 0.00185748 0.214066 178 | -0.377694 0.435254 0.00191691 0.214152 179 | -0.377765 0.435691 0.00200824 0.214201 180 | -0.378608 0.436151 0.00199507 0.214273 181 | -0.379983 0.438908 0.00203276 0.214173 182 | -0.377857 0.436892 0.00217747 0.214315 183 | -0.377858 0.437308 0.00211301 0.214296 184 | -0.37798 0.437774 0.00230524 0.214324 185 | -0.378646 0.439273 0.0021037 0.214387 186 | -0.379037 0.43682 0.00249925 0.214457 187 | -0.37969 0.439415 0.00186533 0.214462 188 | -0.378723 0.437839 0.00192197 0.214479 189 | -0.379068 0.438658 0.00192707 0.214522 190 | -0.379272 0.43979 0.002078 0.214583 191 | -0.379327 0.441363 0.00224745 0.214654 192 | -0.379214 0.439163 0.00220948 0.214719 193 | -0.380008 0.440093 0.00229219 0.214838 194 | -0.380983 0.440023 0.00222119 0.214876 195 | -0.381487 0.441296 0.00217844 0.214971 196 | -0.380114 0.440502 0.00230081 0.21498 197 | -0.381469 0.438432 0.0016057 0.214781 198 | -0.380387 0.433925 0.00241977 0.214665 199 | -0.381555 0.43487 0.00140879 0.21447 200 | -0.38113 0.432874 0.00155654 0.214541 201 | -------------------------------------------------------------------------------- /log/pred_w.summary: -------------------------------------------------------------------------------- 1 | -0.081254 0.0808538 9.77967e-06 0.0447119 2 | -0.0828858 0.0843121 9.77975e-06 0.0447818 3 | -0.0958231 0.0843874 9.77949e-06 0.044795 4 | -0.0965016 0.0881479 9.77931e-06 0.0449593 5 | -0.100523 0.0908292 9.77941e-06 0.0451662 6 | -0.103482 0.0949159 9.78025e-06 0.0454303 7 | -0.105754 0.096545 9.77992e-06 0.0457113 8 | -0.108202 0.101021 9.78079e-06 0.0459989 9 | -0.108431 0.102511 9.77984e-06 0.0462506 10 | -0.109588 0.105311 9.77969e-06 0.0464785 11 | -0.1106 0.106962 9.77891e-06 0.046639 12 | -0.115767 0.112794 9.77967e-06 0.0467457 13 | -0.116089 0.113884 9.78025e-06 0.0469512 14 | -0.117669 0.108594 9.78058e-06 0.0471467 15 | -0.117297 0.119698 9.77943e-06 0.0473357 16 | -0.118396 0.110335 9.77957e-06 0.0475969 17 | -0.119842 0.118699 9.78055e-06 0.0477335 18 | -0.122023 0.116257 9.78027e-06 0.0479365 19 | -0.122411 0.119915 9.78008e-06 0.048121 20 | -0.123357 0.128146 9.77985e-06 0.0482871 21 | -0.124061 0.138319 9.78027e-06 0.0484123 22 | -0.124947 0.131137 9.77996e-06 0.0484238 23 | -0.127681 0.137292 9.78006e-06 0.0486009 24 | -0.128315 0.144763 9.7799e-06 0.0487813 25 | -0.129297 0.138729 9.7803e-06 0.0489562 26 | -0.130316 0.139126 9.78035e-06 0.0490833 27 | -0.130429 0.143619 9.78051e-06 0.0491976 28 | -0.132243 0.134589 9.7802e-06 0.0493182 29 | -0.132309 0.147665 9.78057e-06 0.0494358 30 | -0.13279 0.139945 9.78019e-06 0.0495637 31 | -0.133417 0.149524 9.78065e-06 0.0496739 32 | -0.134566 0.14935 9.7807e-06 0.0497084 33 | -0.13556 0.145415 9.78022e-06 0.0497352 34 | -0.135522 0.148121 9.78027e-06 0.0498339 35 | -0.135124 0.150922 9.78033e-06 0.049921 36 | -0.13572 0.158576 9.78051e-06 0.0500217 37 | -0.135256 0.152586 9.78039e-06 0.050123 38 | -0.134545 0.155204 9.78025e-06 0.050246 39 | -0.136241 0.154255 9.78137e-06 0.0503635 40 | -0.136829 0.158024 9.77978e-06 0.0504337 41 | -0.137264 0.158985 9.78035e-06 0.0505047 42 | -0.137911 0.13999 9.78077e-06 0.0505494 43 | -0.138075 0.151224 9.78124e-06 0.0505916 44 | -0.139012 0.153815 9.78062e-06 0.050663 45 | -0.137896 0.152553 9.78012e-06 0.0507533 46 | -0.139362 0.153504 9.78075e-06 0.0507918 47 | -0.141445 0.154143 9.78044e-06 0.0508292 48 | -0.141088 0.152561 9.78069e-06 0.0508895 49 | -0.141726 0.15472 9.78052e-06 0.0509769 50 | -0.142804 0.156794 9.78025e-06 0.0510765 51 | -0.144219 0.161143 9.78082e-06 0.0511507 52 | -0.145875 0.15881 9.78072e-06 0.0512144 53 | -0.146104 0.156962 9.78116e-06 0.0512761 54 | -0.147383 0.16002 9.7805e-06 0.0513443 55 | -0.147481 0.158439 9.78108e-06 0.051411 56 | -0.14701 0.156532 9.78051e-06 0.051472 57 | -0.147673 0.155412 9.78054e-06 0.0515344 58 | -0.14919 0.158246 9.78028e-06 0.0515132 59 | -0.149123 0.158252 9.7808e-06 0.051517 60 | -0.149029 0.157672 9.78033e-06 0.0515661 61 | -0.149223 0.158952 9.7804e-06 0.0516286 62 | -0.149343 0.159083 9.78084e-06 0.0517054 63 | -0.149305 0.164458 9.78122e-06 0.0517763 64 | -0.145874 0.163691 9.78081e-06 0.051804 65 | -0.146088 0.159891 9.7805e-06 0.0517933 66 | -0.147766 0.161131 9.78089e-06 0.0518645 67 | -0.147591 0.157294 9.77987e-06 0.0519213 68 | -0.14756 0.166554 9.78058e-06 0.0519887 69 | -0.149234 0.162838 9.78062e-06 0.0520357 70 | -0.150573 0.161239 9.78124e-06 0.05209 71 | -0.147432 0.163014 9.78092e-06 0.0521267 72 | -0.148292 0.165183 9.78105e-06 0.0521712 73 | -0.147747 0.166972 9.78092e-06 0.0521586 74 | -0.142947 0.161788 9.78153e-06 0.0522086 75 | -0.143383 0.163507 9.78009e-06 0.0522715 76 | -0.144314 0.167019 9.78056e-06 0.0523224 77 | -0.145947 0.166559 9.78095e-06 0.0523639 78 | -0.145085 0.16518 9.78125e-06 0.0523429 79 | -0.146183 0.162937 9.78118e-06 0.0524055 80 | -0.146649 0.161989 9.78106e-06 0.0524733 81 | -0.146697 0.159908 9.7808e-06 0.0524528 82 | -0.147063 0.163354 9.78097e-06 0.0525075 83 | -0.147451 0.164254 9.78175e-06 0.0525528 84 | -0.147342 0.16316 9.78076e-06 0.0525916 85 | -0.1476 0.16461 9.78161e-06 0.0526348 86 | -0.14734 0.163604 9.78094e-06 0.0526773 87 | -0.147355 0.15769 9.78107e-06 0.0526888 88 | -0.147643 0.161289 9.78082e-06 0.0527351 89 | -0.147787 0.163486 9.78115e-06 0.0527896 90 | -0.148197 0.164064 9.78085e-06 0.0528043 91 | -0.148584 0.162853 9.78133e-06 0.0528579 92 | -0.145339 0.166005 9.78107e-06 0.0528751 93 | -0.145717 0.164472 9.78111e-06 0.0528914 94 | -0.142017 0.164984 9.78091e-06 0.0529412 95 | -0.142317 0.158297 9.78163e-06 0.0529547 96 | -0.142994 0.159593 9.78047e-06 0.0529538 97 | -0.144332 0.161879 9.78109e-06 0.0530025 98 | -0.145164 0.160961 9.78159e-06 0.0530449 99 | -0.145206 0.162107 9.78105e-06 0.0531063 100 | -0.145444 0.169193 9.78078e-06 0.0531535 101 | -0.146776 0.167772 9.77997e-06 0.0531816 102 | -0.147714 0.164641 9.78171e-06 0.0531814 103 | -0.147812 0.164034 9.78111e-06 0.0532149 104 | -0.148418 0.171376 9.78164e-06 0.0532369 105 | -0.148443 0.16916 9.78161e-06 0.0532688 106 | -0.148596 0.169263 9.78013e-06 0.0533176 107 | -0.149716 0.166115 9.78118e-06 0.0533665 108 | -0.142211 0.168795 9.78085e-06 0.0533301 109 | -0.142989 0.166198 9.78047e-06 0.0533477 110 | -0.143205 0.174677 9.78019e-06 0.053343 111 | -0.144136 0.172479 9.78098e-06 0.0533913 112 | -0.141991 0.172168 9.78235e-06 0.0533929 113 | -0.142394 0.171394 9.78081e-06 0.0534534 114 | -0.143844 0.179078 9.78075e-06 0.0535284 115 | -0.144184 0.180151 9.78115e-06 0.0535705 116 | -0.142496 0.18159 9.78146e-06 0.0536257 117 | -0.142158 0.177113 9.78117e-06 0.0536669 118 | -0.142304 0.173933 9.78103e-06 0.0536925 119 | -0.141076 0.170811 9.78119e-06 0.0537126 120 | -0.139911 0.170868 9.78025e-06 0.0537536 121 | -0.140268 0.172938 9.7813e-06 0.0537818 122 | -0.139924 0.16742 9.78176e-06 0.0538549 123 | -0.1392 0.174849 9.78117e-06 0.0538463 124 | -0.136819 0.174401 9.78094e-06 0.0538578 125 | -0.137974 0.174952 9.78101e-06 0.0538519 126 | -0.137758 0.173214 9.7809e-06 0.0538687 127 | -0.138486 0.172571 9.78127e-06 0.053872 128 | -0.139403 0.174694 9.7809e-06 0.0538997 129 | -0.140649 0.172717 9.78092e-06 0.0538953 130 | -0.140684 0.172177 9.78186e-06 0.0539142 131 | -0.141072 0.172884 9.78114e-06 0.0539698 132 | -0.1414 0.174868 9.78091e-06 0.0540112 133 | -0.140197 0.171987 9.78158e-06 0.054033 134 | -0.139508 0.170707 9.78128e-06 0.0540446 135 | -0.140682 0.173965 9.78111e-06 0.0540779 136 | -0.139589 0.172171 9.78036e-06 0.0540877 137 | -0.140656 0.179565 9.7819e-06 0.0540572 138 | -0.140409 0.167694 9.78142e-06 0.0541328 139 | -0.141264 0.167621 9.78228e-06 0.0541698 140 | -0.139908 0.163688 9.78206e-06 0.0541642 141 | -0.140198 0.165251 9.78084e-06 0.0541743 142 | -0.140303 0.172863 9.78114e-06 0.0542081 143 | -0.140984 0.17289 9.78065e-06 0.0541989 144 | -0.141516 0.172567 9.78078e-06 0.0542344 145 | -0.140861 0.173311 9.78327e-06 0.0542583 146 | -0.141072 0.176924 9.78132e-06 0.0542531 147 | -0.141132 0.178876 9.78248e-06 0.0542909 148 | -0.141146 0.178365 9.78156e-06 0.054304 149 | -0.142007 0.176328 9.7824e-06 0.0543284 150 | -0.142304 0.17623 9.78055e-06 0.0543609 151 | -0.142371 0.176012 9.78133e-06 0.0544149 152 | -0.142246 0.172261 9.78065e-06 0.0544483 153 | -0.142996 0.171605 9.78219e-06 0.0544587 154 | -0.143054 0.173566 9.78154e-06 0.0544851 155 | -0.143098 0.174545 9.78205e-06 0.0544946 156 | -0.143477 0.174054 9.78185e-06 0.0545303 157 | -0.143733 0.174865 9.78127e-06 0.0545437 158 | -0.14445 0.174798 9.7814e-06 0.0545685 159 | -0.144426 0.176424 9.78152e-06 0.0545908 160 | -0.142007 0.185206 9.78136e-06 0.0546193 161 | -0.141592 0.183459 9.78159e-06 0.0546547 162 | -0.141729 0.180167 9.78086e-06 0.054665 163 | -0.140796 0.180923 9.78057e-06 0.0547044 164 | -0.142288 0.17729 9.78001e-06 0.0547344 165 | -0.142516 0.177673 9.78058e-06 0.0547624 166 | -0.141434 0.178676 9.78082e-06 0.0547838 167 | -0.143205 0.17905 9.78094e-06 0.0548091 168 | -0.143005 0.179571 9.78156e-06 0.0548043 169 | -0.140812 0.178101 9.78123e-06 0.0548276 170 | -0.14162 0.178515 9.78146e-06 0.054831 171 | -0.14207 0.180138 9.78e-06 0.0548649 172 | -0.141559 0.177567 9.78262e-06 0.0548818 173 | -0.141531 0.177305 9.78217e-06 0.0548796 174 | -0.142505 0.175632 9.78071e-06 0.0548907 175 | -0.141354 0.176356 9.7809e-06 0.054925 176 | -0.141867 0.175242 9.78121e-06 0.0549238 177 | -0.141958 0.175234 9.78121e-06 0.0549605 178 | -0.141934 0.174878 9.78034e-06 0.054992 179 | -0.142543 0.174919 9.78185e-06 0.0550129 180 | -0.142493 0.175258 9.78072e-06 0.0550472 181 | -0.141455 0.178811 9.78169e-06 0.0550226 182 | -0.141996 0.172534 9.78262e-06 0.0550844 183 | -0.141878 0.173383 9.78148e-06 0.0550848 184 | -0.141304 0.173408 9.78062e-06 0.0550988 185 | -0.141291 0.174735 9.78175e-06 0.0551339 186 | -0.140815 0.176207 9.78162e-06 0.0551576 187 | -0.1412 0.178353 9.78129e-06 0.0551892 188 | -0.141329 0.176344 9.78149e-06 0.055215 189 | -0.141662 0.174337 9.78032e-06 0.0552448 190 | -0.141726 0.17441 9.78179e-06 0.0552718 191 | -0.141724 0.174081 9.78158e-06 0.0552988 192 | -0.141914 0.174725 9.7808e-06 0.0553435 193 | -0.14289 0.175262 9.78137e-06 0.055389 194 | -0.142723 0.175389 9.78185e-06 0.0554036 195 | -0.142785 0.175317 9.78026e-06 0.0554419 196 | -0.143171 0.17354 9.78112e-06 0.055451 197 | -0.143316 0.178157 9.77963e-06 0.0554208 198 | -0.143415 0.177461 9.78183e-06 0.0554406 199 | -0.143551 0.180932 9.7813e-06 0.0554145 200 | -0.144365 0.176057 9.78094e-06 0.0554432 201 | -------------------------------------------------------------------------------- /tutorial/mnist_example.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from caffe2.python import core, brew, workspace, visualize, model_helper 4 | # from matplotlib import pyplot 5 | 6 | core.GlobalInit(['caffe2', '--caffe2_log_level=1']) 7 | 8 | # This section preps your image and test set in a leveldb 9 | data_folder = "/Users/lochenchou/CaffeProjects/dataset" 10 | root_folder = "/Users/lochenchou/CaffeProjects/output" 11 | 12 | print("training data folder:" + data_folder) 13 | print("workspace root folder:" + root_folder) 14 | 15 | def AddInput(model, batch_size, db, db_type): 16 | # load the data 17 | data_uint8, label = model.TensorProtosDBInput( 18 | [], ["data_uint8", "label"], batch_size=batch_size, 19 | db=db, db_type=db_type) 20 | # cast the data to float 21 | data = model.Cast(data_uint8, "data", to=core.DataType.FLOAT) 22 | # scale data from [0,255] down to [0,1] 23 | data = model.Scale(data, data, scale=float(1./256)) 24 | # don't need the gradient for the backward pass 25 | data = model.StopGradient(data, data) 26 | return data, label 27 | 28 | def AddLeNetModel(model, data): 29 | # Image size: 28 x 28 -> 24 x 24 30 | conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5) 31 | # Image size: 24 x 24 -> 12 x 12 32 | pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) 33 | # Image size: 12 x 12 -> 8 x 8 34 | conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) 35 | # Image size: 8 x 8 -> 4 x 4 36 | pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) 37 | # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size 38 | fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500) 39 | fc3 = brew.relu(model, fc3, fc3) 40 | pred = brew.fc(model, fc3, 'pred', 500, 10) 41 | softmax = brew.softmax(model, pred, 'softmax') 42 | return softmax 43 | 44 | def AddAccuracy(model, softmax, label): 45 | accuracy = model.Accuracy([softmax, label], "accuracy") 46 | return accuracy 47 | 48 | def AddTrainingOperators(model, softmax, label): 49 | # something very important happens here 50 | xent = model.LabelCrossEntropy([softmax, label], 'xent') 51 | # compute the expected loss 52 | loss = model.AveragedLoss(xent, "loss") 53 | # track the accuracy of the model 54 | AddAccuracy(model, softmax, label) 55 | # use the average loss we just computed to add gradient operators to the model 56 | model.AddGradientOperators([loss]) 57 | # do a simple stochastic gradient descent 58 | ITER = model.param_init_net.ConstantFill([], "ITER", shape=[1], value=0, dtype=core.DataType.INT32) 59 | # ITER = model.Iter("Iter") 60 | model.Iter(ITER, ITER) 61 | # set the learning rate schedule 62 | LR = model.LearningRate( 63 | ITER, "LR", base_lr=-0.1, policy="step", stepsize=1, gamma=0.999 ) 64 | # ONE is a constant value that is used in the gradient update. We only need 65 | # to create it once, so it is explicitly placed in param_init_net. 66 | ONE = model.param_init_net.ConstantFill([], "ONE", shape=[1], value=1.0) 67 | # Now, for each parameter, we do the gradient updates. 68 | for param in model.params: 69 | # Note how we get the gradient of each parameter - ModelHelper keeps 70 | # track of that. 71 | param_grad = model.param_to_grad[param] 72 | # The update is a simple weighted sum: param = param + param_grad * LR 73 | model.WeightedSum([param, ONE, param_grad, LR], param) 74 | # let's checkpoint every 20 iterations, which should probably be fine. 75 | # you may need to delete tutorial_files/tutorial-mnist to re-run the tutorial 76 | # model.Checkpoint([ITER] + model.params, [], db="mnist_lenet_checkpoint_%05d.leveldb", db_type="leveldb", every=20) 77 | 78 | def AddBookkeepingOperators(model): 79 | # Print basically prints out the content of the blob. to_file=1 routes the 80 | # printed output to a file. The file is going to be stored under 81 | # root_folder/[blob name] 82 | model.Print('accuracy', [], to_file=1) 83 | model.Print('loss', [], to_file=1) 84 | # Summarizes the parameters. Different from Print, Summarize gives some 85 | # statistics of the parameter, such as mean, std, min and max. 86 | for param in model.params: 87 | model.Summarize(param, [], to_file=1) 88 | model.Summarize(model.param_to_grad[param], [], to_file=1) 89 | # Now, if we really want to be verbose, we can summarize EVERY blob 90 | # that the model produces; it is probably not a good idea, because that 91 | # is going to take time - summarization do not come for free. For this 92 | # demo, we will only show how to summarize the parameters and their 93 | # gradients. 94 | print("Bookkeeping function created") 95 | 96 | arg_scope = {"order": "NCHW"} 97 | train_model = model_helper.ModelHelper(name="mnist_train", arg_scope=arg_scope) 98 | data, label = AddInput( 99 | train_model, batch_size=64, 100 | db=os.path.join(data_folder, 'mnist-train-nchw-leveldb'), 101 | db_type='leveldb') 102 | softmax = AddLeNetModel(train_model, data) 103 | AddTrainingOperators(train_model, softmax, label) 104 | AddBookkeepingOperators(train_model) 105 | 106 | # Testing model. We will set the batch size to 100, so that the testing 107 | # pass is 100 iterations (10,000 images in total). 108 | # For the testing model, we need the data input part, the main LeNetModel 109 | # part, and an accuracy part. Note that init_params is set False because 110 | # we will be using the parameters obtained from the train model. 111 | test_model = model_helper.ModelHelper( 112 | name="mnist_test", arg_scope=arg_scope, init_params=False) 113 | data, label = AddInput( 114 | test_model, batch_size=100, 115 | db=os.path.join(data_folder, 'mnist-test-nchw-leveldb'), 116 | db_type='leveldb') 117 | softmax = AddLeNetModel(test_model, data) 118 | AddAccuracy(test_model, softmax, label) 119 | 120 | # Deployment model. We simply need the main LeNetModel part. 121 | deploy_model = model_helper.ModelHelper( 122 | name="mnist_deploy", arg_scope=arg_scope, init_params=False) 123 | AddLeNetModel(deploy_model, "data") 124 | # You may wonder what happens with the param_init_net part of the deploy_model. 125 | # No, we will not use them, since during deployment time we will not randomly 126 | # initialize the parameters, but load the parameters from the db. 127 | 128 | print(str(train_model.param_init_net.Proto())[:400] + '\n...') 129 | 130 | with open(os.path.join(root_folder, "train_net.pbtxt"), 'w') as fid: 131 | fid.write(str(train_model.net.Proto())) 132 | with open(os.path.join(root_folder, "train_init_net.pbtxt"), 'w') as fid: 133 | fid.write(str(train_model.param_init_net.Proto())) 134 | with open(os.path.join(root_folder, "test_net.pbtxt"), 'w') as fid: 135 | fid.write(str(test_model.net.Proto())) 136 | with open(os.path.join(root_folder, "test_init_net.pbtxt"), 'w') as fid: 137 | fid.write(str(test_model.param_init_net.Proto())) 138 | with open(os.path.join(root_folder, "deploy_net.pbtxt"), 'w') as fid: 139 | fid.write(str(deploy_model.net.Proto())) 140 | print("Protocol buffers files have been created in your root folder: "+root_folder) 141 | 142 | 143 | # The parameter initialization network only needs to be run once. 144 | workspace.RunNetOnce(train_model.param_init_net) 145 | # creating the network 146 | workspace.CreateNet(train_model.net) 147 | # set the number of iterations and track the accuracy & loss 148 | total_iters = 200 149 | accuracy = np.zeros(total_iters) 150 | loss = np.zeros(total_iters) 151 | # Now, we will manually run the network for 200 iterations. 152 | for i in range(total_iters): 153 | workspace.RunNet(train_model.Proto().name) 154 | accuracy[i] = workspace.FetchBlob('accuracy') 155 | loss[i] = workspace.FetchBlob('loss') 156 | if i % 10 == 0: 157 | print("iter: ", i, "acc: ", accuracy[i]) 158 | 159 | # # After the execution is done, let's plot the values. 160 | # pyplot.plot(loss, 'b') 161 | # pyplot.plot(accuracy, 'r') 162 | # pyplot.legend(('Loss', 'Accuracy'), loc='upper right') 163 | # 164 | # # Let's look at some of the data. 165 | # pyplot.figure() 166 | # data = workspace.FetchBlob('data') 167 | # _ = visualize.NCHW.ShowMultiple(data) 168 | # pyplot.figure() 169 | # softmax = workspace.FetchBlob('softmax') 170 | # _ = pyplot.plot(softmax[0], 'ro') 171 | # pyplot.title('Prediction for the first image') 172 | 173 | # run a test pass on the test net 174 | workspace.RunNetOnce(test_model.param_init_net) 175 | workspace.CreateNet(test_model.net) 176 | test_accuracy = np.zeros(100) 177 | print("-----testing trained model-----") 178 | for i in range(100): 179 | workspace.RunNet(train_model.net.Proto().name) 180 | test_accuracy[i] = workspace.FetchBlob('accuracy') 181 | # After the execution is done, let's plot the values. 182 | # pyplot.plot(test_accuracy, 'r') 183 | # pyplot.title('Acuracy over test batches.') 184 | print('test_accuracy: %f' % test_accuracy.mean()) 185 | 186 | -------------------------------------------------------------------------------- /log/conv1_b_grad.summary: -------------------------------------------------------------------------------- 1 | -0.123297 0.0952789 -0.00905789 0.0516749 2 | -0.148 0.0988459 -0.0070424 0.0632966 3 | -0.111731 0.158864 0.0266406 0.0719923 4 | -0.0782752 0.134444 -0.00232061 0.0506461 5 | -0.0717344 0.147066 0.0149819 0.0534949 6 | -0.1433 0.130745 0.0105148 0.0680256 7 | -0.0904127 0.114347 0.0189087 0.0570256 8 | -0.104942 0.130059 0.0100312 0.05274 9 | -0.0688017 0.106134 0.00932984 0.0463307 10 | -0.16689 0.165308 0.0214115 0.094529 11 | -0.147807 0.11515 0.000141991 0.073693 12 | -0.283631 0.515331 0.0612988 0.208274 13 | -0.0977884 0.161196 0.0177629 0.075768 14 | -0.0974433 0.0646688 -0.0159971 0.0420334 15 | -0.0337508 0.153739 0.0356229 0.0442832 16 | -0.08253 0.0500109 -0.00646752 0.0386171 17 | -0.081317 0.0888573 0.00737459 0.0400407 18 | -0.0704125 0.0979379 0.0123662 0.0494828 19 | -0.0506119 0.0696581 0.0093372 0.032905 20 | -0.0667083 0.0397777 -0.0100862 0.0313469 21 | -0.173194 0.246918 0.046214 0.0842373 22 | -0.0876789 0.0682575 -0.018411 0.0509128 23 | -0.175003 0.102593 -0.00461876 0.0611033 24 | -0.0680606 0.0618996 0.0084415 0.037733 25 | -0.0619482 0.055155 0.00666064 0.0362858 26 | -0.0405486 0.0702223 0.00977558 0.0274056 27 | -0.029882 0.056774 -0.00329171 0.0221156 28 | -0.0609541 0.0559136 0.00152246 0.0302419 29 | -0.196585 0.170877 0.0112516 0.0854072 30 | -0.120355 0.151207 0.0127675 0.069159 31 | -0.0761861 0.0513116 0.00148283 0.0386863 32 | -0.0496707 0.10263 0.00986681 0.0387578 33 | -0.0293083 0.0256057 -0.00321909 0.0153355 34 | -0.0348699 0.0371664 0.00381306 0.0202028 35 | -0.0534009 0.0430036 -0.00795968 0.0283954 36 | -0.0488582 0.0687643 0.0128656 0.0300827 37 | -0.0765428 0.0750885 0.00464685 0.0388469 38 | -0.0740189 0.081001 0.00565019 0.0339052 39 | -0.0695933 0.0676253 0.00122679 0.0335617 40 | -0.11963 0.225782 0.00790258 0.0875359 41 | -0.0927293 0.0966943 0.00728667 0.0509542 42 | -0.0689264 0.0690054 0.000456558 0.0342478 43 | -0.0872081 0.0729972 -0.00868578 0.037768 44 | -0.0334542 0.0313498 0.00299204 0.0188459 45 | -0.0509019 0.091357 0.0144731 0.0363399 46 | -0.029057 0.0351111 -0.00654534 0.0192134 47 | -0.027116 0.0447253 0.00758755 0.0186369 48 | -0.0633475 0.102922 0.0145409 0.0496592 49 | -0.0207996 0.0319953 0.001455 0.0159938 50 | -0.0429148 0.0484791 0.00340174 0.0248378 51 | -0.0380382 0.0811154 0.00596663 0.0272351 52 | -0.0453303 0.0512408 0.00608928 0.0289039 53 | -0.0449092 0.0766693 0.00496506 0.0348252 54 | -0.0647095 0.03966 -0.00368938 0.0288894 55 | -0.0499211 0.0793065 0.014261 0.0306554 56 | -0.0647165 0.0377105 -0.00402616 0.0291671 57 | -0.024333 0.0538417 0.00808491 0.0244919 58 | -0.0490188 0.0313888 -0.000954318 0.0257282 59 | -0.0397819 0.0319963 -0.00014182 0.0196893 60 | -0.0740615 0.056164 0.000706462 0.0269891 61 | -0.0191803 0.0269718 0.0061958 0.0155225 62 | -0.0301704 0.0556717 0.006368 0.0235746 63 | -0.0403245 0.04942 -0.00676567 0.0282971 64 | -0.0686632 0.0517012 -0.00717837 0.0331061 65 | -0.0359101 0.0548319 0.0066978 0.0285231 66 | -0.0559357 0.0616947 0.00150376 0.0344296 67 | -0.0469695 0.0369045 -0.00577298 0.0246922 68 | -0.0892961 0.102737 0.0147886 0.0624522 69 | -0.126661 0.0702349 -0.0137507 0.046822 70 | -0.0343766 0.0621585 0.00681516 0.0253371 71 | -0.0479523 0.0547472 -0.000419751 0.0297241 72 | -0.0236138 0.0434896 0.0025231 0.019458 73 | -0.0560456 0.0384953 -0.00495196 0.0266068 74 | -0.0458086 0.0625796 0.000630883 0.0284109 75 | -0.106195 0.087659 0.00458472 0.0417707 76 | -0.0498195 0.101156 0.00930051 0.0449259 77 | -0.070554 0.0658016 -0.00292244 0.0384351 78 | -0.110024 0.150491 0.0241186 0.0705446 79 | -0.0314869 0.0735033 0.000890042 0.0260811 80 | -0.0683107 0.103865 0.00642946 0.0477101 81 | -0.065873 0.0703889 -0.0116734 0.0405531 82 | -0.0792609 0.0791264 0.0134301 0.0483095 83 | -0.0357753 0.0623344 0.00658864 0.0292218 84 | -0.0710195 0.0358613 -0.0135233 0.0320951 85 | -0.0294522 0.0343507 0.00553583 0.0217254 86 | -0.036164 0.0723853 0.00504152 0.0288133 87 | -0.0358821 0.0484511 0.00189895 0.0212939 88 | -0.0836819 0.0897166 0.0167752 0.0371998 89 | -0.0356512 0.0153203 -0.00317879 0.0132037 90 | -0.0193007 0.0935743 0.0207196 0.026946 91 | -0.0267716 0.0388442 0.00294925 0.0148371 92 | -0.0432204 0.0314364 -0.00136183 0.020271 93 | -0.0953453 0.0677163 -0.00685059 0.0351958 94 | -0.106686 0.124414 0.0184868 0.0601819 95 | -0.0484189 0.0662918 -0.00495169 0.0300764 96 | -0.0562391 0.0479826 0.00634467 0.0305933 97 | -0.0185775 0.0328539 0.00812192 0.0157699 98 | -0.0353053 0.0914106 0.0093508 0.0284201 99 | -0.0621517 0.0626314 -0.00702051 0.0301783 100 | -0.099028 0.0501432 0.000608627 0.0304673 101 | -0.0657334 0.0340231 -0.0109285 0.0305063 102 | -0.103551 0.0868542 -0.00204246 0.0488734 103 | -0.0589394 0.0600402 0.00434759 0.0382547 104 | -0.0698097 0.110649 0.0125533 0.0406856 105 | -0.032787 0.0315501 -5.31394e-06 0.01964 106 | -0.0341958 0.0129393 -0.00270173 0.0115998 107 | -0.0857643 0.0771634 -0.00786846 0.0396997 108 | -0.118646 0.0961106 0.00607476 0.0666471 109 | -0.124226 0.176258 -0.0114677 0.0880133 110 | -0.169629 0.170038 0.0271867 0.100901 111 | -0.0962975 0.105993 -0.021471 0.0554145 112 | -0.0575076 0.0371116 -0.0116575 0.0242072 113 | -0.050728 0.0767254 0.00795695 0.0290205 114 | -0.12381 0.146536 0.0120247 0.0597813 115 | -0.111495 0.075766 -0.0110729 0.0508518 116 | -0.0493972 0.0427558 0.00687754 0.0240782 117 | -0.0483167 0.0894856 0.0125843 0.034059 118 | -0.0456597 0.0368787 0.000832134 0.0203333 119 | -0.0578642 0.047316 -0.00719767 0.0228618 120 | -0.056007 0.103974 0.0185633 0.0456272 121 | -0.119373 0.0823804 -0.0141822 0.0567988 122 | -0.050471 0.0440646 -0.00743374 0.0281929 123 | -0.0808248 0.108473 0.02126 0.0503734 124 | -0.0636914 0.0715273 -0.0018504 0.0336396 125 | -0.059871 0.0420309 0.00585493 0.027287 126 | -0.104753 0.0449021 -0.0228647 0.0427858 127 | -0.0804481 0.0483797 0.000539402 0.0329851 128 | -0.0714053 0.0517834 -0.00662995 0.0348239 129 | -0.0403111 0.0240849 -0.00257084 0.0185338 130 | -0.120844 0.150244 0.0269474 0.0804908 131 | -0.0875719 0.0475202 -0.0123749 0.039353 132 | -0.0830603 0.0583459 -0.00631872 0.0385264 133 | -0.0864633 0.112209 0.0111877 0.0482807 134 | -0.0426862 0.0416046 -0.00561185 0.0211519 135 | -0.0281011 0.0166421 -0.00340731 0.0129112 136 | -0.0741902 0.0358951 -0.0197128 0.0271439 137 | -0.157997 0.0712869 -0.0238452 0.0693164 138 | -0.1783 0.197408 0.0324677 0.100544 139 | -0.0855913 0.0820441 -0.00988122 0.0474606 140 | -0.0915987 0.126611 -0.0170508 0.0600083 141 | -0.0986855 0.0639571 -0.0202565 0.0405206 142 | -0.093474 0.0903152 0.0160267 0.044708 143 | -0.0280795 0.0364828 0.00591348 0.0175788 144 | -0.0906869 0.0841404 -0.0136031 0.0474691 145 | -0.0383341 0.0557701 0.0126765 0.0248098 146 | -0.0691609 0.0378139 -0.00152107 0.0269722 147 | -0.0472986 0.0707425 0.003848 0.0327619 148 | -0.0409808 0.0614393 0.0120125 0.0302016 149 | -0.0866314 0.0504177 -0.013699 0.0349409 150 | -0.0495144 0.0540089 0.00112124 0.0285572 151 | -0.048608 0.0326815 -0.00378805 0.019341 152 | -0.032378 0.0481923 0.00752352 0.0237422 153 | -0.0297394 0.0871677 0.0188166 0.0281264 154 | -0.0178357 0.0282772 0.00613595 0.0141201 155 | -0.0394794 0.0529506 0.00363842 0.0223044 156 | -0.06082 0.0329357 -0.0042414 0.0220902 157 | -0.0425285 0.0312834 0.00081326 0.0142963 158 | -0.0272798 0.0386389 0.00602644 0.019104 159 | -0.0626804 0.0342156 0.00256525 0.0267594 160 | -0.0912934 0.100149 0.0153943 0.0551985 161 | -0.0618424 0.0486998 -0.00708163 0.0291246 162 | -0.0579126 0.0252719 -0.00507273 0.0210814 163 | -0.0253982 0.0321778 -0.00111045 0.0150484 164 | -0.0758055 0.0569683 -0.00700454 0.036936 165 | -0.0223839 0.0214584 0.00355554 0.0117857 166 | -0.030253 0.0471196 0.00563076 0.0237115 167 | -0.0460638 0.0320496 -0.00356008 0.0171934 168 | -0.0800258 0.109292 0.0317863 0.0491766 169 | -0.100797 0.0424384 -0.0159297 0.045791 170 | -0.0412406 0.036027 -0.00188915 0.0223568 171 | -0.0297677 0.0433222 0.0080888 0.0175742 172 | -0.0224159 0.0312589 0.00191677 0.0148797 173 | -0.0220796 0.0311914 0.00097777 0.012758 174 | -0.0352809 0.0437226 -7.27065e-05 0.0256327 175 | -0.0233656 0.0193243 0.000970462 0.0141077 176 | -0.0606434 0.0450781 -0.00913254 0.028886 177 | -0.011436 0.0170215 0.00211737 0.00767183 178 | -0.0428355 0.0416261 -0.00654867 0.0192285 179 | -0.0397121 0.0398971 0.000917903 0.0195533 180 | -0.0113158 0.0272838 0.00288638 0.0110047 181 | -0.120345 0.0942256 -0.00862849 0.0540738 182 | -0.0922104 0.114812 -0.00211548 0.0556549 183 | -0.10735 0.109545 0.00532536 0.0521016 184 | -0.0463313 0.033134 0.000106997 0.0243303 185 | -0.0659228 0.102961 0.0167093 0.0425707 186 | -0.147922 0.0862784 -0.0156493 0.0669774 187 | -0.077783 0.104078 0.0156346 0.0577236 188 | -0.107871 0.144598 0.0139974 0.0693878 189 | -0.0595019 0.0935313 0.00269069 0.0358134 190 | -0.0173447 0.0217497 -0.00123123 0.0109092 191 | -0.0381849 0.0623771 -0.001068 0.0226057 192 | -0.0494885 0.0761008 0.0130767 0.0299694 193 | -0.0113224 0.0200884 0.00225545 0.00868053 194 | -0.0745582 0.0592855 -0.00685453 0.0306316 195 | -0.0177426 0.0315282 0.00460162 0.0136426 196 | -0.0422515 0.0544645 -0.00291111 0.027752 197 | -0.0993869 0.0623647 0.00691358 0.0445975 198 | -0.215999 0.157454 -0.0347657 0.104669 199 | -0.120704 0.142382 0.0176542 0.0721535 200 | -0.0490356 0.0636929 0.0022564 0.0288447 201 | -------------------------------------------------------------------------------- /log/conv2_w.summary: -------------------------------------------------------------------------------- 1 | -0.0790323 0.0792842 -5.41578e-05 0.0447239 2 | -0.0799483 0.0811088 -0.00010353 0.0447358 3 | -0.0821929 0.0813112 -0.000214548 0.0447334 4 | -0.0831059 0.0832142 -0.000238579 0.0447657 5 | -0.0844047 0.0845008 -0.000232602 0.0448084 6 | -0.0855518 0.085319 -0.000170172 0.0448635 7 | -0.0869931 0.0872999 -0.000175191 0.0449217 8 | -0.0874986 0.0886662 -0.000111403 0.0449793 9 | -0.0879142 0.090896 -0.000136746 0.0450224 10 | -0.0890417 0.0925628 -0.00022978 0.0450589 11 | -0.0881351 0.0922816 -0.000254043 0.0450812 12 | -0.0914797 0.0897049 -0.000704851 0.0450935 13 | -0.0913788 0.0918763 -0.000629315 0.0451356 14 | -0.0914552 0.0923479 -0.000609326 0.0451682 15 | -0.0918963 0.0918532 -0.000537248 0.0452074 16 | -0.0924539 0.0928214 -0.000518384 0.0452607 17 | -0.0926163 0.0935139 -0.000484113 0.0452866 18 | -0.0936756 0.0943615 -0.000438946 0.0453264 19 | -0.0936215 0.095039 -0.000443593 0.0453641 20 | -0.0938473 0.0959345 -0.00041359 0.0453936 21 | -0.0959408 0.0961245 -0.000469499 0.045403 22 | -0.100278 0.0971402 -0.000675367 0.0454009 23 | -0.100314 0.0972402 -0.000633564 0.0454389 24 | -0.100565 0.0978395 -0.000539599 0.045478 25 | -0.101086 0.0983195 -0.000557321 0.0455146 26 | -0.100846 0.0991633 -0.000524091 0.0455424 27 | -0.101075 0.0998694 -0.000531472 0.0455658 28 | -0.101726 0.099716 -0.000571441 0.0455886 29 | -0.103427 0.100327 -0.000532263 0.0456094 30 | -0.102136 0.0997981 -0.000663773 0.0456321 31 | -0.101389 0.101382 -0.000608887 0.0456504 32 | -0.10276 0.101099 -0.000642739 0.0456555 33 | -0.103038 0.100175 -0.000641733 0.0456604 34 | -0.103181 0.100486 -0.000649665 0.0456804 35 | -0.102986 0.1013 -0.000619351 0.0456955 36 | -0.103408 0.101929 -0.000604994 0.0457138 37 | -0.103415 0.102442 -0.000621849 0.0457345 38 | -0.104159 0.103728 -0.000648982 0.0457581 39 | -0.103728 0.106073 -0.000580572 0.0457782 40 | -0.104278 0.106677 -0.000719385 0.0457822 41 | -0.10508 0.10568 -0.000747575 0.0457948 42 | -0.105214 0.105546 -0.000839896 0.0457962 43 | -0.106034 0.105933 -0.00083108 0.0458016 44 | -0.105767 0.106671 -0.000825748 0.0458171 45 | -0.10544 0.107062 -0.000813753 0.0458354 46 | -0.105219 0.106196 -0.000788845 0.045844 47 | -0.10506 0.106568 -0.000813521 0.0458505 48 | -0.105991 0.10661 -0.000805501 0.0458632 49 | -0.106445 0.107923 -0.000803658 0.0458825 50 | -0.106371 0.109058 -0.00076987 0.0459051 51 | -0.106644 0.108963 -0.000729185 0.0459211 52 | -0.107264 0.108724 -0.000721893 0.0459338 53 | -0.10858 0.10833 -0.000749192 0.0459462 54 | -0.106802 0.108169 -0.000744234 0.0459582 55 | -0.107357 0.110013 -0.000745078 0.04597 56 | -0.1058 0.110145 -0.000796961 0.0459788 57 | -0.107645 0.109992 -0.000824543 0.0459892 58 | -0.107158 0.110122 -0.000784326 0.0459844 59 | -0.106894 0.109726 -0.000815027 0.045983 60 | -0.107026 0.109711 -0.000792065 0.0459932 61 | -0.107142 0.110327 -0.000762145 0.0460071 62 | -0.10666 0.109948 -0.000791166 0.0460231 63 | -0.10733 0.110581 -0.000745267 0.0460355 64 | -0.107625 0.109039 -0.0008712 0.0460361 65 | -0.108319 0.108621 -0.000878386 0.0460315 66 | -0.108166 0.109222 -0.000831845 0.046044 67 | -0.108608 0.10954 -0.000899684 0.0460537 68 | -0.108926 0.110362 -0.000829754 0.0460687 69 | -0.109051 0.110319 -0.000935716 0.0460767 70 | -0.108798 0.110371 -0.000909908 0.0460889 71 | -0.108943 0.110553 -0.000893102 0.0460968 72 | -0.108613 0.110607 -0.000867166 0.0461073 73 | -0.108606 0.110618 -0.000876163 0.0461035 74 | -0.109291 0.110632 -0.000934258 0.0461126 75 | -0.10933 0.112326 -0.000898715 0.0461215 76 | -0.109878 0.112256 -0.000907363 0.0461294 77 | -0.108629 0.112904 -0.000898347 0.0461392 78 | -0.11025 0.112425 -0.000932983 0.0461312 79 | -0.109625 0.112752 -0.000961038 0.0461445 80 | -0.110359 0.113157 -0.000935661 0.0461595 81 | -0.110987 0.113158 -0.00100486 0.0461537 82 | -0.110748 0.113082 -0.000926026 0.046166 83 | -0.111647 0.112699 -0.000910441 0.0461764 84 | -0.111486 0.112319 -0.000951302 0.0461829 85 | -0.111336 0.113282 -0.00095171 0.0461922 86 | -0.112971 0.113025 -0.000966951 0.0462016 87 | -0.11278 0.11233 -0.00102806 0.0462027 88 | -0.112974 0.113157 -0.000974495 0.0462141 89 | -0.112941 0.11341 -0.000965198 0.0462255 90 | -0.111506 0.114914 -0.000938399 0.0462293 91 | -0.111732 0.114877 -0.000946815 0.0462409 92 | -0.111473 0.11444 -0.000946197 0.046243 93 | -0.112516 0.114284 -0.000992098 0.0462455 94 | -0.112533 0.114207 -0.000979023 0.0462564 95 | -0.112454 0.11378 -0.00103228 0.0462583 96 | -0.112517 0.113574 -0.000991077 0.0462587 97 | -0.11236 0.114888 -0.000976109 0.0462704 98 | -0.111401 0.115428 -0.000931036 0.0462804 99 | -0.111069 0.115543 -0.000953216 0.0462933 100 | -0.111796 0.116457 -0.000875778 0.0463057 101 | -0.111481 0.116514 -0.000884724 0.0463113 102 | -0.11272 0.116792 -0.000984431 0.0463081 103 | -0.113511 0.116928 -0.000967187 0.0463156 104 | -0.112065 0.117429 -0.000876724 0.046323 105 | -0.111198 0.117609 -0.000897234 0.0463299 106 | -0.111714 0.117771 -0.000894505 0.0463408 107 | -0.112406 0.117538 -0.000947905 0.046351 108 | -0.1128 0.116935 -0.000939859 0.0463424 109 | -0.112872 0.117647 -0.00107409 0.0463425 110 | -0.113758 0.117385 -0.000988266 0.0463428 111 | -0.113822 0.116663 -0.00103941 0.0463505 112 | -0.114005 0.116801 -0.00108164 0.0463492 113 | -0.113573 0.118118 -0.00106152 0.0463627 114 | -0.114778 0.12005 -0.000967141 0.0463798 115 | -0.114667 0.119796 -0.000998444 0.0463876 116 | -0.114237 0.120328 -0.000989357 0.0464 117 | -0.114727 0.120243 -0.000999548 0.046409 118 | -0.114708 0.120757 -0.00100821 0.0464147 119 | -0.114616 0.120562 -0.00104212 0.0464181 120 | -0.114932 0.120341 -0.00104574 0.0464269 121 | -0.115531 0.119554 -0.00104313 0.046434 122 | -0.115065 0.119328 -0.00108736 0.0464489 123 | -0.115282 0.121443 -0.00102736 0.0464439 124 | -0.115969 0.120218 -0.00108998 0.046444 125 | -0.116428 0.12076 -0.00104832 0.0464426 126 | -0.116448 0.120026 -0.00106713 0.0464442 127 | -0.116657 0.119577 -0.00107441 0.0464445 128 | -0.117423 0.120532 -0.00103474 0.0464519 129 | -0.117122 0.12048 -0.00103794 0.0464509 130 | -0.117628 0.121296 -0.00104155 0.0464558 131 | -0.117281 0.121617 -0.00108211 0.046468 132 | -0.117192 0.12141 -0.00106412 0.0464777 133 | -0.118268 0.121337 -0.00107558 0.0464825 134 | -0.118208 0.12096 -0.0011073 0.0464842 135 | -0.117221 0.122234 -0.00107513 0.0464917 136 | -0.117541 0.121128 -0.00111772 0.0464918 137 | -0.11649 0.121876 -0.00110411 0.0464848 138 | -0.116217 0.121531 -0.00111053 0.0465009 139 | -0.115978 0.121769 -0.00114661 0.0465083 140 | -0.116312 0.121347 -0.00119812 0.0465051 141 | -0.116583 0.121425 -0.00122458 0.0465061 142 | -0.116602 0.122723 -0.00114736 0.0465161 143 | -0.117499 0.122748 -0.00117604 0.0465132 144 | -0.117909 0.123565 -0.00117742 0.0465219 145 | -0.117655 0.12423 -0.00113413 0.0465279 146 | -0.116373 0.125347 -0.00110574 0.0465272 147 | -0.117332 0.124665 -0.00114416 0.0465354 148 | -0.118093 0.12365 -0.00114308 0.0465376 149 | -0.118959 0.123901 -0.00118265 0.0465427 150 | -0.119298 0.124224 -0.00118376 0.0465488 151 | -0.119023 0.124562 -0.00117305 0.0465617 152 | -0.118856 0.124629 -0.00116775 0.0465693 153 | -0.117988 0.124658 -0.00114412 0.0465713 154 | -0.118414 0.124724 -0.00114457 0.0465772 155 | -0.119317 0.124726 -0.00116269 0.0465784 156 | -0.120631 0.125124 -0.00117468 0.0465862 157 | -0.120053 0.125436 -0.00115441 0.046589 158 | -0.119623 0.126402 -0.0011212 0.0465953 159 | -0.119694 0.126427 -0.00112059 0.0466002 160 | -0.118915 0.127143 -0.00108029 0.046608 161 | -0.118932 0.126683 -0.00110792 0.0466157 162 | -0.119891 0.126222 -0.00112377 0.0466177 163 | -0.119237 0.126517 -0.00111325 0.0466269 164 | -0.120853 0.126091 -0.00116108 0.0466331 165 | -0.120874 0.126228 -0.00114151 0.04664 166 | -0.11969 0.126777 -0.00111574 0.0466454 167 | -0.119809 0.127111 -0.00112022 0.0466505 168 | -0.120093 0.127164 -0.00111292 0.046649 169 | -0.118853 0.127093 -0.00109811 0.046655 170 | -0.119468 0.12682 -0.00111979 0.046655 171 | -0.119594 0.1267 -0.0010965 0.0466636 172 | -0.119116 0.126787 -0.00110425 0.0466671 173 | -0.118717 0.126843 -0.00109908 0.0466665 174 | -0.120164 0.126046 -0.00113057 0.0466678 175 | -0.12005 0.126474 -0.0011156 0.0466759 176 | -0.120898 0.125747 -0.00115025 0.0466745 177 | -0.121065 0.125975 -0.00114348 0.0466833 178 | -0.120255 0.126351 -0.00112154 0.0466911 179 | -0.120798 0.126628 -0.00110285 0.0466961 180 | -0.120296 0.126997 -0.00107968 0.0467044 181 | -0.119737 0.12781 -0.00105142 0.0466978 182 | -0.121426 0.126817 -0.00115104 0.0467103 183 | -0.121078 0.127681 -0.00114004 0.04671 184 | -0.121944 0.128112 -0.00116249 0.0467126 185 | -0.120661 0.128739 -0.00110853 0.0467229 186 | -0.122216 0.128791 -0.0011665 0.0467268 187 | -0.121324 0.128973 -0.00111564 0.0467365 188 | -0.121521 0.12814 -0.00114726 0.0467412 189 | -0.122544 0.12839 -0.00114938 0.0467474 190 | -0.122612 0.128802 -0.00114689 0.0467536 191 | -0.122755 0.128757 -0.00115319 0.0467595 192 | -0.122308 0.128537 -0.00114555 0.0467688 193 | -0.121956 0.129002 -0.00113743 0.0467796 194 | -0.121571 0.129803 -0.00112012 0.046783 195 | -0.121201 0.129793 -0.001093 0.0467925 196 | -0.121662 0.129558 -0.00113701 0.0467934 197 | -0.122287 0.129944 -0.00106613 0.0467847 198 | -0.123085 0.130184 -0.00124716 0.0467805 199 | -0.123068 0.130124 -0.00118379 0.0467746 200 | -0.123026 0.129382 -0.00124247 0.0467795 201 | -------------------------------------------------------------------------------- /log/conv2_b.summary: -------------------------------------------------------------------------------- 1 | -0.00328541 0.00355816 -0.000214426 0.00148015 2 | -0.00551933 0.00398987 -0.000512631 0.00213331 3 | -0.00650371 0.00661022 -0.00125438 0.00309516 4 | -0.00766369 0.00635357 -0.00156499 0.0031515 5 | -0.00843492 0.0093687 -0.00161919 0.00363377 6 | -0.00950747 0.00978676 -0.00124005 0.00391639 7 | -0.00886431 0.00877572 -0.00139001 0.00392003 8 | -0.0100621 0.0114848 -0.00105481 0.00466944 9 | -0.00845887 0.0108433 -0.00130643 0.00425373 10 | -0.0120101 0.0137005 -0.00189665 0.00521793 11 | -0.0125907 0.0100136 -0.00209661 0.00502684 12 | -0.0179804 0.00760936 -0.00515043 0.00577782 13 | -0.0156829 0.0122811 -0.0046982 0.00638761 14 | -0.0171594 0.0119043 -0.00470049 0.00645779 15 | -0.015715 0.0130879 -0.00439144 0.00634643 16 | -0.0149892 0.0130809 -0.00449212 0.00631422 17 | -0.0150346 0.0129594 -0.00440262 0.00637707 18 | -0.0160505 0.0139917 -0.00422235 0.00674395 19 | -0.0164615 0.0142342 -0.00431616 0.00674211 20 | -0.0164953 0.01492 -0.00410754 0.0070128 21 | -0.0208066 0.0166611 -0.00452099 0.00786224 22 | -0.0209828 0.0132663 -0.00598863 0.00737153 23 | -0.0204763 0.0140692 -0.00590738 0.00745489 24 | -0.020652 0.016072 -0.00536174 0.00765899 25 | -0.0195738 0.0156417 -0.00550321 0.00754505 26 | -0.0190068 0.0167315 -0.00531873 0.00758382 27 | -0.0196001 0.0158897 -0.00534284 0.00777592 28 | -0.0184173 0.0141975 -0.00566846 0.00724799 29 | -0.0216754 0.0192334 -0.00529928 0.00841394 30 | -0.0199116 0.0117548 -0.0061356 0.00758559 31 | -0.0209074 0.0150688 -0.00587025 0.00803881 32 | -0.0217278 0.0162351 -0.00613227 0.00832877 33 | -0.0215559 0.0165915 -0.00618903 0.00824138 34 | -0.0213683 0.0159104 -0.00632391 0.00816869 35 | -0.0219706 0.0174506 -0.00608277 0.00848024 36 | -0.022322 0.0166472 -0.00607492 0.00841343 37 | -0.0237533 0.0173502 -0.00618984 0.00866133 38 | -0.0224005 0.0172306 -0.00652138 0.00862149 39 | -0.0242583 0.0194547 -0.00606385 0.00888302 40 | -0.0235515 0.0138886 -0.00701842 0.00868188 41 | -0.0239896 0.0151295 -0.00722727 0.00874575 42 | -0.0233082 0.0141632 -0.00773398 0.00874281 43 | -0.025389 0.0147957 -0.00768972 0.00918099 44 | -0.023736 0.0136421 -0.00764007 0.00893307 45 | -0.0247764 0.0149094 -0.00758601 0.00917469 46 | -0.0252867 0.0150223 -0.00745688 0.0090915 47 | -0.02403 0.0137365 -0.00766691 0.00876603 48 | -0.0251428 0.0157574 -0.00767375 0.00919872 49 | -0.0249141 0.0151505 -0.00765759 0.00906487 50 | -0.025283 0.0159135 -0.007457 0.00916216 51 | -0.0250762 0.016877 -0.00726429 0.00913776 52 | -0.025502 0.0173324 -0.00720323 0.00923549 53 | -0.0259102 0.0162373 -0.00740065 0.00907527 54 | -0.0257529 0.0166579 -0.00742159 0.00922485 55 | -0.0252374 0.0166392 -0.00752838 0.00923417 56 | -0.0254698 0.0167579 -0.00780056 0.00957429 57 | -0.0265905 0.0157505 -0.00801316 0.009498 58 | -0.0270979 0.0153766 -0.00786232 0.00938268 59 | -0.0268674 0.0157104 -0.00803789 0.00948361 60 | -0.0273888 0.0168451 -0.00790771 0.0097587 61 | -0.0274231 0.0170986 -0.00775513 0.00983992 62 | -0.0274897 0.0170322 -0.0079099 0.00981227 63 | -0.0285103 0.0177283 -0.00761077 0.0101921 64 | -0.0289526 0.0156696 -0.00834556 0.0101313 65 | -0.02835 0.0152801 -0.00842625 0.0101356 66 | -0.0277177 0.0156432 -0.00812448 0.0100492 67 | -0.0280471 0.0136309 -0.00862308 0.00988461 68 | -0.0294788 0.0167982 -0.00822077 0.0104805 69 | -0.0293511 0.0144458 -0.00888977 0.0100977 70 | -0.0293243 0.0149766 -0.00872023 0.0100659 71 | -0.0303334 0.0164021 -0.00874341 0.0104681 72 | -0.0294502 0.0156685 -0.00866367 0.0102071 73 | -0.0291316 0.0152464 -0.00875225 0.01017 74 | -0.0296578 0.0150398 -0.00904289 0.0104192 75 | -0.0305751 0.0173141 -0.0087437 0.0107965 76 | -0.0315728 0.0178324 -0.00885728 0.0108366 77 | -0.0310579 0.0163895 -0.00880203 0.0105861 78 | -0.031541 0.0191119 -0.00895596 0.0110796 79 | -0.0304104 0.0168866 -0.00918705 0.010657 80 | -0.0312628 0.0185223 -0.00904063 0.0109877 81 | -0.0314509 0.0163602 -0.00930273 0.0108494 82 | -0.0323718 0.0187929 -0.00889044 0.0111955 83 | -0.0317198 0.0173371 -0.00894381 0.0109628 84 | -0.0314988 0.0155702 -0.0092493 0.0108081 85 | -0.0312605 0.0163096 -0.00932239 0.0108642 86 | -0.0317503 0.0169732 -0.00933778 0.0109641 87 | -0.0308745 0.0157947 -0.00962309 0.0108296 88 | -0.0305306 0.0166183 -0.00932649 0.0109486 89 | -0.0314194 0.0160989 -0.0093233 0.0110277 90 | -0.0304127 0.0160574 -0.00920866 0.0109881 91 | -0.0302441 0.0160282 -0.00925322 0.0109363 92 | -0.0307963 0.0165336 -0.00926654 0.0111846 93 | -0.0295912 0.0144023 -0.0095941 0.0107816 94 | -0.0314737 0.0170156 -0.00943588 0.0113984 95 | -0.0305319 0.0153339 -0.00972905 0.0111102 96 | -0.0316472 0.0172739 -0.00954579 0.0114097 97 | -0.0311228 0.0168772 -0.00946081 0.0112744 98 | -0.031371 0.0173755 -0.00921378 0.0113939 99 | -0.0307523 0.0162576 -0.00946158 0.0111809 100 | -0.0312884 0.0175074 -0.00907325 0.0113902 101 | -0.0309136 0.0161088 -0.00914396 0.0111335 102 | -0.0302829 0.0140797 -0.00965631 0.0108106 103 | -0.0312268 0.0149715 -0.00954119 0.0110316 104 | -0.0318977 0.017025 -0.0091454 0.0114059 105 | -0.0315959 0.0163963 -0.00929744 0.0113181 106 | -0.0313012 0.0158815 -0.00926292 0.011217 107 | -0.0302624 0.0136145 -0.00966317 0.0107836 108 | -0.0328041 0.0173376 -0.00948711 0.0117793 109 | -0.0300033 0.0120783 -0.0102159 0.0108981 110 | -0.0327958 0.0170224 -0.00969552 0.0118637 111 | -0.0318752 0.0143928 -0.010209 0.0114464 112 | -0.030803 0.0134223 -0.0103915 0.0112777 113 | -0.0309923 0.0145001 -0.0103204 0.0112829 114 | -0.0318491 0.0166923 -0.00984738 0.0117199 115 | -0.03021 0.0138534 -0.010154 0.0109682 116 | -0.0307347 0.0149738 -0.010143 0.0112513 117 | -0.0305133 0.0155668 -0.0102971 0.0113066 118 | -0.0299741 0.0152983 -0.0103009 0.0111546 119 | -0.0299577 0.0144709 -0.0105622 0.0111641 120 | -0.0310655 0.0155447 -0.0107027 0.0115036 121 | -0.0301231 0.0138171 -0.0105729 0.0110473 122 | -0.0302407 0.0129714 -0.0109547 0.0110006 123 | -0.0309714 0.0154764 -0.010672 0.0116239 124 | -0.0308745 0.0148243 -0.0109882 0.0116959 125 | -0.0312137 0.01559 -0.0107765 0.0118108 126 | -0.0313591 0.0143322 -0.0108808 0.0116912 127 | -0.0318011 0.0145874 -0.0108514 0.011688 128 | -0.0314702 0.0137788 -0.010658 0.011515 129 | -0.0317081 0.0128002 -0.0107409 0.0114562 130 | -0.0324188 0.0160642 -0.0107422 0.0121305 131 | -0.0315074 0.0143709 -0.0110554 0.0119284 132 | -0.0319136 0.0127676 -0.0110047 0.011758 133 | -0.032885 0.0140489 -0.0109912 0.0121557 134 | -0.0323408 0.0127402 -0.0112277 0.0119467 135 | -0.0313303 0.0134855 -0.0110032 0.0119814 136 | -0.0322182 0.0130179 -0.0112919 0.0119626 137 | -0.031896 0.0119183 -0.0111871 0.0115656 138 | -0.0331092 0.0135052 -0.0111904 0.0122456 139 | -0.0332983 0.0127124 -0.0114951 0.0119725 140 | -0.0321971 0.0118878 -0.0118341 0.0117424 141 | -0.032555 0.0116482 -0.0121087 0.0116394 142 | -0.0332773 0.0124588 -0.0117171 0.0120045 143 | -0.0333907 0.0123516 -0.0118165 0.0119569 144 | -0.0320307 0.0111214 -0.0119133 0.0114592 145 | -0.0317667 0.0115858 -0.0116626 0.0115284 146 | -0.0321813 0.0118089 -0.0115235 0.0116754 147 | -0.0327272 0.0120511 -0.0117458 0.0117676 148 | -0.0325247 0.0128142 -0.0117542 0.0118516 149 | -0.0316374 0.0115843 -0.0119762 0.0115091 150 | -0.0318066 0.0113791 -0.0120414 0.0114526 151 | -0.0323722 0.011129 -0.0120586 0.011362 152 | -0.0318876 0.0115754 -0.0120514 0.0114209 153 | -0.0323606 0.0127319 -0.0119654 0.011571 154 | -0.0326067 0.0131192 -0.0119708 0.0116632 155 | -0.0331923 0.0132869 -0.0120439 0.0118313 156 | -0.0318866 0.0121818 -0.012125 0.0115725 157 | -0.0325568 0.0126285 -0.012065 0.0117145 158 | -0.0327368 0.0124638 -0.0118996 0.0116653 159 | -0.0330626 0.0130654 -0.0119033 0.0118671 160 | -0.0350732 0.0161852 -0.0116115 0.0124824 161 | -0.034704 0.0144579 -0.011776 0.012219 162 | -0.0339692 0.0137002 -0.0119149 0.0119644 163 | -0.0340947 0.0135643 -0.0118938 0.0119507 164 | -0.0325651 0.0126803 -0.0121569 0.0117332 165 | -0.0325787 0.0131786 -0.0120711 0.0118013 166 | -0.0330763 0.0132349 -0.0119705 0.0118043 167 | -0.0332721 0.0128196 -0.0120064 0.0118143 168 | -0.0332912 0.0138925 -0.0119809 0.0117802 169 | -0.0336923 0.0132966 -0.0119506 0.0117805 170 | -0.0334741 0.0130953 -0.0121015 0.0117762 171 | -0.0339204 0.013717 -0.0119513 0.0118603 172 | -0.034568 0.0135972 -0.012073 0.0118699 173 | -0.0346827 0.0133735 -0.0120715 0.0118389 174 | -0.0343048 0.0136875 -0.0122714 0.0118763 175 | -0.0343925 0.0138578 -0.0122484 0.0118994 176 | -0.0340024 0.0124374 -0.0123608 0.0117365 177 | -0.0341831 0.0126579 -0.0123547 0.0117738 178 | -0.0343493 0.0121318 -0.0122998 0.0117767 179 | -0.0334779 0.0117769 -0.0121803 0.0115723 180 | -0.0333594 0.0123931 -0.0121257 0.0116327 181 | -0.0344622 0.0125279 -0.0119192 0.0117968 182 | -0.0335121 0.0108634 -0.0124663 0.0114689 183 | -0.034307 0.0102904 -0.0124041 0.0115714 184 | -0.0338294 0.0101881 -0.012506 0.0115611 185 | -0.0345618 0.0113984 -0.0122088 0.0117302 186 | -0.0335609 0.0111991 -0.0125935 0.0114947 187 | -0.0350249 0.0112871 -0.0123182 0.0118658 188 | -0.0345311 0.0117624 -0.0125529 0.0118199 189 | -0.0347794 0.0120366 -0.0126887 0.012048 190 | -0.0351441 0.0121167 -0.0127096 0.0121232 191 | -0.0352586 0.011674 -0.01278 0.0120926 192 | -0.0342184 0.0116848 -0.0126514 0.0120648 193 | -0.0340308 0.0110117 -0.0125744 0.0119823 194 | -0.0342878 0.0106287 -0.0126088 0.0120563 195 | -0.0344278 0.0106541 -0.0124987 0.0120726 196 | -0.0338139 0.0111245 -0.0126872 0.0119377 197 | -0.0351469 0.0118292 -0.0122369 0.0122752 198 | -0.0336546 0.0110076 -0.0131752 0.0122024 199 | -0.0356203 0.0125531 -0.0127372 0.0127029 200 | -0.0352346 0.011779 -0.0130703 0.0125002 201 | -------------------------------------------------------------------------------- /log/pred_w_grad.summary: -------------------------------------------------------------------------------- 1 | -0.0840193 0.100728 2.42016e-10 0.0166869 2 | -0.127838 0.11432 -1.79062e-09 0.0220995 3 | -0.136163 0.258951 -1.9978e-10 0.0278767 4 | -0.122709 0.0943995 -5.51361e-09 0.018251 5 | -0.0932116 0.129788 3.60503e-09 0.0162815 6 | -0.101282 0.0985534 2.06621e-09 0.0163506 7 | -0.105094 0.0663475 -4.27982e-10 0.0144495 8 | -0.120822 0.069475 1.10096e-09 0.0171893 9 | -0.177372 0.180159 -6.58093e-09 0.0273269 10 | -0.148872 0.273411 -3.10657e-09 0.0307846 11 | -0.181359 0.139379 -7.17189e-09 0.0317015 12 | -0.295364 0.244475 -9.76052e-09 0.0464401 13 | -0.117912 0.17 -1.62083e-09 0.0215476 14 | -0.177484 0.140317 -1.19768e-10 0.0252862 15 | -0.117332 0.0777596 2.34522e-09 0.0167514 16 | -0.0911537 0.103178 1.218e-09 0.0167935 17 | -0.1211 0.0795179 -2.85637e-10 0.0184929 18 | -0.144589 0.0814302 -4.37708e-09 0.0195464 19 | -0.0790321 0.0763484 -1.77079e-09 0.0140753 20 | -0.158694 0.230666 -2.96216e-09 0.0235762 21 | -0.339317 0.292667 -1.59964e-09 0.0417306 22 | -0.136508 0.211102 -5.6203e-10 0.0293937 23 | -0.120123 0.0964501 -3.73782e-09 0.0191443 24 | -0.0983528 0.0478937 7.43777e-10 0.0119822 25 | -0.0757243 0.0755109 6.79187e-10 0.0129784 26 | -0.0737675 0.03088 -2.09335e-10 0.00839158 27 | -0.0665411 0.061475 -1.26027e-09 0.0113934 28 | -0.124916 0.0928701 6.56341e-10 0.0174045 29 | -0.181229 0.151172 -1.73787e-09 0.0238403 30 | -0.122422 0.168093 -5.31072e-09 0.023951 31 | -0.158121 0.175783 2.74181e-10 0.0214145 32 | -0.102116 0.0839936 -1.62496e-09 0.0152275 33 | -0.0596583 0.0519489 6.94878e-10 0.00961868 34 | -0.0724158 0.0685181 6.98512e-10 0.0108054 35 | -0.121318 0.112532 -1.28021e-09 0.0175211 36 | -0.155051 0.124568 5.41313e-11 0.0181147 37 | -0.0645121 0.0621607 -2.57766e-09 0.0119134 38 | -0.113673 0.106536 3.91876e-09 0.0171769 39 | -0.210565 0.142322 1.87288e-09 0.0252676 40 | -0.190784 0.24707 -3.30606e-09 0.0317589 41 | -0.098441 0.0994091 2.23969e-09 0.0171765 42 | -0.180103 0.206565 -2.67763e-09 0.023678 43 | -0.126406 0.0997121 -5.84562e-10 0.0179942 44 | -0.0619683 0.0586415 -3.4885e-10 0.0103719 45 | -0.076514 0.095863 -9.45618e-10 0.0137707 46 | -0.0749698 0.0590163 2.15312e-09 0.0104325 47 | -0.0798876 0.0588725 1.11441e-09 0.0129567 48 | -0.0590761 0.0684673 3.32909e-10 0.0112662 49 | -0.0674159 0.0607378 -6.90999e-10 0.00816554 50 | -0.0429532 0.061639 4.62867e-10 0.00744379 51 | -0.0905302 0.0475776 -1.43181e-09 0.0110878 52 | -0.0918776 0.0953486 -1.36699e-09 0.0128601 53 | -0.0589787 0.0627218 9.35833e-10 0.0120134 54 | -0.0696127 0.145575 -4.64544e-10 0.0162972 55 | -0.149419 0.0740654 -8.12113e-11 0.0159877 56 | -0.141029 0.126388 -1.93559e-09 0.0191412 57 | -0.136821 0.0912847 -7.72253e-10 0.0153127 58 | -0.0685625 0.0833605 -1.08946e-09 0.012313 59 | -0.0610431 0.100662 3.50479e-09 0.0126394 60 | -0.0716818 0.0430599 -1.8876e-09 0.0103677 61 | -0.072434 0.052978 -1.98231e-09 0.00804578 62 | -0.0631623 0.0773666 1.2801e-09 0.00981878 63 | -0.152096 0.119381 -8.31822e-10 0.0194693 64 | -0.144546 0.159374 -2.66363e-09 0.0198695 65 | -0.115867 0.0828114 -6.47131e-09 0.014294 66 | -0.117975 0.153393 3.10821e-09 0.0193009 67 | -0.0797652 0.0863121 1.45093e-09 0.0132329 68 | -0.0991251 0.0845937 -9.38575e-10 0.015098 69 | -0.0840216 0.0546701 3.64362e-10 0.0133436 70 | -0.0575729 0.0461133 7.26222e-10 0.00966343 71 | -0.0733822 0.0531039 -1.16358e-09 0.0102335 72 | -0.0532371 0.055118 5.64405e-10 0.00728981 73 | -0.051488 0.0855746 -2.51271e-10 0.0114229 74 | -0.099634 0.099646 2.0664e-09 0.014701 75 | -0.144658 0.157998 3.70161e-09 0.0216694 76 | -0.134638 0.107569 -2.92428e-09 0.0198459 77 | -0.0825218 0.0515942 -9.4499e-10 0.0117756 78 | -0.128623 0.144283 -1.12299e-09 0.0185958 79 | -0.0674329 0.053029 -7.94604e-10 0.00941607 80 | -0.080518 0.0701331 -2.52632e-09 0.0123951 81 | -0.0968579 0.0813439 -1.57401e-09 0.0137574 82 | -0.092075 0.127835 1.67626e-09 0.0148091 83 | -0.0546449 0.0611493 1.08935e-09 0.00936186 84 | -0.124073 0.0602434 5.65077e-10 0.0129182 85 | -0.068692 0.0413114 3.19501e-09 0.0097099 86 | -0.0537599 0.0572683 -2.23947e-10 0.00856839 87 | -0.0784095 0.079795 -4.26618e-11 0.0103827 88 | -0.0668286 0.071034 -1.26663e-09 0.0118917 89 | -0.0519532 0.0936878 -8.85152e-10 0.0109026 90 | -0.0714522 0.0730715 1.91158e-09 0.0111436 91 | -0.0518033 0.0457537 -2.63657e-10 0.00851169 92 | -0.0985837 0.0732834 1.38761e-09 0.0126335 93 | -0.084567 0.0656223 -3.56552e-10 0.011511 94 | -0.0832615 0.0720793 -2.28127e-10 0.013647 95 | -0.0728908 0.0735415 1.43023e-09 0.00774116 96 | -0.048127 0.0619845 -8.33193e-10 0.00922188 97 | -0.0511568 0.0331149 -4.25814e-10 0.00642757 98 | -0.0748273 0.0874913 8.23777e-10 0.0124454 99 | -0.094842 0.0471349 -1.26553e-09 0.0105593 100 | -0.0783192 0.0378994 -4.17209e-10 0.00891878 101 | -0.0925022 0.0566775 -6.35881e-10 0.011688 102 | -0.0951037 0.153159 2.21972e-09 0.019035 103 | -0.0966463 0.0614162 2.30549e-10 0.0092787 104 | -0.082462 0.080463 -2.27378e-10 0.0127496 105 | -0.0572601 0.0554723 6.18256e-10 0.00822094 106 | -0.0632286 0.0400369 -5.29766e-10 0.00677674 107 | -0.0753765 0.040271 4.15966e-11 0.00999228 108 | -0.128836 0.145788 -1.23866e-09 0.0195549 109 | -0.198722 0.136807 4.02112e-10 0.0230386 110 | -0.129566 0.137963 -4.26819e-10 0.0210408 111 | -0.139087 0.0795291 -3.23025e-09 0.0175092 112 | -0.0773051 0.0723507 -8.07426e-10 0.0107781 113 | -0.0758265 0.0516768 1.50822e-10 0.0110362 114 | -0.119359 0.123853 -1.58735e-09 0.0184125 115 | -0.136467 0.0922238 1.45845e-09 0.016871 116 | -0.0773978 0.061784 3.21447e-10 0.0087234 117 | -0.0445619 0.05033 -1.32951e-09 0.00712178 118 | -0.0378106 0.0492873 9.41195e-10 0.00736107 119 | -0.051221 0.0417558 -6.64359e-10 0.00864912 120 | -0.0732182 0.0649671 5.43929e-10 0.00971588 121 | -0.0829072 0.0707838 -1.3223e-09 0.0104677 122 | -0.0922693 0.0657039 1.36756e-09 0.0123497 123 | -0.153808 0.151471 -3.23419e-10 0.0211026 124 | -0.0801696 0.0933847 -8.69684e-10 0.0136657 125 | -0.0722716 0.102284 6.1802e-10 0.0119827 126 | -0.144264 0.0729731 -3.55532e-11 0.0152096 127 | -0.131791 0.0839142 2.13769e-09 0.013195 128 | -0.0587616 0.0476742 -1.21689e-09 0.00765928 129 | -0.063756 0.057723 2.60072e-11 0.00965004 130 | -0.0896867 0.107097 2.15527e-09 0.0137174 131 | -0.0618948 0.0425215 -9.95118e-11 0.00851901 132 | -0.0963529 0.073521 4.15812e-09 0.0119121 133 | -0.0852159 0.0888295 2.50563e-09 0.0157541 134 | -0.0486426 0.0423622 -7.14662e-10 0.00876219 135 | -0.0655247 0.100042 2.37585e-09 0.012006 136 | -0.0843757 0.0856282 1.06823e-10 0.0138888 137 | -0.141649 0.0742204 -1.66371e-09 0.0165856 138 | -0.116765 0.165319 -1.39978e-08 0.0257223 139 | -0.0654712 0.073607 -3.40834e-09 0.0121386 140 | -0.0825162 0.070537 -1.65357e-09 0.0114668 141 | -0.0676963 0.0886421 3.82774e-10 0.00886404 142 | -0.0877455 0.105293 -5.30708e-10 0.0114317 143 | -0.0690517 0.0648115 -3.18175e-10 0.0107479 144 | -0.0699533 0.0640895 -8.9939e-10 0.0090513 145 | -0.0851791 0.0571094 1.48001e-09 0.0102863 146 | -0.0776769 0.0642966 1.46201e-09 0.0101234 147 | -0.055557 0.0547743 -3.08244e-10 0.00918635 148 | -0.0809281 0.0562269 -2.5277e-09 0.0116243 149 | -0.0609361 0.049339 -7.7181e-10 0.00878957 150 | -0.0918787 0.0738075 -2.42159e-09 0.0134625 151 | -0.0370636 0.0245302 -6.42473e-10 0.0054732 152 | -0.0459254 0.0441329 1.63643e-09 0.00759504 153 | -0.108409 0.0672906 -7.6592e-10 0.0123152 154 | -0.039205 0.032645 2.36009e-10 0.00596347 155 | -0.067171 0.070972 1.03526e-09 0.00919781 156 | -0.0701032 0.0649751 7.58725e-10 0.00940967 157 | -0.063168 0.0650925 3.09478e-10 0.0106215 158 | -0.055994 0.0690108 -2.94484e-10 0.0100457 159 | -0.0619882 0.0488601 3.73475e-10 0.00829659 160 | -0.138762 0.227349 -9.11279e-10 0.0212742 161 | -0.078429 0.0413974 -8.07643e-10 0.0111071 162 | -0.0462305 0.0396921 4.3785e-10 0.00665673 163 | -0.0693232 0.0334883 -2.54928e-09 0.00618467 164 | -0.10538 0.0561213 -1.3029e-09 0.0127752 165 | -0.026255 0.0254737 3.18524e-10 0.0039727 166 | -0.0570157 0.0815529 -1.66986e-10 0.00860723 167 | -0.0912824 0.0621491 3.78035e-10 0.00949985 168 | -0.102087 0.054722 -1.20234e-10 0.0113438 169 | -0.0637714 0.0373297 4.96866e-10 0.00892375 170 | -0.0691358 0.0509331 1.13e-09 0.00669409 171 | -0.0424107 0.0417526 -3.57628e-11 0.00516365 172 | -0.0567943 0.0435578 -5.15883e-10 0.00865315 173 | -0.059511 0.0502774 -9.08005e-10 0.00879547 174 | -0.0540158 0.0823954 -3.43035e-10 0.00929009 175 | -0.0669989 0.0423915 -4.93613e-10 0.00811494 176 | -0.0950934 0.0690548 -2.24655e-10 0.009726 177 | -0.0224289 0.0199427 1.47762e-09 0.00303518 178 | -0.0461574 0.0382481 -5.21669e-10 0.00658589 179 | -0.0559918 0.0576908 -6.75081e-10 0.00955385 180 | -0.0607754 0.0405484 -5.39309e-10 0.0066964 181 | -0.142435 0.0969323 1.46232e-09 0.0176334 182 | -0.122109 0.118692 -2.0527e-09 0.0154356 183 | -0.0832213 0.115927 -3.44175e-09 0.0134376 184 | -0.0409279 0.0368542 -2.2769e-09 0.00731446 185 | -0.0455984 0.0503753 -1.41282e-10 0.00743392 186 | -0.13224 0.0895499 -1.93515e-09 0.0154147 187 | -0.105821 0.114784 4.5764e-10 0.0156052 188 | -0.103144 0.156517 -1.96602e-09 0.0123264 189 | -0.0684621 0.0548373 1.46125e-10 0.0103278 190 | -0.0570409 0.0321867 -5.2727e-10 0.00657888 191 | -0.0619277 0.046675 -2.25915e-09 0.00849878 192 | -0.0964945 0.153416 1.44433e-09 0.0150763 193 | -0.0490439 0.0325936 -1.80076e-09 0.00663661 194 | -0.0749253 0.086247 -2.54945e-09 0.0104439 195 | -0.0404399 0.0476442 -1.12066e-10 0.00664017 196 | -0.112538 0.0597147 1.18075e-09 0.0118515 197 | -0.253848 0.260144 1.21858e-09 0.025029 198 | -0.311498 0.284269 1.57361e-09 0.034245 199 | -0.154658 0.176337 -1.1157e-09 0.0225998 200 | -0.074724 0.0956192 1.7642e-09 0.0121599 201 | -------------------------------------------------------------------------------- /log/pred_b_grad.summary: -------------------------------------------------------------------------------- 1 | -0.0495968 0.0808637 7.45058e-10 0.042473 2 | -0.0937684 0.100097 -1.49012e-09 0.063799 3 | -0.0879051 0.158443 -7.45058e-10 0.0834515 4 | -0.111596 0.0909815 7.45058e-10 0.0606136 5 | -0.0649249 0.113135 -3.72529e-10 0.0502693 6 | -0.0462259 0.083262 -4.65661e-10 0.0381826 7 | -0.0551939 0.0325856 -1.30385e-09 0.0314733 8 | -0.0628285 0.0597957 -2.23517e-09 0.0399451 9 | -0.0726232 0.108558 -9.31323e-10 0.0562253 10 | -0.0886863 0.149402 9.31323e-11 0.0693521 11 | -0.114587 0.0965523 -7.45058e-10 0.0727054 12 | -0.169152 0.164467 5.21541e-09 0.104135 13 | -0.072216 0.150457 -3.58559e-09 0.0590445 14 | -0.105636 0.11818 -3.72529e-09 0.0704718 15 | -0.074895 0.0701327 -3.72529e-10 0.0457971 16 | -0.0480289 0.0847137 1.11759e-09 0.041225 17 | -0.0688243 0.0498062 -1.49012e-09 0.0422035 18 | -0.0694854 0.0585446 -2.23517e-09 0.0412491 19 | -0.0304054 0.0338904 -2.04891e-09 0.0236257 20 | -0.0415958 0.0853576 -1.49012e-09 0.0354343 21 | -0.133354 0.123381 1.49012e-09 0.0726119 22 | -0.0704758 0.106824 -1.49012e-09 0.055174 23 | -0.0833078 0.0745001 -2.04891e-09 0.0487104 24 | -0.043718 0.0269646 -1.30385e-09 0.0241451 25 | -0.0314203 0.0598984 5.58794e-10 0.0254216 26 | -0.0237108 0.0124441 -6.0536e-10 0.00942537 27 | -0.0314937 0.0374456 -9.31323e-11 0.0198586 28 | -0.0672096 0.0370477 -2.70084e-09 0.0312221 29 | -0.0688176 0.0917832 8.3819e-10 0.045407 30 | -0.0641816 0.0905638 -8.84756e-10 0.0466371 31 | -0.0507889 0.0722653 -7.45058e-10 0.0365549 32 | -0.0364205 0.0303288 -1.49012e-09 0.0225587 33 | -0.022351 0.029222 -1.21072e-09 0.0153226 34 | -0.0235598 0.0286405 -3.72529e-10 0.0169903 35 | -0.0603857 0.0501752 -7.45058e-10 0.0314279 36 | -0.046112 0.0541092 -3.72529e-09 0.0311037 37 | -0.0275951 0.0271138 -7.45058e-10 0.0204673 38 | -0.0597203 0.0500071 -2.6077e-09 0.0305782 39 | -0.0714317 0.0543863 -1.86265e-09 0.0377429 40 | -0.0839752 0.140756 -2.98023e-09 0.0571179 41 | -0.0417817 0.0476002 7.45058e-10 0.0294691 42 | -0.0615108 0.0842101 2.98023e-09 0.0373496 43 | -0.0702005 0.0452173 -4.65661e-11 0.0343791 44 | -0.0254896 0.0245412 1.86265e-10 0.0164913 45 | -0.0220504 0.0470941 -2.23517e-09 0.0207821 46 | -0.0185574 0.0162357 2.07219e-09 0.0123129 47 | -0.0400872 0.0263652 -1.30385e-09 0.0219607 48 | -0.0233978 0.0320255 -7.45058e-10 0.0171048 49 | -0.0155885 0.0308409 5.58794e-10 0.0135675 50 | -0.0149467 0.0234635 1.02445e-09 0.0115369 51 | -0.0355174 0.017691 -9.31323e-10 0.0167044 52 | -0.0292946 0.0338553 -9.31323e-10 0.0184706 53 | -0.0266906 0.0257225 1.86265e-10 0.017487 54 | -0.0218644 0.0638752 -7.45058e-10 0.0253646 55 | -0.0524225 0.0286215 -7.45058e-10 0.0235759 56 | -0.0459503 0.0546871 -7.45058e-10 0.0281101 57 | -0.0483384 0.0306531 3.72529e-10 0.0227831 58 | -0.0227863 0.0233954 1.49012e-09 0.0157925 59 | -0.0156202 0.0278871 -5.58794e-10 0.0137139 60 | -0.0285411 0.0220569 -1.86265e-10 0.0173141 61 | -0.0240097 0.0158001 -7.68341e-10 0.0117625 62 | -0.0255526 0.0266943 9.31323e-11 0.0139377 63 | -0.0583829 0.0428218 -2.14204e-09 0.0284609 64 | -0.0612552 0.0690242 1.81608e-09 0.031755 65 | -0.0312223 0.0337329 -1.86265e-09 0.0207539 66 | -0.0448435 0.0657523 0 0.0293776 67 | -0.0348755 0.0380467 6.98492e-10 0.0221592 68 | -0.0427137 0.0411059 1.86265e-10 0.0251406 69 | -0.0363173 0.0276183 3.72529e-10 0.0244358 70 | -0.0190245 0.01433 -1.30385e-09 0.0129665 71 | -0.027658 0.0172971 1.86265e-10 0.0132799 72 | -0.0129151 0.01585 1.76951e-09 0.00921782 73 | -0.0166074 0.0247332 0 0.0134997 74 | -0.0381249 0.0285776 3.72529e-10 0.0182341 75 | -0.0511927 0.0700049 -7.45058e-10 0.0319276 76 | -0.0482522 0.0485577 -3.72529e-10 0.0283507 77 | -0.0282135 0.0217267 0 0.0165262 78 | -0.0355292 0.0532253 -7.45058e-10 0.0264714 79 | -0.0280828 0.0140724 -9.31323e-10 0.0119924 80 | -0.0326378 0.03076 -4.65661e-10 0.0178943 81 | -0.0294826 0.0264441 -4.19095e-10 0.0157138 82 | -0.0282 0.0548844 1.86265e-10 0.0226254 83 | -0.0190085 0.0231773 0 0.013058 84 | -0.0437632 0.029048 -7.21775e-10 0.0199848 85 | -0.0352515 0.0197832 -3.72529e-10 0.0158765 86 | -0.0178075 0.0223212 -1.86265e-10 0.0125367 87 | -0.0186709 0.0238395 -1.11759e-09 0.0118667 88 | -0.0202582 0.0301175 -9.31323e-10 0.0154295 89 | -0.0228258 0.0336698 3.72529e-10 0.0155281 90 | -0.0213095 0.0260125 -3.72529e-10 0.0150109 91 | -0.0138626 0.0150856 -7.45058e-10 0.00893962 92 | -0.0297005 0.0262508 1.86265e-10 0.0182358 93 | -0.0262255 0.0227612 1.67638e-09 0.0157697 94 | -0.0241141 0.0300546 1.11759e-09 0.0182866 95 | -0.0163931 0.0182612 7.91624e-10 0.0108156 96 | -0.0104617 0.0277573 -9.31323e-11 0.0119021 97 | -0.0143046 0.0136949 -1.35042e-09 0.00968887 98 | -0.0249108 0.0309315 -1.11759e-09 0.0167569 99 | -0.0249481 0.0191708 9.31323e-10 0.0144663 100 | -0.0248868 0.013711 4.65661e-10 0.0110518 101 | -0.0364396 0.0189158 -5.12227e-10 0.0167057 102 | -0.033219 0.0666057 -7.45058e-10 0.0298375 103 | -0.0364264 0.0204495 0 0.0145675 104 | -0.0308873 0.0253978 2.04891e-09 0.0178704 105 | -0.020816 0.0220771 7.45058e-10 0.0117205 106 | -0.0150206 0.01722 0 0.00933425 107 | -0.0298836 0.0191037 1.05938e-09 0.0154929 108 | -0.0407681 0.0446303 9.31323e-11 0.0249411 109 | -0.0849344 0.0499099 -2.98023e-09 0.037917 110 | -0.0392151 0.0607492 7.45058e-10 0.0305963 111 | -0.051397 0.0329348 -7.45058e-10 0.0292326 112 | -0.0289638 0.0276897 -1.67638e-09 0.0183082 113 | -0.0395023 0.0220528 -1.86265e-10 0.0185635 114 | -0.0542311 0.0570447 -3.72529e-10 0.0316881 115 | -0.0685099 0.0453059 -3.72529e-10 0.0303329 116 | -0.0248889 0.0194589 7.91624e-10 0.0114139 117 | -0.0117158 0.024705 -1.49012e-09 0.0106412 118 | -0.00858879 0.0215621 -1.53086e-09 0.00872748 119 | -0.0190203 0.0177367 0 0.0124297 120 | -0.0204289 0.0251871 3.72529e-10 0.0124084 121 | -0.0339522 0.0229065 -9.31323e-10 0.0153432 122 | -0.0382552 0.0317901 1.58325e-09 0.0213054 123 | -0.0584589 0.0485941 -7.45058e-10 0.0292376 124 | -0.0240216 0.0318878 7.45058e-10 0.0179521 125 | -0.0174964 0.0321678 7.45058e-10 0.0155677 126 | -0.0468416 0.0213734 1.05938e-09 0.0199431 127 | -0.0409975 0.0179256 4.65661e-10 0.0157723 128 | -0.0204531 0.0165803 -9.31323e-10 0.0106947 129 | -0.0128627 0.0145228 3.25963e-10 0.0100076 130 | -0.0275347 0.0324293 1.30385e-09 0.0191478 131 | -0.0308864 0.0227719 -3.72529e-10 0.0146121 132 | -0.0262731 0.0254541 6.51926e-10 0.0162635 133 | -0.0276087 0.0325139 7.45058e-10 0.0195221 134 | -0.0172888 0.0124209 -1.11759e-09 0.0106011 135 | -0.0224503 0.0265952 -7.45058e-10 0.0167392 136 | -0.0293759 0.0306633 -3.72529e-10 0.0199564 137 | -0.0257838 0.0294467 -1.95578e-09 0.0178849 138 | -0.033545 0.0549919 2.23517e-09 0.0357108 139 | -0.0266961 0.0340529 -9.31323e-11 0.0173347 140 | -0.0313036 0.0259904 -6.98492e-10 0.0167093 141 | -0.0253253 0.0320597 1.86265e-10 0.0144059 142 | -0.0292768 0.0415264 -5.58794e-10 0.0174998 143 | -0.0275278 0.0172203 1.02445e-09 0.0138014 144 | -0.0329477 0.0262598 -6.51926e-10 0.0165742 145 | -0.0354812 0.02479 1.11759e-09 0.0155399 146 | -0.0268574 0.0152965 1.49012e-09 0.0132782 147 | -0.0210924 0.0198872 -1.86265e-10 0.014034 148 | -0.0220606 0.016992 -2.01398e-09 0.011885 149 | -0.0260959 0.0132978 -8.3819e-10 0.0120902 150 | -0.0342428 0.0275152 4.65661e-10 0.020729 151 | -0.0111501 0.00943061 -9.31323e-11 0.00711274 152 | -0.0138895 0.0154625 1.86265e-10 0.0102142 153 | -0.0343217 0.0274724 -3.72529e-10 0.0180949 154 | -0.010511 0.0117991 2.79397e-10 0.0081802 155 | -0.0190828 0.0176413 1.49012e-09 0.0118198 156 | -0.0242626 0.02986 3.72529e-10 0.0161501 157 | -0.0164979 0.0188658 -3.25963e-10 0.0122829 158 | -0.0134281 0.0278102 1.86265e-10 0.0124107 159 | -0.0238198 0.0131195 0 0.0109855 160 | -0.0269104 0.0684335 1.11759e-09 0.0256207 161 | -0.0198069 0.0121101 1.58325e-09 0.0128827 162 | -0.0132514 0.0164128 2.38651e-10 0.0104638 163 | -0.0248112 0.0132468 3.72529e-10 0.0103808 164 | -0.0445343 0.0177167 -1.67638e-09 0.0185624 165 | -0.00829349 0.0103461 -6.8685e-10 0.00569 166 | -0.0129949 0.0292667 9.31323e-11 0.0113462 167 | -0.0352767 0.0156786 -2.04891e-09 0.0146072 168 | -0.0261224 0.0222936 -7.91624e-10 0.0143274 169 | -0.0265293 0.0141369 1.49012e-09 0.0133446 170 | -0.0151712 0.0110441 -2.79397e-10 0.00810512 171 | -0.0149586 0.00995713 8.73115e-10 0.00775656 172 | -0.016586 0.0159588 3.0268e-10 0.0110137 173 | -0.0172685 0.01472 -1.6531e-09 0.00994099 174 | -0.0148662 0.0203123 1.39698e-10 0.0094684 175 | -0.0204177 0.0132708 4.65661e-10 0.00970368 176 | -0.0156781 0.0112539 -1.39698e-09 0.0084266 177 | -0.00753088 0.00402857 2.32831e-10 0.00349792 178 | -0.00868518 0.0122271 -8.14907e-10 0.00728694 179 | -0.0175775 0.0179874 -5.58794e-10 0.0120941 180 | -0.0178206 0.0118479 -1.00117e-09 0.00782237 181 | -0.0449952 0.0309367 1.58325e-09 0.0216771 182 | -0.0299309 0.0403136 -9.31323e-11 0.0211243 183 | -0.0201589 0.0327576 -3.72529e-10 0.0172468 184 | -0.0159121 0.0171495 9.31323e-11 0.00946311 185 | -0.0120477 0.0188757 8.49832e-10 0.00907569 186 | -0.047511 0.0376738 -3.72529e-10 0.022519 187 | -0.0352995 0.0533685 1.49012e-09 0.0226238 188 | -0.0174896 0.0342078 -2.56114e-10 0.0146071 189 | -0.0218792 0.0168588 1.30385e-09 0.0131929 190 | -0.0144356 0.010571 -2.79397e-10 0.00807421 191 | -0.0106055 0.0132189 -3.49246e-10 0.00826351 192 | -0.0364656 0.0482178 -3.72529e-10 0.0210249 193 | -0.0133453 0.00728734 -2.70084e-09 0.00680817 194 | -0.0202631 0.0263921 -1.02445e-09 0.0135801 195 | -0.0113211 0.0135296 3.72529e-10 0.00765861 196 | -0.0337249 0.0159978 -1.16415e-09 0.0137181 197 | -0.0436335 0.0507034 5.70435e-10 0.0241058 198 | -0.112029 0.101927 -2.6077e-09 0.0513404 199 | -0.0443111 0.0772351 5.58794e-09 0.0323202 200 | -0.0222573 0.0292822 2.79397e-10 0.0141373 201 | -------------------------------------------------------------------------------- /log/pred_b.summary: -------------------------------------------------------------------------------- 1 | -0.00807828 0.00495472 0 0.00424305 2 | -0.00707522 0.00973516 9.31323e-11 0.0063076 3 | -0.0111919 0.00932351 2.32831e-10 0.00580082 4 | -0.00664816 0.00827564 1.86265e-10 0.00512129 5 | -0.00828214 0.00877846 9.31323e-11 0.00596113 6 | -0.008672 0.0118137 1.28057e-10 0.00647779 7 | -0.00504469 0.0088871 3.25963e-10 0.00556101 8 | -0.0102739 0.0127535 4.19095e-10 0.00778503 9 | -0.00957042 0.0121238 5.12227e-10 0.00677352 10 | -0.0178681 0.0174438 6.98492e-10 0.0103552 11 | -0.0155238 0.011409 5.58794e-10 0.00945753 12 | -0.0121777 0.0147645 2.79397e-10 0.00845154 13 | -0.0136617 0.0165569 4.65661e-10 0.00930913 14 | -0.0150802 0.0137583 6.51926e-10 0.00978934 15 | -0.0154329 0.0161733 7.45058e-10 0.00999717 16 | -0.0147763 0.0156718 5.58794e-10 0.00955351 17 | -0.0125367 0.0113536 9.31323e-10 0.00908256 18 | -0.0173573 0.0149048 1.02445e-09 0.0105302 19 | -0.014374 0.0116918 1.11759e-09 0.00950175 20 | -0.0192391 0.0156291 1.30385e-09 0.0111369 21 | -0.0215017 0.0154502 1.02445e-09 0.0110604 22 | -0.0146076 0.0119815 1.21072e-09 0.00835597 23 | -0.0116085 0.0131416 1.39698e-09 0.00874413 24 | -0.0134534 0.0174097 1.49012e-09 0.010081 25 | -0.0128291 0.0116169 1.21072e-09 0.00929476 26 | -0.0132524 0.0118702 1.30385e-09 0.00975809 27 | -0.0132175 0.0135757 1.30385e-09 0.00991886 28 | -0.0127798 0.0125111 1.58325e-09 0.00876258 29 | -0.015598 0.0166582 1.67638e-09 0.0117331 30 | -0.0120677 0.0123432 1.95578e-09 0.00845579 31 | -0.0186203 0.0163565 1.86265e-09 0.0110872 32 | -0.015093 0.0155379 1.76951e-09 0.010736 33 | -0.0162966 0.0139386 2.23517e-09 0.0104503 34 | -0.0140194 0.0140039 1.95578e-09 0.00984056 35 | -0.0188642 0.0173113 2.04891e-09 0.0116459 36 | -0.0144162 0.0172638 2.32831e-09 0.0109341 37 | -0.0164279 0.0146509 2.23517e-09 0.0111198 38 | -0.0141026 0.0149072 2.70084e-09 0.0104505 39 | -0.0193331 0.0192932 2.79397e-09 0.0126485 40 | -0.0137307 0.016983 3.07336e-09 0.00950278 41 | -0.013838 0.017149 2.79397e-09 0.0100939 42 | -0.0130003 0.0118351 2.51457e-09 0.00868354 43 | -0.0129436 0.0157989 2.51457e-09 0.00973666 44 | -0.0105258 0.0166859 2.79397e-09 0.00933791 45 | -0.0150065 0.0157476 3.25963e-09 0.0100954 46 | -0.014677 0.0168342 2.98023e-09 0.0100302 47 | -0.0122164 0.0173593 2.98023e-09 0.0096704 48 | -0.0127335 0.015943 3.07336e-09 0.00993629 49 | -0.0118142 0.0162235 2.98023e-09 0.00971555 50 | -0.0113768 0.0168282 2.79397e-09 0.00982457 51 | -0.012528 0.0202033 2.70084e-09 0.0108811 52 | -0.0125827 0.0195418 2.8871e-09 0.0104445 53 | -0.0107493 0.017959 2.8871e-09 0.00982968 54 | -0.0146182 0.0193897 2.8871e-09 0.010812 55 | -0.00992347 0.0184979 2.70084e-09 0.00979938 56 | -0.0148274 0.0184894 2.51457e-09 0.0106238 57 | -0.0102615 0.0174255 2.6077e-09 0.0091279 58 | -0.0118991 0.0193833 2.51457e-09 0.00984079 59 | -0.0117836 0.0195759 2.42144e-09 0.00993605 60 | -0.0130689 0.0189536 2.42144e-09 0.0104413 61 | -0.0126753 0.0192501 2.51457e-09 0.010377 62 | -0.0134209 0.0189027 2.32831e-09 0.0103441 63 | -0.0126177 0.0208823 2.70084e-09 0.0110736 64 | -0.0127801 0.0208123 2.51457e-09 0.0104057 65 | -0.0111491 0.0191694 2.6077e-09 0.00967144 66 | -0.0163364 0.0203387 2.79397e-09 0.0109728 67 | -0.015762 0.0180856 2.6077e-09 0.00987275 68 | -0.0138174 0.022076 2.70084e-09 0.0110817 69 | -0.0126599 0.0200313 2.42144e-09 0.0100582 70 | -0.013996 0.0193074 2.70084e-09 0.0101924 71 | -0.0135694 0.0201998 2.6077e-09 0.0106241 72 | -0.0128618 0.0214015 2.23517e-09 0.0105612 73 | -0.0144802 0.0216504 2.32831e-09 0.0108444 74 | -0.0134923 0.0189966 2.32831e-09 0.010295 75 | -0.0199867 0.0194743 2.23517e-09 0.0117535 76 | -0.0155148 0.021514 2.42144e-09 0.0112562 77 | -0.0175264 0.0223271 2.42144e-09 0.0112735 78 | -0.0151817 0.0206371 2.14204e-09 0.0112593 79 | -0.0160145 0.0193368 2.42144e-09 0.0107716 80 | -0.0152294 0.018636 2.51457e-09 0.0108559 81 | -0.015802 0.0195338 2.42144e-09 0.0107357 82 | -0.0148533 0.0209362 2.51457e-09 0.0110851 83 | -0.0161816 0.0213809 2.42144e-09 0.0109892 84 | -0.0164083 0.0204054 2.42144e-09 0.0103979 85 | -0.0131706 0.0211058 2.51457e-09 0.0101809 86 | -0.0150028 0.0208611 2.51457e-09 0.0105457 87 | -0.0132914 0.0186759 2.70084e-09 0.00968799 88 | -0.0160493 0.0201781 2.79397e-09 0.0107011 89 | -0.0152136 0.0211221 2.98023e-09 0.0105839 90 | -0.0132661 0.0209895 2.8871e-09 0.0103347 91 | -0.0146434 0.020457 3.07336e-09 0.0103648 92 | -0.0127386 0.0220156 3.07336e-09 0.0107078 93 | -0.0120492 0.0207884 2.79397e-09 0.00991515 94 | -0.0147849 0.0208594 2.98023e-09 0.0108712 95 | -0.0141528 0.0191989 2.79397e-09 0.0101095 96 | -0.0133514 0.0200942 2.8871e-09 0.0103385 97 | -0.0139093 0.0209741 2.70084e-09 0.0104338 98 | -0.0167135 0.0202657 2.98023e-09 0.0109678 99 | -0.014454 0.0204478 2.79397e-09 0.0104222 100 | -0.0154294 0.0226995 2.70084e-09 0.0111385 101 | -0.0151669 0.0209897 2.79397e-09 0.0105274 102 | -0.0155476 0.0195392 2.70084e-09 0.00999731 103 | -0.0173923 0.0190747 2.98023e-09 0.0105705 104 | -0.0162835 0.0218582 2.79397e-09 0.0110807 105 | -0.0144095 0.0207083 2.6077e-09 0.0103146 106 | -0.0159582 0.0201083 2.6077e-09 0.0104021 107 | -0.0160125 0.0183919 2.42144e-09 0.00993673 108 | -0.0173482 0.0205215 2.79397e-09 0.0114404 109 | -0.0140944 0.0195135 3.07336e-09 0.0096916 110 | -0.017175 0.0227277 3.1665e-09 0.0116059 111 | -0.0125756 0.020756 3.1665e-09 0.00984444 112 | -0.0149402 0.0204557 3.1665e-09 0.010215 113 | -0.0168832 0.0184862 3.35276e-09 0.0103484 114 | -0.0120447 0.0222527 3.44589e-09 0.0103471 115 | -0.0160829 0.0216972 3.53903e-09 0.0106219 116 | -0.0155486 0.0215198 3.53903e-09 0.0105246 117 | -0.0148082 0.0193222 3.44589e-09 0.00993713 118 | -0.0148371 0.019064 3.72529e-09 0.00984125 119 | -0.0159604 0.0174894 3.72529e-09 0.00970137 120 | -0.014825 0.017361 3.72529e-09 0.00970854 121 | -0.0156052 0.0180607 3.91155e-09 0.00960052 122 | -0.0146418 0.0152469 3.72529e-09 0.00886837 123 | -0.0161022 0.0204159 3.72529e-09 0.0109277 124 | -0.0139803 0.0200465 3.63216e-09 0.0103643 125 | -0.0168189 0.0202061 3.72529e-09 0.0108924 126 | -0.0169379 0.0195957 3.91155e-09 0.0106453 127 | -0.0175196 0.0192281 3.72529e-09 0.010283 128 | -0.0189783 0.0210276 3.72529e-09 0.0107915 129 | -0.0187319 0.0197511 3.72529e-09 0.0105068 130 | -0.0166899 0.0187819 3.72529e-09 0.0105133 131 | -0.0163072 0.0188542 3.91155e-09 0.0103762 132 | -0.015647 0.0198576 4.09782e-09 0.0100497 133 | -0.0176962 0.0186578 3.72529e-09 0.010224 134 | -0.016469 0.017794 3.72529e-09 0.00970135 135 | -0.0187767 0.0197554 3.91155e-09 0.0107612 136 | -0.017433 0.0190944 4.09782e-09 0.0103663 137 | -0.016353 0.0208433 4.09782e-09 0.0102929 138 | -0.0198769 0.0160533 4.09782e-09 0.0104218 139 | -0.0191325 0.0156584 3.91155e-09 0.00998076 140 | -0.0190201 0.0142639 4.09782e-09 0.00961444 141 | -0.0168208 0.014775 3.91155e-09 0.00908254 142 | -0.0160665 0.0173149 3.91155e-09 0.00943755 143 | -0.0148989 0.0170693 3.81842e-09 0.00945749 144 | -0.015639 0.01643 3.91155e-09 0.00902765 145 | -0.0177832 0.01666 3.53903e-09 0.00969739 146 | -0.0164678 0.0189807 3.72529e-09 0.00991681 147 | -0.014647 0.0196731 3.63216e-09 0.0100088 148 | -0.0145023 0.0193218 3.91155e-09 0.00987993 149 | -0.0152439 0.0186176 3.91155e-09 0.00958127 150 | -0.0165243 0.018649 4.09782e-09 0.00987943 151 | -0.015618 0.0183866 4.19095e-09 0.00967822 152 | -0.0165882 0.0171054 3.91155e-09 0.00948514 153 | -0.0148086 0.0168184 4.09782e-09 0.00917863 154 | -0.013983 0.0177186 3.91155e-09 0.00938248 155 | -0.0123488 0.0180627 3.91155e-09 0.00906878 156 | -0.0149033 0.0174707 3.91155e-09 0.00940367 157 | -0.0142474 0.0175959 3.91155e-09 0.00919365 158 | -0.0166218 0.0174342 4.09782e-09 0.00959508 159 | -0.016413 0.0179783 4.09782e-09 0.00975836 160 | -0.0157275 0.0202713 3.91155e-09 0.0103898 161 | -0.0164931 0.0193127 4.09782e-09 0.00992142 162 | -0.0165354 0.017917 3.91155e-09 0.00942741 163 | -0.0144276 0.0184694 4.00469e-09 0.00909689 164 | -0.0152747 0.0170434 4.28408e-09 0.00915834 165 | -0.0153745 0.0171396 4.09782e-09 0.00931226 166 | -0.0142739 0.0175561 4.19095e-09 0.00909608 167 | -0.0156005 0.0164426 4.47035e-09 0.00911334 168 | -0.0158923 0.0163225 4.65661e-09 0.00947912 169 | -0.014166 0.0160647 4.56348e-09 0.00903431 170 | -0.0150976 0.0161743 4.74975e-09 0.00932231 171 | -0.0149981 0.0174349 4.47035e-09 0.00948645 172 | -0.0152524 0.0160913 4.56348e-09 0.0092527 173 | -0.0154321 0.0159809 4.47035e-09 0.00914253 174 | -0.0149268 0.0156942 4.56348e-09 0.00914959 175 | -0.0139677 0.017408 4.47035e-09 0.00931964 176 | -0.0133889 0.0168685 4.56348e-09 0.00915828 177 | -0.0135598 0.0168286 4.37722e-09 0.00918489 178 | -0.0137131 0.0173775 4.56348e-09 0.00913201 179 | -0.0152169 0.0169525 4.37722e-09 0.00935472 180 | -0.0153935 0.0168756 4.47035e-09 0.00947071 181 | -0.0143398 0.0189537 4.47035e-09 0.00988659 182 | -0.01514 0.0155934 4.65661e-09 0.00927542 183 | -0.0168864 0.016787 4.65661e-09 0.00940559 184 | -0.0158953 0.0165194 4.47035e-09 0.00913625 185 | -0.0157839 0.0172167 4.65661e-09 0.00922586 186 | -0.0165094 0.0181593 4.47035e-09 0.0098581 187 | -0.0147961 0.0184818 4.28408e-09 0.00934905 188 | -0.0144893 0.0174684 4.37722e-09 0.00931496 189 | -0.013873 0.0164346 4.28408e-09 0.00929203 190 | -0.0126793 0.0164673 4.28408e-09 0.00920366 191 | -0.0129359 0.015978 4.28408e-09 0.0089131 192 | -0.0135115 0.0163139 4.28408e-09 0.00906193 193 | -0.0138623 0.0163243 4.47035e-09 0.00918525 194 | -0.0149216 0.0168442 4.56348e-09 0.00936482 195 | -0.0139901 0.0174342 4.37722e-09 0.00918674 196 | -0.0146194 0.0164444 4.47035e-09 0.00932358 197 | -0.0145552 0.0185615 4.37722e-09 0.00953389 198 | -0.0134509 0.0186636 4.47035e-09 0.0105384 199 | -0.0143238 0.0202403 4.19095e-09 0.0101265 200 | -0.0146532 0.0178431 3.91155e-09 0.0093639 201 | -------------------------------------------------------------------------------- /log/conv1_w_grad.summary: -------------------------------------------------------------------------------- 1 | -0.0253487 0.0370859 0.00103908 0.0116921 2 | -0.0357647 0.0260542 3.2072e-05 0.0126291 3 | -0.0368733 0.0379506 0.000882987 0.0144793 4 | -0.0457324 0.0318211 -0.00383911 0.0146041 5 | -0.0456705 0.0339547 -0.0057689 0.0147811 6 | -0.0520175 0.0375509 -0.00616615 0.0184234 7 | -0.0613124 0.0372036 -0.00687369 0.0184026 8 | -0.0581267 0.0311194 -0.00519579 0.018022 9 | -0.0347351 0.0322648 -0.00341814 0.0143629 10 | -0.0514968 0.0425931 -0.0023137 0.0154461 11 | -0.0608918 0.0504803 0.00542555 0.014699 12 | -0.0688332 0.0858994 0.000527839 0.0304799 13 | -0.0877417 0.0445746 -0.00260084 0.0183252 14 | -0.049009 0.0259097 -0.0028973 0.0126621 15 | -0.0362865 0.03072 0.000283646 0.0122536 16 | -0.0464451 0.0329639 -0.00464754 0.0170226 17 | -0.0377771 0.0336707 6.466e-05 0.0120405 18 | -0.064916 0.0266243 -0.00616191 0.0138657 19 | -0.0365471 0.0309812 -0.00340433 0.0121802 20 | -0.0332676 0.0223804 -0.00394672 0.0103201 21 | -0.0460036 0.0797955 0.007077 0.0239945 22 | -0.0369628 0.0449832 0.0028621 0.0155396 23 | -0.0558662 0.049885 -0.00443388 0.0187011 24 | -0.0391587 0.0250102 -0.00326742 0.0112443 25 | -0.0404006 0.0224211 -0.00447668 0.0122349 26 | -0.0287891 0.0235236 -0.00211194 0.0091173 27 | -0.025168 0.0130178 -0.00289639 0.00757051 28 | -0.0326777 0.0276901 -0.00218215 0.0119221 29 | -0.0357901 0.0533906 0.00214241 0.0162171 30 | -0.0349995 0.0290794 0.000528702 0.0122327 31 | -0.0236701 0.0243771 -0.00207085 0.00783037 32 | -0.018901 0.0282084 0.00113651 0.0081537 33 | -0.0208281 0.0146647 0.000630497 0.00509248 34 | -0.0233792 0.015033 -0.00286774 0.00708898 35 | -0.015822 0.0215375 0.00098961 0.00742764 36 | -0.0330273 0.0207123 -0.000899351 0.0091731 37 | -0.0250083 0.0135951 -0.00249305 0.00809314 38 | -0.0391279 0.032997 -0.0007034 0.0120312 39 | -0.0559126 0.0426309 -0.0020614 0.0151091 40 | -0.0397595 0.0512255 0.00351605 0.0139576 41 | -0.0262702 0.0192685 -0.00168962 0.00806412 42 | -0.0260203 0.0248198 0.000184082 0.00943561 43 | -0.0216986 0.0236904 0.000693236 0.0083673 44 | -0.0235176 0.0195665 -0.00034883 0.00834971 45 | -0.0363732 0.0273722 -0.00177532 0.0108969 46 | -0.0154025 0.0159113 0.000592527 0.00561401 47 | -0.0278038 0.0267557 -0.00228086 0.00915737 48 | -0.0234872 0.0314177 0.00101574 0.00853022 49 | -0.0200059 0.0165501 -0.00187492 0.00719299 50 | -0.0268086 0.00981238 -0.00352183 0.00606738 51 | -0.0297073 0.0218222 -0.00174242 0.00815611 52 | -0.0172406 0.0183873 -0.000312367 0.00663113 53 | -0.0217216 0.0250615 -0.00208404 0.00688025 54 | -0.0311625 0.0227046 0.000163565 0.00835928 55 | -0.0328102 0.0189629 -0.000725373 0.00779215 56 | -0.0352948 0.0348543 -0.000313405 0.0147451 57 | -0.0189306 0.0115499 -0.000698652 0.00552971 58 | -0.0187638 0.0293074 0.00197118 0.0083417 59 | -0.0184158 0.0178776 -0.00145574 0.00607503 60 | -0.016961 0.0169343 0.000865583 0.00566231 61 | -0.0134083 0.00911944 -0.000530761 0.00419244 62 | -0.0290868 0.0239478 -0.0027055 0.00782824 63 | -0.0183169 0.0320777 0.00267843 0.00867614 64 | -0.0352096 0.0267152 -0.00200024 0.0109563 65 | -0.0226903 0.0222649 0.00238264 0.00773759 66 | -0.0279757 0.0301362 0.000553975 0.0110323 67 | -0.0290277 0.0182722 -0.0013374 0.00787253 68 | -0.0310859 0.0369502 0.00167066 0.0116103 69 | -0.0381403 0.0260421 -0.00367448 0.0110477 70 | -0.0229346 0.0156621 0.000934947 0.00596061 71 | -0.0215259 0.0141885 -0.00155337 0.006618 72 | -0.0163955 0.0145196 -0.00100084 0.00556749 73 | -0.023325 0.0228995 0.000971213 0.00856823 74 | -0.0401182 0.0271686 -0.00163907 0.01252 75 | -0.0209377 0.0299057 0.00110222 0.00841626 76 | -0.031986 0.0171899 -0.00524822 0.00890002 77 | -0.0307599 0.0408214 0.000744041 0.0110514 78 | -0.0269892 0.0410891 0.00417463 0.0136823 79 | -0.0257728 0.0263848 -0.00108057 0.00823617 80 | -0.0213309 0.0247999 0.00108701 0.00779735 81 | -0.0351196 0.041872 -0.00118507 0.0121194 82 | -0.0351066 0.0231498 0.00189523 0.0107779 83 | -0.0212645 0.0140494 -0.00178873 0.00610208 84 | -0.0237401 0.0194581 -0.00260947 0.00889821 85 | -0.0174915 0.0193462 -0.000732925 0.00605356 86 | -0.0239793 0.0191611 -0.00172251 0.0068358 87 | -0.0263312 0.0225052 -0.000582195 0.00811507 88 | -0.035283 0.0206399 -1.02837e-05 0.00903607 89 | -0.0241115 0.0135082 -0.00162933 0.0052793 90 | -0.0195083 0.0209997 0.00301074 0.00673117 91 | -0.016132 0.0160575 0.000442394 0.00503059 92 | -0.0166508 0.022286 -0.000210204 0.00699788 93 | -0.0332302 0.0222954 -0.00237392 0.00920746 94 | -0.0304953 0.0330167 0.000586735 0.0125862 95 | -0.0169645 0.01829 -0.00158513 0.00683631 96 | -0.0262301 0.0229487 0.000428356 0.0087202 97 | -0.0260254 0.0191006 -0.000919025 0.00659045 98 | -0.0227458 0.0244756 0.00158548 0.00795387 99 | -0.0272207 0.0136353 -0.00310285 0.00834745 100 | -0.0245414 0.0174508 -0.00135477 0.00765239 101 | -0.0192791 0.017714 -0.000470577 0.00702911 102 | -0.0365206 0.0471008 -0.000144719 0.0137884 103 | -0.0188917 0.0139753 3.08718e-05 0.00517566 104 | -0.035476 0.0350904 0.00287197 0.0117444 105 | -0.0168124 0.0132578 0.00105942 0.00518681 106 | -0.0135784 0.0117731 -0.000949971 0.00474739 107 | -0.0340154 0.0286368 -0.00234575 0.0108337 108 | -0.0439498 0.0644623 0.00708702 0.0220124 109 | -0.0753853 0.0678455 -0.00362443 0.0219441 110 | -0.0598672 0.0632899 0.00680916 0.0217339 111 | -0.0299825 0.0327015 -0.00168817 0.011697 112 | -0.0140248 0.0186391 0.000783621 0.00608795 113 | -0.0200616 0.0232683 0.000727803 0.0067241 114 | -0.0386324 0.0361075 0.00140004 0.0120787 115 | -0.0414518 0.0262405 -0.00387666 0.0132154 116 | -0.0252485 0.0195861 0.000244203 0.00737104 117 | -0.0232123 0.0244454 -0.000628817 0.00787387 118 | -0.0175813 0.0153581 -0.00144431 0.00553566 119 | -0.0205298 0.0216524 -8.97381e-05 0.00691774 120 | -0.0252098 0.0225552 -0.000370254 0.00803756 121 | -0.0371513 0.0321061 -0.00273264 0.0120359 122 | -0.0316632 0.0185242 -0.00287439 0.009172 123 | -0.0278744 0.0370151 0.002788 0.0120774 124 | -0.0342155 0.0313746 -0.000167185 0.0101616 125 | -0.0192575 0.0264544 0.00287638 0.0073967 126 | -0.0299563 0.0345119 -0.00197424 0.00999227 127 | -0.0242528 0.0288878 0.0034525 0.00994487 128 | -0.0243747 0.0138951 -0.00131881 0.00655408 129 | -0.0185882 0.0190664 -0.000215577 0.00685468 130 | -0.0247876 0.0382443 0.00192048 0.0122486 131 | -0.0332053 0.0122655 -0.00495958 0.00867075 132 | -0.0363803 0.0325997 -0.0022606 0.0118538 133 | -0.0397852 0.0558252 0.00799318 0.0190781 134 | -0.0178727 0.0160954 -0.00211118 0.00640139 135 | -0.0167455 0.0193416 -0.000365591 0.00603884 136 | -0.0307343 0.0325515 -0.00140922 0.0109513 137 | -0.0374488 0.0467684 -0.000557879 0.014494 138 | -0.0714998 0.0624803 0.0076709 0.0252988 139 | -0.0249226 0.0166108 -0.00369443 0.00766233 140 | -0.0265899 0.0196942 -0.000933938 0.00726328 141 | -0.0163269 0.0116787 -0.00159284 0.00529922 142 | -0.0218896 0.033397 0.00452525 0.0114198 143 | -0.0234568 0.0300428 0.000127581 0.00873698 144 | -0.0337561 0.0255788 -0.0027504 0.00996854 145 | -0.0157174 0.0164636 0.00215858 0.00496697 146 | -0.0275074 0.018654 -0.000614962 0.00837941 147 | -0.0249376 0.0279266 -0.000383574 0.010418 148 | -0.0266738 0.0222888 0.00228334 0.00783264 149 | -0.0268529 0.0215159 -0.00392006 0.00889058 150 | -0.01442 0.0148776 -0.00109902 0.00536397 151 | -0.0170395 0.00971986 -0.00253563 0.0053645 152 | -0.0101169 0.0159923 0.00185295 0.00496261 153 | -0.0246779 0.0224698 -0.000548119 0.00684031 154 | -0.0143153 0.0132698 -0.000658937 0.00458165 155 | -0.0185342 0.0176587 -0.0010067 0.00622819 156 | -0.0219992 0.0166642 -0.00276733 0.00664357 157 | -0.0183556 0.0153596 -9.0475e-07 0.00587745 158 | -0.0284826 0.0167294 0.000620241 0.00767842 159 | -0.0167566 0.0160402 0.00138211 0.00602804 160 | -0.0361121 0.0587658 0.00950456 0.021536 161 | -0.0368653 0.02242 -0.00200335 0.0102772 162 | -0.0169302 0.00873674 -0.00167336 0.00413267 163 | -0.0181095 0.0080054 -0.00158707 0.00462322 164 | -0.0300417 0.0294692 -0.00544251 0.011387 165 | -0.0116808 0.00469996 -0.00102124 0.00269103 166 | -0.0232791 0.0189706 9.05746e-05 0.00671123 167 | -0.0158317 0.0135931 -0.000663098 0.00540278 168 | -0.0204966 0.0190584 0.000576584 0.0069331 169 | -0.0231434 0.018886 -0.00102525 0.00765318 170 | -0.0162464 0.0139041 -0.000477745 0.00474544 171 | -0.0132276 0.00984505 -0.000741583 0.00374542 172 | -0.0177759 0.0254525 0.00186454 0.00631543 173 | -0.0123422 0.0144196 0.00176489 0.00494149 174 | -0.0253174 0.0218699 -0.00194632 0.00689206 175 | -0.0259978 0.0180437 -0.00253033 0.00641153 176 | -0.0239278 0.0212807 -0.00152946 0.00805065 177 | -0.0121645 0.00714993 -0.000286857 0.00320002 178 | -0.0184417 0.0120607 -0.000710046 0.00530917 179 | -0.0147124 0.0156123 -0.00109256 0.00477541 180 | -0.0180186 0.0147426 0.000157671 0.00489515 181 | -0.0339472 0.0400115 -0.0004517 0.0122258 182 | -0.0509954 0.0392033 -0.00173606 0.0159711 183 | -0.0283245 0.0307775 0.000774026 0.00958167 184 | -0.0158349 0.00838742 -0.0023108 0.00450137 185 | -0.0266278 0.0256084 0.00242519 0.00927865 186 | -0.0447855 0.0295498 -0.00476446 0.0129499 187 | -0.0371522 0.0537688 0.00764324 0.0163214 188 | -0.0394793 0.0246677 -0.00068352 0.00872635 189 | -0.0156286 0.02158 -6.16013e-05 0.00648312 190 | -0.0141827 0.00597445 -0.00182528 0.00354076 191 | -0.0211504 0.0132278 -0.00205133 0.00576383 192 | -0.0261219 0.027064 0.00046025 0.00838801 193 | -0.0157611 0.0131324 -0.0010034 0.00566798 194 | -0.017414 0.0162655 0.000862157 0.00595479 195 | -0.0216502 0.01063 0.000519576 0.00471364 196 | -0.0275955 0.0283359 -0.00148884 0.00870831 197 | -0.040666 0.0533699 0.00846549 0.0156106 198 | -0.079768 0.0816258 -0.00992412 0.0244932 199 | -0.0343358 0.0586887 0.0123369 0.0171823 200 | -0.0243968 0.0280745 -0.00180483 0.00938295 201 | -------------------------------------------------------------------------------- /log/conv2_b_grad.summary: -------------------------------------------------------------------------------- 1 | -0.0356173 0.032887 0.00214641 0.0148163 2 | -0.0532917 0.0367665 0.00298802 0.0195608 3 | -0.0435959 0.0902317 0.0074398 0.0264684 4 | -0.0320178 0.0458416 0.00311851 0.0165679 5 | -0.0303026 0.0394657 0.000544726 0.0166634 6 | -0.0393392 0.0313817 -0.00381415 0.0161249 7 | -0.0293439 0.0235782 0.00151007 0.012655 8 | -0.0345627 0.0227684 -0.00337893 0.0137253 9 | -0.042493 0.0370191 0.00253895 0.016579 10 | -0.0288598 0.0583153 0.00596158 0.0213864 11 | -0.0417211 0.0590672 0.00202175 0.0231489 12 | -0.0826831 0.133993 0.030907 0.0438094 13 | -0.0488368 0.0375395 -0.00458145 0.0187823 14 | -0.0358866 0.0400677 2.32529e-05 0.0147469 15 | -0.0247049 0.0299162 -0.00313722 0.0136873 16 | -0.0237925 0.0356162 0.00102297 0.010667 17 | -0.0236058 0.0214112 -0.000910346 0.0105862 18 | -0.0309965 0.0295055 -0.00183545 0.0134474 19 | -0.0221727 0.0207551 0.000956152 0.00780867 20 | -0.0269592 0.0194216 -0.00212837 0.0102548 21 | -0.0619821 0.0735886 0.00422229 0.0271475 22 | -0.0195573 0.0708064 0.015003 0.0196688 23 | -0.036446 0.0389621 -0.000831401 0.0177739 24 | -0.0365761 0.0105058 -0.00558901 0.00801261 25 | -0.016086 0.0291446 0.00145052 0.00955783 26 | -0.0111852 0.00870172 -0.00189342 0.00498964 27 | -0.0155138 0.0155822 0.000247737 0.00671103 28 | -0.0231886 0.028869 0.00334866 0.010374 29 | -0.0518413 0.0400078 -0.00380046 0.0197301 30 | -0.0351152 0.0770648 0.00861799 0.0203078 31 | -0.0341843 0.0209161 -0.00273701 0.00937792 32 | -0.0135434 0.0182238 0.00270542 0.00924064 33 | -0.0104681 0.00909243 0.000586603 0.00448438 34 | -0.0100748 0.0100551 0.00139548 0.00442203 35 | -0.0233075 0.0127305 -0.00249727 0.00796794 36 | -0.0232631 0.0198032 -8.14539e-05 0.00962177 37 | -0.012758 0.0192476 0.00119255 0.00818453 38 | -0.017124 0.0298078 0.00344394 0.00937909 39 | -0.0244994 0.0193171 -0.00475742 0.00950667 40 | -0.0300598 0.0579342 0.00993551 0.020762 41 | -0.0211482 0.0262107 0.00217598 0.0109853 42 | -0.0167373 0.0372028 0.00528456 0.0110835 43 | -0.0230703 0.0217225 -0.000462128 0.0107443 44 | -0.0172732 0.0120552 -0.000518798 0.00562314 45 | -0.0141944 0.0108824 -0.000565521 0.00702028 46 | -0.0153892 0.00823885 -0.00135213 0.00556546 47 | -0.0184691 0.0180673 0.00220142 0.00761267 48 | -0.0212029 0.0187016 7.18255e-05 0.00887485 49 | -0.0117474 0.0105477 -0.00016973 0.00521231 50 | -0.0123131 0.010022 -0.0021088 0.00432693 51 | -0.014677 0.0151902 -0.00202803 0.0065128 52 | -0.0184443 0.011793 -0.00064315 0.00561174 53 | -0.0123815 0.0214006 0.00208172 0.00819289 54 | -0.0188923 0.0199286 0.000221001 0.00887659 55 | -0.0134692 0.0146452 0.00112831 0.00703936 56 | -0.017422 0.0229666 0.00287868 0.00870142 57 | -0.0175031 0.0203804 0.00225075 0.00729358 58 | -0.0174971 0.00906127 -0.0015986 0.0051338 59 | -0.0079665 0.0155712 0.00186245 0.00563377 60 | -0.0146694 0.0112291 -0.00138223 0.00628852 61 | -0.00890379 0.00520651 -0.00162191 0.00374752 62 | -0.0111716 0.0158121 0.00164675 0.00474778 63 | -0.0268002 0.0151298 -0.00318593 0.00877967 64 | -0.0108483 0.0275008 0.00783379 0.0095323 65 | -0.021299 0.012267 0.000861141 0.00719246 66 | -0.0248263 0.0203947 -0.00322369 0.00895025 67 | -0.010674 0.0215185 0.00533174 0.00610589 68 | -0.0339032 0.0153249 -0.00430635 0.0106216 69 | -0.0193611 0.0265053 0.00716814 0.00976693 70 | -0.0117842 0.0134415 -0.00181845 0.00545906 71 | -0.0153036 0.0130632 0.000248903 0.00707016 72 | -0.010128 0.0147203 -0.000856956 0.00519553 73 | -0.0100112 0.016365 0.000952901 0.00595503 74 | -0.0155192 0.0263861 0.00312972 0.00912141 75 | -0.0245155 0.0160447 -0.00322505 0.00913794 76 | -0.0295137 0.0171536 0.0012256 0.010279 77 | -0.0225594 0.0182428 -0.000596791 0.00998295 78 | -0.0294333 0.0328353 0.00166423 0.0148144 79 | -0.0122356 0.0240835 0.00250099 0.00664506 80 | -0.022672 0.00923449 -0.00158625 0.0080115 81 | -0.0145689 0.023446 0.00284231 0.00914937 82 | -0.0264073 0.00999606 -0.0044754 0.00760356 83 | -0.0155908 0.0158186 0.000579894 0.00694938 84 | -0.010531 0.0192185 0.00332267 0.00635655 85 | -0.0149466 0.0114557 0.000795766 0.00541477 86 | -0.0138172 0.0153681 0.000167785 0.00671159 87 | -0.00955356 0.01481 0.00311261 0.00493074 88 | -0.01718 0.0136138 -0.00323905 0.00724344 89 | -0.0112427 0.00971574 -3.48617e-05 0.00572686 90 | -0.0197369 0.0169536 -0.00125443 0.00872657 91 | -0.00410105 0.0086135 0.000488142 0.00295278 92 | -0.0128723 0.0129795 0.000146031 0.00635481 93 | -0.014503 0.0233914 0.00359496 0.00862344 94 | -0.0287096 0.021568 -0.00173819 0.0103752 95 | -0.0103563 0.0184938 0.00322401 0.00549848 96 | -0.021356 0.0122771 -0.00201733 0.00642199 97 | -0.0132611 0.0109263 -0.000936439 0.00553834 98 | -0.0144478 0.00977706 -0.00272475 0.00595094 99 | -0.00927115 0.0135766 0.00273598 0.00576479 100 | -0.0162367 0.00615953 -0.00429192 0.00555208 101 | -0.0144029 0.015473 0.000782343 0.00588046 102 | -0.0280014 0.0309529 0.00567394 0.0129777 103 | -0.0189213 0.011215 -0.00127617 0.0067963 104 | -0.03093 0.0241086 -0.00439194 0.0105127 105 | -0.00902607 0.0131787 0.00168884 0.00514034 106 | -0.00668145 0.00579434 -0.000383852 0.00323267 107 | -0.0126435 0.0252323 0.00445484 0.00867959 108 | -0.04148 0.0283164 -0.00196152 0.0154855 109 | -0.0327724 0.0608268 0.00812771 0.0185248 110 | -0.0551929 0.0311739 -0.00580932 0.0182903 111 | -0.0152594 0.0302751 0.00573813 0.0101249 112 | -0.0120141 0.0143399 0.00204146 0.00549609 113 | -0.012389 0.0182115 -0.000796594 0.0065178 114 | -0.0245709 0.0162364 -0.00530147 0.00976261 115 | -0.0193398 0.033327 0.00343953 0.0135428 116 | -0.0125823 0.00977007 -0.000123462 0.00531872 117 | -0.00910786 0.0154379 0.00173307 0.00596909 118 | -0.00991979 0.00927 4.22873e-05 0.00434666 119 | -0.00574505 0.0146689 0.00294365 0.00435946 120 | -0.0129894 0.017217 0.00158378 0.00742425 121 | -0.0214299 0.02702 -0.00146467 0.0112571 122 | -0.0100576 0.0152554 0.00431398 0.00569364 123 | -0.0283294 0.0198701 -0.00319791 0.0110267 124 | -0.0115493 0.0188471 0.00358037 0.00691511 125 | -0.0115604 0.00954082 -0.00239944 0.00507398 126 | -0.017432 0.0221017 0.00118288 0.00869272 127 | -0.0124285 0.0179725 -0.000332912 0.00739145 128 | -0.0143616 0.0151674 -0.002199 0.00698081 129 | -0.0112537 0.0119446 0.000942923 0.00411986 130 | -0.0371729 0.0333706 1.48499e-05 0.0145169 131 | -0.0121734 0.0193043 0.00357146 0.00796251 132 | -0.020474 0.018296 -0.000578975 0.00987202 133 | -0.0335166 0.0223738 -0.000153683 0.0107112 134 | -0.00825737 0.014965 0.00270435 0.00443261 135 | -0.012605 0.00621069 -0.00256997 0.00482801 136 | -0.0191831 0.0181203 0.00330704 0.00722359 137 | -0.02938 0.0256389 -0.00120177 0.013247 138 | -0.0575176 0.0393754 3.8214e-05 0.0210242 139 | -0.0142706 0.0237647 0.00350119 0.00879947 140 | -0.0126682 0.0181017 0.00390027 0.00662994 141 | -0.00724048 0.014379 0.00316194 0.00496006 142 | -0.0285905 0.016156 -0.00451399 0.00929157 143 | -0.0194265 0.0151499 0.00114716 0.00702725 144 | -0.021625 0.0314978 0.00111808 0.0118315 145 | -0.0159052 0.0139717 -0.0028991 0.00614248 146 | -0.0196981 0.0176704 -0.0016096 0.00795603 147 | -0.0113127 0.0221326 0.00257608 0.00806972 148 | -0.0135945 0.0138162 9.63495e-05 0.00623888 149 | -0.0106285 0.0209741 0.00257769 0.0075183 150 | -0.0153706 0.0183596 0.000757346 0.00672253 151 | -0.00716611 0.00796235 0.000200481 0.00379171 152 | -0.0130294 0.00788902 -8.4756e-05 0.00410675 153 | -0.0173565 0.0170403 -0.00100194 0.00758089 154 | -0.00914156 0.00651118 6.34551e-05 0.003206 155 | -0.00769297 0.0107004 0.000853718 0.00474384 156 | -0.0152629 0.0184209 0.000948051 0.00721918 157 | -0.0100781 0.00957322 -0.000702848 0.00478104 158 | -0.0150092 0.00890482 -0.0019373 0.00552415 159 | -0.0127775 0.0128358 4.40479e-05 0.00595307 160 | -0.0366136 0.0235969 -0.00342479 0.0126481 161 | -0.00848738 0.0202922 0.001932 0.0068057 162 | -0.0141697 0.0136666 0.00163352 0.00547867 163 | -0.00904286 0.0100912 -0.000248611 0.00414647 164 | -0.0189577 0.0251715 0.00310095 0.0106531 165 | -0.0058776 0.00306362 -0.00101244 0.00222474 166 | -0.0150759 0.0108704 -0.00118732 0.00629596 167 | -0.0084041 0.0104247 0.000423983 0.0047565 168 | -0.015758 0.0143695 -0.000301678 0.00736523 169 | -0.0149939 0.0165853 -0.000359141 0.00791242 170 | -0.0070143 0.0110475 0.00178937 0.00397943 171 | -0.0078937 0.0095216 -0.00178269 0.00371389 172 | -0.00785463 0.00933247 0.00144533 0.00428585 173 | -0.00921241 0.010213 -1.69117e-05 0.0039418 174 | -0.00875821 0.0170517 0.00237857 0.00606438 175 | -0.00877376 0.00805755 -0.000273913 0.00373229 176 | -0.00810792 0.016939 0.00133986 0.00458075 177 | -0.00400453 0.00435789 -7.17885e-05 0.00193059 178 | -0.0112766 0.00965553 -0.000656932 0.00432891 179 | -0.0104233 0.0108774 -0.00142905 0.00467732 180 | -0.00756876 0.00498407 -0.000653913 0.00304149 181 | -0.0305067 0.0234754 -0.00247494 0.0117158 182 | -0.0242533 0.0359294 0.00656415 0.0137988 183 | -0.0201582 0.0284392 -0.000746856 0.0108812 184 | -0.0110658 0.0113658 0.00122457 0.0050745 185 | -0.0207195 0.0101211 -0.00357599 0.00668391 186 | -0.0186353 0.030801 0.00463322 0.0130458 187 | -0.0390588 0.0195556 -0.00331949 0.0121738 188 | -0.0171448 0.0254129 0.00283331 0.0110644 189 | -0.0112209 0.0137093 0.00164095 0.00648587 190 | -0.00531556 0.00648617 0.000252398 0.00267885 191 | -0.00955346 0.00995326 0.000851944 0.00431134 192 | -0.0140075 0.0144335 -0.0015576 0.00710461 193 | -0.00659145 0.00816445 -0.000935147 0.00275518 194 | -0.0139373 0.0189781 0.000418049 0.00671411 195 | -0.011261 0.00728212 -0.00133792 0.00441033 196 | -0.0102501 0.0215929 0.0022935 0.00656808 197 | -0.0301468 0.0253192 -0.00548446 0.0126675 198 | -0.0509476 0.0626078 0.0114393 0.0242715 199 | -0.0514287 0.0317039 -0.0053459 0.0184918 200 | -0.0196902 0.0236614 0.00406915 0.00838161 201 | -------------------------------------------------------------------------------- /log/conv2_w_grad.summary: -------------------------------------------------------------------------------- 1 | -0.0375197 0.0336978 0.000336821 0.00740039 2 | -0.0412951 0.0355541 0.000494702 0.00888242 3 | -0.0508186 0.0629736 0.00111352 0.010627 4 | -0.0331149 0.0339145 0.000241275 0.00769889 5 | -0.0351586 0.0318197 -6.00643e-05 0.00796084 6 | -0.0461717 0.0362674 -0.00062807 0.00813397 7 | -0.0327746 0.0306067 5.05467e-05 0.0072161 8 | -0.0487291 0.0338241 -0.000642977 0.00693184 9 | -0.0378462 0.0393576 0.000255702 0.00834057 10 | -0.0465961 0.0523261 0.000939701 0.00990913 11 | -0.0541585 0.0502315 0.000245322 0.0100068 12 | -0.0843571 0.129544 0.00456252 0.0190634 13 | -0.0559222 0.0360175 -0.000765252 0.00868568 14 | -0.0374046 0.0423563 -0.000202711 0.00695236 15 | -0.0436402 0.0277383 -0.000731692 0.00660687 16 | -0.0344962 0.0298772 -0.00019165 0.00632314 17 | -0.0344419 0.0253919 -0.0003486 0.00608679 18 | -0.0376377 0.0344245 -0.000459893 0.00666615 19 | -0.0306092 0.0305006 4.7355e-05 0.00511802 20 | -0.0393302 0.0326888 -0.000306082 0.00737469 21 | -0.0862131 0.0850224 0.00057097 0.013816 22 | -0.0537685 0.0830136 0.00210451 0.0121715 23 | -0.0499221 0.0451544 -0.000427803 0.00826221 24 | -0.0275082 0.0187907 -0.000962487 0.00466179 25 | -0.0262579 0.0260689 0.000181715 0.00503579 26 | -0.0304596 0.0160014 -0.000341023 0.00361369 27 | -0.0179477 0.024841 7.58007e-05 0.0038322 28 | -0.0320231 0.030489 0.00041106 0.00575717 29 | -0.0407969 0.0394366 -0.000403309 0.00815828 30 | -0.0384258 0.0704533 0.00135511 0.00877082 31 | -0.0312 0.024404 -0.000566109 0.0055564 32 | -0.0265222 0.0324358 0.000349554 0.00529192 33 | -0.0152099 0.0172062 -1.04401e-05 0.00315548 34 | -0.0159169 0.0138726 8.20954e-05 0.00315405 35 | -0.0231201 0.0224088 -0.000313957 0.00428267 36 | -0.0334828 0.0251509 -0.000148808 0.0051481 37 | -0.023272 0.0198005 0.000174885 0.00405508 38 | -0.0264467 0.0372103 0.000281823 0.00558156 39 | -0.042353 0.0427642 -0.00071133 0.00771135 40 | -0.0555749 0.0603312 0.00144486 0.0103522 41 | -0.0271404 0.0316075 0.000293688 0.00569375 42 | -0.0334742 0.0437275 0.000962812 0.00669617 43 | -0.0256337 0.0297386 -9.2063e-05 0.00490265 44 | -0.0215081 0.0234159 -5.56752e-05 0.00409529 45 | -0.0265541 0.0215321 -0.000125552 0.0045248 46 | -0.0239306 0.0175467 -0.000260765 0.00409042 47 | -0.0241047 0.0289688 0.000258676 0.00427829 48 | -0.0242953 0.0182271 -8.41456e-05 0.00430922 49 | -0.0178731 0.0193636 -1.93282e-05 0.00349628 50 | -0.0145386 0.0156652 -0.000355185 0.00283344 51 | -0.017239 0.0187326 -0.000428197 0.00359502 52 | -0.0252717 0.0208448 -7.68098e-05 0.00404221 53 | -0.0198595 0.0247218 0.000287819 0.00441795 54 | -0.027294 0.0326112 -5.23095e-05 0.00429965 55 | -0.0247041 0.0245961 8.92802e-06 0.00445311 56 | -0.0298793 0.0386558 0.000548696 0.00583676 57 | -0.0210005 0.0310186 0.000291995 0.00402368 58 | -0.0307109 0.0231442 -0.00042614 0.0043201 59 | -0.0205705 0.0220219 0.00032566 0.00403563 60 | -0.0205886 0.0153477 -0.000243813 0.00325457 61 | -0.0146496 0.0106231 -0.00031805 0.00247412 62 | -0.0178445 0.025771 0.000308813 0.00344357 63 | -0.0326685 0.032817 -0.00048887 0.00557436 64 | -0.0292728 0.0434476 0.00134266 0.00650373 65 | -0.0258945 0.028374 7.66252e-05 0.00442298 66 | -0.0321611 0.022206 -0.000497182 0.00514944 67 | -0.0184261 0.0258623 0.000725394 0.00394358 68 | -0.0431782 0.0233898 -0.000748475 0.00546792 69 | -0.0232481 0.0304136 0.00113534 0.00540376 70 | -0.0189582 0.0177305 -0.00027682 0.00350485 71 | -0.0196845 0.0173574 -0.000180401 0.00375638 72 | -0.0147238 0.0139531 -0.000278762 0.00291471 73 | -0.0262002 0.0220741 9.67635e-05 0.00414262 74 | -0.0275981 0.0320102 0.000625663 0.00579077 75 | -0.0328018 0.0251139 -0.000383215 0.00542006 76 | -0.0366417 0.0401802 9.33338e-05 0.00647565 77 | -0.0324571 0.0273443 -9.74126e-05 0.00529464 78 | -0.0348101 0.0389966 0.000374495 0.00651339 79 | -0.0215407 0.0284663 0.000303646 0.0037211 80 | -0.0375599 0.0239972 -0.000274896 0.00458804 81 | -0.0281383 0.0433296 0.000750365 0.00633812 82 | -0.0381272 0.0245332 -0.000855764 0.00491064 83 | -0.031809 0.0175592 -0.000169285 0.00368688 84 | -0.0208265 0.0246862 0.000444419 0.00416873 85 | -0.0210169 0.0182048 4.41948e-06 0.00362978 86 | -0.0193964 0.0247761 0.000166089 0.00399259 87 | -0.0206784 0.0245473 0.000666709 0.00446488 88 | -0.024725 0.0321716 -0.00058493 0.00459118 89 | -0.0222941 0.0198909 -0.000101623 0.0034962 90 | -0.0318447 0.0233281 -0.000293292 0.00476682 91 | -0.0195734 0.0174242 9.22507e-05 0.00311269 92 | -0.0177429 0.0166957 -6.91427e-06 0.00357981 93 | -0.0205745 0.0290931 0.000503839 0.00457249 94 | -0.0447086 0.0319884 -0.000143658 0.00552485 95 | -0.0149896 0.0202978 0.000585681 0.00346567 96 | -0.0225443 0.0211129 -0.000453539 0.00383999 97 | -0.0179671 0.0171624 -0.000164948 0.00301779 98 | -0.0249673 0.0229154 -0.000497121 0.00476419 99 | -0.0170752 0.0251135 0.000244897 0.00365501 100 | -0.024619 0.0187002 -0.000855849 0.00392332 101 | -0.0215426 0.0264032 9.89115e-05 0.00424936 102 | -0.0327962 0.054073 0.00110414 0.00771365 103 | -0.0244551 0.0231892 -0.000191112 0.00329544 104 | -0.0416645 0.0307304 -0.00100381 0.00607061 105 | -0.0171621 0.0186163 0.000227775 0.00356272 106 | -0.0130724 0.0119229 -3.02906e-05 0.0023908 107 | -0.0177157 0.0237586 0.000594245 0.0041091 108 | -0.051447 0.0525436 -8.9579e-05 0.00877088 109 | -0.0460825 0.0753946 0.00149703 0.00941378 110 | -0.0606486 0.0469476 -0.000958111 0.00919101 111 | -0.02463 0.0329675 0.000571499 0.00583807 112 | -0.0192495 0.0215021 0.000472307 0.00378443 113 | -0.0230567 0.0207303 -0.000225182 0.00372263 114 | -0.041692 0.0309134 -0.00105787 0.00631194 115 | -0.026762 0.0416456 0.000351179 0.00611117 116 | -0.0176064 0.0146158 -0.000102031 0.00328855 117 | -0.0152989 0.0207454 0.000114661 0.00291199 118 | -0.0165148 0.0170964 9.74076e-05 0.0029803 119 | -0.0178024 0.0214246 0.000382006 0.00332003 120 | -0.0234004 0.0210471 4.08724e-05 0.00364332 121 | -0.0270257 0.0396663 -2.94842e-05 0.0049356 122 | -0.0236514 0.0249533 0.000499663 0.00440706 123 | -0.0451843 0.0327597 -0.000678568 0.00631748 124 | -0.0266461 0.0308819 0.000708899 0.00484263 125 | -0.0270854 0.0193632 -0.000472048 0.00411598 126 | -0.0249343 0.0297244 0.000213294 0.00488876 127 | -0.0250643 0.0262415 8.26874e-05 0.00557525 128 | -0.0179754 0.0179161 -0.000450829 0.00328979 129 | -0.0206923 0.0196565 3.63062e-05 0.00428333 130 | -0.0449333 0.0463554 4.11112e-05 0.00696628 131 | -0.0195735 0.0245494 0.000462449 0.00401647 132 | -0.0311705 0.0246631 -0.000205248 0.00490591 133 | -0.0411998 0.0383022 0.000130842 0.00715314 134 | -0.0149823 0.0175671 0.000362699 0.00333553 135 | -0.0207776 0.0214112 -0.000368216 0.00397566 136 | -0.0258644 0.0329071 0.000488011 0.00527621 137 | -0.0400031 0.0357 -0.000156113 0.00680967 138 | -0.0892567 0.0751283 7.36351e-05 0.0109246 139 | -0.0194384 0.0338147 0.000414743 0.0051071 140 | -0.0177551 0.0225942 0.000592504 0.00435056 141 | -0.0143718 0.0171321 0.000304683 0.00270168 142 | -0.0321197 0.0212916 -0.00089005 0.00519571 143 | -0.0314822 0.0265487 0.000330947 0.00470114 144 | -0.0291542 0.0220309 1.58426e-05 0.00432898 145 | -0.0181487 0.0180274 -0.0005004 0.00355358 146 | -0.03034 0.0250327 -0.00032854 0.00428036 147 | -0.0225607 0.0264828 0.000444981 0.00457108 148 | -0.0230604 0.02015 -1.25119e-05 0.00428114 149 | -0.0184109 0.0270557 0.000459377 0.00432943 150 | -0.0233198 0.0265708 1.28846e-05 0.00421178 151 | -0.0118245 0.0110808 -0.000124617 0.00241132 152 | -0.0187728 0.014453 -6.16917e-05 0.0027186 153 | -0.0241883 0.0219111 -0.00027534 0.00409778 154 | -0.0116162 0.0102122 5.2312e-06 0.00229445 155 | -0.0171495 0.0188072 0.000211599 0.00331975 156 | -0.0154487 0.025039 0.000140121 0.00353561 157 | -0.0197666 0.0199559 -0.000237124 0.00356683 158 | -0.0220038 0.0172906 -0.000389032 0.00379002 159 | -0.0161162 0.0209727 -7.18051e-06 0.00326429 160 | -0.0529096 0.0428648 -0.000472966 0.00972594 161 | -0.0253546 0.0285873 0.000324531 0.00499939 162 | -0.0140175 0.0135479 0.000186497 0.00245802 163 | -0.0142834 0.0160057 -0.000123897 0.00219281 164 | -0.0297504 0.0352227 0.000563682 0.00594175 165 | -0.0102974 0.00542838 -0.000230826 0.00144499 166 | -0.0244746 0.0149247 -0.000304283 0.00327671 167 | -0.0199611 0.0167455 5.28961e-05 0.00289263 168 | -0.0255889 0.0237101 -8.63221e-05 0.00430814 169 | -0.0266106 0.0225305 -0.000175427 0.00412687 170 | -0.0132075 0.0129272 0.000257042 0.00238478 171 | -0.0107899 0.0101416 -0.00027637 0.00195484 172 | -0.0175749 0.0169942 9.19917e-05 0.00354227 173 | -0.0200333 0.0144553 -6.14097e-05 0.00330177 174 | -0.0161293 0.0224322 0.00037485 0.00384591 175 | -0.0161883 0.0169723 -0.000178415 0.00301593 176 | -0.0177717 0.0184183 0.000413243 0.00364492 177 | -0.0093983 0.0069845 -8.09376e-05 0.0013031 178 | -0.018137 0.0127053 -0.000261951 0.00267386 179 | -0.0163088 0.0182829 -0.00022371 0.00304205 180 | -0.0152384 0.0114495 -0.00027737 0.00257733 181 | -0.0494551 0.0372052 -0.00033868 0.00642107 182 | -0.0294613 0.0501035 0.00119512 0.00709455 183 | -0.0259302 0.0408053 -0.000132095 0.0052285 184 | -0.013613 0.0179783 0.000269932 0.00304185 185 | -0.0263509 0.0213785 -0.000649301 0.00428471 186 | -0.0336124 0.0496 0.000698202 0.00673959 187 | -0.06288 0.0419215 -0.000613138 0.00775233 188 | -0.0215725 0.0392338 0.000381556 0.00480551 189 | -0.020236 0.0160825 2.55468e-05 0.00372531 190 | -0.0125553 0.0126116 -3.00687e-05 0.00229286 191 | -0.021565 0.0227787 7.62657e-05 0.00340703 192 | -0.0230315 0.0291042 -9.25443e-05 0.00481056 193 | -0.0134918 0.0160542 -9.85139e-05 0.00262814 194 | -0.0230491 0.0267972 -0.000210148 0.0036959 195 | -0.0143431 0.0109523 -0.000329684 0.00256342 196 | -0.0276 0.0389737 0.000535458 0.00495532 197 | -0.0645636 0.0432036 -0.000863301 0.00888822 198 | -0.0601918 0.0888093 0.00220699 0.0131153 199 | -0.0700359 0.0500359 -0.000773356 0.00900743 200 | -0.0296556 0.0340088 0.000716856 0.00489968 201 | -------------------------------------------------------------------------------- /tutorial/MNIST-light.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "# EXPLAIN WHAT MNIST IS (GIVE EXAMPLE) (SHOW GRAPH STRUCTURE) (MORE VISUALS IN GENERAL)" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "# MNIST\n", 19 | "\n", 20 | "In this tutorial, we will show you how to train an actual CNN model, albeit small. We will be using the old good MNIST dataset and the LeNet model, with a slight change that the sigmoid activations are replaced with ReLUs.\n", 21 | "\n", 22 | "We will use the model helper - that helps us to deal with parameter initializations naturally.\n", 23 | "\n", 24 | "First, let's import the necessities." 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "%matplotlib inline\n", 34 | "from matplotlib import pyplot\n", 35 | "import numpy as np\n", 36 | "import os\n", 37 | "import shutil\n", 38 | "from IPython import display\n", 39 | "\n", 40 | "from caffe2.python import core, model_helper, net_drawer, workspace, visualize, brew\n", 41 | "\n", 42 | "# If you would like to see some really detailed initializations,\n", 43 | "# you can change --caffe2_log_level=0 to --caffe2_log_level=-1\n", 44 | "core.GlobalInit(['caffe2', '--caffe2_log_level=0'])\n" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "# This section preps your image and test set in a leveldb\n", 54 | "current_folder = os.getcwd()\n", 55 | "\n", 56 | "data_folder = os.path.join(current_folder, 'tutorial_data', 'mnist')\n", 57 | "root_folder = os.path.join(current_folder, 'tutorial_files', 'mnist')\n", 58 | "image_file_train = os.path.join(data_folder, \"train-images-idx3-ubyte\")\n", 59 | "label_file_train = os.path.join(data_folder, \"train-labels-idx1-ubyte\")\n", 60 | "image_file_test = os.path.join(data_folder, \"t10k-images-idx3-ubyte\")\n", 61 | "label_file_test = os.path.join(data_folder, \"t10k-labels-idx1-ubyte\")\n", 62 | "\n", 63 | "# Get the dataset if it is missing\n", 64 | "def DownloadDataset(url, path):\n", 65 | " import requests, zipfile, StringIO\n", 66 | " print \"Downloading... \", url, \" to \", path\n", 67 | " r = requests.get(url, stream=True)\n", 68 | " z = zipfile.ZipFile(StringIO.StringIO(r.content))\n", 69 | " z.extractall(path)\n", 70 | "\n", 71 | "def GenerateDB(image, label, name):\n", 72 | " name = os.path.join(data_folder, name)\n", 73 | " print 'DB: ', name\n", 74 | " if not os.path.exists(name):\n", 75 | " syscall = \"/usr/local/binaries/make_mnist_db --channel_first --db leveldb --image_file \" + image + \" --label_file \" + label + \" --output_file \" + name\n", 76 | " print \"Creating database with: \", syscall\n", 77 | " os.system(syscall)\n", 78 | " else:\n", 79 | " print \"Database exists already. Delete the folder if you have issues/corrupted DB, then rerun this.\"\n", 80 | " if os.path.exists(os.path.join(name, \"LOCK\")):\n", 81 | " print \"Deleting the pre-existing lock file\"\n", 82 | " os.remove(os.path.join(name, \"LOCK\"))\n", 83 | "\n", 84 | "if not os.path.exists(data_folder):\n", 85 | " os.makedirs(data_folder)\n", 86 | "if not os.path.exists(label_file_train):\n", 87 | " DownloadDataset(\"https://s3.amazonaws.com/caffe2/datasets/mnist/mnist.zip\", data_folder)\n", 88 | " \n", 89 | "if os.path.exists(root_folder):\n", 90 | " print(\"Looks like you ran this before, so we need to cleanup those old files...\")\n", 91 | " shutil.rmtree(root_folder)\n", 92 | " \n", 93 | "os.makedirs(root_folder)\n", 94 | "workspace.ResetWorkspace(root_folder)\n", 95 | "\n", 96 | "# (Re)generate the levledb database (known to get corrupted...) \n", 97 | "GenerateDB(image_file_train, label_file_train, \"mnist-train-nchw-leveldb\")\n", 98 | "GenerateDB(image_file_test, label_file_test, \"mnist-test-nchw-leveldb\")\n", 99 | "\n", 100 | " \n", 101 | "print(\"training data folder:\" + data_folder)\n", 102 | "print(\"workspace root folder:\" + root_folder)\n" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "def AddInput(model, batch_size, db, db_type):\n", 112 | " # load the data\n", 113 | " data_uint8, label = model.TensorProtosDBInput(\n", 114 | " [], [\"data_uint8\", \"label\"], batch_size=batch_size,\n", 115 | " db=db, db_type=db_type)\n", 116 | " # cast the data to float\n", 117 | " data = model.Cast(data_uint8, \"data\", to=core.DataType.FLOAT)\n", 118 | " # scale data from [0,255] down to [0,1]\n", 119 | " data = model.Scale(data, data, scale=float(1./256))\n", 120 | " # don't need the gradient for the backward pass\n", 121 | " data = model.StopGradient(data, data)\n", 122 | " return data, label" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [ 131 | "def AddLeNetModel(model, data):\n", 132 | " # Image size: 28 x 28 -> 24 x 24\n", 133 | " conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)\n", 134 | " # Image size: 24 x 24 -> 12 x 12\n", 135 | " pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)\n", 136 | " # Image size: 12 x 12 -> 8 x 8\n", 137 | " conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)\n", 138 | " # Image size: 8 x 8 -> 4 x 4\n", 139 | " pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)\n", 140 | " # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size\n", 141 | " fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)\n", 142 | " fc3 = brew.relu(model, fc3, fc3)\n", 143 | " pred = brew.fc(model, fc3, 'pred', 500, 10)\n", 144 | " softmax = brew.softmax(model, pred, 'softmax')\n", 145 | " return softmax" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "def AddAccuracy(model, softmax, label):\n", 155 | " accuracy = brew.accuracy(model, [softmax, label], \"accuracy\")\n", 156 | " return accuracy" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": null, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "def AddTrainingOperators(model, softmax, label):\n", 166 | " # something very important happens here\n", 167 | " xent = model.LabelCrossEntropy([softmax, label], 'xent')\n", 168 | " # compute the expected loss\n", 169 | " loss = model.AveragedLoss(xent, \"loss\")\n", 170 | " # track the accuracy of the model\n", 171 | " AddAccuracy(model, softmax, label)\n", 172 | " # use the average loss we just computed to add gradient operators to the model\n", 173 | " model.AddGradientOperators([loss])\n", 174 | " # do a simple stochastic gradient descent\n", 175 | " ITER = brew.iter(model, \"iter\")\n", 176 | " # set the learning rate schedule\n", 177 | " LR = model.LearningRate(\n", 178 | " ITER, \"LR\", base_lr=-0.1, policy=\"step\", stepsize=1, gamma=0.999 )\n", 179 | " # ONE is a constant value that is used in the gradient update. We only need\n", 180 | " # to create it once, so it is explicitly placed in param_init_net.\n", 181 | " ONE = model.param_init_net.ConstantFill([], \"ONE\", shape=[1], value=1.0)\n", 182 | " # Now, for each parameter, we do the gradient updates.\n", 183 | " for param in model.params:\n", 184 | " # Note how we get the gradient of each parameter - CNNModelHelper keeps\n", 185 | " # track of that.\n", 186 | " param_grad = model.param_to_grad[param]\n", 187 | " # The update is a simple weighted sum: param = param + param_grad * LR\n", 188 | " model.WeightedSum([param, ONE, param_grad, LR], param)\n", 189 | " # let's checkpoint every 20 iterations, which should probably be fine.\n", 190 | " # you may need to delete tutorial_files/tutorial-mnist to re-run the tutorial\n", 191 | " model.Checkpoint([ITER] + model.params, [],\n", 192 | " db=\"mnist_lenet_checkpoint_%05d.leveldb\",\n", 193 | " db_type=\"leveldb\", every=20)" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": null, 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "arg_scope = {\"order\": \"NCHW\"}\n", 203 | "train_model = model_helper.ModelHelper(name=\"mnist_train\", arg_scope=arg_scope)\n", 204 | "data, label = AddInput(\n", 205 | " train_model, batch_size=64,\n", 206 | " db=os.path.join(data_folder, 'mnist-train-nchw-leveldb'),\n", 207 | " db_type='leveldb')\n", 208 | "softmax = AddLeNetModel(train_model, data)\n", 209 | "AddTrainingOperators(train_model, softmax, label)\n", 210 | "\n", 211 | "# Testing model. We will set the batch size to 100, so that the testing\n", 212 | "# pass is 100 iterations (10,000 images in total).\n", 213 | "# For the testing model, we need the data input part, the main LeNetModel\n", 214 | "# part, and an accuracy part. Note that init_params is set False because\n", 215 | "# we will be using the parameters obtained from the train model.\n", 216 | "test_model = model_helper.ModelHelper(\n", 217 | " name=\"mnist_test\", arg_scope=arg_scope, init_params=False)\n", 218 | "data, label = AddInput(\n", 219 | " test_model, batch_size=100,\n", 220 | " db=os.path.join(data_folder, 'mnist-test-nchw-leveldb'),\n", 221 | " db_type='leveldb')\n", 222 | "softmax = AddLeNetModel(test_model, data)\n", 223 | "AddAccuracy(test_model, softmax, label)\n", 224 | "\n", 225 | "# Deployment model. We simply need the main LeNetModel part.\n", 226 | "deploy_model = model_helper.ModelHelper(\n", 227 | " name=\"mnist_deploy\", arg_scope=arg_scope, init_params=False)\n", 228 | "AddLeNetModel(deploy_model, \"data\")" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "graph = net_drawer.GetPydotGraphMinimal(\n", 238 | " train_model.net.Proto().op, \"mnist\", rankdir=\"LR\", minimal_dependency=True)\n", 239 | "display.Image(graph.create_png(), width=800)" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": null, 245 | "metadata": {}, 246 | "outputs": [], 247 | "source": [ 248 | "with open(os.path.join(root_folder, \"train_net.pbtxt\"), 'w') as fid:\n", 249 | " fid.write(str(train_model.net.Proto()))\n", 250 | "with open(os.path.join(root_folder, \"train_init_net.pbtxt\"), 'w') as fid:\n", 251 | " fid.write(str(train_model.param_init_net.Proto()))\n", 252 | "with open(os.path.join(root_folder, \"test_net.pbtxt\"), 'w') as fid:\n", 253 | " fid.write(str(test_model.net.Proto()))\n", 254 | "with open(os.path.join(root_folder, \"test_init_net.pbtxt\"), 'w') as fid:\n", 255 | " fid.write(str(test_model.param_init_net.Proto()))\n", 256 | "with open(os.path.join(root_folder, \"deploy_net.pbtxt\"), 'w') as fid:\n", 257 | " fid.write(str(deploy_model.net.Proto()))\n", 258 | "print(\"Protocol buffers files have been created in your root folder: \"+root_folder)" 259 | ] 260 | }, 261 | { 262 | "cell_type": "code", 263 | "execution_count": null, 264 | "metadata": {}, 265 | "outputs": [], 266 | "source": [ 267 | "# The parameter initialization network only needs to be run once.\n", 268 | "workspace.RunNetOnce(train_model.param_init_net)\n", 269 | "# creating the network\n", 270 | "workspace.CreateNet(train_model.net)\n", 271 | "# set the number of iterations and track the accuracy & loss\n", 272 | "total_iters = 200\n", 273 | "accuracy = np.zeros(total_iters)\n", 274 | "loss = np.zeros(total_iters)\n", 275 | "# Now, we will manually run the network for 200 iterations. \n", 276 | "for i in range(total_iters):\n", 277 | " workspace.RunNet(train_model.net.Proto().name)\n", 278 | " accuracy[i] = workspace.FetchBlob('accuracy')\n", 279 | " loss[i] = workspace.FetchBlob('loss')\n", 280 | "# After the execution is done, let's plot the values.\n", 281 | "pyplot.plot(loss, 'b')\n", 282 | "pyplot.plot(accuracy, 'r')\n", 283 | "pyplot.legend(('Loss', 'Accuracy'), loc='upper right')" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": null, 289 | "metadata": {}, 290 | "outputs": [], 291 | "source": [ 292 | "# Let's look at some of the data.\n", 293 | "pyplot.figure()\n", 294 | "data = workspace.FetchBlob('data')\n", 295 | "_ = visualize.NCHW.ShowMultiple(data)\n", 296 | "pyplot.figure()\n", 297 | "softmax = workspace.FetchBlob('softmax')\n", 298 | "_ = pyplot.plot(softmax[0], 'ro')\n", 299 | "pyplot.title('Prediction for the first image')" 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": null, 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "# run a test pass on the test net\n", 309 | "workspace.RunNetOnce(test_model.param_init_net)\n", 310 | "workspace.CreateNet(test_model.net)\n", 311 | "test_accuracy = np.zeros(100)\n", 312 | "for i in range(100):\n", 313 | " workspace.RunNet(test_model.net.Proto().name)\n", 314 | " test_accuracy[i] = workspace.FetchBlob('accuracy')\n", 315 | "# After the execution is done, let's plot the values.\n", 316 | "pyplot.plot(test_accuracy, 'r')\n", 317 | "pyplot.title('Acuracy over test batches.')\n", 318 | "print('test_accuracy: %f' % test_accuracy.mean())" 319 | ] 320 | }, 321 | { 322 | "cell_type": "code", 323 | "execution_count": null, 324 | "metadata": { 325 | "collapsed": true 326 | }, 327 | "outputs": [], 328 | "source": [] 329 | } 330 | ], 331 | "metadata": { 332 | "kernelspec": { 333 | "display_name": "Python 2", 334 | "language": "python", 335 | "name": "python2" 336 | }, 337 | "language_info": { 338 | "codemirror_mode": { 339 | "name": "ipython", 340 | "version": 2 341 | }, 342 | "file_extension": ".py", 343 | "mimetype": "text/x-python", 344 | "name": "python", 345 | "nbconvert_exporter": "python", 346 | "pygments_lexer": "ipython2", 347 | "version": "2.7.12" 348 | } 349 | }, 350 | "nbformat": 4, 351 | "nbformat_minor": 1 352 | } 353 | -------------------------------------------------------------------------------- /tutorial/MNIST_BNN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# MNIST-BNN\n", 8 | "\n", 9 | "In this tutorial, we will show you how to train an actual CNN model, albeit small. We will be using the good ol' MNIST dataset and the LeNet model, with a slight change that the sigmoid activations are replaced with ReLUs.\n", 10 | "\n", 11 | "We will use the model helper - that helps us to deal with parameter initializations naturally.\n", 12 | "\n", 13 | "First, let's import the necessities." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 27, 19 | "metadata": {}, 20 | "outputs": [ 21 | { 22 | "name": "stdout", 23 | "output_type": "stream", 24 | "text": [ 25 | "Necessities imported!\n" 26 | ] 27 | } 28 | ], 29 | "source": [ 30 | "%matplotlib inline\n", 31 | "from matplotlib import pyplot\n", 32 | "import numpy as np\n", 33 | "import os\n", 34 | "import shutil\n", 35 | "\n", 36 | "\n", 37 | "from caffe2.python import core, model_helper, net_drawer, workspace, visualize, brew\n", 38 | "\n", 39 | "# If you would like to see some really detailed initializations,\n", 40 | "# you can change --caffe2_log_level=0 to --caffe2_log_level=-1\n", 41 | "core.GlobalInit(['caffe2', '--caffe2_log_level=0'])\n", 42 | "print(\"Necessities imported!\")" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "We will track statistics during the training time and store these on disk in a local folder. We need to set up a data folder for the data and a root folder for the stats. You should already have these folders, and in the data folder the MNIST dataset should be setup as a leveldb database for both the training set and the test set for this tutorial. \n", 50 | "\n", 51 | "If these folders are missing then you will need to [download the MNIST dataset](Models_and_Datasets.ipynb), g/unzip the dataset and labels, then find the binaries in `/caffe2/build/caffe2/binaries/` or in `/usr/local/binaries/` and run the following, however the code block below will attempt to do this for you, so try that first.\n", 52 | "\n", 53 | "```\n", 54 | "./make_mnist_db --channel_first --db leveldb --image_file ~/Downloads/train-images-idx3-ubyte --label_file ~/Downloads/train-labels-idx1-ubyte --output_file ~/caffe2/caffe2/python/tutorials/tutorial_data/mnist/mnist-train-nchw-leveldb\n", 55 | "\n", 56 | "./make_mnist_db --channel_first --db leveldb --image_file ~/Downloads/t10k-images-idx3-ubyte --label_file ~/Downloads/t10k-labels-idx1-ubyte --output_file ~/caffe2/caffe2/python/tutorials/tutorial_data/mnist/mnist-test-nchw-leveldb\n", 57 | "```" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": 28, 63 | "metadata": {}, 64 | "outputs": [ 65 | { 66 | "name": "stdout", 67 | "output_type": "stream", 68 | "text": [ 69 | "/home/ko/caffe2_notebooks\n" 70 | ] 71 | } 72 | ], 73 | "source": [ 74 | "current_folder = os.path.join(os.path.expanduser('~'), 'caffe2_notebooks')\n", 75 | "print(current_folder)" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 29, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "Looks like you ran this before, so we need to cleanup those old files...\n", 88 | "DB: /home/ko/caffe2_notebooks/tutorial_data/mnist/mnist-train-nchw-leveldb\n", 89 | "Database exists already. Delete the folder if you have issues/corrupted DB, then return this.\n", 90 | "DB: /home/ko/caffe2_notebooks/tutorial_data/mnist/mnist-test-nchw-leveldb\n", 91 | "Database exists already. Delete the folder if you have issues/corrupted DB, then return this.\n", 92 | "training data folder:/home/ko/caffe2_notebooks/tutorial_data/mnist\n", 93 | "workspace root folder:/home/ko/caffe2_notebooks/tutorial_files/tutorial_mnist\n" 94 | ] 95 | } 96 | ], 97 | "source": [ 98 | "# This section preps your image and test set in a leveldb\n", 99 | "current_folder = os.path.join(os.path.expanduser('~'), 'caffe2_notebooks')\n", 100 | "\n", 101 | "data_folder = os.path.join(current_folder, 'tutorial_data', 'mnist')\n", 102 | "root_folder = os.path.join(current_folder, 'tutorial_files', 'tutorial_mnist')\n", 103 | "image_file_train = os.path.join(data_folder, \"train-images-idx3-ubyte\")\n", 104 | "label_file_train = os.path.join(data_folder, \"train-labels-idx1-ubyte\")\n", 105 | "image_file_test = os.path.join(data_folder, \"t10k-images-idx3-ubyte\")\n", 106 | "label_file_test = os.path.join(data_folder, \"t10k-labels-idx1-ubyte\")\n", 107 | "\n", 108 | "# Get the dataset if it is missing\n", 109 | "def DownloadDataset(url, path):\n", 110 | " import requests, zipfile, StringIO\n", 111 | " print \"Downloading... \", url, \" to \", path\n", 112 | " r = requests.get(url, stream=True)\n", 113 | " z = zipfile.ZipFile(StringIO.StringIO(r.content))\n", 114 | " z.extractall(path)\n", 115 | "\n", 116 | "def GenerateDB(image, label, name):\n", 117 | " name = os.path.join(data_folder, name)\n", 118 | " print 'DB: ', name\n", 119 | " if not os.path.exists(name):\n", 120 | " #syscall = \"/usr/local/binaries/make_mnist_db --channel_first --db leveldb --image_file \" + image + \" --label_file \" + label + \" --output_file \" + name\n", 121 | " syscall = \"/usr/local/bin/make_mnist_db --channel_first --db leveldb --image_file \" + image + \" --label_file \" + label + \" --output_file \" + name\n", 122 | " # print \"Creating database with: \", syscall\n", 123 | " os.system(syscall)\n", 124 | " else:\n", 125 | " print \"Database exists already. Delete the folder if you have issues/corrupted DB, then return this.\"\n", 126 | " if os.path.exists(os.path.join(name, \"LOCK\")):\n", 127 | " # print \"Deleting the pre-existing lock file\"\n", 128 | " os.remove(os.path.join(name, \"LOCK\"))\n", 129 | "\n", 130 | "if not os.path.exists(data_folder):\n", 131 | " os.makedirs(data_folder)\n", 132 | "if not os.path.exists(label_file_train):\n", 133 | " DownloadDataset(\"https://s3.amazonaws.com/caffe2/datasets/mnist/mnist.zip\", data_folder)\n", 134 | " \n", 135 | "if os.path.exists(root_folder):\n", 136 | " print(\"Looks like you ran this before, so we need to cleanup those old files...\")\n", 137 | " shutil.rmtree(root_folder)\n", 138 | " \n", 139 | "os.makedirs(root_folder)\n", 140 | "workspace.ResetWorkspace(root_folder)\n", 141 | "\n", 142 | "# (Re)generate the leveldb database (known to get corrupted...) \n", 143 | "GenerateDB(image_file_train, label_file_train, \"mnist-train-nchw-leveldb\")\n", 144 | "GenerateDB(image_file_test, label_file_test, \"mnist-test-nchw-leveldb\")\n", 145 | "\n", 146 | " \n", 147 | "print(\"training data folder:\" + data_folder)\n", 148 | "print(\"workspace root folder:\" + root_folder)" 149 | ] 150 | }, 151 | { 152 | "cell_type": "markdown", 153 | "metadata": {}, 154 | "source": [ 155 | "We will be using the `ModelHelper` class to represent our main model and using `brew` module and `Operators` to build our model. `brew` module has a set of wrapper functions that automatically separates the parameter intialization and the actual computation into two networks. Under the hood, a `ModelHelper` object has two underlying nets, `param_init_net` and `net`, that keeps record of the initialization network and the main network respectively.\n", 156 | "\n", 157 | "For the sake of modularity, we will separate the model to multiple different parts:\n", 158 | "\n", 159 | " (1) The data input part (AddInput function)\n", 160 | " (2) The main computation part (AddLeNetModel function)\n", 161 | " (3) The training part - adding gradient operators, update, etc. (AddTrainingOperators function)\n", 162 | " (4) The bookkeeping part, where we just print out statistics for inspection. (AddBookkeepingOperators function)\n", 163 | " \n", 164 | "`AddInput` will load the data from a DB. We store MNIST data in pixel values, so after batching this will give us data with shape `(batch_size, num_channels, width, height)`, in this case `[batch_size, 1, 28, 28]` of data type *uint8* and a label with shape `[batch_size]` of data type *int*.\n", 165 | " \n", 166 | "Since we are going to do float computations, we will cast the data to the *float* data type.\n", 167 | "For better numerical stability, instead of representing data in [0, 255] range, we will scale them down to [0, 1].\n", 168 | "Note that we are doing in-place computation for this operator: we don't need the pre-scale data.\n", 169 | "Now, when computing the backward pass, we will not need the gradient computation for the backward pass. `StopGradient` does exactly that: in the forward pass it does nothing and in the backward pass all it does is to tell the gradient generator \"the gradient does not need to pass through me\".\n", 170 | " " 171 | ] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": 30, 176 | "metadata": { 177 | "collapsed": true 178 | }, 179 | "outputs": [], 180 | "source": [ 181 | "def AddInput(model, batch_size, db, db_type):\n", 182 | " # load the data\n", 183 | " data_uint8, label = model.TensorProtosDBInput(\n", 184 | " [], [\"data_uint8\", \"label\"], batch_size=batch_size,\n", 185 | " db=db, db_type=db_type)\n", 186 | " # cast the data to float\n", 187 | " data = model.Cast(data_uint8, \"data\", to=core.DataType.FLOAT)\n", 188 | " # scale data from [0,255] down to [0,1]\n", 189 | " data = model.Scale(data, data, scale=float(1./256))\n", 190 | " # don't need the gradient for the backward pass\n", 191 | " data = model.StopGradient(data, data)\n", 192 | " return data, label" 193 | ] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "metadata": {}, 198 | "source": [ 199 | "At this point we need to take a look at the predictions coming out of the network at convert them into probabilities.\n", 200 | "\"What's the probability that this number we're looking at is a 5\", or \"is this a 7\", and so forth. \n", 201 | "\n", 202 | "The results will be conformed into a range between 0 and 1 such that the closer you are to 1 the more likely the number matches the prediction. The process that we can use to do this is available in LeNet and will provide us the *softmax* prediction. The `AddLeNetModel` function below will output the `softmax`. However, in this case, it does much more than the softmax - it is the computed model with its convoluted layers, as well as the softmax.\n", 203 | "\n", 204 | "[An explanation of kernels in image processing](https://en.wikipedia.org/wiki/Kernel_%28image_processing%29) might be useful for more info on why `kernel=5` is used in the convolutional layers below. `dim_in` is the number of input channels and `dim_out` is the number of output channels. \n", 205 | "\n", 206 | "As you can see below `conv1` has 1 channel coming in (`dim_in`) and 20 going out (`dim_out`), whereas `conv2` has 20 going in and 50 going out and `fc3` has 50 going in and 500 going out. The images are transformed to smaller sizes along each convolution.d\n", 207 | "\n", 208 | "TODO: include image of the model below" 209 | ] 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "## BinaryNet Function " 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 31, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "# The neurons' activations binarization function\n", 225 | "# It behaves like the sign function during forward propagation\n", 226 | "# And like:\n", 227 | "# hard_tanh(x) = 2*hard_sigmoid(x)-1 \n", 228 | "# during back propagation\n", 229 | "\n", 230 | "#def hard_sigmoid(X):\n", 231 | " # return net.Clip( ['X'], ['X'], min= 0., max = 1.)\n", 232 | "#clip((x+1.)/2.,0,1)\n", 233 | "\n", 234 | "def binary_tanh_unit(x):\n", 235 | " return 2.*round(hard_sigmoid(x))-1.\n", 236 | "\n", 237 | "def binary_sigmoid_unit(x):\n", 238 | " return round(hard_sigmoid(x))\n", 239 | "\n", 240 | "#X = np.random.randn(2, 3).astype(np.float32)\n", 241 | "#print(\"Generated X from numpy:\\n{}\".format(X))\n", 242 | "#workspace.FeedBlob(\"X\", X)\n", 243 | "\n", 244 | "#X = hard_sigmoid(X)\n", 245 | "#workspace.RunNetOnce(net)\n", 246 | "#print('X after clip...')\n", 247 | "#print(workspace.FetchBlob('X'))" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "metadata": {}, 253 | "source": [ 254 | "## Custom Helper Function " 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 32, 260 | "metadata": { 261 | "collapsed": true 262 | }, 263 | "outputs": [], 264 | "source": [ 265 | "def binary_fc(model, blob_in, blob_out, dim_in, dim_out, binary, **kwargs):\n", 266 | " \"\"\"\n", 267 | " fc with binarization.\n", 268 | " \"\"\"\n", 269 | "\n", 270 | "#brew.Register(binary_fc)\n", 271 | "#brew.binary_fc(model, blob_in, blob_out, dim_in, dim_out, binary)" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "metadata": {}, 277 | "source": [ 278 | "## Test clip op " 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 33, 284 | "metadata": {}, 285 | "outputs": [ 286 | { 287 | "name": "stdout", 288 | "output_type": "stream", 289 | "text": [ 290 | "---------------\n", 291 | "Current network proto:\n", 292 | "\n", 293 | "name: \"my_first_net_2\"\n", 294 | "\n", 295 | "Generated X from numpy:\n", 296 | "[[ 0.76915532 0.77813929 0.47178778]\n", 297 | " [-0.72258306 0.49754134 -0.10331368]]\n", 298 | "Current blobs in the workspace: [u'X']\n", 299 | "Workspace has blob 'X'? True\n", 300 | "Fetched X:\n", 301 | "[[ 0.76915532 0.77813929 0.47178778]\n", 302 | " [-0.72258306 0.49754134 -0.10331368]]\n", 303 | "X after clip...\n", 304 | "[[ 0.76915532 0.77813929 0.47178778]\n", 305 | " [ 0. 0.49754134 0. ]]\n" 306 | ] 307 | } 308 | ], 309 | "source": [ 310 | "\n", 311 | "print('---------------')\n", 312 | "net = core.Net(\"my_first_net\")\n", 313 | "print(\"Current network proto:\\n\\n{}\".format(net.Proto()))\n", 314 | "\n", 315 | "X = np.random.randn(2, 3).astype(np.float32)\n", 316 | "print(\"Generated X from numpy:\\n{}\".format(X))\n", 317 | "workspace.FeedBlob(\"X\", X)\n", 318 | "\n", 319 | "print(\"Current blobs in the workspace: {}\".format(workspace.Blobs()))\n", 320 | "print(\"Workspace has blob 'X'? {}\".format(workspace.HasBlob(\"X\")))\n", 321 | "print(\"Fetched X:\\n{}\".format(workspace.FetchBlob(\"X\")))\n", 322 | "\n", 323 | "X = net.Clip( ['X'], ['X'], min= 0., max = 1.)\n", 324 | "workspace.RunNetOnce(net)\n", 325 | "print('X after clip...')\n", 326 | "print(workspace.FetchBlob('X'))\n", 327 | "\n", 328 | "\n", 329 | "#X = net.Clip( [\"X\"], [\"X\"], min= 0., max = 1.)\n", 330 | "#print(\"New network proto:\\n\\n{}\".format(net.Proto()))\n", 331 | "\n", 332 | "#print(\"Type of X is: {}\".format(type(X)))\n", 333 | "#print(\"The blob name is: {}\".format(str(X)))\n", 334 | "#print(\"Current network proto:\\n\\n{}\".format(net.Proto()))\n", 335 | "\n", 336 | "#workspace.ResetWorkspace()\n", 337 | "#print(\"Current blobs in the workspace: {}\".format(workspace.Blobs()))\n", 338 | "#workspace.CreateNet(net)\n", 339 | "#workspace.RunNet(net.Proto().name)\n", 340 | "#print(\"Blobs in the workspace after execution: {}\".format(workspace.Blobs()))\n", 341 | "\n", 342 | "#W = net.GaussianFill([], [\"W\"], mean=0.0, std=1.0, shape=[5, 3], run_once=0)\n", 343 | "#b = net.ConstantFill([], [\"b\"], shape=[5,], value=1.0, run_once=0)\n", 344 | "\n", 345 | "#op = core.CreateOperator(\n", 346 | "# \"Clip\",\n", 347 | "# 0, # minimum value, under which element is replaced by min\n", 348 | "# 1, \n", 349 | "# [\"X\"],\n", 350 | "# [\"Y\"],\n", 351 | "#)\n", 352 | "\n", 353 | "#print(\"Type of the created op is: {}\".format(type(op)))\n", 354 | "#print(\"Content:\\n\")\n", 355 | "#print(str(op))\n" 356 | ] 357 | }, 358 | { 359 | "cell_type": "code", 360 | "execution_count": 34, 361 | "metadata": {}, 362 | "outputs": [ 363 | { 364 | "data": { 365 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWUAAAA7CAYAAACjdpuIAAAABmJLR0QA/wD/AP+gvaeTAAASuUlE\nQVR4nO3deVzU9b7H8dew75uKYKLiltsBjgqhGZo67oC0YKXick0NTfN2hDilebFIPJ0UxVy61Um8\nCdiJAxp21OOSZip4kuWUG+GGoqmxKLLO3D9wRghGB2RmfgPf5+PB4yG/38zv+5l5jB/e8/1tMqVS\nqUQQBEGQBBNDFyAIgiA8IJqyIAiChIimLAiCICFm+hikoqKCb7/9lsrKSn0MV0/nzp0ZMmSI3seV\nqvKaKvZe/pEapcLQpQhaGNihB13sOhi6DMm4fPkyx44dM8jY/v7+eHh46HwcnTflzMxMZs2aRW5u\nrq6H0mj69OnExcXh7OxssBqkYu/lH5m+f42hyxC09Hz3oXw64nVDl2FwSqWSTZs2ERkZSWlpqUFq\nsLe3JzY2lvnz5yOTyXQ2js6mL6qqqoiNjWXo0KE4OTlx9uxZlEql3n/S09M5cOAAffv2JSUlRVcv\n12hUi4RsVMQ3Grhw4QJyuZxFixYRHh5OeXm53vtIZWUlb7/9NosXLyYgIIBz587p7PXqpClnZWXh\n5+dHdHQ0K1eu5NChQ/Tq1UsXQz3S+PHjycnJITg4mOeee47Q0FBu3bplkFoEQdCeUqlky5YteHl5\nUVhYyNGjR1m1ahWWlpZ6r8Xc3JzIyEgyMjK4c+cOPj4+xMbGolC0/B/NFm3KqnTs6+uLnZ0dp06d\nIjIyEhMTw+5PdHJyYvPmzaSnp/PDDz/Qv39/kZoFQcJU6XjBggWEh4dz8uRJfH19DV0W3t7enDhx\nguXLl7Ns2TKGDx/e4qm5xbqllNKxJiI1C4K0SSkda6Lr1PzYTVmq6VgTkZoFQZqkmo410VVqfqzO\naQzpWBORmgVBGowhHWuii9TcrKZsbOlYE5GaBcGwjC0da9KSqbnJXdSY07EmIjULgn4ZczrWpKVS\ns9ZNubWkY01EahYE/Wgt6ViTx03NWnXU1piONRGpWRB0ozWmY00eJzU/tCm39nSsiUjNgtCyWns6\n1qQ5qVljd21L6VgTkZoF4fG0pXSsSVNTc4OmXF1dzdtvv83gwYNxcXEhNze3TaRjTVSpOTU1lSNH\njuDl5UV6erqhyxIEybt06RKjR49mwYIFLFy4sM2kY028vb05duwYixcv5p133kEul3Pp0qUGj2vQ\naXft2kVMTAxxcXHs27cPT09PvRQsdUFBQeTm5uLn50dISIihyxEEyYuKiiIvL4+jR48SExPTptKx\nJpaWlsTExHD06FHy8vKIiopq8JgGTVl1zePw8HCdXp7OGLm4uPDyyy8b5LrQgmBsqqqq8PPza9Pp\nWBNfX1/8/PyoqqpqsK5tzkkIOuVq7UiIpz9/8hbfKAShqSTVlA8dOkRoaCgymQyZTMagQYNISEhQ\nr9+/fz/jxo1DJpMRFBREcnKyAattW2TICPH0J0keweHJq0gZ+2cSRy/lwyGzWeIVzPt+0wB40ukJ\nInye5/NnFzOl57B62/hX4Hus9J3aonX1dHTn9T9MatFtquye+C7juwwCYExnH/ZOilavkyFjeu9n\n+WLkGywbNIX1w+byYo+n1evNTEyJ9n2FTrYuOqlN0MzY+4hebgelreHDhxMQEIClpSXbtm3DwsKC\nadOmqdePHDmS+Ph4IiIiWLVqlZhe0ZP2Vg787dnFPGHXjlcPxnPy1zyUKDGRyXix+9Os8p/BNxcz\nAThTVMA7JxKY01feYDsX79ygvKbh17XmGubej5lPjiL8u40tts26PO3dyC+5Xvtvh47kl15Xr4vw\neY5pvUfwTOpbFFXcxcnSlsPBq2hn5cCm/+ymWlHD2uw01g2byzsntnGh9IZOahQaMvY+IqmkDCCT\nydiyZQsDBw7k2LFjbN++Xb0uMTERBwcHSb6R+lZTU4O7uzsjRozgs88+o6ioSCfjmMhkfDn6Twxw\n6cqotHfI/PU8SpQAKJRKkvKOMH3/GmzMH+zE0dR4Zx9Yx/v/bplU0sepM5sDwon44XMqFdUtss26\nbMws6WjjqG6mnvYPmrKHXXuW+oTw+Zl9FFXcBaCo4i5fnNnPu4Om4GJpB8Dtijus+vHvJI5eiq25\nVYvXqC/5+fk4OTkREhLC119/TXl5uaFLeiRj7iOSa8oA1tbWfPXVV9jZ2fH6669z9epVMjIy+Pjj\nj9m4caMk30h9q66uprCwkO+++45XX32VDh06EBgYSHJyMvfu3WuxcQK7+uHn2os12ancrrjT6GOO\nXPuJf+Tr72aWJjIZW4YvYNu5gxprelyeDh25evc25TW1O3W7OXQkv6S2QYf2GIaZiSmHrta/7+R3\n13KxNrMk7MmR6mW5ty+SX3q9xadt9OnmzZsUFxezc+dOXnjhBdq1a8eMGTPYs2cPNTU1hi5PI2Pt\nI5KavqjL09OTtWvXMmfOHF566SVKS0tJS0vD2tra0KVJiuoeYgqFgt27d5Oeno6ZmRmBgYHMmDGD\nsWPHYmFh0eztB3bzA+DQtYff+DbtwgmN60xlJgR182Osxx/pau/KhG+i8XXtSXC3pwjq9hRjdr3L\nh0Nm8bRbXwrLfiPmxx0P3d74LoPwateNP/3wWb3lDhY2LPUOoUapwMLUjL5OHvxcdJnVp76muKJM\nqzHn9hvLav+Z6m0WzX6QsMZ5DGRTwGv8+2YeAAV3b9cb/8rd2hOLBrh0rbf8XwVZxPrPZH3OrnpT\nIMZG1YDLysr48ssv2bp1Kw4ODgQHBxMWFsaoUaMk1+iMsY9IMimrzJ49mwkTJnD48GFGjx6tl9t7\nG7OamhoUCgWVlZWkpaURFBREu3btCAsLY9++fc26xmsPBzcAfilpfjOpUSrI+PU8L/UMoL2VAyYy\nGQ4WNvxXXzkedu1ZOGAC63N38cbR/8XNxpmtI5fg3/FJjdt7znMIAD/e/EW9zM7cmgNB73O3uoIV\nmdv58/EE5h6KZ6zHQA4FxeBsaavVmJ+d3ofbF2H85VQKn57ei9sXYTyRMIsqRQ19EsNx+yIMSxNz\nAPXUhcpv93/vateh3vITN85hKjMhpLt/s99Dqamurp0yKikpISkpCblcjru7O4sXL+bIkSMGrq4+\nY+sjkk3KKi4uLlhZWREXF8fUqVPx8fExdEkA7Nixw6DjN3Z8Y2Pr79y5Q2JiIgkJCbi6ujJ0shxF\nrzJMXGy0Gkd1N2UbMwtKKsuaXW/BnQenp9coFey7kkXB3dv0cHBjReZ2qhS1KczO3Ir4YfOY228s\nx66faXRbfq69KaksUz8HYIlXED0c3PjbmX3qZb+Wl/CXUylsCniNN7yCWJ7x5aPHPHCGamro7dSJ\nAwU5lNdU0duuA7fKSyks+w2AkirV+6CsV5dSWfu7hWn9/1Y37hUDMKRjHyC1Se/b5e9z2HHTsJ+1\nvLy8h65XHbd//fp1Nm7cyLp16+jduzcWFhb07NlTHyU+klT7SGMk3ZTXrl2LlZUVCQkJvPjii0yd\nOpXMzExJfPUIDQ01dAlaUzXoGzdu8I8t/4flpH5YhXhp9dwzRVcY3KEnvR2foLCs+TsTlb9rYACK\n+w2/bnPddyULeJDQG+Nq7cj1e/VrUSXr0qr6O6GOFv4M1DZybcb8duIK2lnZ42HXAT/XXiwYMAFb\nMyucLW3JeP6v7LqYwdmiqwzp2AdHC1vK69ThZGkLwLX7zVul+P4fM1drR42vSZMjq7ezv/TTJj/P\nUFSftbNnzwJgZ2dnyHIAafeRxkh2+mLPnj2kpKQQHx/PCy+8wCuvvMJPP/1ERESEoUsDHszlGupH\n2z3gqvnk9u3bs2jRIt5P3oTVZO0aMsCR+03N11U/F6NSNbCCu5ov/KRQKjCVmTRYBtDld1MHqpRa\nUqV552fdMcd9s4Jn/hGFhYkZg776b3z//ibbzh3k09N78f37m/xPZiI/F10BwM3Gud523GycABok\nfFWCltH0+daXU2MM/lk7cULz/H5dqs9ajx49ePfddxk3bpzBpwqk3kcaI8mmfPbsWRYuXEhSUpL6\nfPn169fj7OxMfHw8u3fvNnCF0mZubo5MJsPW1pbQ0FDS0tK4du0acXFx9B3sTVN6Q/L5I2Tdymd+\n//ENmpCKlak5r/Qa3iK1qxLywauadywW3ivC0aL+9Mv3hacBGOvxx3rLO9u1q91eQY7WY3Z36Mi1\nstuUVVfc/92NX0oK1Y9POn+YksoynnHvV287Ae4DqFLUsCPv+3rLVQm68J5uDls0JFUj7tChA/Pn\nz+fw4cOcP3+eFStWYG9vb9DajLWPSK4pFxQUMGbMGJYuXYq7u7t6uYuLC5GRkQDMnDnzkfNcbY2p\nqSkmJiaYm5sjl8tJSkri9u3bJCQkEBgYiJlZ82aqapQK5h7aQEV1Jf+cuILAbn6YmZgCYG1mSYB7\nf5LHRHKu+Kr6OdZmtf8BrEwfHPVhZ177VdHevOFctk/72oteyZDx+h8mkXUrn89P72vwOJXvr/2M\nnbm1epsAcTk7Of3bFeb1G6tOrABz+o7h+I2zfPLzP7Ues6eje73X092hY70dnb9V3OGj7FRm9Rld\n53VZM/PJkXx4KqVBym9nVducjl0/rfE1GRNz89odnXZ2dkyZMoW9e/dSWFhIXFwcw4YNe8Sz9cOY\n+4ik5pQ/+eQTVq9ezcWLF8nOziYrKwtvb28AMjIyuHjxIlA7NzpixAjefPNN3njjDUOWLAmmpqbI\n5XKmTZtGcHBwi8/jnSkqwD8lgjl95Uzv/Szv+02jrKqCamUN/7z8IzP3r1UfL9zN3pXX+k8Aak+y\neK3/eFIvHOfVvmOA2q/4CwZMZOuZ/ertT+rqy/x+47E0NedmeQkT0qPrzfn+3vbz3zGt9wj8XHux\nvyAbgHvVFYzetZwInxA2PhPOT79dokap4HZ5KUG732uwvYeNWduUr6kf292+flIGiMveya3yUv46\ndDZX7tykp6M7cTk72XrmQIN6/V2fRKFU8vUvP2j9nkuVtbU1ISEhTJ06Fblcrm7QUmLsfUSmVE14\n3ZecnMyUKVP43WLhPim9P0uWLKFPnz7qA/q1kZJ/jFkH4nRcmXYynv8rvRw74fTZy01+7o4xkeQV\nX+Ot41v1NmZzJcqXcuNeMYuObGnycyd7+vO3ZxfroCrtFRcXExERwciRIwkMDMTGRrsjd1Q7w6V2\nbQmp0PT+SCopC02zZs0aQ5dgMAsOb+LbiStYk53W4EgMKfFz7UVPB3dePRhv6FKazdHRkc2bNxu6\njDZDcnPKQttha1Z7PQhny6ZPt9y4V8z0/WuIeWo6NmbaXzz9ccZsKncbF970DiH42/cpfcjRH4JQ\nl2jKgt7ZmluxfNBL6starvafyVP3jyNuiv/cvsTKk0nq+Wp9jKktcxNTXur5DHMOrn/o4X2C8Hsa\npy+USqXkzmOXAinMJRu7u1XlRJ9MJPpk4mNv60LpDeJydup1TG1UKWpYk920s/dao+ac2t9WKBSK\nRu992mBJly5dMDMzY9SoUeTn5+ulOGORlpbGkiVL6NSpk6FLEQTJ6969O6mpqURFRVFRUWHociSj\noqKCqKgoUlNTG+0lDZqyv78/J0+epKioiAEDBjz0VthtRVFREfPmzSM4OJhhw4aRnZ1t6JIEQfI+\n+OADNmzYwIYNGxg4cKDWZwa2ZllZWfj7+7Nu3Tree+89PvroowaPaXRO2cvLi+PHj7N8+XKWL19O\nQEAA586d03nBUpSens6AAQNIS0sjJSWF5ORkrQ8/E4S2TCaTMXfuXHJycujUqRNPP/00b731VptM\nzVVVVcTGxuLr64u9vT1ZWVlERkZqN32hYm5uTmRkJBkZGZSVleHj49OmUrMqHU+cOJGhQ4eSm5vL\n5MmTDV2WIBidrl27smfPHjZs2MDHH3/c5lJzVlYWfn5+REdHs3LlSg4ePPjQq+c98uiLtpiaRToW\nhJbVFlNzU9JxXVodEtdWUrNIx4KgW20lNTc1HdfVpOOUW3NqFulYEPSjNafm5qbjupp88khrS80i\nHQuCYbS21Pw46biuZp/R1xpSs0jHgmBYrSE1t0Q6ruuxTrM21tQs0rEgSIuxpuaWSsd1tci1L4wp\nNYt0LAjSZEypuaXTcV0tdkEiqadmkY4FwThIPTXrIh3X1eJXiZNiahbpWBCMixRTsy7TcV0N7jzS\nkjIzM5k1axYXLlxg2bJleHp66moojdLS0ti2bRthYWGsXbsWZ+fGb/7ZVkjpziPCo0nhziOGplQq\n2bx5MxEREXh4eBAVFaW+Eaq+VFRU8MEHH3D58mVWr17NvHnzdHYVTZ3eeWTw4MFkZmYSHR3NsmXL\nqKys1OVwjercuTM7d+5k0qRJeh9bitxsnDA1MaFGItNKwsN1snExdAkGJ5PJmD9/PuPGjWPu3LlM\nnz7dIHXI5XK++eYbunXrptNxdJqUBUEQhKYRdx4RBEGQENGUBUEQJEQ0ZUEQBAkxA3YYughBEASh\n1v8D55L6HPhbticAAAAASUVORK5CYII=\n", 366 | "text/plain": [ 367 | "" 368 | ] 369 | }, 370 | "execution_count": 34, 371 | "metadata": { 372 | "image/png": { 373 | "width": 800 374 | } 375 | }, 376 | "output_type": "execute_result" 377 | } 378 | ], 379 | "source": [ 380 | "from caffe2.python import net_drawer\n", 381 | "from IPython import display\n", 382 | "graph = net_drawer.GetPydotGraph(net, rankdir=\"LR\")\n", 383 | "display.Image(graph.create_png(), width=800)" 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": 35, 389 | "metadata": {}, 390 | "outputs": [], 391 | "source": [ 392 | "def AddMLP(model, data):\n", 393 | " '''\n", 394 | " This part is the standard LeNet model: from data to the softmax prediction.\n", 395 | " \n", 396 | " For each convolutional layer we specify dim_in - number of input channels\n", 397 | " and dim_out - number or output channels. Also each Conv and MaxPool layer changes the\n", 398 | " image size. For example, kernel of size 5 reduces each side of an image by 4.\n", 399 | "\n", 400 | " While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides\n", 401 | " each side in half.\n", 402 | " '''\n", 403 | " ## ------------------ ko --------------\n", 404 | " # ko modified. goal: porting theano version BNN to caffe2 version\n", 405 | " # Image size: 28 x 28 -> 28 x 28\n", 406 | " #drop1 = model.dropout(data, 'drop1', ratio=0.5, is_test=0)\n", 407 | " # Image size: 28 x 28 -> 28 x 28\n", 408 | " #fc2 = model.FC(drop1, 'fc2', )\n", 409 | " #bn2 = model.spatial_bn(fc2,'bn2',) \n", 410 | " #nonlinear2 =\n", 411 | " #drop2=\n", 412 | " # NxCxHxW: 1x1x28x28 -> 1x1x28x28\n", 413 | " #drop1 = brew.dropout(model, data, 'drop1', ratio=0.5, is_test=0)\n", 414 | " # NxCxHxW: 1x1x28x28 -> 20x1x28x28\n", 415 | " #fc2 = brew.fc(model, drop1, 'fc2', dim_in=1 * 28 * 28, dim_out=20)\n", 416 | " #bn2 = brew.spatial_bn(model, fc2, 'bn2',dim_in = 20) \n", 417 | " \n", 418 | " \n", 419 | " num_units = 4096\n", 420 | " drop1 = brew.dropout(model, data, 'drop1', ratio=0.5, is_test=0)\n", 421 | " # NxCxHxW: 1x1x28x28 -> 4096x1x28x28\n", 422 | " fc2 = brew.fc(model, drop1, 'fc2', dim_in=1 * 28 * 28, dim_out=num_units)\n", 423 | " \n", 424 | " print(\"Current blobs in the workspace: {}\".format(workspace.Blobs()))\n", 425 | " \n", 426 | " # NxCxHxW: 1x1x28x28 -> 4096x1x28x28\n", 427 | " bn2 = brew.spatial_bn(model, fc2, 'bn2', num_units, epsilon=1e-3,\n", 428 | " momentum=0.1,\n", 429 | " is_test=is_test\n", 430 | " )\n", 431 | " \n", 432 | " relu2 = brew.relu(model, bn2, 'relu2')\n", 433 | " fc3 = brew.fc(model, relu2,'fc3', dim_in=num_units, dim_out=num_units)\n", 434 | " bn3 = brew.spatial_bn(model, fc3, 'bn3',dim_in = num_units)\n", 435 | " relu3 = brew.relu(model, bn3,'relu3')\n", 436 | " fc4 = brew.fc(model, relu3, 'fc4', dim_in=num_units, dim_out=num_units)\n", 437 | " bn4 = brew.spatial_bn(model, fc4, 'bn4',dim_in = num_units)\n", 438 | " relu4 = brew.relu(model, bn4, 'relu4') \n", 439 | " fc5 = brew.fc(model, relu4, 'fc5', dim_in=num_units, dim_out=10)\n", 440 | " bn5 = brew.spatial_bn(model, fc2, 'bn5',dim_in = 10)\n", 441 | " softmax = brew.softmax(model, bn5, 'softmax')\n", 442 | " return softmax\n", 443 | " \n", 444 | " \n", 445 | " " 446 | ] 447 | }, 448 | { 449 | "cell_type": "markdown", 450 | "metadata": {}, 451 | "source": [ 452 | "The `AddAccuracy` function below adds an accuracy operator to the model. We will use this in the next function to keep track of the model's accuracy." 453 | ] 454 | }, 455 | { 456 | "cell_type": "code", 457 | "execution_count": 36, 458 | "metadata": { 459 | "collapsed": true 460 | }, 461 | "outputs": [], 462 | "source": [ 463 | "def AddAccuracy(model, softmax, label):\n", 464 | " \"\"\"Adds an accuracy op to the model\"\"\"\n", 465 | " accuracy = brew.accuracy(model, [softmax, label], \"accuracy\")\n", 466 | " return accuracy" 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "The next function, `AddTrainingOperators`, adds training operators to the model. \n", 474 | "\n", 475 | "In the first step, we apply an Operator, `LabelCrossEntropy`, that computes the cross entropy between the input and the label set. This operator is almost always used after getting a softmax and before computing the model's loss. It's going to take in the `[softmax, label]` array along with a label, `'xent'` for \"Cross Entropy\".\n", 476 | "\n", 477 | " xent = model.LabelCrossEntropy([softmax, label], 'xent')\n", 478 | " \n", 479 | "`AveragedLoss` will take in the cross entropy and return the average of the losses found in the cross entropy.\n", 480 | "\n", 481 | " loss = model.AveragedLoss(xent, \"loss\")\n", 482 | "\n", 483 | "For bookkeeping purposes, we will also compute the accuracy of the model by invoking the AddAccuracy function like so:\n", 484 | "\n", 485 | " AddAccuracy(model, softmax, label)\n", 486 | "\n", 487 | "The next line is the key part of the training model: we add all the gradient operators to the model. The gradient is computed with respect to the loss that we computed above.\n", 488 | "\n", 489 | " model.AddGradientOperators([loss])\n", 490 | " " 491 | ] 492 | }, 493 | { 494 | "cell_type": "markdown", 495 | "metadata": {}, 496 | "source": [ 497 | "The next handful of lines support a very simple stochastic gradient descent. \n", 498 | "--- TODO(jiayq): We are working on wrapping these SGD operations in a cleaner fashion, and we will update this when it is ready. For now, you can see how we basically express the SGD algorithms with basic operators. \n", 499 | "It isn't necessary to fully understand this part at the moment, but we'll walk you through the process anyway.\n", 500 | "\n", 501 | "`Iter` operator is a counter for the number of iterations we run in the training. We use `brew.iter` helper function to add it to model\n", 502 | "\n", 503 | " ITER = brew.iter(model, \"iter\")\n", 504 | "\n", 505 | "We do a simple learning rate schedule where lr = base_lr * (t ^ gamma)\n", 506 | "Note that we are doing minimization, so the base_lr is negative so we are going the DOWNHILL direction. \n", 507 | "\n", 508 | " LR = model.LearningRate(\n", 509 | " ITER, \"LR\", base_lr=-0.1, policy=\"step\", stepsize=1, gamma=0.999 )\n", 510 | "ONE is a constant value that is used in the gradient update. We only need to create it once, so it is explicitly placed in param_init_net.\n", 511 | "\n", 512 | " ONE = model.param_init_net.ConstantFill([], \"ONE\", shape=[1], value=1.0)\n", 513 | "\n", 514 | "Now, for each parameter, we do the gradient updates. Note how we get the gradient of each parameter - `ModelHelper` keeps track of that. The update is a simple weighted sum: param = param + param_grad * LR\n", 515 | "\n", 516 | " for param in model.params:\n", 517 | " param_grad = model.param_to_grad[param]\n", 518 | " model.WeightedSum([param, ONE, param_grad, LR], param) \n", 519 | " \n", 520 | "We will need to checkpoint the parameters of the model periodically. This is achieved via the Checkpoint operator. It also takes in a parameter \"every\" so that we don't checkpoint way too often. In this case, we will say let's checkpoint every 20 iterations, which should probably be fine.\n", 521 | "\n", 522 | " model.Checkpoint([ITER] + model.params, [],\n", 523 | " db=\"mnist_lenet_checkpoint_%05d.leveldb\",\n", 524 | " db_type=\"leveldb\", every=20)" 525 | ] 526 | }, 527 | { 528 | "cell_type": "code", 529 | "execution_count": 37, 530 | "metadata": { 531 | "collapsed": true 532 | }, 533 | "outputs": [], 534 | "source": [ 535 | "def AddTrainingOperators(model, softmax, label):\n", 536 | " \"\"\"Adds training operators to the model.\"\"\"\n", 537 | " xent = model.LabelCrossEntropy([softmax, label], 'xent')\n", 538 | " # compute the expected loss\n", 539 | " loss = model.AveragedLoss(xent, \"loss\")\n", 540 | " # track the accuracy of the model\n", 541 | " AddAccuracy(model, softmax, label)\n", 542 | " # use the average loss we just computed to add gradient operators to the model\n", 543 | " model.AddGradientOperators([loss])\n", 544 | " # do a simple stochastic gradient descent\n", 545 | " ITER = brew.iter(model, \"iter\")\n", 546 | " # set the learning rate schedule\n", 547 | " LR = model.LearningRate(\n", 548 | " ITER, \"LR\", base_lr=-0.1, policy=\"step\", stepsize=1, gamma=0.999 )\n", 549 | " # ONE is a constant value that is used in the gradient update. We only need\n", 550 | " # to create it once, so it is explicitly placed in param_init_net.\n", 551 | " ONE = model.param_init_net.ConstantFill([], \"ONE\", shape=[1], value=1.0)\n", 552 | " # Now, for each parameter, we do the gradient updates.\n", 553 | " for param in model.params:\n", 554 | " # Note how we get the gradient of each parameter - ModelHelper keeps\n", 555 | " # track of that.\n", 556 | " param_grad = model.param_to_grad[param]\n", 557 | " # The update is a simple weighted sum: param = param + param_grad * LR\n", 558 | " model.WeightedSum([param, ONE, param_grad, LR], param)" 559 | ] 560 | }, 561 | { 562 | "cell_type": "markdown", 563 | "metadata": {}, 564 | "source": [ 565 | "The following function, `AddBookkeepingOperations`, adds a few bookkeeping operators that we can inspect later. These operators do not affect the training procedure: they only collect statistics and prints them to file or to logs." 566 | ] 567 | }, 568 | { 569 | "cell_type": "code", 570 | "execution_count": 38, 571 | "metadata": { 572 | "collapsed": true 573 | }, 574 | "outputs": [], 575 | "source": [ 576 | "def AddBookkeepingOperators(model):\n", 577 | " \"\"\"This adds a few bookkeeping operators that we can inspect later.\n", 578 | " \n", 579 | " These operators do not affect the training procedure: they only collect\n", 580 | " statistics and prints them to file or to logs.\n", 581 | " \"\"\" \n", 582 | " # Print basically prints out the content of the blob. to_file=1 routes the\n", 583 | " # printed output to a file. The file is going to be stored under\n", 584 | " # root_folder/[blob name]\n", 585 | " model.Print('accuracy', [], to_file=1)\n", 586 | " model.Print('loss', [], to_file=1)\n", 587 | " # Summarizes the parameters. Different from Print, Summarize gives some\n", 588 | " # statistics of the parameter, such as mean, std, min and max.\n", 589 | " for param in model.params:\n", 590 | " model.Summarize(param, [], to_file=1)\n", 591 | " model.Summarize(model.param_to_grad[param], [], to_file=1)\n", 592 | " # Now, if we really want to be verbose, we can summarize EVERY blob\n", 593 | " # that the model produces; it is probably not a good idea, because that\n", 594 | " # is going to take time - summarization do not come for free. For this\n", 595 | " # demo, we will only show how to summarize the parameters and their\n", 596 | " # gradients." 597 | ] 598 | }, 599 | { 600 | "cell_type": "markdown", 601 | "metadata": {}, 602 | "source": [ 603 | "Now, let's actually create the models for training and testing. If you are seeing WARNING messages below, don't be alarmed. The functions we established earlier are now going to be executed. Remember the four steps that we're doing:\n", 604 | "\n", 605 | " (1) data input \n", 606 | " (2) main computation\n", 607 | " (3) training \n", 608 | " (4) bookkeeping\n", 609 | " \n", 610 | "Before we can do the data input though we need to define our training model. We will basically need every piece of the components we defined above. In this example, we're using NCHW storage order on the mnist_train dataset. " 611 | ] 612 | }, 613 | { 614 | "cell_type": "code", 615 | "execution_count": 39, 616 | "metadata": {}, 617 | "outputs": [ 618 | { 619 | "name": "stdout", 620 | "output_type": "stream", 621 | "text": [ 622 | "Current blobs in the workspace: [u'X']\n" 623 | ] 624 | }, 625 | { 626 | "ename": "KeyError", 627 | "evalue": "BlobReference(\"bn2_s\")", 628 | "output_type": "error", 629 | "traceback": [ 630 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 631 | "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", 632 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;31m#softmax = AddLeNetModel(train_model, data)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0msoftmax\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAddMLP\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mAddTrainingOperators\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msoftmax\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0mAddBookkeepingOperators\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 633 | "\u001b[0;32m\u001b[0m in \u001b[0;36mAddTrainingOperators\u001b[0;34m(model, softmax, label)\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;31m# Note how we get the gradient of each parameter - ModelHelper keeps\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;31m# track of that.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0mparam_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_to_grad\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 23\u001b[0m \u001b[0;31m# The update is a simple weighted sum: param = param + param_grad * LR\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWeightedSum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mONE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_grad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLR\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 634 | "\u001b[0;31mKeyError\u001b[0m: BlobReference(\"bn2_s\")" 635 | ] 636 | } 637 | ], 638 | "source": [ 639 | "arg_scope = {\"order\": \"NCHW\"}\n", 640 | "train_model = model_helper.ModelHelper(name=\"mnist_train\", arg_scope=arg_scope)\n", 641 | "is_test = False\n", 642 | "data, label = AddInput(\n", 643 | " train_model, batch_size=64,\n", 644 | " db=os.path.join(data_folder, 'mnist-train-nchw-leveldb'),\n", 645 | " db_type='leveldb')\n", 646 | "#softmax = AddLeNetModel(train_model, data)\n", 647 | "softmax = AddMLP(train_model, data)\n", 648 | "AddTrainingOperators(train_model, softmax, label)\n", 649 | "AddBookkeepingOperators(train_model)\n", 650 | "\n", 651 | "# Testing model. We will set the batch size to 100, so that the testing\n", 652 | "# pass is 100 iterations (10,000 images in total).\n", 653 | "# For the testing model, we need the data input part, the main LeNetModel\n", 654 | "# part, and an accuracy part. Note that init_params is set False because\n", 655 | "# we will be using the parameters obtained from the train model.\n", 656 | "test_model = model_helper.ModelHelper(\n", 657 | " name=\"mnist_test\", arg_scope=arg_scope, init_params=False)\n", 658 | "data, label = AddInput(\n", 659 | " test_model, batch_size=100,\n", 660 | " db=os.path.join(data_folder, 'mnist-test-nchw-leveldb'),\n", 661 | " db_type='leveldb')\n", 662 | "softmax = AddLeNetModel(test_model, data)\n", 663 | "\n", 664 | "AddAccuracy(test_model, softmax, label)\n", 665 | "\n", 666 | "# Deployment model. We simply need the main LeNetModel part.\n", 667 | "deploy_model = model_helper.ModelHelper(\n", 668 | " name=\"mnist_deploy\", arg_scope=arg_scope, init_params=False)\n", 669 | "AddLeNetModel(deploy_model, \"data\")\n", 670 | "# You may wonder what happens with the param_init_net part of the deploy_model.\n", 671 | "# No, we will not use them, since during deployment time we will not randomly\n", 672 | "# initialize the parameters, but load the parameters from the db.\n" 673 | ] 674 | }, 675 | { 676 | "cell_type": "markdown", 677 | "metadata": { 678 | "collapsed": true 679 | }, 680 | "source": [ 681 | "Now, let's take a look what the training and deploy models look like, using the simple graph visualization tool that Caffe2 has. If the following command fails for you, it might be because that the machine you run on does not have graphviz installed. You can usually install that by:\n", 682 | "\n", 683 | "```sudo yum install graphviz```\n", 684 | "\n", 685 | "If the graph looks too small, right click and open the image in a new tab for better inspection." 686 | ] 687 | }, 688 | { 689 | "cell_type": "code", 690 | "execution_count": null, 691 | "metadata": { 692 | "scrolled": true 693 | }, 694 | "outputs": [], 695 | "source": [ 696 | "from IPython import display\n", 697 | "graph = net_drawer.GetPydotGraph(train_model.net.Proto().op, \"mnist\", rankdir=\"LR\")\n", 698 | "display.Image(graph.create_png(), width=800)" 699 | ] 700 | }, 701 | { 702 | "cell_type": "markdown", 703 | "metadata": {}, 704 | "source": [ 705 | "Now, the graph above shows everything that is happening in the training phase: the white nodes are the blobs, and the green rectangular nodes are the operators being run. You may have noticed the massive parallel lines like train tracks: these are dependencies from the blobs generated in the forward pass to their backward operators.\n", 706 | "\n", 707 | "Let's display the graph in a more minimal way by showing only the necessary dependencies and only showing the operators. If you read carefully, you can see that the left half of the graph is the forward pass, the right half of the graph is the backward pass, and on the very right there are a set of parameter update and summarization operators." 708 | ] 709 | }, 710 | { 711 | "cell_type": "code", 712 | "execution_count": null, 713 | "metadata": {}, 714 | "outputs": [], 715 | "source": [ 716 | "graph = net_drawer.GetPydotGraphMinimal(\n", 717 | " train_model.net.Proto().op, \"mnist\", rankdir=\"LR\", minimal_dependency=True)\n", 718 | "display.Image(graph.create_png(), width=800)" 719 | ] 720 | }, 721 | { 722 | "cell_type": "markdown", 723 | "metadata": {}, 724 | "source": [ 725 | "Now, when we run the network, one way is to directly run it from Python. Remember as we are running the network, we can periodically pull blobs from the network - Let's first show how we do this.\n", 726 | "\n", 727 | "Before, that, let's re-iterate the fact that, the ModelHelper class has not executed anything yet. All it does is to *declare* the network, which is basically creating the protocol buffers. For example, we will show a portion of the serialized protocol buffer for the training models' param init net." 728 | ] 729 | }, 730 | { 731 | "cell_type": "code", 732 | "execution_count": null, 733 | "metadata": {}, 734 | "outputs": [], 735 | "source": [ 736 | "print(str(train_model.param_init_net.Proto())[:400] + '\\n...')" 737 | ] 738 | }, 739 | { 740 | "cell_type": "markdown", 741 | "metadata": {}, 742 | "source": [ 743 | "We will also dump all the protocol buffers to disk so you can easily inspect them. As you may have noticed, these protocol buffers are much like the old good caffe's network definitions." 744 | ] 745 | }, 746 | { 747 | "cell_type": "code", 748 | "execution_count": null, 749 | "metadata": {}, 750 | "outputs": [], 751 | "source": [ 752 | "with open(os.path.join(root_folder, \"train_net.pbtxt\"), 'w') as fid:\n", 753 | " fid.write(str(train_model.net.Proto()))\n", 754 | "with open(os.path.join(root_folder, \"train_init_net.pbtxt\"), 'w') as fid:\n", 755 | " fid.write(str(train_model.param_init_net.Proto()))\n", 756 | "with open(os.path.join(root_folder, \"test_net.pbtxt\"), 'w') as fid:\n", 757 | " fid.write(str(test_model.net.Proto()))\n", 758 | "with open(os.path.join(root_folder, \"test_init_net.pbtxt\"), 'w') as fid:\n", 759 | " fid.write(str(test_model.param_init_net.Proto()))\n", 760 | "with open(os.path.join(root_folder, \"deploy_net.pbtxt\"), 'w') as fid:\n", 761 | " fid.write(str(deploy_model.net.Proto()))\n", 762 | "print(\"Protocol buffers files have been created in your root folder: \" + root_folder)" 763 | ] 764 | }, 765 | { 766 | "cell_type": "markdown", 767 | "metadata": {}, 768 | "source": [ 769 | "Next we will run the training procedure. We will drive all the computation in Python here, however you can also write a plan out to disk so that you can completely train stuff in C++. We'll leave discussion on that route for another tutorial.\n", 770 | "\n", 771 | "Please note that this process will take a while to run. Keep an eye on the asterisk (In [\\*]) or other IPython indicators that the code block is still running.\n", 772 | "\n", 773 | "First we must initialize the network with:\n", 774 | "\n", 775 | " workspace.RunNetOnce(train_model.param_init_net)\n", 776 | " \n", 777 | "Since we are going to run the main network multiple times, we first create the network which puts the actual network generated from the protobuf into the workspace.\n", 778 | "\n", 779 | " workspace.CreateNet(train_model.net)\n", 780 | "\n", 781 | "We will set the number of iterations that we'll run the network to 200 and create two numpy arrays to record the accuracy and loss for each iteration.\n", 782 | "\n", 783 | " total_iters = 200\n", 784 | " accuracy = np.zeros(total_iters)\n", 785 | " loss = np.zeros(total_iters)\n", 786 | "\n", 787 | "With the network and tracking of accuracy and loss setup we can now loop the 200 interations calling `workspace.RunNet` and passing the name of the network `train_model.net.Proto().name`. On each iteration we calculate the accuracy and loss with `workspace.FetchBlob('accuracy')` and `workspace.FetchBlob('loss')`.\n", 788 | "\n", 789 | " for i in range(total_iters):\n", 790 | " workspace.RunNet(train_model.net.Proto().name)\n", 791 | " accuracy[i] = workspace.FetchBlob('accuracy')\n", 792 | " loss[i] = workspace.FetchBlob('loss')\n", 793 | "\n", 794 | "Finally, we can plot the results using `pyplot`." 795 | ] 796 | }, 797 | { 798 | "cell_type": "code", 799 | "execution_count": null, 800 | "metadata": { 801 | "scrolled": true 802 | }, 803 | "outputs": [], 804 | "source": [ 805 | "# The parameter initialization network only needs to be run once.\n", 806 | "workspace.RunNetOnce(train_model.param_init_net)\n", 807 | "# creating the network\n", 808 | "workspace.CreateNet(train_model.net, overwrite=True)\n", 809 | "# set the number of iterations and track the accuracy & loss\n", 810 | "total_iters = 200\n", 811 | "accuracy = np.zeros(total_iters)\n", 812 | "loss = np.zeros(total_iters)\n", 813 | "# Now, we will manually run the network for 200 iterations. \n", 814 | "for i in range(total_iters):\n", 815 | " workspace.RunNet(train_model.net)\n", 816 | " accuracy[i] = workspace.FetchBlob('accuracy')\n", 817 | " loss[i] = workspace.FetchBlob('loss')\n", 818 | "# After the execution is done, let's plot the values.\n", 819 | "pyplot.plot(loss, 'b')\n", 820 | "pyplot.plot(accuracy, 'r')\n", 821 | "pyplot.legend(('Loss', 'Accuracy'), loc='upper right')" 822 | ] 823 | }, 824 | { 825 | "cell_type": "markdown", 826 | "metadata": {}, 827 | "source": [ 828 | "Now we can sample some of the data and predictions. " 829 | ] 830 | }, 831 | { 832 | "cell_type": "code", 833 | "execution_count": null, 834 | "metadata": { 835 | "collapsed": true 836 | }, 837 | "outputs": [], 838 | "source": [ 839 | "# Let's look at some of the data.\n", 840 | "pyplot.figure()\n", 841 | "data = workspace.FetchBlob('data')\n", 842 | "_ = visualize.NCHW.ShowMultiple(data)\n", 843 | "pyplot.figure()\n", 844 | "softmax = workspace.FetchBlob('softmax')\n", 845 | "_ = pyplot.plot(softmax[0], 'ro')\n", 846 | "pyplot.title('Prediction for the first image')" 847 | ] 848 | }, 849 | { 850 | "cell_type": "code", 851 | "execution_count": null, 852 | "metadata": { 853 | "collapsed": true 854 | }, 855 | "outputs": [], 856 | "source": [ 857 | "# Convolutions for this mini-batch\n", 858 | "pyplot.figure()\n", 859 | "conv = workspace.FetchBlob('conv1')\n", 860 | "shape = list(conv.shape)\n", 861 | "shape[1] = 1\n", 862 | "# We can look into any channel. This of it as a feature model learned\n", 863 | "conv = conv[:,15,:,:].reshape(shape)\n", 864 | "\n", 865 | "_ = visualize.NCHW.ShowMultiple(conv)" 866 | ] 867 | }, 868 | { 869 | "cell_type": "markdown", 870 | "metadata": {}, 871 | "source": [ 872 | "Remember that we created the test net? We will run the test pass and report the test accuracy here. Note that although test_model will be using the parameters obtained from train_model, test_model.param_init_net must still be run to initialize the input data.\n", 873 | "In this run, we only need to track the accuracy and we're also only going to run 100 iterations." 874 | ] 875 | }, 876 | { 877 | "cell_type": "code", 878 | "execution_count": null, 879 | "metadata": { 880 | "collapsed": true 881 | }, 882 | "outputs": [], 883 | "source": [ 884 | "# run a test pass on the test net\n", 885 | "workspace.RunNetOnce(test_model.param_init_net)\n", 886 | "workspace.CreateNet(test_model.net, overwrite=True)\n", 887 | "test_accuracy = np.zeros(100)\n", 888 | "for i in range(100):\n", 889 | " workspace.RunNet(test_model.net.Proto().name)\n", 890 | " test_accuracy[i] = workspace.FetchBlob('accuracy')\n", 891 | "# After the execution is done, let's plot the values.\n", 892 | "pyplot.plot(test_accuracy, 'r')\n", 893 | "pyplot.title('Acuracy over test batches.')\n", 894 | "print('test_accuracy: %f' % test_accuracy.mean())" 895 | ] 896 | }, 897 | { 898 | "cell_type": "code", 899 | "execution_count": null, 900 | "metadata": { 901 | "collapsed": true 902 | }, 903 | "outputs": [], 904 | "source": [ 905 | "print(\"Current blobs in the workspace: {}\".format(workspace.Blobs()))" 906 | ] 907 | }, 908 | { 909 | "cell_type": "code", 910 | "execution_count": null, 911 | "metadata": { 912 | "collapsed": true 913 | }, 914 | "outputs": [], 915 | "source": [ 916 | "print(str(train_model.param_init_net.Proto())[:400] + '\\n...')" 917 | ] 918 | }, 919 | { 920 | "cell_type": "markdown", 921 | "metadata": {}, 922 | "source": [ 923 | "This concludes the MNIST tutorial. We hope this tutorial highlighted some of Caffe2's features and how easy it is to create a simple CNN." 924 | ] 925 | } 926 | ], 927 | "metadata": { 928 | "kernelspec": { 929 | "display_name": "Python 2", 930 | "language": "python", 931 | "name": "python2" 932 | }, 933 | "language_info": { 934 | "codemirror_mode": { 935 | "name": "ipython", 936 | "version": 2 937 | }, 938 | "file_extension": ".py", 939 | "mimetype": "text/x-python", 940 | "name": "python", 941 | "nbconvert_exporter": "python", 942 | "pygments_lexer": "ipython2", 943 | "version": "2.7.12" 944 | } 945 | }, 946 | "nbformat": 4, 947 | "nbformat_minor": 1 948 | } 949 | --------------------------------------------------------------------------------