├── .gitignore ├── README.md ├── c ├── ebnn.h └── tests │ ├── minunit.h │ ├── onelayer.c │ ├── onelayer.h │ ├── onelayer_compare.h │ ├── twolayer.c │ ├── twolayer.h │ ├── twolayer_compare.h │ ├── unit.c │ └── unit_compare.h ├── chainer_ext ├── __init__.py ├── functions.py └── weight_clip.py ├── chainer_sequential ├── .gitignore ├── README.md ├── __init__.py ├── binary │ ├── __init__.py │ ├── functions │ │ ├── __init__.py │ │ ├── function_binary_convolution_2d.py │ │ ├── function_binary_linear.py │ │ └── function_bst.py │ ├── links │ │ ├── __init__.py │ │ ├── link_batch_normalization.py │ │ ├── link_binary_conv_BN_BST.py │ │ ├── link_binary_conv_pool_BN_BST.py │ │ ├── link_binary_convolution.py │ │ ├── link_binary_linear.py │ │ ├── link_binary_linear_BN_BST.py │ │ ├── link_binary_linear_BN_softmax_layer.py │ │ ├── link_binary_linear_softmax_layer.py │ │ ├── link_bst.py │ │ ├── link_conv_BN_BST.py │ │ ├── link_conv_pool_BN_BST.py │ │ ├── link_linear_BN_BST.py │ │ ├── link_pool.py │ │ └── link_softmax_cross_entropy.py │ └── utils │ │ ├── __init__.py │ │ └── binary_util.py ├── binary_function.py ├── binary_link.py ├── chain.py ├── function.py ├── link.py ├── multiinputchain.py ├── multiinputsequential.py ├── sequential.py ├── test.py ├── util.py └── weightnorm │ ├── __init__.py │ ├── convolution_2d.py │ ├── deconvolution_2d.py │ └── linear.py ├── datasets ├── __init__.py ├── datasets.py ├── mnist.py └── util │ ├── mvmc.py │ └── process_mvmc.py ├── deepopt ├── __init__.py ├── chooser.py ├── deepopt.py └── trainer.py ├── elaas ├── __init__.py ├── elaas.py └── family │ ├── __init__.py │ ├── binary.py │ ├── binary_base_fixed.py │ ├── binary_cloud.py │ ├── binary_float.py │ ├── ebnn.py │ ├── float.py │ ├── float_float.py │ ├── multi_input.py │ ├── multi_input_edge.py │ ├── multi_input_edge_with_dropout.py │ ├── simple.py │ └── single_input.py ├── examples ├── binary_hybrid_analysis.py ├── comparison.py ├── dropout.py ├── multiview.py ├── simple.py ├── test.py ├── test_binary.py ├── test_binary_fixed.py ├── test_binary_float.py ├── test_float_float.py ├── test_multiinput.py ├── test_multiinput_edge.py └── test_multiinput_edge_with_dropout.py ├── exp ├── exp_combining.py ├── exp_combining_avg_pool.py ├── exp_combining_avg_pool_concat.py ├── exp_combining_concat.py ├── exp_combining_max_pool.py ├── exp_combining_max_pool_concat.py ├── exp_comm.py ├── exp_increasing.py ├── exp_individual.py ├── exp_individual_single.py ├── exp_mem.py ├── exp_model_size.py ├── exp_threshold.py ├── exp_weights.py ├── run_all.sh ├── run_combining.sh ├── run_increasing.sh ├── run_individual.sh ├── run_individual_single.sh ├── run_missing.py └── run_threshod.sh ├── experiments ├── mnist_dropout.py └── mnist_dropout_nofire.py ├── plots.ipynb ├── requirements.txt ├── setup.py ├── tests ├── conv.py ├── linear.py └── slice.py └── visualize ├── __init__.py ├── data ├── add_cam.csv ├── comm.csv ├── entropy.csv ├── mem.csv └── missing_cam.csv ├── generate_figures.py └── visualize.py /.gitignore: -------------------------------------------------------------------------------- 1 | _* 2 | !__init__.py 3 | *.ipynb 4 | .ipynb_checkpoints 5 | ebnn 6 | *.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | For a more recent implementation please take a look at the pytorch branch. 2 | 3 | # Distributed Deep Neural Networks over the Cloud, the Edge and End Devices 4 | 5 | We propose distributed deep neural networks (DDNNs) over distributed computing hierarchies, consisting of the cloud, the edge (fog) and end devices. While being able to accommodate inference of a deep neural network (DNN) in the cloud, a DDNN also allows fast and localized inference using shallow portions of the neural network at the edge and end devices. Due to its distributed nature, DDNNs enhance data privacy and system fault tolerance for DNN applications. When supported by a scalable distributed computing hierarchy, a DDNN can scale up in neural network size and scale out in geographical span. In implementing a DDNN, we map sections of a DNN onto a distributed computing hierarchy. By jointly training these sections, we minimize communication and resource usage for devices and maximize usefulness of extracted features which are utilized in the cloud. As a proof of concept, we show a DDNN can exploit geographical diversity of sensors to improve recognition accuracy and reduce communication cost. In our experiment, compared with the traditional method of offloading raw sensor data to be processed in the cloud, DDNN locally processes most sensor data on end devices and is able to reduce the communication cost by a factor of over 20x. 6 | 7 | ## Dependencies 8 | 9 | This library is dependent on Python 2.7+ and [Chainer](http://chainer.org/). Please install Chainer 1.17+ before starting. 10 | 11 | ``` 12 | pip install chainer 13 | ``` 14 | 15 | ## Quick Start 16 | 17 | ## Paper 18 | 19 | Our paper is available [here]() 20 | 21 | If you use this model or codebase, please cite: 22 | ```bibtex 23 | @article{teerapittayanondistributed, 24 | title={Distributed Deep Neural Networks over the Cloud, the Edge and End Devices}, 25 | author={Teerapittayanon, Surat and McDanel, Bradley and Kung, HT}, 26 | jornal={Proceedings of the 37th IEEE International Conference on Distributed Computing Systems}, 27 | year={2017} 28 | } 29 | ``` 30 | 31 | ## License 32 | 33 | Copyright (c) 2017 Bradley McDanel, Surat Teerapittayanon, HT Kung, Harvard University 34 | 35 | Permission is hereby granted, free of charge, to any person obtaining a copy 36 | of this software and associated documentation files (the "Software"), to deal 37 | in the Software without restriction, including without limitation the rights 38 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 39 | copies of the Software, and to permit persons to whom the Software is 40 | furnished to do so, subject to the following conditions: 41 | 42 | The above copyright notice and this permission notice shall be included in all 43 | copies or substantial portions of the Software. 44 | 45 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 46 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 47 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 48 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 49 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 50 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 51 | SOFTWARE. 52 | -------------------------------------------------------------------------------- /c/tests/minunit.h: -------------------------------------------------------------------------------- 1 | #define mu_assert(message, test) do { if (!(test)) return message; } while (0) 2 | #define mu_run_test(test) do { char *message = test(); tests_run++; if (message) return message; } while (0) 3 | extern int tests_run; 4 | -------------------------------------------------------------------------------- /c/tests/onelayer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "minunit.h" 3 | #include "onelayer.h" 4 | #include "onelayer_compare.h" 5 | 6 | int tests_run = 0; 7 | char output_buf[1000]; 8 | int errors = 0; 9 | 10 | static char* test_l1() { 11 | char output_msg[] = "Output Mismatch: Layer 1.\nComputed[%d]=%d, Actual[%d]=%d\n"; 12 | int i; 13 | 14 | l_conv_pool_bn_bst0(x_in, temp1); 15 | 16 | for (i = 0; i < 196; ++i) { 17 | sprintf(output_buf, output_msg, i, temp1[i], i, inter1[i]); 18 | mu_assert(output_buf, temp1[i] == inter1[i]); 19 | } 20 | 21 | return 0; 22 | } 23 | 24 | static char* test_l2() { 25 | char output_msg[] = "Output Mismatch: Layer 2.\nComputed: %d, Actual: %d\n"; 26 | uint8_t output[1]; 27 | 28 | blinear_layer(inter1, l_b_linear_bn_softmax1_bl_W, output, 29 | l_b_linear_bn_softmax1_bl_b, l_b_linear_bn_softmax1_bn_gamma, 30 | l_b_linear_bn_softmax1_bn_beta, l_b_linear_bn_softmax1_bn_mean, 31 | l_b_linear_bn_softmax1_bn_std, 1, 1568, 10); 32 | 33 | sprintf(output_buf, output_msg, output[0], 7); 34 | mu_assert(output_buf, output[0] == 7); 35 | 36 | return 0; 37 | } 38 | 39 | static char* test_endtoend() { 40 | char output_msg[] = "Output Mismatch: Layer 3.\nComputed: %d, Actual: %d\n"; 41 | uint8_t output[1]; 42 | 43 | ebnn_compute(x_in, output); 44 | 45 | sprintf(output_buf, output_msg, output[0], 7); 46 | mu_assert(output_buf, output[0] == 7); 47 | 48 | return 0; 49 | } 50 | 51 | static char* all_tests() { 52 | mu_run_test(test_l1); 53 | mu_run_test(test_l2); 54 | mu_run_test(test_endtoend); 55 | return 0; 56 | } 57 | 58 | int main () { 59 | char *result = all_tests(); 60 | if (result != 0) { 61 | printf("%s\n", result); 62 | } 63 | else { 64 | printf("ALL TESTS PASSED\n"); 65 | } 66 | printf("Tests run: %d\n", tests_run); 67 | 68 | return 0; 69 | } 70 | -------------------------------------------------------------------------------- /c/tests/onelayer_compare.h: -------------------------------------------------------------------------------- 1 | float x_in[784] = {0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.011764707,0.070588239,0.070588239,0.070588239,0.49411768,0.53333336,0.68627453,0.10196079,0.65098041,1.0,0.96862751,0.49803925,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.11764707,0.14117648,0.36862746,0.60392159,0.66666669,0.99215692,0.99215692,0.99215692,0.99215692,0.99215692,0.88235301,0.67450982,0.99215692,0.94901967,0.76470596,0.25098041,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.19215688,0.9333334,0.99215692,0.99215692,0.99215692,0.99215692,0.99215692,0.99215692,0.99215692,0.99215692,0.98431379,0.36470589,0.32156864,0.32156864,0.21960786,0.15294118,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.070588239,0.8588236,0.99215692,0.99215692,0.99215692,0.99215692,0.99215692,0.77647066,0.71372551,0.96862751,0.9450981,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.3137255,0.61176473,0.41960788,0.99215692,0.99215692,0.80392164,0.043137256,0.0,0.16862746,0.60392159,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.054901965,0.0039215689,0.60392159,0.99215692,0.35294119,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.54509807,0.99215692,0.74509805,0.0078431377,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.043137256,0.74509805,0.99215692,0.27450982,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.13725491,0.9450981,0.88235301,0.627451,0.42352945,0.0039215689,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.31764707,0.94117653,0.99215692,0.99215692,0.4666667,0.098039225,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.17647059,0.72941178,0.99215692,0.99215692,0.58823532,0.10588236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.062745102,0.36470589,0.98823535,0.99215692,0.73333335,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.97647065,0.99215692,0.97647065,0.25098041,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.18039216,0.50980395,0.71764708,0.99215692,0.99215692,0.81176478,0.0078431377,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.15294118,0.58039218,0.89803928,0.99215692,0.99215692,0.99215692,0.98039222,0.71372551,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.094117656,0.44705886,0.86666673,0.99215692,0.99215692,0.99215692,0.99215692,0.78823537,0.30588236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.090196081,0.25882354,0.83529419,0.99215692,0.99215692,0.99215692,0.99215692,0.77647066,0.31764707,0.0078431377,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.070588239,0.67058825,0.8588236,0.99215692,0.99215692,0.99215692,0.99215692,0.76470596,0.3137255,0.035294119,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.21568629,0.67450982,0.88627458,0.99215692,0.99215692,0.99215692,0.99215692,0.95686281,0.52156866,0.043137256,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.53333336,0.99215692,0.99215692,0.99215692,0.83137262,0.52941179,0.51764709,0.062745102,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0}; 2 | uint8_t inter1[196] = {0,0,0,0,0,0,0,0,0,0,0,0,63,252,0,31,255,192,63,255,252,3,255,255,192,63,255,252,3,255,255,128,63,255,224,3,255,240,0,31,255,0,0,127,224,0,7,255,128,0,63,252,0,1,255,192,0,15,252,0,0,127,192,0,3,252,0,14,255,192,7,255,252,1,255,255,192,31,255,240,1,255,252,0,63,255,0,3,255,224,0,63,248,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,223,255,252,3,255,252,3,255,255,192,127,255,255,127,255,255,247,255,255,255,255,255,255,255,255,255,255,247,255,255,255,63,255,255,253,255,255,255,207,255,255,252,255,255,255,159,255,255,243,255,255,252,127,255,255,31,255,255,199,255,255,241,255,255,253,255,255,255,255,255,255,255,255,255,255,255,255,255}; 3 | float y_out[10] = {-0.47949642,-0.38772577,-0.19543217,0.56020212,-0.19370732,-1.4620155,-0.0064702593,0.93141794,-0.35823631,-1.3799701}; 4 | -------------------------------------------------------------------------------- /c/tests/twolayer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "minunit.h" 3 | #include "twolayer.h" 4 | #include "twolayer_compare.h" 5 | 6 | int tests_run = 0; 7 | char output_buf[1000]; 8 | int errors = 0; 9 | 10 | static char* test_l1() { 11 | char output_msg[] = "Output Mismatch: Layer 1.\nComputed[%d]=%d, Actual[%d]=%d\n"; 12 | int i; 13 | 14 | l_conv_pool_bn_bst0(x_in, temp1); 15 | 16 | for (i = 0; i < 196; ++i) { 17 | sprintf(output_buf, output_msg, i, temp1[i], i, inter1[i]); 18 | mu_assert(output_buf, temp1[i] == inter1[i]); 19 | } 20 | 21 | return 0; 22 | } 23 | 24 | static char* test_l2() { 25 | char output_msg[] = "Output Mismatch: Layer 2.\nComputed[%d]=%d, Actual[%d]=%d\n"; 26 | int i; 27 | 28 | l_b_conv_pool_bn_bst1(inter1, temp1); 29 | 30 | for (i = 0; i < 196; ++i) { 31 | sprintf(output_buf, output_msg, i, temp1[i], i, inter2[i]); 32 | mu_assert(output_buf, temp1[i] == inter2[i]); 33 | } 34 | 35 | return 0; 36 | } 37 | 38 | static char* test_l3() { 39 | char output_msg[] = "Output Mismatch: Layer 3.\nComputed: %d, Actual: %d\n"; 40 | uint8_t output[1]; 41 | 42 | blinear_layer(inter2, l_b_linear_bn_softmax2_bl_W, output, 43 | l_b_linear_bn_softmax2_bl_b, l_b_linear_bn_softmax2_bn_gamma, 44 | l_b_linear_bn_softmax2_bn_beta, l_b_linear_bn_softmax2_bn_mean, 45 | l_b_linear_bn_softmax2_bn_std, 1, 1568, 10); 46 | 47 | sprintf(output_buf, output_msg, output[0], 7); 48 | mu_assert(output_buf, output[0] == 7); 49 | 50 | return 0; 51 | } 52 | 53 | static char* test_endtoend() { 54 | char output_msg[] = "Output Mismatch: Layer 3.\nComputed: %d, Actual: %d\n"; 55 | uint8_t output[1]; 56 | 57 | ebnn_compute(x_in, output); 58 | 59 | sprintf(output_buf, output_msg, output[0], 7); 60 | mu_assert(output_buf, output[0] == 7); 61 | 62 | return 0; 63 | } 64 | 65 | static char* all_tests() { 66 | mu_run_test(test_l1); 67 | mu_run_test(test_l2); 68 | mu_run_test(test_l3); 69 | mu_run_test(test_endtoend); 70 | return 0; 71 | } 72 | 73 | int main () { 74 | char *result = all_tests(); 75 | if (result != 0) { 76 | printf("%s\n", result); 77 | } 78 | else { 79 | printf("ALL TESTS PASSED\n"); 80 | } 81 | printf("Tests run: %d\n", tests_run); 82 | 83 | return 0; 84 | } 85 | -------------------------------------------------------------------------------- /chainer_ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/chainer_ext/__init__.py -------------------------------------------------------------------------------- /chainer_ext/functions.py: -------------------------------------------------------------------------------- 1 | from chainer import function,cuda 2 | import chainer.functions as F 3 | class Entropy(function.Function): 4 | def forward(self, x): 5 | xp = cuda.get_array_module(*x) 6 | y = x[0] * xp.log(x[0]+1e-9) 7 | return -xp.sum(y,1), 8 | 9 | def backward(self, x, gy): 10 | return gy, 11 | 12 | def entropy(x): 13 | return Entropy()(x) -------------------------------------------------------------------------------- /chainer_ext/weight_clip.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | from chainer import cuda 3 | 4 | class WeightClip(object): 5 | 6 | """Optimizer hook function for weight clip manipulation. 7 | This hook function clips a parameter to [low, high]. 8 | It can be used in a binary weight network. 9 | Args: 10 | low (float): low value for the weight clip. 11 | high (float): high value for the weight clip. 12 | Attributes: 13 | low (float): low value for the weight clip. 14 | high (float): low value for the weight clip. 15 | """ 16 | name = 'WeightClip' 17 | 18 | def __init__(self, low=-1.0, high=1.0): 19 | self.low=low 20 | self.high=high 21 | 22 | def __call__(self, opt): 23 | if cuda.available: 24 | kernel = cuda.elementwise( 25 | 'T low, T high', 26 | 'T p', 27 | 'p = (p < low) ? low : (p > high) ? high : p', 28 | 'weight_clip') 29 | 30 | for link in opt.target.links(): 31 | # only apply to binary layers 32 | if getattr(link,'cname',False): 33 | for param in link.params(): 34 | p = param.data 35 | with cuda.get_device(p) as dev: 36 | if int(dev) == -1: 37 | numpy.clip(p, self.low, self.high) 38 | else: 39 | kernel(self.low, self.high, p) -------------------------------------------------------------------------------- /chainer_sequential/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /chainer_sequential/README.md: -------------------------------------------------------------------------------- 1 | ChainerをKerasっぽく書きたいという思いから作りました。 2 | 3 | ただしKerasを使ったことがないのであくまでそれっぽいというだけです。 4 | 5 | 内部的にはChainerのLinkやFunctionをユーザーが定義したとおりの順に並べて実行します。 6 | 7 | ## Requirements 8 | - Chainer 1.17 9 | 10 | ## Usage 11 | 12 | ``` 13 | from link import Linear, BatchNormalization 14 | from function import Activation 15 | from chain import Chain 16 | 17 | x = np.random.normal(scale=1, size=(128, 28*28)).astype(np.float32) 18 | x = Variable(x) 19 | 20 | model = Sequential(weight_initializer="GlorotNormal", weight_init_std=0.05) 21 | model.add(Linear(28*28, 500, use_weightnorm=True)) 22 | model.add(BatchNormalization(500)) 23 | model.add(Activation("relu")) 24 | model.add(Linear(None, 500, use_weightnorm=True)) 25 | model.add(BatchNormalization(500)) 26 | model.add(Activation("relu")) 27 | model.add(Linear(500, 28*28, use_weightnorm=True)) 28 | model.build() 29 | 30 | chain = Chain() 31 | chain.add_sequence(model) 32 | chain.setup_optimizers("adam", 0.001, momentum=0.9, weight_decay=0.000001, gradient_clipping=10) 33 | 34 | for i in xrange(100): 35 | y = chain(x) 36 | loss = F.mean_squared_error(x, y) 37 | chain.backprop(loss) 38 | print float(loss.data) 39 | 40 | chain.save("model") 41 | ``` 42 | 43 | ## Getting chainer.links.Link objects 44 | 45 | ``` 46 | model = Sequential() 47 | ... 48 | model.build() 49 | 50 | for i, link in enumerate(model.links): 51 | if isinstance(link, chainer.link.Link): 52 | ... 53 | ``` 54 | 55 | ## JSON 56 | 57 | ``` 58 | json_str = model.to_json() 59 | model.from_json(json_str) 60 | ``` 61 | 62 | ## Adding activation function 63 | 64 | ``` 65 | model.add(function.Activation("relu")) 66 | ``` 67 | 68 | or 69 | 70 | ``` 71 | model.add(function.relu()) 72 | ``` 73 | 74 | ## Dropout 75 | 76 | ``` 77 | model.add(function.dropout()) 78 | ``` 79 | 80 | ## Adding gaussian noise 81 | 82 | ``` 83 | model.add(function.gaussian_noise(std=0.5)) 84 | ``` 85 | 86 | ## Weight Normalization 87 | 88 | ``` 89 | model.add(link.Linear(500, 500, use_weightnorm=True)) 90 | model.add(link.Convolution2D(64, 128, ksize=4, stride=2, pad=0, use_weightnorm=True)) 91 | model.add(link.Deconvolution2D(64, 32, ksize=4, stride=2, pad=1, use_weightnorm=True)) 92 | ``` 93 | 94 | ## Initializing weights 95 | 96 | ``` 97 | model = Sequential(weight_initializer="Normal", weight_init_std=0.05) 98 | model = Sequential(weight_initializer="GlorotNormal", weight_init_std=0.05) 99 | model = Sequential(weight_initializer="HeNormal", weight_init_std=0.05) 100 | ``` 101 | 102 | ## DCGAN 103 | 104 | ``` 105 | import util 106 | 107 | image_width = 96 108 | 109 | disciminator = Sequential() 110 | disciminator.add(Convolution2D(3, 64, ksize=4, stride=2, pad=1)) 111 | disciminator.add(Activation("elu")) 112 | disciminator.add(Convolution2D(64, 128, ksize=4, stride=2, pad=1)) 113 | disciminator.add(Activation("elu")) 114 | disciminator.add(Convolution2D(128, 256, ksize=4, stride=2, pad=1)) 115 | disciminator.add(Activation("elu")) 116 | disciminator.add(Linear(None, 1)) 117 | disciminator.add(sigmoid()) 118 | disciminator.build() 119 | 120 | # compute projection width 121 | input_width = util.get_in_size_of_deconv_layers(image_width, num_layers=3, ksize=4, stride=2) 122 | 123 | # compute required paddings 124 | paddings = util.get_paddings_of_deconv_layers(image_width, num_layers=3, ksize=4, stride=2) 125 | 126 | generator = Sequential() 127 | generator.add(Linear(100, 64 * input_width ** 2)) 128 | generator.add(BatchNormalization(64 * input_width ** 2)) 129 | generator.add(Activation("relu")) 130 | generator.add(reshape((-1, 64, input_width, input_width))) 131 | generator.add(Deconvolution2D(64, 32, ksize=4, stride=2, pad=paddings.pop(0))) 132 | generator.add(BatchNormalization(32)) 133 | generator.add(Activation("relu")) 134 | generator.add(Deconvolution2D(32, 16, ksize=4, stride=2, pad=paddings.pop(0))) 135 | generator.add(BatchNormalization(16)) 136 | generator.add(Activation("relu")) 137 | generator.add(Deconvolution2D(16, 3, ksize=4, stride=2, pad=paddings.pop(0))) 138 | generator.build() 139 | ``` -------------------------------------------------------------------------------- /chainer_sequential/__init__.py: -------------------------------------------------------------------------------- 1 | import link 2 | import function 3 | import util 4 | import chain 5 | from sequential import * 6 | 7 | def from_json(str): 8 | seq = Sequential() 9 | seq.from_json(str) 10 | return seq 11 | 12 | def from_dict(dict_array): 13 | seq = Sequential() 14 | seq.from_dict(dict_array) 15 | return seq -------------------------------------------------------------------------------- /chainer_sequential/binary/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/chainer_sequential/binary/__init__.py -------------------------------------------------------------------------------- /chainer_sequential/binary/functions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/chainer_sequential/binary/functions/__init__.py -------------------------------------------------------------------------------- /chainer_sequential/binary/functions/function_binary_linear.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | from chainer import cuda 4 | from chainer import function 5 | from chainer.utils import type_check 6 | 7 | def _kern(): 8 | return cuda.elementwise( 9 | 'T x', 'T y', 10 | 'y = x >= 0 ? 1 : -1', 11 | 'binarize') 12 | 13 | def _as_mat(x): 14 | if x.ndim == 2: 15 | return x 16 | return x.reshape(len(x), -1) 17 | 18 | class BinaryLinearFunction(function.Function): 19 | 20 | def check_type_forward(self, in_types): 21 | n_in = in_types.size() 22 | type_check.expect(2 <= n_in, n_in <= 3) 23 | x_type, w_type = in_types[:2] 24 | 25 | type_check.expect( 26 | x_type.dtype == numpy.float32, 27 | w_type.dtype == numpy.float32, 28 | x_type.ndim >= 2, 29 | w_type.ndim == 2, 30 | type_check.prod(x_type.shape[1:]) == w_type.shape[1], 31 | ) 32 | if n_in.eval() == 3: 33 | b_type = in_types[2] 34 | type_check.expect( 35 | b_type.dtype == numpy.float32, 36 | b_type.ndim == 1, 37 | b_type.shape[0] == w_type.shape[0], 38 | ) 39 | 40 | def forward_cpu(self, inputs): 41 | x = _as_mat(inputs[0]) 42 | W = inputs[1] 43 | Wb = numpy.where(W>=0, 1, -1).astype(x.dtype, copy=False) 44 | y = x.dot(Wb.T).astype(x.dtype, copy=False) 45 | if len(inputs) == 3: 46 | b = inputs[2] 47 | y += b 48 | return y, 49 | 50 | def forward_gpu(self, inputs): 51 | x = _as_mat(inputs[0]) 52 | W = inputs[1] 53 | Wb = _kern()(W) 54 | y = x.dot(Wb.T).astype(x.dtype, copy=False) 55 | if len(inputs) == 3: 56 | b = inputs[2] 57 | y += b 58 | return y, 59 | 60 | 61 | def backward_cpu(self, inputs, grad_outputs): 62 | x = _as_mat(inputs[0]) 63 | W = inputs[1] 64 | Wb = numpy.where(W>=0, 1, -1).astype(numpy.float32, copy=False) 65 | gy = grad_outputs[0] 66 | 67 | gx = gy.dot(Wb).astype(x.dtype, copy=False).reshape(inputs[0].shape) 68 | gW = gy.T.dot(x).astype(x.dtype, copy=False) 69 | if len(inputs) == 3: 70 | gb = gy.sum(0) 71 | return gx, gW, gb 72 | else: 73 | return gx, gW 74 | 75 | def backward_gpu(self, inputs, grad_outputs): 76 | x = _as_mat(inputs[0]) 77 | W = inputs[1] 78 | Wb = _kern()(W) 79 | gy = grad_outputs[0] 80 | 81 | gx = gy.dot(Wb).astype(x.dtype, copy=False).reshape(inputs[0].shape) 82 | gW = gy.T.dot(x).astype(x.dtype, copy=False) 83 | if len(inputs) == 3: 84 | gb = gy.sum(0) 85 | return gx, gW, gb 86 | else: 87 | return gx, gW 88 | 89 | 90 | def binary_linear(x, W, b=None): 91 | """Binary Linear function, or affine transformation. 92 | 93 | It accepts two or three arguments: an input minibatch ``x``, a weight 94 | matrix ``W``, and optionally a bias vector ``b``. It computes 95 | :math:`Y = xW^\\top + b`. 96 | 97 | Args: 98 | x (~chainer.Variable): Input variable. Its first dimension is assumed 99 | to be the *minibatch dimension*. The other dimensions are treated 100 | as concatenated one dimension whose size must be ``N``. 101 | W (~chainer.Variable): Weight variable of shape ``(M, N)``. 102 | b (~chainer.Variable): Bias variable (optional) of shape ``(M,)``.. 103 | 104 | Returns: 105 | ~chainer.Variable: Output variable. 106 | 107 | .. seealso:: :class:`~chainer.links.Linear` 108 | 109 | """ 110 | if b is None: 111 | return BinaryLinearFunction()(x, W) 112 | else: 113 | return BinaryLinearFunction()(x, W, b) 114 | -------------------------------------------------------------------------------- /chainer_sequential/binary/functions/function_bst.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | 3 | from chainer import cuda 4 | from chainer import function 5 | from chainer.utils import type_check 6 | 7 | 8 | class BST(function.Function): 9 | 10 | """Binary with Straight Thourgh estimator Unit.""" 11 | 12 | def __init__(self): 13 | pass 14 | 15 | def check_type_forward(self, in_types): 16 | type_check.expect(in_types.size() == 1) 17 | x_type, = in_types 18 | 19 | type_check.expect( 20 | x_type.dtype == numpy.float32, 21 | ) 22 | 23 | def forward_cpu(self, x): 24 | y = x[0] 25 | y = numpy.where(y>=0, 1, -1).astype(numpy.float32, copy=False) 26 | return y, 27 | 28 | def forward_gpu(self, x): 29 | y = cuda.elementwise( 30 | 'T x', 'T y', 31 | 'y = x >= 0 ? 1 : -1', 'bst_fwd')( 32 | x[0]) 33 | return y, 34 | 35 | def backward_cpu(self, x, gy): 36 | gx = gy[0].copy() 37 | zero_indices = numpy.abs(x[0]) > 1 38 | gx[zero_indices] = 0 39 | return gx, 40 | 41 | def backward_gpu(self, x, gy): 42 | gx = cuda.elementwise( 43 | 'T x, T gy', 'T gx', 44 | 'gx = abs(x) > 1 ? 0 : gy', 'bst_bwd')( 45 | x[0], gy[0]) 46 | return gx, 47 | 48 | 49 | def bst(x): 50 | """Binary with Straight Thourgh estimator Unit function. 51 | 52 | This function is expressed as 53 | 54 | .. math:: 55 | f(x) = \\left \\{ \\begin{array}{ll} 56 | 1 & {\\rm if}~ x \\ge 0 \\\\ 57 | -1 & {\\rm if}~ x < 0, 58 | \\end{array} \\right. 59 | 60 | See: http://arxiv.org/abs/1511.07289 61 | 62 | Args: 63 | x (~chainer.Variable): Input variable. 64 | 65 | Returns: 66 | ~chainer.Variable: Output variable. 67 | 68 | """ 69 | return BST()(x) 70 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/__init__.py: -------------------------------------------------------------------------------- 1 | from chainer import link 2 | 3 | class CLink(object): 4 | def generate_c(self): 5 | raise NotImplementedError("Not implemented. This link cannot be exported as c.") 6 | 7 | def param_mem(self): 8 | raise NotImplementedError("Not implemented. This link cannot be exported as c.") 9 | 10 | def temp_mem(self): 11 | raise NotImplementedError("Not implemented. This link cannot be exported as c.") 12 | 13 | from link_bst import BST 14 | from link_pool import Pool2D 15 | from link_batch_normalization import BatchNormalization 16 | from link_binary_convolution import BinaryConvolution2D 17 | from link_binary_linear import BinaryLinear 18 | from link_softmax_cross_entropy import SoftmaxCrossEntropy 19 | 20 | from link_linear_BN_BST import LinearBNBST 21 | from link_binary_linear_BN_BST import BinaryLinearBNBST 22 | 23 | from link_binary_linear_softmax_layer import BinaryLinearSoftmax 24 | from link_binary_linear_BN_softmax_layer import BinaryLinearBNSoftmax 25 | 26 | from link_conv_BN_BST import ConvBNBST 27 | from link_binary_conv_BN_BST import BinaryConvBNBST 28 | 29 | from link_conv_pool_BN_BST import ConvPoolBNBST 30 | from link_binary_conv_pool_BN_BST import BinaryConvPoolBNBST 31 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_batch_normalization.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from chainer.links import batch_normalization as CBN 4 | 5 | class BatchNormalization(CBN.BatchNormalization): 6 | def __init__(self, size, **kwargs): 7 | super(BatchNormalization, self).__init__(size, **kwargs) 8 | self.cname = "l_bn" 9 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_binary_conv_BN_BST.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import chainer 5 | import numpy as np 6 | 7 | from ..links import CLink 8 | from ..links import BinaryConvolution2D 9 | from ..links import BatchNormalization 10 | from ..links import BST 11 | from ..utils import binary_util as bu 12 | 13 | class BinaryConvBNBST(chainer.Chain, CLink): 14 | def __init__(self, in_channels, out_channels, ksize=3, stride=1, pad=0): 15 | super(BinaryConvBNBST, self).__init__( 16 | bconv=BinaryConvolution2D(in_channels, out_channels, ksize=ksize, stride=stride, pad=pad), 17 | bn=BatchNormalization(out_channels), 18 | bst=BST() 19 | ) 20 | self.cname = "l_b_conv_bn_bst" 21 | 22 | def __call__(self, h, test=False): 23 | #self.inp_shape = h.data.shape 24 | h = self.bst(self.bn(self.bconv(h), test)) 25 | return h 26 | 27 | def generate_c(self, link_idx, inp_shape): 28 | #if not hasattr(self,'inp_shape'): 29 | # raise Exception("no input shape found") 30 | # return "" 31 | w, h = inp_shape[2:4] 32 | name = self.cname + str(link_idx) 33 | text = [] 34 | m = 1 35 | sw, sh = self.bconv.stride 36 | pw, ph = self.bconv.pad 37 | pl_w, pl_h = 1, 1 38 | pl_sw, pl_sh = 1, 1 39 | pl_pw, pl_ph = 0, 0 40 | 41 | # Bconv 42 | l = self.bconv 43 | lname = name + '_' + l.name 44 | for p in l.params(): 45 | pname = p.name 46 | if pname == 'W': 47 | num_f, n, kw, kh = p.data.shape 48 | bin_data = bu.binarize_real(p.data).reshape(p.data.shape[0]*p.data.shape[1], -1) 49 | text += [bu.np_to_uint8C(bin_data, lname+'_'+pname, 'row_major', pad='1')] 50 | elif pname == 'b': 51 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 52 | 53 | # BatchNormalization bn 54 | l = self.bn 55 | lName = l.name 56 | lname=name+'_'+lName 57 | for p in l.params(): 58 | pname=p.name 59 | if pname == 'gamma': 60 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 61 | elif pname == 'beta': 62 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 63 | for p in l._persistent: 64 | pname=p 65 | persistent = l.__dict__[p] 66 | if pname == 'avg_mean': 67 | text += [bu.np_to_floatC(persistent, lname+'_mean', 'row_major')] 68 | elif pname == 'avg_var': 69 | text += [bu.np_to_floatC(np.sqrt(persistent, dtype=persistent.dtype), lname+'_std', 'row_major')] 70 | 71 | text = "\n".join(text)+'\n' 72 | ftext = "void {name}(uint8_t* input, uint8_t* output){{\n" 73 | ftext += " bconv_layer(input, {name}_bconv_W, output, {name}_bconv_b, {name}_bn_gamma, {name}_bn_beta, {name}_bn_mean, {name}_bn_std, {m}, {num_f}, {w}, {h}, {n}, {kw}, {kh}, {sw}, {sh}, {pw}, {ph}, {pl_w}, {pl_h}, {pl_sw}, {pl_sh}, {pl_pw}, {pl_ph});\n}}\n\n" 74 | ftext = ftext.format(name=name, m=m, n=n, w=w, h=h, num_f=num_f, kw=kw, 75 | kh=kh, sw=sw, sh=sh, pw=pw, ph=ph, pl_w=pl_w, 76 | pl_h=pl_h, pl_sw=pl_sw, pl_sh=pl_sh, pl_pw=pl_pw, 77 | pl_ph=pl_ph) 78 | text += ftext 79 | 80 | return text 81 | 82 | def param_mem(self): 83 | mem = 0. 84 | l = self.bconv 85 | for p in self.bconv.params(): 86 | if p.name == 'W': 87 | num_f, n, kw, kh = p.data.shape 88 | #Filters 89 | mem += num_f*n*kh*math.ceil(kw/8.) 90 | #Bias + BN 91 | mem += 5*num_f*32 92 | 93 | return mem 94 | 95 | def temp_mem(self, inp_shape): 96 | #TODO: UPDATE 97 | m, n, w, h = inp_shape 98 | sw, sh = self.bconv.stride 99 | for p in self.bconv.params(): 100 | if p.name == 'W': 101 | _, _, kw, kh = p.data.shape 102 | break 103 | 104 | res_w = (w - kw + 8) / 8 105 | res_h = h - kh + 1 106 | 107 | return m*n*res_w*res_h 108 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_binary_conv_pool_BN_BST.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import numpy as np 5 | import chainer 6 | 7 | from ..links import CLink 8 | from ..links import BinaryConvolution2D 9 | from ..links import Pool2D 10 | from ..links import BatchNormalization 11 | from ..links import BST 12 | from ..utils import binary_util as bu 13 | 14 | class BinaryConvPoolBNBST(chainer.Chain, CLink): 15 | def __init__(self, in_channels, out_channels, ksize=3, stride=1, pad=0, pksize=3, pstride=2, ppad=0): 16 | super(BinaryConvPoolBNBST, self).__init__( 17 | bconv=BinaryConvolution2D(in_channels, out_channels, ksize=ksize, stride=stride, pad=pad), 18 | pool=Pool2D(pksize,pstride,ppad), 19 | bn=BatchNormalization(out_channels), 20 | bst=BST() 21 | ) 22 | self.cname = "l_b_conv_pool_bn_bst" 23 | self.pksize = pksize 24 | self.ppad = ppad 25 | 26 | def __call__(self, h, test=False): 27 | h = self.bconv(h) 28 | h = self.pool(h) 29 | h = self.bn(h, test) 30 | h = self.bst(h) 31 | return h 32 | 33 | def generate_c(self, link_idx, inp_shape): 34 | #if not hasattr(self,'inp_shape'): 35 | # raise Exception("no input shape found") 36 | # return "" 37 | name = self.cname + str(link_idx) 38 | text = [] 39 | m = 1 40 | w, h = inp_shape[2:4] 41 | sw, sh = self.bconv.stride 42 | pw, ph = self.bconv.pad 43 | pl_w, pl_h = self.pksize, self.pksize 44 | pl_sw, pl_sh = self.pool.stride, self.pool.stride 45 | pl_pw, pl_ph = self.ppad, self.ppad 46 | 47 | # Bconv 48 | l = self.bconv 49 | lname = name + '_' + l.name 50 | for p in l.params(): 51 | pname = p.name 52 | if pname == 'W': 53 | num_f, n, kw, kh = p.data.shape 54 | bin_data = bu.binarize_real(p.data).reshape(p.data.shape[0]*p.data.shape[1], -1) 55 | text += [bu.np_to_uint8C(bin_data, lname+'_'+pname, 'row_major', pad='1')] 56 | elif pname == 'b': 57 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 58 | 59 | # BatchNormalization bn 60 | l = self.bn 61 | lName = l.name 62 | lname=name+'_'+lName 63 | for p in l.params(): 64 | pname=p.name 65 | if pname == 'gamma': 66 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 67 | elif pname == 'beta': 68 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 69 | for p in l._persistent: 70 | pname=p 71 | persistent = l.__dict__[p] 72 | if pname == 'avg_mean': 73 | text += [bu.np_to_floatC(persistent, lname+'_mean', 'row_major')] 74 | elif pname == 'avg_var': 75 | text += [bu.np_to_floatC(np.sqrt(persistent, dtype=persistent.dtype), lname+'_std', 'row_major')] 76 | 77 | text = "\n".join(text)+'\n' 78 | ftext = "void {name}(uint8_t* input, uint8_t* output){{\n" 79 | ftext += " bconv_layer(input, {name}_bconv_W, output, {name}_bconv_b, {name}_bn_gamma, {name}_bn_beta, {name}_bn_mean, {name}_bn_std, {m}, {num_f}, {w}, {h}, {n}, {kw}, {kh}, {sw}, {sh}, {pw}, {ph}, {pl_w}, {pl_h}, {pl_sw}, {pl_sh}, {pl_pw}, {pl_ph});\n}}\n\n" 80 | ftext = ftext.format(name=name, m=m, n=n, w=w, h=h, num_f=num_f, kw=kw, 81 | kh=kh, sw=sw, sh=sh, pw=pw, ph=ph, pl_w=pl_w, 82 | pl_h=pl_h, pl_sw=pl_sw, pl_sh=pl_sh, pl_pw=pl_pw, 83 | pl_ph=pl_ph) 84 | text += ftext 85 | 86 | 87 | return text 88 | 89 | def param_mem(self): 90 | mem = 0. 91 | l = self.bconv 92 | for p in self.bconv.params(): 93 | if p.name == 'W': 94 | num_f, n, kw, kh = p.data.shape 95 | #Filters 96 | mem += num_f*n*kh*math.ceil(kw/8.) 97 | #Bias + BN 98 | mem += 5*num_f*32 99 | 100 | return mem 101 | 102 | def temp_mem(self, inp_shape): 103 | #TODO: UPDATE 104 | m, n, w, h = inp_shape 105 | sw, sh = self.bconv.stride 106 | for p in self.bconv.params(): 107 | if p.name == 'W': 108 | _, _, kw, kh = p.data.shape 109 | break 110 | 111 | res_w = (w - kw + 8) / 8 112 | res_h = h - kh + 1 113 | 114 | return m*n*res_w*res_h 115 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_binary_convolution.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import numpy 4 | 5 | from chainer import link 6 | from ..functions import function_binary_convolution_2d 7 | 8 | def _pair(x): 9 | if hasattr(x, '__getitem__'): 10 | return x 11 | return x, x 12 | 13 | class BinaryConvolution2D(link.Link): 14 | 15 | """Two-dimensional binarized convolutional layer. 16 | 17 | This link wraps the :func:`function_binary_convolution_2d.binary_convolution_2d` function and 18 | holds the filter weight and bias vector as parameters. 19 | 20 | Args: 21 | in_channels (int): Number of channels of input arrays. 22 | out_channels (int): Number of channels of output arrays. 23 | ksize (int or pair of ints): Size of filters (a.k.a. kernels). 24 | ``ksize=k`` and ``ksize=(k, k)`` are equivalent. 25 | stride (int or pair of ints): Stride of filter applications. 26 | ``stride=s`` and ``stride=(s, s)`` are equivalent. 27 | pad (int or pair of ints): Spatial padding width for input arrays. 28 | ``pad=p`` and ``pad=(p, p)`` are equivalent. 29 | wscale (float): Scaling factor of the initial weight. 30 | bias (float): Initial bias value. 31 | nobias (bool): If ``True``, then this link does not use the bias term. 32 | use_cudnn (bool): If ``True``, then this link uses cuDNN if available. 33 | initialW (4-D array): Initial weight value. If ``None``, then this 34 | function uses to initialize ``wscale``. 35 | initial_bias (1-D array): Initial bias value. If ``None``, then this 36 | function uses to initialize ``bias``. 37 | 38 | .. seealso:: 39 | See :func:`chainer.functions.convolution_2d` for the definition of 40 | two-dimensional convolution. 41 | 42 | Attributes: 43 | W (~chainer.Variable): Weight parameter. 44 | b (~chainer.Variable): Bias parameter. 45 | 46 | """ 47 | def __init__(self, in_channels, out_channels, ksize, stride=1, pad=0, 48 | wscale=1, bias=0, nobias=False, use_cudnn=True, 49 | initialW=None, initial_bias=None): 50 | self.cname = "l_b_conv" 51 | kh, kw = _pair(ksize) 52 | self.stride = _pair(stride) 53 | self.pad = _pair(pad) 54 | self.use_cudnn = use_cudnn 55 | 56 | W_shape = (out_channels, in_channels, kh, kw) 57 | super(BinaryConvolution2D, self).__init__(W=W_shape) 58 | 59 | if initialW is not None: 60 | self.W.data[...] = initialW 61 | else: 62 | std = wscale * numpy.sqrt(1. / (kh * kw * in_channels)) 63 | self.W.data[...] = numpy.random.normal(0, std, W_shape) 64 | 65 | if nobias: 66 | self.b = None 67 | else: 68 | self.add_param('b', out_channels) 69 | if initial_bias is None: 70 | initial_bias = bias 71 | self.b.data[...] = initial_bias 72 | 73 | def __call__(self, x): 74 | """Applies the convolution layer. 75 | 76 | Args: 77 | x (~chainer.Variable): Input image. 78 | 79 | Returns: 80 | ~chainer.Variable: Output of the convolution. 81 | 82 | """ 83 | return function_binary_convolution_2d.binary_convolution_2d( 84 | x, self.W, self.b, self.stride, self.pad, self.use_cudnn) 85 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_binary_linear.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import numpy 4 | from chainer import link 5 | 6 | from ..functions import function_binary_linear 7 | import math 8 | from chainer import initializers 9 | from chainer import cuda 10 | 11 | class BinaryLinear(link.Link): 12 | """Binary Linear layer (a.k.a. binary fully-connected layer). 13 | 14 | This is a link that wraps the :func:`~chainer.functions.linear` function, 15 | and holds a weight matrix ``W`` and optionally a bias vector ``b`` as 16 | parameters. 17 | 18 | The weight matrix ``W`` is initialized with i.i.d. Gaussian samples, each 19 | of which has zero mean and deviation :math:`\\sqrt{1/\\text{in_size}}`. The 20 | bias vector ``b`` is of size ``out_size``. Each element is initialized with 21 | the ``bias`` value. If ``nobias`` argument is set to True, then this link 22 | does not hold a bias vector. 23 | 24 | Args: 25 | in_size (int): Dimension of input vectors. 26 | out_size (int): Dimension of output vectors. 27 | wscale (float): Scaling factor of the weight matrix. 28 | bias (float): Initial bias value. 29 | nobias (bool): If True, then this function does not use the bias. 30 | initialW (2-D array): Initial weight value. If ``None``, then this 31 | function uses to initialize ``wscale``. 32 | initial_bias (1-D array): Initial bias value. If ``None``, then this 33 | function uses to initialize ``bias``. 34 | 35 | .. seealso:: :func:`~chainer.functions.linear` 36 | 37 | Attributes: 38 | W (~chainer.Variable): Weight parameter. 39 | b (~chainer.Variable): Bias parameter. 40 | 41 | """ 42 | def __init__(self, in_size, out_size, wscale=1, bias=0, nobias=False, 43 | initialW=None, initial_bias=None): 44 | super(BinaryLinear, self).__init__() 45 | self.cname = "l_b_linear" 46 | 47 | # For backward compatibility 48 | self.initialW = initialW 49 | self.wscale = wscale 50 | 51 | self.out_size = out_size 52 | # For backward compatibility, the scale of weights is proportional to 53 | # the square root of wscale. 54 | self._W_initializer = initializers._get_initializer( 55 | initialW, math.sqrt(wscale)) 56 | 57 | if in_size is None: 58 | self.add_uninitialized_param('W') 59 | else: 60 | self._initialize_params(in_size) 61 | 62 | if nobias: 63 | self.b = None 64 | else: 65 | if initial_bias is None: 66 | initial_bias = bias 67 | bias_initializer = initializers._get_initializer(initial_bias) 68 | self.add_param('b', out_size, initializer=bias_initializer) 69 | 70 | # def __init__(self, in_size, out_size, wscale=1, bias=0, nobias=False, 71 | # initialW=None, initial_bias=None): 72 | # super(BinaryLinear, self).__init__(W=(out_size, in_size)) 73 | # self.cname = "l_b_linear" 74 | # 75 | # if initialW is None: 76 | # initialW = numpy.random.normal( 77 | # 0, wscale * numpy.sqrt(1. / in_size), (out_size, in_size)) 78 | # self.W.data[...] = initialW 79 | # 80 | # if nobias: 81 | # self.b = None 82 | # else: 83 | # self.add_param('b', out_size) 84 | # if initial_bias is None: 85 | # initial_bias = bias 86 | # self.b.data[...] = initial_bias 87 | 88 | def _initialize_params(self, in_size): 89 | self.add_param('W', (self.out_size, in_size), 90 | initializer=self._W_initializer) 91 | def __call__(self, x): 92 | """Applies the linear layer. 93 | 94 | Args: 95 | x (~chainer.Variable): Batch of input vectors. 96 | 97 | Returns: 98 | ~chainer.Variable: Output of the linear layer. 99 | 100 | """ 101 | if self.has_uninitialized_params: 102 | with cuda.get_device(self._device_id): 103 | self._initialize_params(x.size // len(x.data)) 104 | return function_binary_linear.binary_linear(x, self.W, self.b) 105 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_binary_linear_BN_BST.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import chainer 5 | import numpy as np 6 | 7 | from ..links import CLink 8 | from ..links import BinaryLinear 9 | from ..links import BatchNormalization 10 | from ..links import BST 11 | from ..utils import binary_util as bu 12 | 13 | class BinaryLinearBNBST(chainer.Chain, CLink): 14 | def __init__(self, in_channels, out_channels): 15 | super(BinaryLinearBNBST, self).__init__( 16 | bl=BinaryLinear(in_channels, out_channels), 17 | bn=BatchNormalization(out_channels), 18 | bst=BST() 19 | ) 20 | self.cname = "l_b_linear_bn_bst" 21 | 22 | def __call__(self, h, test=False): 23 | #self.inp_shape = h.data.shape 24 | h = self.bst(self.bn(self.bl(h), test=test)) 25 | return h 26 | 27 | def generate_c(self, link_idx, inp_shape): 28 | name = self.cname + str(link_idx) 29 | text = [] 30 | 31 | # BinaryLinear bl 32 | l = self.bl 33 | lName = l.name 34 | lname=name+'_'+lName 35 | for p in l.params(): 36 | pname=p.name 37 | if pname == 'W': 38 | text += [bu.np_to_uint8C(bu.binarize_real(p.data.T), lname+'_'+pname, 'col_major', pad='1')] 39 | num_classes = p.data.shape[0] 40 | fc_size = p.data.shape[1] 41 | elif pname == 'b': 42 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 43 | 44 | # BatchNormalization bn 45 | l = self.bn 46 | lName = l.name 47 | lname=name+'_'+lName 48 | for p in l.params(): 49 | pname=p.name 50 | if pname == 'gamma': 51 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 52 | elif pname == 'beta': 53 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 54 | for p in l._persistent: 55 | pname=p 56 | persistent = l.__dict__[p] 57 | if pname == 'avg_mean': 58 | text += [bu.np_to_floatC(persistent, lname+'_mean', 'row_major')] 59 | elif pname == 'avg_var': 60 | text += [bu.np_to_floatC(np.sqrt(persistent, dtype=persistent.dtype), lname+'_std', 'row_major')] 61 | 62 | text = "\n".join(text)+'\n' 63 | m = 1 64 | n = fc_size 65 | k = num_classes 66 | ftext = "void {name}(uint8_t* input, uint8_t* output){{\n" 67 | ftext += " fused_linear_layer(input, {name}_bl_W, output, {name}_bl_b, {name}_bn_gamma, {name}_bn_beta, {name}_bn_mean, {name}_bn_std, {m}, {n}, {k}); \n}}\n\n" 68 | ftext = ftext.format(name=name, m=m, n=n, k=k) 69 | text += ftext 70 | 71 | return text 72 | 73 | def param_mem(self): 74 | mem = 0. 75 | for p in self.bl.params(): 76 | if p.name == 'W': 77 | m, w = p.data.shape 78 | mem += m*math.ceil(w / 8.) 79 | #Bias + BN 80 | mem += 5*m*32 81 | 82 | return mem 83 | 84 | def temp_mem(self, inp_shape): 85 | m = inp_shape[0] 86 | w = np.prod(inp_shape[1:]) 87 | res_w = math.ceil(w / 8.) 88 | return m*res_w 89 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_binary_linear_BN_softmax_layer.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import chainer 5 | import chainer.functions as F 6 | import numpy as np 7 | 8 | from ..links import CLink 9 | from ..links import BinaryLinear 10 | from ..links import BatchNormalization 11 | from ..links import SoftmaxCrossEntropy 12 | from ..utils import binary_util as bu 13 | 14 | class BinaryLinearBNSoftmax(chainer.Chain, CLink): 15 | def __init__(self, in_channels, out_channels): 16 | super(BinaryLinearBNSoftmax, self).__init__( 17 | bl=BinaryLinear(in_channels, out_channels), 18 | bn=BatchNormalization(out_channels), 19 | sm=SoftmaxCrossEntropy() 20 | ) 21 | self.cname = "l_b_linear_bn_softmax" 22 | 23 | def __call__(self, h, t=None, test=False): 24 | #self.inp_shape = h.data.shape 25 | h = self.bn(self.bl(h), test) 26 | if t is not None: 27 | self.accuracy = F.accuracy(h,t) 28 | loss = self.sm(h,t) 29 | return loss 30 | return h 31 | 32 | def generate_c(self, link_idx, inp_shape): 33 | name = self.cname + str(link_idx) 34 | text = [] 35 | 36 | # BinaryLinear bl 37 | l = self.bl 38 | lName = l.name 39 | lname=name+'_'+lName 40 | for p in l.params(): 41 | pname=p.name 42 | if pname == 'W': 43 | text += [bu.np_to_uint8C(bu.binarize_real(p.data.T), lname+'_'+pname, 'col_major', pad='1')] 44 | num_classes = p.data.shape[0] 45 | fc_size = p.data.shape[1] 46 | elif pname == 'b': 47 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 48 | 49 | # BatchNormalization bn 50 | l = self.bn 51 | lName = l.name 52 | lname=name+'_'+lName 53 | for p in l.params(): 54 | pname=p.name 55 | if pname == 'gamma': 56 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 57 | elif pname == 'beta': 58 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 59 | for p in l._persistent: 60 | pname=p 61 | persistent = l.__dict__[p] 62 | if pname == 'avg_mean': 63 | text += [bu.np_to_floatC(persistent, lname+'_mean', 'row_major')] 64 | elif pname == 'avg_var': 65 | text += [bu.np_to_floatC(np.sqrt(persistent, dtype=persistent.dtype), lname+'_std', 'row_major')] 66 | 67 | text = "\n".join(text)+'\n' 68 | m = 1 69 | n = fc_size 70 | k = num_classes 71 | 72 | ftext = "void {name}(uint8_t* input, uint8_t* output){{\n" 73 | ftext += " blinear_layer(input, {name}_bl_W, output, {name}_bl_b, {name}_bn_gamma, {name}_bn_beta, {name}_bn_mean, {name}_bn_std, {m}, {n}, {k}); \n}}\n\n" 74 | ftext = ftext.format(name=name, m=m, n=n, k=k) 75 | text += ftext 76 | 77 | return text 78 | 79 | def param_mem(self): 80 | mem = 0. 81 | for p in self.bl.params(): 82 | if p.name == 'W': 83 | m, w = p.data.shape 84 | mem += m*math.ceil(w/8.) 85 | #Bias + BN 86 | mem += 5*m*32 87 | 88 | 89 | return mem 90 | 91 | def temp_mem(self, inp_shape): 92 | m = inp_shape[0] 93 | w = np.prod(inp_shape[1:]) 94 | res_w = math.ceil(w/8.) 95 | return m*res_w 96 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_binary_linear_softmax_layer.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import chainer 4 | import chainer.functions as F 5 | import numpy as np 6 | 7 | from ..links import CLink 8 | from ..links import BinaryLinear 9 | from ..links import BatchNormalization 10 | from ..links import SoftmaxCrossEntropy 11 | from ..utils import binary_util as bu 12 | import math 13 | 14 | class BinaryLinearSoftmax(chainer.Chain, CLink): 15 | def __init__(self, in_channels, out_channels): 16 | super(BinaryLinearSoftmax, self).__init__( 17 | bl=BinaryLinear(in_channels, out_channels), 18 | sm=SoftmaxCrossEntropy() 19 | ) 20 | self.cname = "l_b_linear_softmax" 21 | 22 | def __call__(self, h, t=None): 23 | h = self.bl(h) 24 | if t is not None: 25 | self.accuracy = F.accuracy(h,t) 26 | loss = self.sm(h,t) 27 | return loss 28 | return h 29 | 30 | def generate_c(self, link_idx, inp_shape): 31 | name = self.cname + str(link_idx) 32 | text = [] 33 | 34 | # BinaryLinear bl 35 | l = self.bl 36 | lName = l.name 37 | lname=name+'_'+lName 38 | for p in l.params(): 39 | pname=p.name 40 | if pname == 'W': 41 | text += [bu.np_to_uint8C(bu.binarize_real(p.data.T), lname+'_'+pname, 'col_major', pad='1')] 42 | num_classes = p.data.shape[0] 43 | fc_size = p.data.shape[1] 44 | elif pname == 'b': 45 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 46 | 47 | text = "\n".join(text) 48 | m = 1 49 | n = fc_size 50 | k = num_classes 51 | 52 | ftext = "void {name}(uint8_t* input, uint8_t* output){{\n" 53 | ftext += " blinear_layer(input, {name}_bl_W, output, {name}_bl_b, {m}, {n}, {k}); \n}}\n\n" 54 | ftext = ftext.format(name=name, m=m, n=n, k=k) 55 | text += ftext 56 | 57 | return text 58 | 59 | def param_mem(self): 60 | mem = 0. 61 | for p in self.bl.params(): 62 | if p.name == 'W': 63 | w, h = p.data.shape 64 | mem += math.ceil(h/8.)*w 65 | #Bias + BN 66 | mem += 5*h*32 67 | 68 | return mem 69 | 70 | def temp_mem(self, inp_shape): 71 | m = inp_shape[0] 72 | w = np.prod(inp_shape[1:]) 73 | res_w = math.ceil(w/8.) 74 | return m*res_w 75 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_bst.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from chainer import link 4 | 5 | from ..functions import function_bst 6 | 7 | class BST(link.Link): 8 | def __init__(self): 9 | super(BST, self).__init__() 10 | self.cname = "l_bst" 11 | 12 | def __call__(self, x): 13 | """Applies the binary activation. 14 | 15 | Args: 16 | x (~chainer.Variable): Batch of input vectors. 17 | 18 | Returns: 19 | ~chainer.Variable: Output of the binary activation. 20 | 21 | """ 22 | return function_bst.bst(x) 23 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_conv_BN_BST.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import chainer 5 | import numpy as np 6 | 7 | from ..links import CLink 8 | from ..links import BinaryConvolution2D 9 | from ..links import BatchNormalization 10 | from ..links import BST 11 | from ..utils import binary_util as bu 12 | 13 | class ConvBNBST(chainer.Chain, CLink): 14 | def __init__(self, in_channels, out_channels, ksize=3, stride=1, pad=0): 15 | super(ConvBNBST, self).__init__( 16 | bconv=BinaryConvolution2D(in_channels, out_channels, ksize=ksize, stride=stride, pad=pad), 17 | bn=BatchNormalization(out_channels), 18 | bst=BST() 19 | ) 20 | self.cname = "l_conv_bn_bst" 21 | 22 | def __call__(self, h, test=False): 23 | #self.inp_shape = h.data.shape 24 | h = self.bst(self.bn(self.bconv(h), test)) 25 | return h 26 | 27 | def generate_c(self, link_idx, inp_shape): 28 | #if not hasattr(self,'inp_shape'): 29 | # raise Exception("no input shape found") 30 | # return "" 31 | w, h = inp_shape[2:4] 32 | name = self.cname + str(link_idx) 33 | text = [] 34 | m = 1 35 | sw, sh = self.bconv.stride 36 | pw, ph = self.bconv.pad 37 | pl_w, pl_h = 1, 1 38 | pl_sw, pl_sh = 1, 1 39 | pl_pw, pl_ph = 0, 0 40 | 41 | # Bconv 42 | l = self.bconv 43 | lname = name + '_' + l.name 44 | for p in l.params(): 45 | pname = p.name 46 | if pname == 'W': 47 | num_f, n, kw, kh = p.data.shape 48 | bin_data = bu.binarize_real(p.data).reshape(p.data.shape[0], -1) 49 | text += [bu.np_to_uint8C(bin_data, lname+'_'+pname, 'row_major', pad='1')] 50 | elif pname == 'b': 51 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 52 | 53 | # BatchNormalization bn 54 | l = self.bn 55 | lName = l.name 56 | lname=name+'_'+lName 57 | for p in l.params(): 58 | pname=p.name 59 | if pname == 'gamma': 60 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 61 | elif pname == 'beta': 62 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 63 | for p in l._persistent: 64 | pname=p 65 | persistent = l.__dict__[p] 66 | if pname == 'avg_mean': 67 | text += [bu.np_to_floatC(persistent, lname+'_mean', 'row_major')] 68 | elif pname == 'avg_var': 69 | text += [bu.np_to_floatC(np.sqrt(persistent, dtype=persistent.dtype), lname+'_std', 'row_major')] 70 | text = "\n".join(text)+'\n' 71 | ftext = "void {name}(float* input, uint8_t* output){{\n" 72 | ftext += " fconv_layer(input, {name}_bconv_W, output, {name}_bconv_b, {name}_bn_gamma, {name}_bn_beta, {name}_bn_mean, {name}_bn_std, {m}, {num_f}, {w}, {h}, {n}, {kw}, {kh}, {sw}, {sh}, {pw}, {ph}, {pl_w}, {pl_h}, {pl_sw}, {pl_sh}, {pl_pw}, {pl_ph});\n}}\n\n" 73 | ftext = ftext.format(name=name, m=m, n=n, w=w, h=h, num_f=num_f, kw=kw, 74 | kh=kh, sw=sw, sh=sh, pw=pw, ph=ph, pl_w=pl_w, 75 | pl_h=pl_h, pl_sw=pl_sw, pl_sh=pl_sh, pl_pw=pl_pw, 76 | pl_ph=pl_ph) 77 | text += ftext 78 | 79 | return text 80 | 81 | def param_mem(self): 82 | mem = 0. 83 | l = self.bconv 84 | for p in self.bconv.params(): 85 | if p.name == 'W': 86 | num_f, n, kw, kh = p.data.shape 87 | #Filters 88 | mem += num_f*n*kh*math.ceil(kw/8.) 89 | #Bias + BN 90 | mem += 5*num_f*32 91 | 92 | return mem 93 | 94 | def temp_mem(self, inp_shape): 95 | #TODO: UPDATE 96 | m, n, w, h = inp_shape 97 | sw, sh = self.bconv.stride 98 | for p in self.bconv.params(): 99 | if p.name == 'W': 100 | _, _, kw, kh = p.data.shape 101 | break 102 | 103 | res_w = (w - kw + 8) / 8 104 | res_h = h - kh + 1 105 | 106 | return m*n*res_w*res_h 107 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_conv_pool_BN_BST.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import chainer 5 | import numpy as np 6 | 7 | from ..links import CLink 8 | from ..links import BinaryConvolution2D 9 | from ..links import Pool2D 10 | from ..links import BatchNormalization 11 | from ..links import BST 12 | from ..utils import binary_util as bu 13 | 14 | class ConvPoolBNBST(chainer.Chain, CLink): 15 | def __init__(self, in_channels, out_channels, ksize=3, stride=1, pad=0, pksize=3, pstride=2, ppad=0): 16 | super(ConvPoolBNBST, self).__init__( 17 | bconv=BinaryConvolution2D(in_channels, out_channels, ksize=ksize, stride=stride, pad=pad), 18 | pool=Pool2D(pksize,pstride,ppad), 19 | bn=BatchNormalization(out_channels), 20 | bst=BST() 21 | ) 22 | self.cname = "l_conv_pool_bn_bst" 23 | self.pksize = pksize 24 | self.ppad = ppad 25 | 26 | def __call__(self, h, test=False): 27 | #self.inp_shape = h.data.shape 28 | h = self.bconv(h) 29 | h = self.pool(h) 30 | h = self.bn(h, test) 31 | h = self.bst(h) 32 | return h 33 | 34 | def generate_c(self, link_idx, inp_shape): 35 | #if not hasattr(self,'inp_shape'): 36 | # raise Exception("no input shape found") 37 | # return "" 38 | w, h = inp_shape[2:4] 39 | name = self.cname + str(link_idx) 40 | text = [] 41 | m = 1 42 | w, h = inp_shape[2:4] 43 | sw, sh = self.bconv.stride 44 | pw, ph = self.bconv.pad 45 | pl_w, pl_h = self.pksize, self.pksize 46 | pl_sw, pl_sh = self.pool.stride, self.pool.stride 47 | pl_pw, pl_ph = self.ppad, self.ppad 48 | 49 | 50 | # Bconv 51 | l = self.bconv 52 | lname = name + '_' + l.name 53 | for p in l.params(): 54 | pname = p.name 55 | if pname == 'W': 56 | num_f, n, kw, kh = p.data.shape 57 | bin_data = bu.binarize_real(p.data).reshape(p.data.shape[0], -1) 58 | text += [bu.np_to_uint8C(bin_data, lname+'_'+pname, 'row_major', pad='1')] 59 | elif pname == 'b': 60 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 61 | 62 | # BatchNormalization bn 63 | l = self.bn 64 | lName = l.name 65 | lname=name+'_'+lName 66 | for p in l.params(): 67 | pname=p.name 68 | if pname == 'gamma': 69 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 70 | elif pname == 'beta': 71 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 72 | for p in l._persistent: 73 | pname=p 74 | persistent = l.__dict__[p] 75 | if pname == 'avg_mean': 76 | text += [bu.np_to_floatC(persistent, lname+'_mean', 'row_major')] 77 | elif pname == 'avg_var': 78 | text += [bu.np_to_floatC(np.sqrt(persistent, dtype=persistent.dtype), lname+'_std', 'row_major')] 79 | 80 | text = "\n".join(text) + "\n" 81 | ftext = "void {name}(float* input, uint8_t* output){{\n" 82 | ftext += " fconv_layer(input, {name}_bconv_W, output, {name}_bconv_b, {name}_bn_gamma, {name}_bn_beta, {name}_bn_mean, {name}_bn_std, {m}, {num_f}, {w}, {h}, {n}, {kw}, {kh}, {sw}, {sh}, {pw}, {ph}, {pl_w}, {pl_h}, {pl_sw}, {pl_sh}, {pl_pw}, {pl_ph});\n}}\n\n" 83 | ftext = ftext.format(name=name, m=m, n=n, w=w, h=h, num_f=num_f, kw=kw, 84 | kh=kh, sw=sw, sh=sh, pw=pw, ph=ph, pl_w=pl_w, 85 | pl_h=pl_h, pl_sw=pl_sw, pl_sh=pl_sh, pl_pw=pl_pw, 86 | pl_ph=pl_ph) 87 | text += ftext 88 | 89 | 90 | return text 91 | 92 | 93 | def param_mem(self): 94 | mem = 0. 95 | l = self.bconv 96 | for p in self.bconv.params(): 97 | if p.name == 'W': 98 | num_f, n, kw, kh = p.data.shape 99 | #Filters 100 | mem += num_f*n*kh*math.ceil(kw/8.) 101 | #Bias + BN 102 | mem += 5*num_f*32 103 | 104 | return mem 105 | 106 | def temp_mem(self, inp_shape): 107 | #TODO: UPDATE 108 | m, n, w, h = inp_shape 109 | sw, sh = self.bconv.stride 110 | for p in self.bconv.params(): 111 | if p.name == 'W': 112 | _, _, kw, kh = p.data.shape 113 | break 114 | 115 | res_w = (w - kw + 8) / 8 116 | res_h = h - kh + 1 117 | 118 | return m*n*res_w*res_h 119 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_linear_BN_BST.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import math 3 | 4 | import chainer 5 | import numpy as np 6 | 7 | from ..links import CLink 8 | from ..links import BinaryLinear 9 | from ..links import BatchNormalization 10 | from ..links import BST 11 | from ..utils import binary_util as bu 12 | 13 | class LinearBNBST(chainer.Chain, CLink): 14 | def __init__(self, in_channels, out_channels): 15 | super(LinearBNBST, self).__init__( 16 | bl=BinaryLinear(in_channels, out_channels), 17 | bn=BatchNormalization(out_channels), 18 | bst=BST() 19 | ) 20 | self.cname = "l_b_linear_bn_bst" 21 | 22 | def __call__(self, h, test=False): 23 | #self.inp_shape = h.data.shape 24 | h = self.bst(self.bn(self.bl(h), test=test)) 25 | return h 26 | 27 | def generate_c(self, link_idx, inp_shape): 28 | name = self.cname + str(link_idx) 29 | text = [] 30 | 31 | # BinaryLinear bl 32 | l = self.bl 33 | lName = l.name 34 | lname=name+'_'+lName 35 | for p in l.params(): 36 | pname=p.name 37 | if pname == 'W': 38 | text += [bu.np_to_uint8C(bu.binarize_real(p.data.T), lname+'_'+pname, 'col_major', pad='1')] 39 | num_classes = p.data.shape[0] 40 | fc_size = p.data.shape[1] 41 | elif pname == 'b': 42 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 43 | 44 | # BatchNormalization bn 45 | l = self.bn 46 | lName = l.name 47 | lname=name+'_'+lName 48 | for p in l.params(): 49 | pname=p.name 50 | if pname == 'gamma': 51 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 52 | elif pname == 'beta': 53 | text += [bu.np_to_floatC(p.data, lname+'_'+pname, 'row_major')] 54 | for p in l._persistent: 55 | pname=p 56 | persistent = l.__dict__[p] 57 | if pname == 'avg_mean': 58 | text += [bu.np_to_floatC(persistent, lname+'_mean', 'row_major')] 59 | elif pname == 'avg_var': 60 | text += [bu.np_to_floatC(np.sqrt(persistent, dtype=persistent.dtype), lname+'_std', 'row_major')] 61 | 62 | text = "\n".join(text)+'\n' 63 | m = 1 64 | n = fc_size 65 | k = num_classes 66 | ftext = "void {name}(float* input, uint8_t* output){{\n" 67 | ftext += " fused_float_linear_layer(input, {name}_bl_W, output, {name}_bl_b, {name}_bn_gamma, {name}_bn_beta, {name}_bn_mean, {name}_bn_std, {m}, {n}, {k}); \n}}\n\n" 68 | ftext = ftext.format(name=name, m=m, n=n, k=k) 69 | text += ftext 70 | 71 | return text 72 | 73 | def param_mem(self): 74 | mem = 0. 75 | for p in self.bl.params(): 76 | if p.name == 'W': 77 | m, w = p.data.shape 78 | mem += m*math.ceil(w / 8.) 79 | #Bias + BN 80 | mem += 5*m*32 81 | 82 | return mem 83 | 84 | def temp_mem(self, inp_shape): 85 | m = inp_shape[0] 86 | w = np.prod(inp_shape[1:]) 87 | res_w = math.ceil(w / 8.) 88 | return m*res_w 89 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_pool.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from chainer import link 3 | 4 | import chainer.functions as F 5 | 6 | class Pool2D(link.Link): 7 | def __init__(self, kern=3, stride=2, pad=0): 8 | self.kern = kern 9 | self.stride = stride 10 | self.pad = pad 11 | super(Pool2D, self).__init__() 12 | self.cname = "l_pool" 13 | 14 | def __call__(self, x): 15 | return F.max_pooling_2d(x, self.kern, stride=self.stride, pad=self.pad, cover_all=False) 16 | -------------------------------------------------------------------------------- /chainer_sequential/binary/links/link_softmax_cross_entropy.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from chainer import link 4 | from chainer.functions import softmax_cross_entropy 5 | 6 | class SoftmaxCrossEntropy(link.Link): 7 | def __init__(self): 8 | super(SoftmaxCrossEntropy, self).__init__() 9 | self.cname = "l_softmax_cross_entropy" 10 | 11 | def __call__(self, x, t): 12 | """Applies the softmax cross entropy. 13 | 14 | Args: 15 | x (~chainer.Variable): Batch of input vectors. 16 | 17 | Returns: 18 | ~chainer.Variable: Output of the softmax cross entropy. 19 | 20 | """ 21 | loss = softmax_cross_entropy(x,t) 22 | return loss 23 | -------------------------------------------------------------------------------- /chainer_sequential/binary/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/chainer_sequential/binary/utils/__init__.py -------------------------------------------------------------------------------- /chainer_sequential/binary_function.py: -------------------------------------------------------------------------------- 1 | import math 2 | from chainer import cuda, Variable 3 | from chainer import functions as F 4 | from .binary.functions import function_bst 5 | 6 | class Function(object): 7 | def __call__(self, x): 8 | raise NotImplementedError() 9 | 10 | def from_dict(self, dict): 11 | for attr, value in dict.iteritems(): 12 | setattr(self, attr, value) 13 | 14 | def to_dict(self): 15 | dict = {} 16 | for attr, value in self.__dict__.iteritems(): 17 | dict[attr] = value 18 | return dict 19 | 20 | class BinaryActivation(object): 21 | def __init__(self, nonlinearity="bst"): 22 | self.nonlinearity = nonlinearity 23 | 24 | def to_function(self): 25 | if self.nonlinearity.lower() == "bst": 26 | return bst() 27 | raise NotImplementedError() 28 | 29 | class bst(Function): 30 | def __init__(self): 31 | self._function = "bst" 32 | 33 | def __call__(self, x): 34 | return function_bst.bst(x) 35 | 36 | -------------------------------------------------------------------------------- /chainer_sequential/multiinputchain.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | from scipy.stats import entropy 5 | import chainer 6 | import sequential 7 | import multiinputsequential 8 | from chainer import optimizers, serializers, Variable 9 | from chainer import functions as F 10 | from chainer import reporter 11 | from chainer.functions.activation.softmax import softmax 12 | from chainer.functions.evaluation import accuracy 13 | from chainer.functions.loss import softmax_cross_entropy 14 | from chainer import link 15 | from chain import Chain 16 | 17 | class MultiInputChain(Chain): 18 | 19 | def __init__(self, **kwargs): 20 | super(MultiInputChain,self).__init__(**kwargs) 21 | 22 | def recursive_add_sequence(self, link, i, prefix=''): 23 | for j, tlink in enumerate(link.links): 24 | if isinstance(tlink, chainer.link.Link): 25 | self.add_link("{}link_{}_{}".format(prefix, i,j), tlink) 26 | elif isinstance(tlink, sequential.Sequential): 27 | self.recursive_add_sequence(tlink, str(i)+"_"+str(j)) 28 | 29 | def add_sequence(self, sequence): 30 | if isinstance(sequence, sequential.Sequential) == False: 31 | raise Exception() 32 | if sequence.local is not None: 33 | link = sequence.local 34 | if isinstance(link, chainer.link.Link): 35 | self.add_link("local_link_{}".format(0), link) 36 | elif isinstance(link, sequential.Sequential): 37 | self.recursive_add_sequence(link, 0, prefix='local_') 38 | for i, link in enumerate(sequence.inputs): 39 | if isinstance(link, chainer.link.Link): 40 | self.add_link("input_link_{}".format(i), link) 41 | elif isinstance(link, sequential.Sequential): 42 | self.recursive_add_sequence(link, i, prefix='input_') 43 | for i, link in enumerate(sequence.links): 44 | if isinstance(link, chainer.link.Link): 45 | self.add_link("link_{}".format(i), link) 46 | #print(link.name, link) 47 | elif isinstance(link, sequential.Sequential): 48 | self.recursive_add_sequence(link, i) 49 | 50 | self.sequence = sequence 51 | self.test = False 52 | 53 | def __call__(self, *args): 54 | split = len(args)//2 55 | x = args[:split] 56 | t = args[split:] 57 | 58 | return self.evaluate(x,t) -------------------------------------------------------------------------------- /chainer_sequential/test.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from chainer import Variable 3 | from chainer import functions as F 4 | from sequential import Sequential 5 | import link 6 | import function 7 | import util 8 | import chain 9 | 10 | # Linear test 11 | x = np.random.normal(scale=1, size=(2, 28*28)).astype(np.float32) 12 | x = Variable(x) 13 | 14 | seq = Sequential(weight_initializer="GlorotNormal", weight_init_std=0.05) 15 | seq.add(link.Linear(28*28, 500)) 16 | seq.add(link.BatchNormalization(500)) 17 | seq.add(link.Linear(500, 500)) 18 | seq.add(function.Activation("clipped_relu")) 19 | seq.add(link.Linear(500, 500, use_weightnorm=True)) 20 | seq.add(function.Activation("crelu")) # crelu outputs 2x 21 | seq.add(link.BatchNormalization(1000)) 22 | seq.add(link.Linear(1000, 500)) 23 | seq.add(function.Activation("elu")) 24 | seq.add(link.Linear(500, 500, use_weightnorm=True)) 25 | seq.add(function.Activation("hard_sigmoid")) 26 | seq.add(link.BatchNormalization(500)) 27 | seq.add(link.Linear(500, 500)) 28 | seq.add(function.Activation("leaky_relu")) 29 | seq.add(link.Linear(500, 500, use_weightnorm=True)) 30 | seq.add(function.Activation("relu")) 31 | seq.add(link.BatchNormalization(500)) 32 | seq.add(link.Linear(500, 500)) 33 | seq.add(function.Activation("sigmoid")) 34 | seq.add(link.Linear(500, 500, use_weightnorm=True)) 35 | seq.add(function.Activation("softmax")) 36 | seq.add(link.BatchNormalization(500)) 37 | seq.add(link.Linear(500, 500)) 38 | seq.add(function.Activation("softplus")) 39 | seq.add(link.Linear(500, 500, use_weightnorm=True)) 40 | seq.add(function.Activation("tanh")) 41 | seq.add(link.Linear(500, 10)) 42 | seq.build() 43 | 44 | y = seq(x) 45 | print y.data.shape 46 | 47 | # Conv test 48 | x = np.random.normal(scale=1, size=(2, 3, 96, 96)).astype(np.float32) 49 | x = Variable(x) 50 | 51 | seq = Sequential(weight_initializer="GlorotNormal", weight_init_std=0.05) 52 | seq.add(link.Convolution2D(3, 64, ksize=4, stride=2, pad=0)) 53 | seq.add(link.BatchNormalization(64)) 54 | seq.add(function.Activation("relu")) 55 | seq.add(link.Convolution2D(64, 128, ksize=4, stride=2, pad=0)) 56 | seq.add(link.BatchNormalization(128)) 57 | seq.add(function.Activation("relu")) 58 | seq.add(link.Convolution2D(128, 256, ksize=4, stride=2, pad=0)) 59 | seq.add(link.BatchNormalization(256)) 60 | seq.add(function.Activation("relu")) 61 | seq.add(link.Convolution2D(256, 512, ksize=4, stride=2, pad=0)) 62 | seq.add(link.BatchNormalization(512)) 63 | seq.add(function.Activation("relu")) 64 | seq.add(link.Convolution2D(512, 1024, ksize=4, stride=2, pad=0)) 65 | seq.add(link.BatchNormalization(1024)) 66 | seq.add(function.Activation("relu")) 67 | seq.add(link.Linear(None, 10, use_weightnorm=True)) 68 | seq.add(function.softmax()) 69 | seq.build() 70 | 71 | y = seq(x) 72 | print y.data.shape 73 | 74 | # Deconv test 75 | x = np.random.normal(scale=1, size=(2, 100)).astype(np.float32) 76 | x = Variable(x) 77 | 78 | image_size = 96 79 | # compute projection width 80 | input_size = util.get_in_size_of_deconv_layers(image_size, num_layers=3, ksize=4, stride=2) 81 | # compute required paddings 82 | paddings = util.get_paddings_of_deconv_layers(image_size, num_layers=3, ksize=4, stride=2) 83 | 84 | seq = Sequential(weight_initializer="GlorotNormal", weight_init_std=0.05) 85 | seq.add(link.Linear(100, 64 * input_size ** 2)) 86 | seq.add(link.BatchNormalization(64 * input_size ** 2)) 87 | seq.add(function.Activation("relu")) 88 | seq.add(function.reshape((-1, 64, input_size, input_size))) 89 | seq.add(link.Deconvolution2D(64, 32, ksize=4, stride=2, pad=paddings.pop(0), use_weightnorm=True)) 90 | seq.add(link.BatchNormalization(32)) 91 | seq.add(function.Activation("relu")) 92 | seq.add(link.Deconvolution2D(32, 16, ksize=4, stride=2, pad=paddings.pop(0), use_weightnorm=True)) 93 | seq.add(link.BatchNormalization(16)) 94 | seq.add(function.Activation("relu")) 95 | seq.add(link.Deconvolution2D(16, 3, ksize=4, stride=2, pad=paddings.pop(0), use_weightnorm=True)) 96 | seq.build() 97 | 98 | y = seq(x) 99 | print y.data.shape 100 | 101 | # train test 102 | x = np.random.normal(scale=1, size=(128, 28*28)).astype(np.float32) 103 | x = Variable(x) 104 | 105 | seq = Sequential(weight_initializer="GlorotNormal", weight_init_std=0.05) 106 | seq.add(link.Linear(28*28, 500, use_weightnorm=True)) 107 | seq.add(link.BatchNormalization(500)) 108 | seq.add(function.Activation("relu")) 109 | seq.add(link.Linear(None, 500, use_weightnorm=True)) 110 | seq.add(link.BatchNormalization(500)) 111 | seq.add(function.Activation("relu")) 112 | seq.add(link.Linear(500, 28*28, use_weightnorm=True)) 113 | seq.build() 114 | 115 | chain = chain.Chain() 116 | chain.add_sequence(seq) 117 | chain.setup_optimizers("adam", 0.001, momentum=0.9, weight_decay=0.000001, gradient_clipping=10) 118 | 119 | for i in xrange(100): 120 | y = chain(x) 121 | loss = F.mean_squared_error(x, y) 122 | chain.backprop(loss) 123 | print float(loss.data) 124 | 125 | chain.save("model") -------------------------------------------------------------------------------- /chainer_sequential/util.py: -------------------------------------------------------------------------------- 1 | def get_conv_outsize(in_size, ksize, stride, padding, cover_all=False, d=1): 2 | dk = ksize + (ksize - 1) * (d - 1) 3 | if cover_all: 4 | return (in_size + padding * 2 - dk + stride - 1) // stride + 1 5 | else: 6 | return (in_size + padding * 2 - dk) // stride + 1 7 | 8 | def get_conv_padding(in_size, ksize, stride): 9 | pad2 = stride - (in_size - ksize) % stride 10 | if pad2 % stride == 0: 11 | return 0 12 | if pad2 % 2 == 1: 13 | return pad2 14 | return pad2 / 2 15 | 16 | def get_deconv_padding(in_size, out_size, ksize, stride, cover_all=False): 17 | if cover_all: 18 | return (stride * (in_size - 1) + ksize - stride + 1 - out_size) // 2 19 | else: 20 | return (stride * (in_size - 1) + ksize - out_size) // 2 21 | 22 | def get_deconv_outsize(in_size, ksize, stride, padding, cover_all=False): 23 | if cover_all: 24 | return stride * (in_size - 1) + ksize - stride + 1 - 2 * padding 25 | else: 26 | return stride * (in_size - 1) + ksize - 2 * padding 27 | 28 | def get_deconv_insize(out_size, ksize, stride, padding, cover_all=False): 29 | if cover_all: 30 | return (out_size - ksize + stride - 1 + 2 * padding) // stride + 1 31 | else: 32 | return (out_size - ksize + 2 * padding) // stride + 1 33 | 34 | def get_paddings_of_deconv_layers(target_size, num_layers, ksize, stride): 35 | # compute required deconv paddings 36 | paddings = [] 37 | deconv_out_sizes = [target_size] 38 | deconv_out_sizes.append(get_conv_outsize(deconv_out_sizes[-1], ksize, stride, get_conv_padding(deconv_out_sizes[-1], ksize, stride))) 39 | deconv_out_sizes.append(get_conv_outsize(deconv_out_sizes[-1], ksize, stride, get_conv_padding(deconv_out_sizes[-1], ksize, stride))) 40 | deconv_out_sizes.append(get_conv_outsize(deconv_out_sizes[-1], ksize, stride, get_conv_padding(deconv_out_sizes[-1], ksize, stride))) 41 | # target_size of hidden layer must be an even number 42 | for i, size in enumerate(deconv_out_sizes[1:-1]): 43 | if size % 2 == 1: 44 | deconv_out_sizes[i+1] = size + 1 45 | 46 | deconv_out_sizes.reverse() 47 | paddings = [] 48 | for i, (in_size, target_size) in enumerate(zip(deconv_out_sizes[:-1], deconv_out_sizes[1:])): 49 | paddings.append(get_deconv_padding(in_size, target_size, ksize, stride)) 50 | 51 | return paddings 52 | 53 | def get_in_size_of_deconv_layers(target_size, num_layers, ksize, stride): 54 | # compute required deconv paddings 55 | paddings = [] 56 | deconv_out_sizes = [target_size] 57 | deconv_out_sizes.append(get_conv_outsize(deconv_out_sizes[-1], ksize, stride, get_conv_padding(deconv_out_sizes[-1], ksize, stride))) 58 | deconv_out_sizes.append(get_conv_outsize(deconv_out_sizes[-1], ksize, stride, get_conv_padding(deconv_out_sizes[-1], ksize, stride))) 59 | deconv_out_sizes.append(get_conv_outsize(deconv_out_sizes[-1], ksize, stride, get_conv_padding(deconv_out_sizes[-1], ksize, stride))) 60 | return deconv_out_sizes[-1] -------------------------------------------------------------------------------- /chainer_sequential/weightnorm/__init__.py: -------------------------------------------------------------------------------- 1 | from linear import Linear 2 | from convolution_2d import Convolution2D 3 | from deconvolution_2d import Deconvolution2D -------------------------------------------------------------------------------- /chainer_sequential/weightnorm/linear.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | from chainer import cuda, Variable 4 | from chainer import initializers 5 | from chainer import link 6 | from chainer import function 7 | from chainer.utils import array 8 | from chainer.utils import type_check 9 | from chainer.functions.connection import linear 10 | 11 | def _as_mat(x): 12 | if x.ndim == 2: 13 | return x 14 | return x.reshape(len(x), -1) 15 | 16 | def get_norm(W): 17 | xp = cuda.get_array_module(W) 18 | norm = xp.sqrt(xp.sum(W ** 2, axis=1)) + 1e-9 19 | norm = norm.reshape((-1, 1)) 20 | return norm 21 | 22 | class LinearFunction(linear.LinearFunction): 23 | 24 | def check_type_forward(self, in_types): 25 | n_in = in_types.size() 26 | type_check.expect(3 <= n_in, n_in <= 4) 27 | x_type, w_type, g_type = in_types[:3] 28 | 29 | type_check.expect( 30 | x_type.dtype.kind == "f", 31 | w_type.dtype.kind == "f", 32 | g_type.dtype.kind == "f", 33 | x_type.ndim >= 2, 34 | w_type.ndim == 2, 35 | g_type.ndim == 2, 36 | type_check.prod(x_type.shape[1:]) == w_type.shape[1], 37 | ) 38 | 39 | if n_in.eval() == 4: 40 | b_type = in_types[3] 41 | type_check.expect( 42 | b_type.dtype == x_type.dtype, 43 | b_type.ndim == 1, 44 | b_type.shape[0] == w_type.shape[0], 45 | ) 46 | 47 | def forward(self, inputs): 48 | x = _as_mat(inputs[0]) 49 | V = inputs[1] 50 | g = inputs[2] 51 | xp = cuda.get_array_module(V) 52 | 53 | self.normV = get_norm(V) 54 | self.normalizedV = V / self.normV 55 | self.W = g * self.normalizedV 56 | 57 | y = x.dot(self.W.T).astype(x.dtype, copy=False) 58 | if len(inputs) == 4: 59 | b = inputs[3] 60 | y += b 61 | return y, 62 | 63 | def backward(self, inputs, grad_outputs): 64 | x = _as_mat(inputs[0]) 65 | V = inputs[1] 66 | g = inputs[2] 67 | W = self.W 68 | xp = cuda.get_array_module(x) 69 | 70 | gy = grad_outputs[0] 71 | gx = gy.dot(W).astype(x.dtype, copy=False).reshape(inputs[0].shape) 72 | gW = gy.T.dot(x).astype(W.dtype, copy=False) 73 | 74 | gg = xp.sum(gW * self.normalizedV, axis=1, keepdims=True).astype(g.dtype, copy=False) 75 | gV = g * (gW - gg * self.normalizedV) / self.normV 76 | gV = gV.astype(V.dtype, copy=False) 77 | 78 | if len(inputs) == 4: 79 | gb = gy.sum(0) 80 | return gx, gV, gg, gb 81 | else: 82 | return gx, gV, gg 83 | 84 | def linear(x, V, g, b=None): 85 | if b is None: 86 | return LinearFunction()(x, V, g) 87 | else: 88 | return LinearFunction()(x, V, g, b) 89 | 90 | class Linear(link.Link): 91 | 92 | def __init__(self, in_size, out_size, wscale=1, bias=0, nobias=False, initialV=None, dtype=np.float32): 93 | super(Linear, self).__init__() 94 | 95 | self.initialV = initialV 96 | self.wscale = wscale 97 | self.nobias = nobias 98 | self.dtype = dtype 99 | self.out_size = out_size 100 | 101 | if in_size is None: 102 | self.add_uninitialized_param("V") 103 | else: 104 | self._initialize_weight(in_size) 105 | 106 | if nobias: 107 | self.b = None 108 | else: 109 | self.add_uninitialized_param("b") 110 | 111 | self.add_uninitialized_param("g") 112 | 113 | def _initialize_weight(self, in_size): 114 | self.add_param("V", (self.out_size, in_size), initializer=initializers._get_initializer(self.initialV, math.sqrt(self.wscale))) 115 | 116 | def _initialize_params(self, t): 117 | xp = cuda.get_array_module(t) 118 | self.mean_t = xp.mean(t, axis=0) 119 | self.std_t = xp.sqrt(xp.var(t, axis=0)) 120 | g = 1 / self.std_t 121 | b = -self.mean_t / self.std_t 122 | 123 | # print "g <- {}, b <- {}".format(g, b) 124 | 125 | if self.nobias == False: 126 | self.add_param("b", self.out_size, initializer=initializers.Constant(b, self.dtype)) 127 | self.add_param("g", (self.out_size, 1), initializer=initializers.Constant(g.reshape(-1, 1), self.dtype)) 128 | 129 | def _get_W_data(self): 130 | V = self.V.data 131 | xp = cuda.get_array_module(V) 132 | norm = get_norm(V) 133 | V = V / norm 134 | return self.g.data * V 135 | 136 | def __call__(self, x): 137 | if hasattr(self, "V") == False: 138 | with cuda.get_device(self._device_id): 139 | self._initialize_weight(x.size // len(x.data)) 140 | 141 | if hasattr(self, "b") == False or hasattr(self, "g") == False: 142 | xp = cuda.get_array_module(x.data) 143 | t = linear(x, self.V, Variable(xp.full((self.out_size, 1), 1.0).astype(x.dtype))) # compute output with g = 1 and without bias 144 | self._initialize_params(t.data) 145 | return (t - self.mean_t) / self.std_t 146 | 147 | return linear(x, self.V, self.g, self.b) -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/datasets/__init__.py -------------------------------------------------------------------------------- /datasets/mnist.py: -------------------------------------------------------------------------------- 1 | import chainer 2 | import numpy as np 3 | from chainer.datasets import TupleDataset 4 | 5 | def get_mnist(): 6 | train, test = chainer.datasets.get_mnist(ndim=3) 7 | 8 | train_data = [t for t in train] 9 | test_data = [t for t in test] 10 | 11 | train_data = np.array(train_data) 12 | test_data = np.array(test_data) 13 | 14 | train_data = np.expand_dims(train_data, 1) 15 | test_data = np.expand_dims(test_data, 1) 16 | 17 | train_xs = train_data[:,:,0].T 18 | train_ys = train_data[:,:,1].T 19 | 20 | test_xs = test_data[:,:,0].T 21 | test_ys = test_data[:,:,1].T 22 | 23 | train = TupleDataset(*(train_xs.tolist() + train_ys.tolist())) 24 | test = TupleDataset(*(test_xs.tolist() + test_ys.tolist())) 25 | 26 | return train,test 27 | -------------------------------------------------------------------------------- /datasets/util/process_mvmc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import argparse 4 | import numpy as np 5 | import copy 6 | 7 | from PIL import Image 8 | 9 | def details(filename): 10 | class_name = filename.split('/')[-1].split('_')[0] 11 | cam = filename.split('/')[-1].split('_')[1] 12 | cam = int(cam) 13 | oid = filename.split('/')[-1].split('_')[2].split('.')[0] 14 | oid = int(oid) 15 | return {'class': class_name, 'cam': cam, 'oid': oid, 'path': filename} 16 | 17 | 18 | parser = argparse.ArgumentParser(description='Dataset directory') 19 | parser.add_argument('-d', '--data_dir') 20 | 21 | args = parser.parse_args() 22 | classes = ['car', 'bus', 'person'] 23 | image_dir = os.path.join(args.data_dir, 'annotated') 24 | 25 | xs, ys = [], [] 26 | for i in range(1, 242): 27 | root = os.path.join(image_dir, str(i)) 28 | imgs = glob.glob(os.path.join(root, '*')) 29 | if len(imgs) == 0: 30 | continue 31 | 32 | dets = [] 33 | for img in imgs: 34 | try: 35 | det = details(img) 36 | except: 37 | continue 38 | 39 | if det['path'].split('.')[0][-1] == '_': 40 | continue 41 | dets.append(details(img)) 42 | 43 | res = [] 44 | while(len(dets) > 0): 45 | x, y = [], [] 46 | root = copy.copy(dets[0]) 47 | for cam in range(6): 48 | idxs = [] 49 | sample = [] 50 | cam_found = False 51 | for i, det in enumerate(dets): 52 | if det['cam'] == cam and det['oid'] == root['oid'] and det['class'] == root['class']: 53 | sample.append(det) 54 | im = np.array(Image.open(det['path'])).astype(np.float32) 55 | im = np.rollaxis(im, 2, 0) 56 | im /= im.max() 57 | x.append(im) 58 | y.append(classes.index(det['class'])) 59 | idxs.append(i) 60 | cam_found = True 61 | if not cam_found: 62 | x.append(np.zeros((3,32,32))) 63 | y.append(len(classes)) 64 | 65 | # print(sample) 66 | dets = [i for j, i in enumerate(dets) if j not in idxs] 67 | 68 | xs.append(x) 69 | ys.append(y) 70 | 71 | xs = np.array(xs, dtype=np.float32) 72 | ys = np.array(ys, dtype=np.int32) 73 | 74 | np.savez_compressed('MVMC.npz', X=xs, y=ys) 75 | -------------------------------------------------------------------------------- /deepopt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/deepopt/__init__.py -------------------------------------------------------------------------------- /deepopt/chooser.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | import numpy as np 3 | import scipy.stats as sps 4 | 5 | def get_max_epoch(do, new_point): 6 | X = np.array(do.X) 7 | for i, param in enumerate(do.params): 8 | if param == 'nepochs': 9 | continue 10 | X = X[X[:, i] == new_point[param]] 11 | 12 | return np.max(X[:, do.params.index('nepochs')]) 13 | 14 | def compute_ts(do): 15 | X_samples = np.array(do.X_samples) 16 | 17 | with warnings.catch_warnings(record=True) as w: 18 | warnings.simplefilter('always') 19 | y_pred, sigma = do.gp.predict(X_samples, return_std=True) 20 | X = np.array(do.X) 21 | 22 | ts = [] 23 | for x in X_samples: 24 | x = dict(zip(do.params, x)) 25 | epochs = [] 26 | for xi in X: 27 | xi = dict(zip(do.params, xi)) 28 | same_model = True 29 | for k in x.keys(): 30 | if x[k] != xi[k] and k != 'nepochs': 31 | same_model = False 32 | 33 | if same_model: 34 | epochs.append(xi['nepochs']) 35 | 36 | start_epoch = np.max(epochs) 37 | t = max(1, x['nepochs'] - start_epoch) 38 | ts.append(t) 39 | ts = np.array(ts) 40 | 41 | return ts 42 | 43 | 44 | 45 | class SimpleChooser(object): 46 | def __init__(self, beta=1): 47 | self.beta = beta 48 | def choose(self, do, y_pred, sigma): 49 | return np.argmax(u+self.beta*v) 50 | 51 | class EIChooser(object): 52 | def __init__(self): 53 | pass 54 | 55 | def get_ei(self, do, y_pred, sigma, eps=1e-9, ts=None): 56 | ei_values = np.min(do.y, axis=0) 57 | y_pred = np.expand_dims(y_pred,1) 58 | sigma = np.expand_dims(sigma,1) 59 | func_s = sigma 60 | u = (ei_values - y_pred) / (func_s+eps) 61 | ncdf = sps.norm.cdf(u) 62 | npdf = sps.norm.pdf(u) 63 | ei = np.mean(func_s*( u*ncdf + npdf),axis=1) 64 | 65 | if ts is not None: 66 | ei = ei / ts 67 | 68 | return ei 69 | 70 | def choose(self, do, y_pred, sigma, ts=None, norm_k=1, eps=1e-9, return_values=False): 71 | ei = self.get_ei(do, y_pred, sigma, eps, ts) 72 | max_idx = np.argmax(ei) 73 | if return_values: 74 | return max_idx, ei 75 | else: 76 | return max_idx 77 | 78 | 79 | class GridChooser(object): 80 | def __init__(self): 81 | self.init = False 82 | 83 | def choose_gen(self, do, y_pred, sigma, return_values=False): 84 | X_samples = np.array(do.X_samples) 85 | epoch_idx = do.params.index('nepochs') 86 | max_epoch = np.max(X_samples[:, epoch_idx]) 87 | while True: 88 | for idx, x in enumerate(do.X_samples): 89 | if x[epoch_idx] == max_epoch: 90 | if return_values: 91 | yield idx, np.zeros(len(y_pred)) 92 | else: 93 | yield idx 94 | 95 | def choose(self, do, y_pred, sigma, return_values=False): 96 | if not self.init: 97 | self.gen = self.choose_gen(do, y_pred, sigma, return_values=return_values) 98 | self.init = True 99 | 100 | return self.gen.next() 101 | 102 | 103 | class EpochChooser(EIChooser): 104 | def __init__(self, k=5, **kwargs): 105 | super(EpochChooser, self).__init__(**kwargs) 106 | self.k = k 107 | 108 | def choose(self, do, y_pred, sigma, return_values=False): 109 | traces = do.get_traces() 110 | traces = [t for t in traces if t['action'] == 'add_point'] 111 | if len(traces) == 0: 112 | curr_point = None 113 | else: 114 | curr_point = dict(zip(do.params, traces[-1]['x'])) 115 | 116 | X_samples = np.array(do.X_samples) 117 | X = np.array(do.X) 118 | ts = compute_ts(do) 119 | ei = self.get_ei(do, y_pred, sigma, ts=ts) 120 | ei = np.array(ei) 121 | top_idxs = np.argsort(ei)[::-1][:self.k] 122 | max_epoch, best_idx = 0, 0 123 | for idx in top_idxs: 124 | new_point = dict(zip(do.params, X_samples[idx])) 125 | if curr_point is not None: 126 | same_model = True 127 | for key in new_point.keys(): 128 | if curr_point[key] != new_point[key] and key != 'nepochs': 129 | same_model = False 130 | 131 | if same_model: 132 | best_idx = idx 133 | break 134 | 135 | epoch = get_max_epoch(do, new_point) 136 | if epoch > max_epoch: 137 | best_idx = idx 138 | 139 | if return_values: 140 | return best_idx, ei 141 | else: 142 | return best_idx 143 | -------------------------------------------------------------------------------- /elaas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/elaas/__init__.py -------------------------------------------------------------------------------- /elaas/family/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/elaas/family/__init__.py -------------------------------------------------------------------------------- /elaas/family/binary.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.chain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.function import * 7 | from chainer_sequential.link import * 8 | from chainer_sequential.binary_link import * 9 | from chainer import functions as F 10 | 11 | class BinaryFamily: 12 | def __init__(self, folder="_models/binary", prefix=None, input_dims=1, output_dims=10): 13 | self.folder = folder 14 | self.prefix = prefix 15 | self.input_dims = input_dims 16 | self.output_dims = output_dims 17 | 18 | def get_configurable_params(self): 19 | return ["nfilters_embeded", "nlayers_embeded", "lr"] 20 | 21 | def generate_model(self, **kwargs): 22 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 23 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 24 | 25 | model = Sequential() 26 | for i in range(nlayers_embeded): 27 | if i == 0: 28 | nfilters = self.input_dims 29 | model.add(ConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 30 | # model.add(ConvBNBST(nfilters, nfilters_embeded, 3, 1, 1)) 31 | else: 32 | nfilters = nfilters_embeded 33 | model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 34 | # model.add(BinaryConvBNBST(nfilters, nfilters_embeded, 3, 1, 1)) 35 | 36 | model.add(BinaryLinearBNSoftmax(None, self.output_dims)) 37 | model.build() 38 | return model 39 | 40 | def load_chain_model(self, **kwargs): 41 | name = self.get_name(**kwargs) 42 | path = '{}/{}'.format(self.folder,name) 43 | epoch = int(kwargs.get("nepochs",2)) 44 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 45 | 46 | chain, model = self.setup_chain_model(**kwargs) 47 | S.load_npz(fn, chain) 48 | return chain, model 49 | 50 | def setup_chain_model(self, **kwargs): 51 | model = self.generate_model(**kwargs) 52 | 53 | branchweight = kwargs.get("branchweight", 3) 54 | lr = kwargs.get("lr", 0.001) 55 | 56 | chain = Chain(branchweight=branchweight) 57 | chain.add_sequence(model) 58 | chain.setup_optimizers('adam', lr) 59 | return chain, model 60 | 61 | def get_name(self, **kwargs): 62 | if self.prefix is not None: 63 | name = "{}_".format(self.prefix) 64 | else: 65 | name = "" 66 | for k,v in kwargs.iteritems(): 67 | if k=='nepochs' or k=='ent_T': 68 | continue 69 | name = "{}_{}_{}".format(name, k, v) 70 | return name 71 | 72 | def train_model(self, trainset, testset, **kwargs): 73 | chain, model = self.setup_chain_model(**kwargs) 74 | 75 | nepochs = int(kwargs.get("nepochs", 2)) 76 | name = self.get_name(**kwargs) 77 | 78 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 79 | testset, nepoch=nepochs, resume=True) 80 | trainer.run() 81 | 82 | return trainer, model, chain 83 | -------------------------------------------------------------------------------- /elaas/family/binary_cloud.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.chain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.function import * 7 | from chainer_sequential.link import * 8 | from chainer_sequential.binary_link import * 9 | from chainer import functions as F 10 | 11 | class BinaryCloudFamily: 12 | def __init__(self, folder="_models/binary_cloud", prefix=None, input_dims=1, output_dims=10): 13 | self.folder = folder 14 | self.prefix = prefix 15 | self.input_dims = input_dims 16 | self.output_dims = output_dims 17 | 18 | def get_configurable_params(self): 19 | return ["nfilters_embeded", "nlayers_embeded", "nfilters_cloud", "nlayers_cloud", "branchweight", "lr", "ent_T"] 20 | 21 | def generate_model(self, **kwargs): 22 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 23 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 24 | nfilters_cloud = int(kwargs.get("nfilters_cloud", 1)) 25 | nlayers_cloud = int(kwargs.get("nlayers_cloud", 1)) 26 | 27 | model = Sequential() 28 | for i in range(nlayers_embeded): 29 | if i == 0: 30 | nfilters = self.input_dims 31 | model.add(ConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 32 | else: 33 | nfilters = nfilters_embeded 34 | model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 35 | 36 | branch = Sequential() 37 | branch.add(BinaryLinearBNSoftmax(None, self.output_dims)) 38 | model.add(branch) 39 | 40 | for i in range(nlayers_cloud): 41 | if i == 0: 42 | nfilters = nfilters_embeded 43 | else: 44 | nfilters = nfilters_cloud 45 | model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 46 | model.add(BinaryLinearBNSoftmax(None, self.output_dims)) 47 | model.build() 48 | return model 49 | 50 | def load_chain_model(self, **kwargs): 51 | name = self.get_name(**kwargs) 52 | path = '{}/{}'.format(self.folder,name) 53 | epoch = int(kwargs.get("nepochs",2)) 54 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 55 | 56 | chain, model = self.setup_chain_model(**kwargs) 57 | S.load_npz(fn, chain) 58 | return chain, model 59 | 60 | def setup_chain_model(self, **kwargs): 61 | model = self.generate_model(**kwargs) 62 | 63 | branchweight = kwargs.get("branchweight", 3) 64 | ent_T = kwargs.get("ent_T", None) 65 | lr = kwargs.get("lr", 0.001) 66 | 67 | chain = Chain(branchweight=branchweight, ent_T=ent_T) 68 | chain.add_sequence(model) 69 | chain.setup_optimizers('adam', lr) 70 | return chain, model 71 | 72 | def get_name(self, **kwargs): 73 | if self.prefix is not None: 74 | name = "{}_".format(self.prefix) 75 | else: 76 | name = "" 77 | for k,v in kwargs.iteritems(): 78 | if k=='nepochs' or k=='ent_T': 79 | continue 80 | name = "{}_{}_{}".format(name, k, v) 81 | return name 82 | 83 | def train_model(self, trainset, testset, **kwargs): 84 | chain, model = self.setup_chain_model(**kwargs) 85 | 86 | nepochs = int(kwargs.get("nepochs", 2)) 87 | name = self.get_name(**kwargs) 88 | 89 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 90 | testset, nepoch=nepochs, resume=True) 91 | trainer.run() 92 | 93 | return trainer, model 94 | -------------------------------------------------------------------------------- /elaas/family/binary_float.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.chain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.function import * 7 | from chainer_sequential.link import * 8 | from chainer_sequential.binary_link import * 9 | from chainer import functions as F 10 | 11 | class BinaryFloatFamily: 12 | def __init__(self, folder="_models/binary_float", prefix=None, input_dims=1, output_dims=10): 13 | self.folder = folder 14 | self.prefix = prefix 15 | self.input_dims = input_dims 16 | self.output_dims = output_dims 17 | 18 | def get_configurable_params(self): 19 | return ["nfilters_embeded", "nlayers_embeded", "nfilters_cloud", "nlayers_cloud", "branchweight", "lr", "ent_T"] 20 | 21 | def generate_model(self, **kwargs): 22 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 23 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 24 | nfilters_cloud = int(kwargs.get("nfilters_cloud", 1)) 25 | nlayers_cloud = int(kwargs.get("nlayers_cloud", 1)) 26 | 27 | model = Sequential() 28 | for i in range(nlayers_embeded): 29 | if i == 0: 30 | nfilters = self.input_dims 31 | model.add(ConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 32 | else: 33 | nfilters = nfilters_embeded 34 | model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 35 | 36 | branch = Sequential() 37 | branch.add(BinaryLinearBNSoftmax(None, self.output_dims)) 38 | model.add(branch) 39 | 40 | # float branch 41 | for i in range(nlayers_cloud): 42 | if i == 0: 43 | nfilters = nfilters_embeded 44 | else: 45 | nfilters = nfilters_cloud 46 | model.add(Convolution2D(nfilters, nfilters_cloud, 3, 1, 1)) 47 | model.add(max_pooling_2d(3,1,1)) 48 | model.add(BatchNormalization(nfilters_cloud)) 49 | model.add(Activation('bst')) 50 | # Note: should we move pool to before batch norm like in binary? 51 | model.add(Linear(None, self.output_dims)) 52 | model.add(BatchNormalization(self.output_dims)) 53 | 54 | model.build() 55 | return model 56 | 57 | def load_chain_model(self, **kwargs): 58 | name = self.get_name(**kwargs) 59 | path = '{}/{}'.format(self.folder,name) 60 | epoch = int(kwargs.get("nepochs",2)) 61 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 62 | 63 | chain, model = self.setup_chain_model(**kwargs) 64 | S.load_npz(fn, chain) 65 | return chain, model 66 | 67 | def setup_chain_model(self, **kwargs): 68 | model = self.generate_model(**kwargs) 69 | 70 | branchweight = kwargs.get("branchweight", 3) 71 | ent_T = kwargs.get("ent_T", None) 72 | lr = kwargs.get("lr", 0.001) 73 | 74 | chain = Chain(branchweight=branchweight, ent_T=ent_T) 75 | chain.add_sequence(model) 76 | chain.setup_optimizers('adam', lr) 77 | return chain, model 78 | 79 | def get_name(self, **kwargs): 80 | if self.prefix is not None: 81 | name = "{}_".format(self.prefix) 82 | else: 83 | name = "" 84 | for k,v in kwargs.iteritems(): 85 | if k=='nepochs' or k=='ent_T': 86 | continue 87 | name = "{}_{}_{}".format(name, k, float(v)) 88 | return name 89 | 90 | def train_model(self, trainset, testset, **kwargs): 91 | chain, model = self.setup_chain_model(**kwargs) 92 | 93 | nepochs = int(kwargs.get("nepochs", 2)) 94 | name = self.get_name(**kwargs) 95 | 96 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 97 | testset, nepoch=nepochs, resume=True) 98 | trainer.run() 99 | 100 | return trainer, model 101 | -------------------------------------------------------------------------------- /elaas/family/ebnn.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.multiinputchain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.multiinputsequential import MultiInputSequential 7 | from chainer_sequential.function import * 8 | from chainer_sequential.link import * 9 | from chainer_sequential.binary_link import * 10 | from chainer import functions as F 11 | 12 | class EBNNFamily: 13 | def __init__(self, folder="_models/ebnn", prefix=None, input_dims=3, output_dims=3, batchsize=10): 14 | self.folder = folder 15 | self.prefix = prefix 16 | self.input_dims = input_dims 17 | self.output_dims = output_dims 18 | self.batchsize = batchsize 19 | 20 | def get_configurable_params(self): 21 | return ["nfilters_embeded", "nlayers_embeded", "nfilters_embeded_last", "branchweight", "lr", "ent_T"] 22 | 23 | def generate_model(self, **kwargs): 24 | nfilters_embeded_last = int(kwargs.get("nfilters_embeded_last", 1)) 25 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 26 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 27 | 28 | input_model = Sequential() 29 | for i in range(nlayers_embeded): 30 | if i == 0: 31 | nfilters = self.input_dims 32 | input_model.add(ConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 33 | elif i == nlayers_embeded-1: 34 | nfilters = nfilters_embeded 35 | input_model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded_last, 3, 1, 1, 3, 1, 1)) 36 | else: 37 | nfilters = nfilters_embeded 38 | input_model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 39 | input_model.add(BinaryLinearBNSoftmax(None, self.output_dims)) 40 | 41 | input_model.build() 42 | return input_model 43 | 44 | def load_chain_model(self, **kwargs): 45 | name = self.get_name(**kwargs) 46 | path = '{}/{}'.format(self.folder,name) 47 | epoch = int(kwargs.get("nepochs",2)) 48 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 49 | 50 | chain, model = self.setup_chain_model(**kwargs) 51 | S.load_npz(fn, chain) 52 | return chain, model 53 | 54 | def setup_chain_model(self, **kwargs): 55 | model = self.generate_model(**kwargs) 56 | 57 | branchweight = kwargs.get("branchweight", 3) 58 | ent_T = kwargs.get("ent_T", None) 59 | lr = kwargs.get("lr", 0.001) 60 | 61 | chain = Chain(branchweight=branchweight, ent_T=ent_T) 62 | chain.add_sequence(model) 63 | chain.setup_optimizers('adam', lr) 64 | return chain, model 65 | 66 | def get_name(self, **kwargs): 67 | if self.prefix is not None: 68 | name = "{}_".format(self.prefix) 69 | else: 70 | name = "" 71 | for k,v in kwargs.iteritems(): 72 | if k=='nepochs' or k=='ent_T': 73 | #if k=='nepochs': 74 | continue 75 | name = "{}_{}_{}".format(name, k, float(v)) 76 | return name 77 | 78 | def train_model(self, trainset, testset, **kwargs): 79 | chain, model = self.setup_chain_model(**kwargs) 80 | 81 | nepochs = int(kwargs.get("nepochs", 2)) 82 | ent_T = kwargs.get("ent_T", None) 83 | name = self.get_name(**kwargs) 84 | 85 | reports = [ 86 | 'epoch', 87 | 'main/loss', 88 | 'main/accuracy', 89 | 'validation/main/accuracy', 90 | 'validation/main/memory' 91 | ] 92 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 93 | testset, batchsize=self.batchsize, nepoch=nepochs, resume=True, reports=reports) 94 | trainer.run() 95 | 96 | return trainer, model, chain 97 | -------------------------------------------------------------------------------- /elaas/family/float.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.chain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.function import * 7 | from chainer_sequential.link import * 8 | from chainer_sequential.binary_link import * 9 | from chainer import functions as F 10 | 11 | class FloatFamily: 12 | def __init__(self, folder="_models/float", prefix=None, input_dims=1, output_dims=10): 13 | self.folder = folder 14 | self.prefix = prefix 15 | self.input_dims = input_dims 16 | self.output_dims = output_dims 17 | 18 | def get_configurable_params(self): 19 | return ["nfilters_embeded", "nlayers_embeded", "lr"] 20 | 21 | def generate_model(self, **kwargs): 22 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 23 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 24 | 25 | model = Sequential() 26 | for i in range(nlayers_embeded): 27 | if i == 0: 28 | nfilters = self.input_dims 29 | else: 30 | nfilters = nfilters_embeded 31 | model.add(Convolution2D(nfilters, nfilters_embeded, 3, 1, 1)) 32 | model.add(max_pooling_2d(3,1,1)) 33 | model.add(BatchNormalization(nfilters_embeded)) 34 | model.add(Activation('bst')) 35 | 36 | model.add(Linear(None, self.output_dims)) 37 | 38 | model.build() 39 | return model 40 | 41 | def load_chain_model(self, **kwargs): 42 | name = self.get_name(**kwargs) 43 | path = '{}/{}'.format(self.folder,name) 44 | epoch = int(kwargs.get("nepochs",2)) 45 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 46 | 47 | chain, model = self.setup_chain_model(**kwargs) 48 | S.load_npz(fn, chain) 49 | return chain, model 50 | 51 | def setup_chain_model(self, **kwargs): 52 | model = self.generate_model(**kwargs) 53 | 54 | branchweight = kwargs.get("branchweight", 3) 55 | lr = kwargs.get("lr", 0.001) 56 | 57 | chain = Chain(branchweight=branchweight) 58 | chain.add_sequence(model) 59 | chain.setup_optimizers('adam', lr) 60 | return chain, model 61 | 62 | def get_name(self, **kwargs): 63 | if self.prefix is not None: 64 | name = "{}_".format(self.prefix) 65 | else: 66 | name = "" 67 | for k,v in kwargs.iteritems(): 68 | if k=='nepochs' or k=='ent_T': 69 | continue 70 | name = "{}_{}_{}".format(name, k, v) 71 | return name 72 | 73 | def train_model(self, trainset, testset, **kwargs): 74 | chain, model = self.setup_chain_model(**kwargs) 75 | 76 | nepochs = int(kwargs.get("nepochs", 2)) 77 | name = self.get_name(**kwargs) 78 | 79 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 80 | testset, nepoch=nepochs, resume=True) 81 | trainer.run() 82 | 83 | return trainer, model, chain 84 | -------------------------------------------------------------------------------- /elaas/family/float_float.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.chain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.function import * 7 | from chainer_sequential.link import * 8 | from chainer_sequential.binary_link import * 9 | from chainer import functions as F 10 | 11 | class FloatFloatFamily: 12 | def __init__(self, folder="_models/float_float", prefix=None, input_dims=1, output_dims=10): 13 | self.folder = folder 14 | self.prefix = prefix 15 | self.input_dims = input_dims 16 | self.output_dims = output_dims 17 | 18 | def get_configurable_params(self): 19 | return ["nfilters_embeded", "nlayers_embeded", "nfilters_cloud", "nlayers_cloud", "branchweight", "lr", "ent_T"] 20 | 21 | def generate_model(self, **kwargs): 22 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 23 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 24 | nfilters_cloud = int(kwargs.get("nfilters_cloud", 1)) 25 | nlayers_cloud = int(kwargs.get("nlayers_cloud", 1)) 26 | 27 | model = Sequential() 28 | for i in range(nlayers_embeded): 29 | if i == 0: 30 | nfilters = self.input_dims 31 | else: 32 | nfilters = nfilters_embeded 33 | model.add(Convolution2D(nfilters, nfilters_embeded, 3, 1, 1)) 34 | model.add(max_pooling_2d(3,1,1)) 35 | model.add(BatchNormalization(nfilters_embeded)) 36 | model.add(Activation('bst')) 37 | 38 | branch = Sequential() 39 | branch.add(Linear(None, self.output_dims)) 40 | branch.add(BatchNormalization(self.output_dims)) 41 | 42 | model.add(branch) 43 | 44 | # float branch 45 | for i in range(nlayers_cloud): 46 | if i == 0: 47 | nfilters = nfilters_embeded 48 | else: 49 | nfilters = nfilters_cloud 50 | model.add(Convolution2D(nfilters, nfilters_cloud, 3, 1, 1)) 51 | model.add(max_pooling_2d(3,1,1)) 52 | model.add(BatchNormalization(nfilters_cloud)) 53 | model.add(Activation('bst')) 54 | # Note: should we move pool to before batch norm like in binary? 55 | model.add(Linear(None, self.output_dims)) 56 | model.add(BatchNormalization(self.output_dims)) 57 | 58 | model.build() 59 | return model 60 | 61 | def load_chain_model(self, **kwargs): 62 | name = self.get_name(**kwargs) 63 | path = '{}/{}'.format(self.folder,name) 64 | epoch = int(kwargs.get("nepochs",2)) 65 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 66 | 67 | chain, model = self.setup_chain_model(**kwargs) 68 | S.load_npz(fn, chain) 69 | return chain, model 70 | 71 | def setup_chain_model(self, **kwargs): 72 | model = self.generate_model(**kwargs) 73 | 74 | branchweight = kwargs.get("branchweight", 3) 75 | ent_T = kwargs.get("ent_T", None) 76 | lr = kwargs.get("lr", 0.001) 77 | 78 | chain = Chain(branchweight=branchweight, ent_T=ent_T) 79 | chain.add_sequence(model) 80 | chain.setup_optimizers('adam', lr) 81 | return chain, model 82 | 83 | def get_name(self, **kwargs): 84 | if self.prefix is not None: 85 | name = "{}_".format(self.prefix) 86 | else: 87 | name = "" 88 | for k,v in kwargs.iteritems(): 89 | if k=='nepochs' or k=='ent_T': 90 | continue 91 | name = "{}_{}_{}".format(name, k, float(v)) 92 | return name 93 | 94 | def train_model(self, trainset, testset, **kwargs): 95 | chain, model = self.setup_chain_model(**kwargs) 96 | 97 | nepochs = int(kwargs.get("nepochs", 2)) 98 | name = self.get_name(**kwargs) 99 | 100 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 101 | testset, nepoch=nepochs, resume=True) 102 | trainer.run() 103 | 104 | return trainer, model 105 | -------------------------------------------------------------------------------- /elaas/family/simple.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.chain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.function import * 7 | from chainer_sequential.link import * 8 | from chainer_sequential.binary_link import * 9 | from chainer import functions as F 10 | 11 | class SimpleHybridFamily: 12 | def __init__(self, folder="_models/simple_hybrid", prefix=None, input_dims=1, output_dims=10): 13 | self.folder = folder 14 | self.prefix = prefix 15 | self.input_dims = input_dims 16 | self.output_dims = output_dims 17 | 18 | def get_configurable_params(self): 19 | return ["nfilters_embeded", "nlayers_embeded", "nfilters_cloud", "nlayers_cloud", "branchweight", "lr", "ent_T"] 20 | 21 | def generate_model(self, **kwargs): 22 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 23 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 24 | nfilters_cloud = int(kwargs.get("nfilters_cloud", 1)) 25 | nlayers_cloud = int(kwargs.get("nlayers_cloud", 1)) 26 | 27 | model = Sequential() 28 | for i in range(nlayers_embeded): 29 | if i == 0: 30 | nfilters = self.input_dims 31 | model.add(ConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 32 | else: 33 | nfilters = nfilters_embeded 34 | model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 1, 1)) 35 | 36 | branch = Sequential() 37 | branch.add(BinaryLinearBNSoftmax(None, self.output_dims)) 38 | model.add(branch) 39 | 40 | for i in range(nlayers_cloud): 41 | if i == 0: 42 | nfilters = nfilters_embeded 43 | else: 44 | nfilters = nfilters_cloud 45 | model.add(Convolution2D(nfilters, nfilters_cloud, 3, 1, 1)) 46 | model.add(BatchNormalization(nfilters_cloud)) 47 | model.add(Activation('relu')) 48 | model.add(max_pooling_2d(3,1,1)) 49 | model.add(Linear(None, self.output_dims)) 50 | model.build() 51 | return model 52 | 53 | def load_chain_model(self, **kwargs): 54 | name = self.get_name(**kwargs) 55 | path = '{}/{}'.format(self.folder,name) 56 | epoch = int(kwargs.get("nepochs",2)) 57 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 58 | 59 | chain, model = self.setup_chain_model(**kwargs) 60 | S.load_npz(fn, chain) 61 | return chain, model 62 | 63 | def setup_chain_model(self, **kwargs): 64 | model = self.generate_model(**kwargs) 65 | 66 | branchweight = kwargs.get("branchweight", 3) 67 | ent_T = kwargs.get("ent_T", None) 68 | lr = kwargs.get("lr", 0.001) 69 | 70 | chain = Chain(branchweight=branchweight, ent_T=ent_T) 71 | chain.add_sequence(model) 72 | chain.setup_optimizers('adam', lr) 73 | return chain, model 74 | 75 | def get_name(self, **kwargs): 76 | if self.prefix is not None: 77 | name = "{}_".format(self.prefix) 78 | else: 79 | name = "" 80 | for k,v in kwargs.iteritems(): 81 | # not saving params at test time in a different folder 82 | if k=='nepochs' or k=='ent_T': 83 | continue 84 | name = "{}_{}_{}".format(name, k, v) 85 | return name 86 | 87 | def train_model(self, trainset, testset, **kwargs): 88 | chain, model = self.setup_chain_model(**kwargs) 89 | 90 | nepochs = int(kwargs.get("nepochs", 2)) 91 | name = self.get_name(**kwargs) 92 | 93 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 94 | testset, nepoch=nepochs, resume=True) 95 | trainer.run() 96 | 97 | return trainer, model 98 | -------------------------------------------------------------------------------- /elaas/family/single_input.py: -------------------------------------------------------------------------------- 1 | from chainer_sequential.multiinputchain import Chain 2 | from deepopt.trainer import Trainer 3 | import chainer 4 | import chainer.serializers as S 5 | from chainer_sequential.sequential import Sequential 6 | from chainer_sequential.multiinputsequential import MultiInputSequential 7 | from chainer_sequential.function import * 8 | from chainer_sequential.link import * 9 | from chainer_sequential.binary_link import * 10 | from chainer import functions as F 11 | 12 | class SingleInputFamily: 13 | def __init__(self, folder="_models/single_input", prefix=None, input_dims=3, output_dims=3, batchsize=10): 14 | self.folder = folder 15 | self.prefix = prefix 16 | self.input_dims = input_dims 17 | self.output_dims = output_dims 18 | self.batchsize = batchsize 19 | 20 | def get_configurable_params(self): 21 | return ["nfilters_embeded", "nlayers_embeded", "nfilters_embeded_last", "nfilters_cloud", "nlayers_cloud", "branchweight", "lr", "ent_T"] 22 | 23 | def generate_model(self, **kwargs): 24 | nfilters_embeded_last = int(kwargs.get("nfilters_embeded_last", 1)) 25 | nfilters_embeded = int(kwargs.get("nfilters_embeded", 1)) 26 | nlayers_embeded = int(kwargs.get("nlayers_embeded", 1)) 27 | nfilters_cloud = int(kwargs.get("nfilters_cloud", 1)) 28 | nlayers_cloud = int(kwargs.get("nlayers_cloud", 1)) 29 | nfilters_edge = int(kwargs.get("nfilters_edge", 1)) 30 | nlayers_edge = int(kwargs.get("nlayers_edge", 1)) 31 | 32 | input_model = Sequential() 33 | for i in range(nlayers_embeded): 34 | if i == 0: 35 | nfilters = self.input_dims 36 | input_model.add(ConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 2, 1)) 37 | elif i == nlayers_embeded-1: 38 | nfilters = nfilters_embeded 39 | input_model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded_last, 3, 1, 1, 3, 2, 1)) 40 | else: 41 | nfilters = nfilters_embeded 42 | input_model.add(BinaryConvPoolBNBST(nfilters, nfilters_embeded, 3, 1, 1, 3, 2, 1)) 43 | input_model.add(BinaryLinearBNSoftmax(None, self.output_dims)) 44 | 45 | input_model.build() 46 | return input_model 47 | 48 | def load_chain_model(self, **kwargs): 49 | name = self.get_name(**kwargs) 50 | path = '{}/{}'.format(self.folder,name) 51 | epoch = int(kwargs.get("nepochs",2)) 52 | fn = "{}/chain_snapshot_epoch_{:06}".format(path,epoch) 53 | 54 | chain, model = self.setup_chain_model(**kwargs) 55 | S.load_npz(fn, chain) 56 | return chain, model 57 | 58 | def setup_chain_model(self, **kwargs): 59 | model = self.generate_model(**kwargs) 60 | 61 | branchweight = kwargs.get("branchweight", 3) 62 | ent_T = kwargs.get("ent_T", None) 63 | lr = kwargs.get("lr", 0.001) 64 | 65 | chain = Chain(branchweight=branchweight, ent_T=ent_T) 66 | chain.add_sequence(model) 67 | chain.setup_optimizers('adam', lr) 68 | return chain, model 69 | 70 | def get_name(self, **kwargs): 71 | if self.prefix is not None: 72 | name = "{}_".format(self.prefix) 73 | else: 74 | name = "" 75 | for k,v in kwargs.iteritems(): 76 | if k=='nepochs' or k=='ent_T': 77 | #if k=='nepochs': 78 | continue 79 | name = "{}_{}_{}".format(name, k, float(v)) 80 | return name 81 | 82 | def train_model(self, trainset, testset, **kwargs): 83 | chain, model = self.setup_chain_model(**kwargs) 84 | 85 | nepochs = int(kwargs.get("nepochs", 2)) 86 | ent_T = kwargs.get("ent_T", None) 87 | name = self.get_name(**kwargs) 88 | 89 | reports = [ 90 | 'epoch', 91 | 'validation/main/branch0accuracy', 92 | 'validation/main/branch1accuracy', 93 | 'validation/main/branch2accuracy', 94 | 'validation/main/branch3accuracy', 95 | 'validation/main/branch4accuracy', 96 | 'validation/main/branch5accuracy', 97 | 'validation/main/branch6accuracy', 98 | 'validation/main/branch7accuracy', 99 | 'validation/main/branch8accuracy', 100 | 'validation/main/accuracy', 101 | 'validation/main/communication0', 102 | 'validation/main/communication1', 103 | 'validation/main/memory', 104 | 'validation/main/ent_T', 105 | 'validation/main/branch0exit', 106 | 'validation/main/branch1exit', 107 | 'validation/main/branch2exit' 108 | ] 109 | trainer = Trainer('{}/{}'.format(self.folder,name), chain, trainset, 110 | testset, batchsize=self.batchsize, nepoch=nepochs, resume=True, reports=reports) 111 | trainer.run() 112 | 113 | return trainer, model, chain 114 | -------------------------------------------------------------------------------- /examples/binary_hybrid_analysis.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import sys 4 | sys.path.append('..') 5 | import argparse 6 | 7 | import chainer 8 | 9 | from elaas.elaas import Collection 10 | from elaas.family.simple import SimpleHybridFamily 11 | from elaas.family.binary import BinaryFamily 12 | from visualize import visualize 13 | import deepopt.chooser 14 | 15 | def max_acc(trace): 16 | acc = 0 17 | best_idx = 0 18 | for i, t in enumerate(trace): 19 | if t['action'] == 'add_point': 20 | acc = max(acc, t['y']) 21 | best_idx = i 22 | return acc, best_idx 23 | 24 | parser = argparse.ArgumentParser(description='Hybrid Example') 25 | parser.add_argument('-s', '--save_dir', default='_models') 26 | parser.add_argument('--iters', type=int, default=100) 27 | parser.add_argument('-e', '--epochs', type=int, default=20) 28 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 29 | parser.add_argument('-v', '--verbose', action='store_true') 30 | args = parser.parse_args() 31 | 32 | train, test = chainer.datasets.get_mnist(ndim=3) 33 | 34 | 35 | hybrid = Collection('simple_hybrid', args.save_dir, nepochs=args.epochs, verbose=args.verbose) 36 | hybrid.set_model_family(SimpleHybridFamily) 37 | hybrid.add_trainset(train) 38 | hybrid.add_testset(test) 39 | hybrid.set_searchspace( 40 | nfilters_embeded=[64], 41 | nlayers_embeded=[2], 42 | nfilters_cloud=[64], 43 | nlayers_cloud=[2], 44 | lr=[1e-3], 45 | branchweight=[.5], 46 | ent_T=[0., 0.005, 0.001, 0.1, 100] 47 | ) 48 | hybrid.set_chooser(deepopt.chooser.GridChooser()) 49 | 50 | 51 | 52 | binary_traces = binary.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 53 | idx, acc = max_acc(binary_traces) 54 | print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 55 | -------------------------------------------------------------------------------- /examples/comparison.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.simple import SimpleHybridFamily 10 | from elaas.family.binary import BinaryFamily 11 | from elaas.family.float import FloatFamily 12 | from elaas.family.multi_input_edge_with_dropout import MultiInputEdgeDropoutFamily 13 | from visualize import visualize 14 | import deepopt.chooser 15 | import matplotlib 16 | matplotlib.rcParams['font.size'] = 20.0 17 | import matplotlib.pyplot as plt 18 | 19 | 20 | def max_acc(trace): 21 | acc = 0 22 | best_idx = 0 23 | for i, t in enumerate(trace): 24 | if t['action'] == 'add_point': 25 | acc = max(acc, t['y']) 26 | best_idx = i 27 | return acc, best_idx 28 | 29 | 30 | model_dict = { 31 | "binary": BinaryFamily, 32 | "float": FloatFamily 33 | } 34 | 35 | def train_model(args, model_type, nfilters): 36 | trainer = Collection(model_type, args.save_dir, nepochs=args.epochs, verbose=args.verbose) 37 | trainer.set_model_family(model_dict[model_type]) 38 | train, test = chainer.datasets.get_mnist(ndim=3) 39 | data_shape = train._datasets[0].shape[1:] 40 | trainer.add_trainset(train) 41 | trainer.add_testset(test) 42 | trainer.set_searchspace( 43 | nfilters_embeded=[nfilters], 44 | nlayers_embeded=[2], 45 | lr=[1e-3] 46 | ) 47 | res = trainer.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 48 | return max_acc(res)[0] 49 | 50 | 51 | 52 | parser = argparse.ArgumentParser(description='Training Simple eBNN model') 53 | parser.add_argument('-s', '--save_dir', default='_models') 54 | parser.add_argument('-c', '--c_file', default=os.path.join('c', 'simple.h')) 55 | parser.add_argument('--inter_file', default=os.path.join('c', 'inter_simple.h')) 56 | parser.add_argument('-i', '--iters', type=int, default=10) 57 | parser.add_argument('-e', '--epochs', type=int, default=20) 58 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 59 | parser.add_argument('-v', '--verbose', action='store_true') 60 | parser.add_argument('--gen_inter', action='store_true') 61 | args = parser.parse_args() 62 | 63 | names = ['float', 'binary'] 64 | accs = {name: [] for name in names} 65 | mem = {name: [] for name in names} 66 | for i in [1,2,3,4,5]: 67 | acc = train_model(args, 'float', i) 68 | print(acc) 69 | accs['float'].append(acc*100) 70 | mem['float'].append(i*32*32*9) 71 | 72 | print("====") 73 | 74 | binary_accs = [] 75 | binary_mem = [] 76 | for i in [1,3,5,10,20,40,80,160]: 77 | acc = train_model(args, 'binary', i) 78 | print(acc) 79 | accs['binary'].append(acc*100) 80 | mem['binary'].append(i*32*9) 81 | 82 | 83 | #plot code 84 | linewidth = 4 85 | ms = 8 86 | colors = {'binary': '#FF944D', 'float': '#FF8F80'} 87 | styles = {'binary': '-o', 'float': '-.o'} 88 | plt.figure(figsize=(8, 6.5)) 89 | for name in names: 90 | plt.plot(mem[name], accs[name], styles[name], 91 | linewidth=linewidth, ms=ms, color=colors[name], 92 | label=name) 93 | plt.xlabel('Memory (bits)') 94 | plt.ylabel('Classification Accuracy (%)') 95 | plt.legend(loc=0, prop={'size': 14}) 96 | plt.tight_layout() 97 | plt.grid() 98 | plt.savefig("comparison_2layer.png") 99 | plt.clf() 100 | -------------------------------------------------------------------------------- /examples/dropout.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.simple import SimpleHybridFamily 10 | from elaas.family.binary import BinaryFamily 11 | from elaas.family.float import FloatFamily 12 | from elaas.family.multi_input_edge_with_dropout import MultiInputEdgeDropoutFamily 13 | from visualize import visualize 14 | import deepopt.chooser 15 | import matplotlib 16 | matplotlib.rcParams['font.size'] = 20.0 17 | import matplotlib.pyplot as plt 18 | 19 | 20 | def max_acc(trace): 21 | acc = 0 22 | best_idx = 0 23 | for i, t in enumerate(trace): 24 | if t['action'] == 'add_point': 25 | acc = max(acc, t['y']) 26 | best_idx = i 27 | return acc, best_idx 28 | 29 | 30 | model_dict = { 31 | "binary": BinaryFamily, 32 | "float": FloatFamily, 33 | "dropout": MultiInputEdgeDropoutFamily 34 | } 35 | 36 | def train_model(args, model_type, nfilters): 37 | train, test = chainer.datasets.get_cifar10(ndim=3) 38 | # train, test = chainer.datasets.get_mnist(ndim=3) 39 | 40 | trainer = Collection(model_type, args.save_dir, nepochs=args.epochs, verbose=args.verbose) 41 | trainer.set_model_family(model_dict[model_type], ninputs=1, resume=False, input_dims=train._datasets[0].shape[1], 42 | merge_function="max_pool_concat", drop_comm_train=0.5, drop_comm_test=0.5) 43 | trainer.add_trainset(train) 44 | trainer.add_testset(test) 45 | trainer.set_searchspace( 46 | nfilters_embeded=[nfilters], 47 | nlayers_embeded=[2], 48 | nfilters_cloud=[32], 49 | nlayers_cloud=[2], 50 | lr=[1e-3], 51 | branchweight=[.1], 52 | ent_T=[100] 53 | ) 54 | res = trainer.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 55 | return max_acc(res)[0] 56 | 57 | parser = argparse.ArgumentParser(description='Training Simple eBNN model') 58 | parser.add_argument('-s', '--save_dir', default='_models') 59 | parser.add_argument('-c', '--c_file', default=os.path.join('c', 'simple.h')) 60 | parser.add_argument('--inter_file', default=os.path.join('c', 'inter_simple.h')) 61 | parser.add_argument('-i', '--iters', type=int, default=10) 62 | parser.add_argument('-e', '--epochs', type=int, default=20) 63 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 64 | parser.add_argument('-v', '--verbose', action='store_true') 65 | parser.add_argument('--gen_inter', action='store_true') 66 | args = parser.parse_args() 67 | 68 | acc = train_model(args, 'dropout', 8) 69 | 70 | #plot code 71 | # linewidth = 4 72 | # ms = 8 73 | # colors = {'binary': '#FF944D', 'float': '#FF8F80'} 74 | # styles = {'binary': '-o', 'float': '-.o'} 75 | # plt.figure(figsize=(8, 6.5)) 76 | # for name in names: 77 | # plt.plot(mem[name], accs[name], styles[name], 78 | # linewidth=linewidth, ms=ms, color=colors[name], 79 | # label=name) 80 | # plt.xlabel('Memory (bits)') 81 | # plt.ylabel('Classification Accuracy (%)') 82 | # plt.legend(loc=0, prop={'size': 14}) 83 | # plt.tight_layout() 84 | # plt.grid() 85 | # plt.savefig("comparison_2layer.png") 86 | # plt.clf() 87 | -------------------------------------------------------------------------------- /examples/multiview.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import sys 4 | sys.path.append('..') 5 | import argparse 6 | 7 | from datasets import datasets 8 | 9 | #if cam is not specified, will return all cams 10 | train, test = datasets.get_mvmc() 11 | -------------------------------------------------------------------------------- /examples/simple.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.simple import SimpleHybridFamily 10 | from elaas.family.binary import BinaryFamily 11 | from visualize import visualize 12 | import deepopt.chooser 13 | 14 | parser = argparse.ArgumentParser(description='Training Simple eBNN model') 15 | parser.add_argument('-s', '--save_dir', default='_models') 16 | parser.add_argument('-c', '--c_file', default=os.path.join('c', 'simple.h')) 17 | parser.add_argument('--inter_file', default=os.path.join('c', 'inter_simple.h')) 18 | parser.add_argument('-i', '--iters', type=int, default=10) 19 | parser.add_argument('-e', '--epochs', type=int, default=20) 20 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 21 | parser.add_argument('-v', '--verbose', action='store_true') 22 | parser.add_argument('--gen_inter', action='store_true') 23 | args = parser.parse_args() 24 | 25 | #Setup model type (e.g. Binary) 26 | trainer = Collection('binary', args.save_dir, nepochs=args.epochs, verbose=args.verbose) 27 | trainer.set_model_family(BinaryFamily) 28 | 29 | #Dataset 30 | train, test = chainer.datasets.get_mnist(ndim=3) 31 | data_shape = train._datasets[0].shape[1:] 32 | trainer.add_trainset(train) 33 | trainer.add_testset(test) 34 | 35 | #Model parameters 36 | trainer.set_searchspace( 37 | nfilters_embeded=[2], 38 | nlayers_embeded=[1], 39 | lr=[1e-3] 40 | ) 41 | 42 | #Train model 43 | trainer.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 44 | 45 | # generate eBNN C library 46 | trainer.generate_c(args.c_file, data_shape) 47 | if args.gen_inter: 48 | inter = trainer.generate_inter_results_c(args.inter_file, train._datasets[0][:1]) 49 | -------------------------------------------------------------------------------- /examples/test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.simple import SimpleHybridFamily 10 | from visualize import visualize 11 | import deepopt.chooser 12 | 13 | parser = argparse.ArgumentParser(description='Hybrid Example') 14 | parser.add_argument('-s', '--save_dir', default='_models') 15 | parser.add_argument('--iters', type=int, default=100) 16 | parser.add_argument('-e', '--epochs', type=int, default=20) 17 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 18 | parser.add_argument('-v', '--verbose', action='store_true') 19 | args = parser.parse_args() 20 | 21 | mnist = Collection('simple_hybrid', args.save_dir, nepochs=args.epochs, verbose=args.verbose) 22 | mnist.set_model_family(SimpleHybridFamily) 23 | 24 | train, test = chainer.datasets.get_mnist(ndim=3) 25 | mnist.add_trainset(train) 26 | mnist.add_testset(test) 27 | 28 | mnist.set_searchspace( 29 | nfilters_embeded=[1], 30 | nlayers_embeded=[2], 31 | nfilters_cloud=[1], 32 | nlayers_cloud=[1], 33 | lr=[1e-3], 34 | branchweight=[.1], 35 | ent_T=[0.0001, 0.001, 0.01, 0.1] 36 | ) 37 | 38 | # mnist.set_searchspace( 39 | # nfilters_embeded=[64, 128], 40 | # nlayers_embeded=[1, 2], 41 | # nfilters_cloud=[128], 42 | # nlayers_cloud=[2,4], 43 | # lr=[1e-3,1e-4, 1e-5], 44 | # branchweight=[.1], 45 | # ent_T=[0.0, 0.01, 0.1] 46 | # ) 47 | 48 | def constraintfn(**kwargs): 49 | #TODO: change to memory cost 50 | return True 51 | 52 | mnist.set_constraints(constraintfn) 53 | 54 | # switch chooser 55 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 56 | 57 | # currently optimize based on the validation accuracy of the main model 58 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 59 | visualize.min_error(traces, args.save_dir) 60 | visualize.embed_memory_err(mnist.do, traces, args.save_dir) 61 | visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 62 | 63 | # generate c 64 | # mnist.generate_c((1,28,28)) 65 | 66 | # generate container 67 | # mnist.generate_container() 68 | 69 | # get traces for the collection 70 | # mnist = Collection('simple_hybrid', save_dir) 71 | # traces = mnist.get_do_traces() 72 | -------------------------------------------------------------------------------- /examples/test_binary.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.simple import SimpleHybridFamily 10 | from elaas.family.binary import BinaryFamily 11 | from visualize import visualize 12 | import deepopt.chooser 13 | 14 | parser = argparse.ArgumentParser(description='Hybrid Example') 15 | parser.add_argument('-s', '--save_dir', default='_models') 16 | parser.add_argument('--iters', type=int, default=100) 17 | parser.add_argument('-e', '--epochs', type=int, default=20) 18 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 19 | parser.add_argument('-v', '--verbose', action='store_true') 20 | args = parser.parse_args() 21 | 22 | mnist = Collection('binary', args.save_dir, nepochs=args.epochs, verbose=args.verbose) 23 | mnist.set_model_family(BinaryFamily) 24 | 25 | train, test = chainer.datasets.get_mnist(ndim=3) 26 | mnist.add_trainset(train) 27 | mnist.add_testset(test) 28 | 29 | # mnist.set_searchspace( 30 | # nfilters_embeded=[5, 10], 31 | # nlayers_embeded=[1, 2], 32 | # nfilters_cloud=[5], 33 | # nlayers_cloud=[1], 34 | # lr=[1e-3], 35 | # branchweight=[.1], 36 | # ent_T=[0.1, 0.2, 0.4] 37 | # ) 38 | 39 | mnist.set_searchspace( 40 | nfilters_embeded=[4, 8, 16], 41 | nlayers_embeded=[2], 42 | nfilters_cloud=[16], 43 | nlayers_cloud=[3], 44 | lr=[1e-3, 1e-4], 45 | branchweight=[.1], 46 | ent_T=[0.5, 0.6] 47 | ) 48 | 49 | def constraintfn(**kwargs): 50 | #TODO: change to memory cost 51 | return True 52 | 53 | mnist.set_constraints(constraintfn) 54 | 55 | # switch chooser 56 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 57 | 58 | # currently optimize based on the validation accuracy of the main model 59 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 60 | visualize.min_error(traces, args.save_dir) 61 | visualize.embed_memory_err(mnist.do, traces, args.save_dir) 62 | visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 63 | 64 | # generate c 65 | # mnist.generate_c((1,28,28)) 66 | 67 | # generate container 68 | # mnist.generate_container() 69 | 70 | # get traces for the collection 71 | # mnist = Collection('simple_hybrid', save_dir) 72 | # traces = mnist.get_do_traces() 73 | -------------------------------------------------------------------------------- /examples/test_binary_fixed.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.binary_base_fixed import BinaryBaseFixedFamily 10 | from visualize import visualize 11 | import deepopt.chooser 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=80) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | args = parser.parse_args() 30 | 31 | mnist = Collection('binary_base_fixed_cifar10', args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 32 | mnist.set_model_family(BinaryBaseFixedFamily) 33 | 34 | #train, test = chainer.datasets.get_mnist(ndim=3) 35 | train, test = chainer.datasets.get_cifar10(ndim=3) 36 | 37 | #from chainer.datasets.sub_dataset import SubDataset 38 | #train = SubDataset(train, 0, 500) 39 | #test = SubDataset(train, 0, 500) 40 | 41 | mnist.add_trainset(train) 42 | mnist.add_testset(test) 43 | 44 | mnist.set_searchspace( 45 | pretrain_nepochs=[20], 46 | nfilters_embeded=[64], 47 | nlayers_embeded=[2], 48 | nfilters_cloud=[64], 49 | nlayers_cloud=[2], 50 | lr=[1e-3], 51 | branchweight=[.1], 52 | ent_T=[100] 53 | ) 54 | 55 | # mnist.set_searchspace( 56 | # nfilters_embeded=[64, 128], 57 | # nlayers_embeded=[1, 2], 58 | # nfilters_cloud=[128], 59 | # nlayers_cloud=[2,4], 60 | # lr=[1e-3,1e-4, 1e-5], 61 | # branchweight=[.1], 62 | # ent_T=[0.0, 0.01, 0.1] 63 | # ) 64 | 65 | def constraintfn(**kwargs): 66 | #TODO: change to memory cost 67 | return True 68 | 69 | mnist.set_constraints(constraintfn) 70 | 71 | # switch chooser 72 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 73 | 74 | # currently optimize based on the validation accuracy of the main model 75 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 76 | idx, acc = max_acc(traces) 77 | print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 78 | 79 | # visualize.min_error(traces, args.save_dir) 80 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 81 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 82 | 83 | # generate c 84 | # mnist.generate_c((1,28,28)) 85 | 86 | # generate container 87 | # mnist.generate_container() 88 | 89 | # get traces for the collection 90 | # mnist = Collection('simple_hybrid', save_dir) 91 | # traces = mnist.get_do_traces() 92 | -------------------------------------------------------------------------------- /examples/test_binary_float.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.binary_float import BinaryFloatFamily 10 | from visualize import visualize 11 | import deepopt.chooser 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | args = parser.parse_args() 30 | 31 | mnist = Collection('binary_float_cifar10', args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 32 | mnist.set_model_family(BinaryFloatFamily) 33 | 34 | #train, test = chainer.datasets.get_mnist(ndim=3) 35 | train, test = chainer.datasets.get_cifar10(ndim=3) 36 | 37 | #from chainer.datasets.sub_dataset import SubDataset 38 | #train = SubDataset(train, 0, 500) 39 | #test = SubDataset(train, 0, 500) 40 | 41 | mnist.add_trainset(train) 42 | mnist.add_testset(test) 43 | 44 | mnist.set_searchspace( 45 | nfilters_embeded=[64], 46 | nlayers_embeded=[2], 47 | nfilters_cloud=[64], 48 | nlayers_cloud=[2], 49 | lr=[1e-3], 50 | branchweight=[.1], 51 | ent_T=[100] 52 | ) 53 | 54 | # mnist.set_searchspace( 55 | # nfilters_embeded=[64, 128], 56 | # nlayers_embeded=[1, 2], 57 | # nfilters_cloud=[128], 58 | # nlayers_cloud=[2,4], 59 | # lr=[1e-3,1e-4, 1e-5], 60 | # branchweight=[.1], 61 | # ent_T=[0.0, 0.01, 0.1] 62 | # ) 63 | 64 | def constraintfn(**kwargs): 65 | #TODO: change to memory cost 66 | return True 67 | 68 | mnist.set_constraints(constraintfn) 69 | 70 | # switch chooser 71 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 72 | 73 | # currently optimize based on the validation accuracy of the main model 74 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 75 | idx, acc = max_acc(traces) 76 | print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 77 | 78 | # visualize.min_error(traces, args.save_dir) 79 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 80 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 81 | 82 | # generate c 83 | # mnist.generate_c((1,28,28)) 84 | 85 | # generate container 86 | # mnist.generate_container() 87 | 88 | # get traces for the collection 89 | # mnist = Collection('simple_hybrid', save_dir) 90 | # traces = mnist.get_do_traces() 91 | -------------------------------------------------------------------------------- /examples/test_float_float.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | 8 | from elaas.elaas import Collection 9 | from elaas.family.float_float import FloatFloatFamily 10 | from visualize import visualize 11 | import deepopt.chooser 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | args = parser.parse_args() 30 | 31 | mnist = Collection('float_float_cifar10', args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 32 | mnist.set_model_family(FloatFloatFamily) 33 | 34 | #train, test = chainer.datasets.get_mnist(ndim=3) 35 | train, test = chainer.datasets.get_cifar10(ndim=3) 36 | 37 | #from chainer.datasets.sub_dataset import SubDataset 38 | #train = SubDataset(train, 0, 500) 39 | #test = SubDataset(train, 0, 500) 40 | 41 | mnist.add_trainset(train) 42 | mnist.add_testset(test) 43 | 44 | mnist.set_searchspace( 45 | nfilters_embeded=[64], 46 | nlayers_embeded=[2], 47 | nfilters_cloud=[64], 48 | nlayers_cloud=[2], 49 | lr=[1e-3], 50 | branchweight=[.1], 51 | ent_T=[100] 52 | ) 53 | 54 | # mnist.set_searchspace( 55 | # nfilters_embeded=[64, 128], 56 | # nlayers_embeded=[1, 2], 57 | # nfilters_cloud=[128], 58 | # nlayers_cloud=[2,4], 59 | # lr=[1e-3,1e-4, 1e-5], 60 | # branchweight=[.1], 61 | # ent_T=[0.0, 0.01, 0.1] 62 | # ) 63 | 64 | def constraintfn(**kwargs): 65 | #TODO: change to memory cost 66 | return True 67 | 68 | mnist.set_constraints(constraintfn) 69 | 70 | # switch chooser 71 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 72 | 73 | # currently optimize based on the validation accuracy of the main model 74 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 75 | idx, acc = max_acc(traces) 76 | print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 77 | 78 | # visualize.min_error(traces, args.save_dir) 79 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 80 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 81 | 82 | # generate c 83 | # mnist.generate_c((1,28,28)) 84 | 85 | # generate container 86 | # mnist.generate_container() 87 | 88 | # get traces for the collection 89 | # mnist = Collection('simple_hybrid', save_dir) 90 | # traces = mnist.get_do_traces() 91 | -------------------------------------------------------------------------------- /examples/test_multiinput.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input import MultiInputFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_{}'.format(args.ncams), args.save_dir, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputFamily,ninputs=ncams) 36 | 37 | train, test = get_mvmc_flatten(range(ncams)) 38 | #train, test = chainer.datasets.get_cifar10(ndim=3) 39 | #print(train[1]) 40 | #print(len(train[0])) 41 | #from chainer.datasets.sub_dataset import SubDataset 42 | #train = SubDataset(train, 0, 500) 43 | #test = SubDataset(train, 0, 500) 44 | 45 | mnist.add_trainset(train) 46 | mnist.add_testset(test) 47 | 48 | mnist.set_searchspace( 49 | nfilters_embeded=[64], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[64], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[.1], 55 | ent_T=[100] 56 | ) 57 | 58 | # mnist.set_searchspace( 59 | # nfilters_embeded=[64, 128], 60 | # nlayers_embeded=[1, 2], 61 | # nfilters_cloud=[128], 62 | # nlayers_cloud=[2,4], 63 | # lr=[1e-3,1e-4, 1e-5], 64 | # branchweight=[.1], 65 | # ent_T=[0.0, 0.01, 0.1] 66 | # ) 67 | 68 | def constraintfn(**kwargs): 69 | #TODO: change to memory cost 70 | return True 71 | 72 | mnist.set_constraints(constraintfn) 73 | 74 | # switch chooser 75 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 76 | 77 | # currently optimize based on the validation accuracy of the main model 78 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 79 | idx, acc = max_acc(traces) 80 | print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 81 | 82 | # visualize.min_error(traces, args.save_dir) 83 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 84 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 85 | 86 | # generate c 87 | # mnist.generate_c((1,28,28)) 88 | 89 | # generate container 90 | # mnist.generate_container() 91 | 92 | # get traces for the collection 93 | # mnist = Collection('simple_hybrid', save_dir) 94 | # traces = mnist.get_do_traces() 95 | -------------------------------------------------------------------------------- /examples/test_multiinput_edge.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_edge_mpmp_{}'.format(args.ncams), args.save_dir, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=ncams) 36 | 37 | train, test = get_mvmc_flatten(range(ncams)) 38 | #train, test = chainer.datasets.get_cifar10(ndim=3) 39 | #print(train[1]) 40 | #print(len(train[0])) 41 | #from chainer.datasets.sub_dataset import SubDataset 42 | #train = SubDataset(train, 0, 500) 43 | #test = SubDataset(train, 0, 500) 44 | 45 | mnist.add_trainset(train) 46 | mnist.add_testset(test) 47 | 48 | mnist.set_searchspace( 49 | nfilters_embeded=[64], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[64], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[.1], 55 | ent_T=[100] 56 | ) 57 | 58 | # mnist.set_searchspace( 59 | # nfilters_embeded=[64, 128], 60 | # nlayers_embeded=[1, 2], 61 | # nfilters_cloud=[128], 62 | # nlayers_cloud=[2,4], 63 | # lr=[1e-3,1e-4, 1e-5], 64 | # branchweight=[.1], 65 | # ent_T=[0.0, 0.01, 0.1] 66 | # ) 67 | 68 | def constraintfn(**kwargs): 69 | #TODO: change to memory cost 70 | return True 71 | 72 | mnist.set_constraints(constraintfn) 73 | 74 | # switch chooser 75 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 76 | 77 | # currently optimize based on the validation accuracy of the main model 78 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 79 | idx, acc = max_acc(traces) 80 | print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 81 | 82 | # visualize.min_error(traces, args.save_dir) 83 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 84 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 85 | 86 | # generate c 87 | # mnist.generate_c((1,28,28)) 88 | 89 | # generate container 90 | # mnist.generate_container() 91 | 92 | # get traces for the collection 93 | # mnist = Collection('simple_hybrid', save_dir) 94 | # traces = mnist.get_do_traces() 95 | -------------------------------------------------------------------------------- /examples/test_multiinput_edge_with_dropout.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge_with_dropout import MultiInputEdgeDropoutFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_edge_dropout_mpcc_{}'.format(args.ncams), args.save_dir, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputEdgeDropoutFamily,ninputs=ncams,resume=False,merge_function="max_pool_concat",drop_comm_train=0.5,drop_comm_test=0.5, input_dims=3) 36 | 37 | # train, test = get_mvmc_flatten(range(ncams)) 38 | train, test = chainer.datasets.get_cifar10(ndim=3) 39 | # train, test = chainer.datasets.get_mnist(ndim=3) 40 | # assert False 41 | 42 | #from chainer.datasets.sub_dataset import SubDataset 43 | #train = SubDataset(train, 0, 500) 44 | #test = SubDataset(train, 0, 500) 45 | 46 | mnist.add_trainset(train) 47 | mnist.add_testset(test) 48 | 49 | mnist.set_searchspace( 50 | nfilters_embeded=[32], 51 | nlayers_embeded=[2], 52 | nfilters_cloud=[32], 53 | nlayers_cloud=[2], 54 | lr=[1e-3], 55 | branchweight=[.1], 56 | ent_T=[100] 57 | ) 58 | 59 | def constraintfn(**kwargs): 60 | #TODO: change to memory cost 61 | return True 62 | 63 | mnist.set_constraints(constraintfn) 64 | 65 | # switch chooser 66 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 67 | 68 | # currently optimize based on the validation accuracy of the main model 69 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 70 | #idx, acc = max_acc(traces) 71 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 72 | -------------------------------------------------------------------------------- /exp/exp_combining.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0,1,2,3,4,5") 30 | parser.add_argument('-m', '--merge', type=str, default="max_pool_concat") 31 | 32 | args = parser.parse_args() 33 | merge = args.merge 34 | 35 | args.ncams = [int(i) for i in args.ncams.split(",")] 36 | mnist = Collection('binary_combining_{}_{}'.format(merge, reduce(lambda x,y: str(x)+str(y), args.ncams)), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 37 | 38 | ncams = args.ncams 39 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function=merge) 40 | 41 | train, test = get_mvmc_flatten_eval(ncams) 42 | #print("train",len(train)) 43 | #print("test",len(test)) 44 | #import sys 45 | #sys.exit(0) 46 | #train, test = chainer.datasets.get_cifar10(ndim=3) 47 | #print(train[1]) 48 | #print(len(train[0])) 49 | #from chainer.datasets.sub_dataset import SubDataset 50 | #train = SubDataset(train, 0, 500) 51 | #test = SubDataset(train, 0, 500) 52 | 53 | mnist.add_trainset(train) 54 | mnist.add_testset(test) 55 | 56 | mnist.set_searchspace( 57 | nfilters_embeded_last=[16], 58 | nfilters_embeded=[16], 59 | nlayers_embeded=[1], 60 | #nfilters_edge=[16], 61 | #nlayers_edge=[2], 62 | nfilters_cloud=[16], 63 | nlayers_cloud=[2], 64 | lr=[1e-3], 65 | branchweight=[1], 66 | ent_T=[0.8] 67 | ) 68 | 69 | #mnist.set_searchspace( 70 | # nfilters_embeded_last=[8], 71 | # nfilters_embeded=[8], 72 | # nlayers_embeded=[2], 73 | # nfilters_edge=[16], 74 | # nlayers_edge=[1], 75 | # nfilters_cloud=[16], 76 | # nlayers_cloud=[2], 77 | # lr=[1e-3], 78 | # branchweight=[1], 79 | # ent_T=[0.8] 80 | #) 81 | 82 | # mnist.set_searchspace( 83 | # nfilters_embeded=[64, 128], 84 | # nlayers_embeded=[1, 2], 85 | # nfilters_cloud=[128], 86 | # nlayers_cloud=[2,4], 87 | # lr=[1e-3,1e-4, 1e-5], 88 | # branchweight=[.1], 89 | # ent_T=[0.0, 0.01, 0.1] 90 | # ) 91 | 92 | def constraintfn(**kwargs): 93 | #TODO: change to memory cost 94 | return True 95 | 96 | mnist.set_constraints(constraintfn) 97 | 98 | # switch chooser 99 | mnist.set_chooser(deepopt.chooser.GridChooser()) 100 | 101 | # currently optimize based on the validation accuracy of the main model 102 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 103 | #idx, acc = max_acc(traces) 104 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 105 | 106 | # visualize.min_error(traces, args.save_dir) 107 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 108 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 109 | 110 | # generate c 111 | # mnist.generate_c((1,28,28)) 112 | 113 | # generate container 114 | # mnist.generate_container() 115 | 116 | # get traces for the collection 117 | # mnist = Collection('simple_hybrid', save_dir) 118 | # traces = mnist.get_do_traces() 119 | -------------------------------------------------------------------------------- /exp/exp_combining_avg_pool.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_edge_apap_{}'.format(args.ncams), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=ncams,batchsize=400,merge_function="avg_pool") 36 | 37 | train, test = get_mvmc_flatten(range(ncams)) 38 | #train, test = chainer.datasets.get_cifar10(ndim=3) 39 | #print(train[1]) 40 | #print(len(train[0])) 41 | #from chainer.datasets.sub_dataset import SubDataset 42 | #train = SubDataset(train, 0, 500) 43 | #test = SubDataset(train, 0, 500) 44 | 45 | mnist.add_trainset(train) 46 | mnist.add_testset(test) 47 | 48 | mnist.set_searchspace( 49 | nfilters_embeded=[16], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[16], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[1], 55 | ent_T=[.7] 56 | ) 57 | 58 | # mnist.set_searchspace( 59 | # nfilters_embeded=[64, 128], 60 | # nlayers_embeded=[1, 2], 61 | # nfilters_cloud=[128], 62 | # nlayers_cloud=[2,4], 63 | # lr=[1e-3,1e-4, 1e-5], 64 | # branchweight=[.1], 65 | # ent_T=[0.0, 0.01, 0.1] 66 | # ) 67 | 68 | def constraintfn(**kwargs): 69 | #TODO: change to memory cost 70 | return True 71 | 72 | mnist.set_constraints(constraintfn) 73 | 74 | # switch chooser 75 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 76 | 77 | # currently optimize based on the validation accuracy of the main model 78 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 79 | #idx, acc = max_acc(traces) 80 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 81 | 82 | # visualize.min_error(traces, args.save_dir) 83 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 84 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 85 | 86 | # generate c 87 | # mnist.generate_c((1,28,28)) 88 | 89 | # generate container 90 | # mnist.generate_container() 91 | 92 | # get traces for the collection 93 | # mnist = Collection('simple_hybrid', save_dir) 94 | # traces = mnist.get_do_traces() 95 | -------------------------------------------------------------------------------- /exp/exp_combining_avg_pool_concat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_edge_apcc_{}'.format(args.ncams), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=ncams,batchsize=400,merge_function="avg_pool_concat") 36 | 37 | train, test = get_mvmc_flatten(range(ncams)) 38 | #train, test = chainer.datasets.get_cifar10(ndim=3) 39 | #print(train[1]) 40 | #print(len(train[0])) 41 | #from chainer.datasets.sub_dataset import SubDataset 42 | #train = SubDataset(train, 0, 500) 43 | #test = SubDataset(train, 0, 500) 44 | 45 | mnist.add_trainset(train) 46 | mnist.add_testset(test) 47 | 48 | mnist.set_searchspace( 49 | nfilters_embeded=[16], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[16], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[1], 55 | ent_T=[.7] 56 | ) 57 | 58 | # mnist.set_searchspace( 59 | # nfilters_embeded=[64, 128], 60 | # nlayers_embeded=[1, 2], 61 | # nfilters_cloud=[128], 62 | # nlayers_cloud=[2,4], 63 | # lr=[1e-3,1e-4, 1e-5], 64 | # branchweight=[.1], 65 | # ent_T=[0.0, 0.01, 0.1] 66 | # ) 67 | 68 | def constraintfn(**kwargs): 69 | #TODO: change to memory cost 70 | return True 71 | 72 | mnist.set_constraints(constraintfn) 73 | 74 | # switch chooser 75 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 76 | 77 | # currently optimize based on the validation accuracy of the main model 78 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 79 | #idx, acc = max_acc(traces) 80 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 81 | 82 | # visualize.min_error(traces, args.save_dir) 83 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 84 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 85 | 86 | # generate c 87 | # mnist.generate_c((1,28,28)) 88 | 89 | # generate container 90 | # mnist.generate_container() 91 | 92 | # get traces for the collection 93 | # mnist = Collection('simple_hybrid', save_dir) 94 | # traces = mnist.get_do_traces() 95 | -------------------------------------------------------------------------------- /exp/exp_combining_concat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_edge_cccc_{}'.format(args.ncams), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=ncams,batchsize=400,merge_function="concat") 36 | 37 | train, test = get_mvmc_flatten(range(ncams)) 38 | #train, test = chainer.datasets.get_cifar10(ndim=3) 39 | #print(train[1]) 40 | #print(len(train[0])) 41 | #from chainer.datasets.sub_dataset import SubDataset 42 | #train = SubDataset(train, 0, 500) 43 | #test = SubDataset(train, 0, 500) 44 | 45 | mnist.add_trainset(train) 46 | mnist.add_testset(test) 47 | 48 | mnist.set_searchspace( 49 | nfilters_embeded=[16], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[16], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[1], 55 | ent_T=[.7] 56 | ) 57 | 58 | # mnist.set_searchspace( 59 | # nfilters_embeded=[64, 128], 60 | # nlayers_embeded=[1, 2], 61 | # nfilters_cloud=[128], 62 | # nlayers_cloud=[2,4], 63 | # lr=[1e-3,1e-4, 1e-5], 64 | # branchweight=[.1], 65 | # ent_T=[0.0, 0.01, 0.1] 66 | # ) 67 | 68 | def constraintfn(**kwargs): 69 | #TODO: change to memory cost 70 | return True 71 | 72 | mnist.set_constraints(constraintfn) 73 | 74 | # switch chooser 75 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 76 | 77 | # currently optimize based on the validation accuracy of the main model 78 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 79 | #idx, acc = max_acc(traces) 80 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 81 | 82 | # visualize.min_error(traces, args.save_dir) 83 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 84 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 85 | 86 | # generate c 87 | # mnist.generate_c((1,28,28)) 88 | 89 | # generate container 90 | # mnist.generate_container() 91 | 92 | # get traces for the collection 93 | # mnist = Collection('simple_hybrid', save_dir) 94 | # traces = mnist.get_do_traces() 95 | -------------------------------------------------------------------------------- /exp/exp_combining_max_pool.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_edge_mpmp_{}'.format(args.ncams), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=ncams,batchsize=400,merge_function="max_pool") 36 | 37 | train, test = get_mvmc_flatten(range(ncams)) 38 | #train, test = chainer.datasets.get_cifar10(ndim=3) 39 | #print(train[1]) 40 | #print(len(train[0])) 41 | #from chainer.datasets.sub_dataset import SubDataset 42 | #train = SubDataset(train, 0, 500) 43 | #test = SubDataset(train, 0, 500) 44 | 45 | mnist.add_trainset(train) 46 | mnist.add_testset(test) 47 | 48 | mnist.set_searchspace( 49 | nfilters_embeded=[16], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[16], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[1], 55 | ent_T=[.7] 56 | ) 57 | 58 | # mnist.set_searchspace( 59 | # nfilters_embeded=[64, 128], 60 | # nlayers_embeded=[1, 2], 61 | # nfilters_cloud=[128], 62 | # nlayers_cloud=[2,4], 63 | # lr=[1e-3,1e-4, 1e-5], 64 | # branchweight=[.1], 65 | # ent_T=[0.0, 0.01, 0.1] 66 | # ) 67 | 68 | def constraintfn(**kwargs): 69 | #TODO: change to memory cost 70 | return True 71 | 72 | mnist.set_constraints(constraintfn) 73 | 74 | # switch chooser 75 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 76 | 77 | # currently optimize based on the validation accuracy of the main model 78 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 79 | #idx, acc = max_acc(traces) 80 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 81 | 82 | # visualize.min_error(traces, args.save_dir) 83 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 84 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 85 | 86 | # generate c 87 | # mnist.generate_c((1,28,28)) 88 | 89 | # generate container 90 | # mnist.generate_container() 91 | 92 | # get traces for the collection 93 | # mnist = Collection('simple_hybrid', save_dir) 94 | # traces = mnist.get_do_traces() 95 | -------------------------------------------------------------------------------- /exp/exp_combining_max_pool_concat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=40) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=2) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=int, default=6) 30 | args = parser.parse_args() 31 | 32 | mnist = Collection('multiinput_edge_mpcc_{}'.format(args.ncams), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 33 | 34 | ncams = args.ncams 35 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=ncams,batchsize=400,merge_function="max_pool_concat") 36 | 37 | train, test = get_mvmc_flatten(range(ncams)) 38 | #train, test = chainer.datasets.get_cifar10(ndim=3) 39 | #print(train[1]) 40 | #print(len(train[0])) 41 | #from chainer.datasets.sub_dataset import SubDataset 42 | #train = SubDataset(train, 0, 500) 43 | #test = SubDataset(train, 0, 500) 44 | 45 | mnist.add_trainset(train) 46 | mnist.add_testset(test) 47 | 48 | mnist.set_searchspace( 49 | nfilters_embeded=[16], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[16], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[1], 55 | ent_T=[.7] 56 | ) 57 | 58 | # mnist.set_searchspace( 59 | # nfilters_embeded=[64, 128], 60 | # nlayers_embeded=[1, 2], 61 | # nfilters_cloud=[128], 62 | # nlayers_cloud=[2,4], 63 | # lr=[1e-3,1e-4, 1e-5], 64 | # branchweight=[.1], 65 | # ent_T=[0.0, 0.01, 0.1] 66 | # ) 67 | 68 | def constraintfn(**kwargs): 69 | #TODO: change to memory cost 70 | return True 71 | 72 | mnist.set_constraints(constraintfn) 73 | 74 | # switch chooser 75 | mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 76 | 77 | # currently optimize based on the validation accuracy of the main model 78 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 79 | #idx, acc = max_acc(traces) 80 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 81 | 82 | # visualize.min_error(traces, args.save_dir) 83 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 84 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 85 | 86 | # generate c 87 | # mnist.generate_c((1,28,28)) 88 | 89 | # generate container 90 | # mnist.generate_container() 91 | 92 | # get traces for the collection 93 | # mnist = Collection('simple_hybrid', save_dir) 94 | # traces = mnist.get_do_traces() 95 | -------------------------------------------------------------------------------- /exp/exp_comm.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0,1,2,3,4,5") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | mnist = Collection('exp_comm_percent_{}'.format(reduce(lambda x,y: str(x)+str(y), args.ncams)), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 34 | 35 | ncams = args.ncams 36 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function="max_pool_concat") 37 | 38 | train, test = get_mvmc_flatten_eval(ncams) 39 | #print("train",len(train)) 40 | #print("test",len(test)) 41 | #import sys 42 | #sys.exit(0) 43 | #train, test = chainer.datasets.get_cifar10(ndim=3) 44 | #print(train[1]) 45 | #print(len(train[0])) 46 | #from chainer.datasets.sub_dataset import SubDataset 47 | #train = SubDataset(train, 0, 500) 48 | #test = SubDataset(train, 0, 500) 49 | 50 | mnist.add_trainset(train) 51 | mnist.add_testset(test) 52 | 53 | mnist.set_searchspace( 54 | target_branch0exit=[0.70], 55 | nfilters_embeded_last=[8], 56 | nfilters_embeded=[4], 57 | nlayers_embeded=[2], 58 | #nfilters_edge=[16], 59 | #nlayers_edge=[2], 60 | nfilters_cloud=[16], 61 | nlayers_cloud=[2], 62 | lr=[1e-3], 63 | branchweight=[1], 64 | ent_T=[0.8] 65 | ) 66 | 67 | # mnist.set_searchspace( 68 | # nfilters_embeded=[64, 128], 69 | # nlayers_embeded=[1, 2], 70 | # nfilters_cloud=[128], 71 | # nlayers_cloud=[2,4], 72 | # lr=[1e-3,1e-4, 1e-5], 73 | # branchweight=[.1], 74 | # ent_T=[0.0, 0.01, 0.1] 75 | # ) 76 | 77 | def constraintfn(**kwargs): 78 | #TODO: change to memory cost 79 | return True 80 | 81 | mnist.set_constraints(constraintfn) 82 | 83 | # switch chooser 84 | mnist.set_chooser(deepopt.chooser.GridChooser()) 85 | 86 | # currently optimize based on the validation accuracy of the main model 87 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 88 | #idx, acc = max_acc(traces) 89 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 90 | 91 | # visualize.min_error(traces, args.save_dir) 92 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 93 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 94 | 95 | # generate c 96 | # mnist.generate_c((1,28,28)) 97 | 98 | # generate container 99 | # mnist.generate_container() 100 | 101 | # get traces for the collection 102 | # mnist = Collection('simple_hybrid', save_dir) 103 | # traces = mnist.get_do_traces() 104 | -------------------------------------------------------------------------------- /exp/exp_increasing.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | mnist = Collection('binary_multiinput_increasing_mpcc_{}'.format(reduce(lambda x,y: str(x)+str(y), args.ncams)), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 34 | 35 | ncams = args.ncams 36 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function="max_pool_concat") 37 | 38 | train, test = get_mvmc_flatten_eval(ncams) 39 | #print("train",len(train)) 40 | #print("test",len(test)) 41 | #import sys 42 | #sys.exit(0) 43 | #train, test = chainer.datasets.get_cifar10(ndim=3) 44 | #print(train[1]) 45 | #print(len(train[0])) 46 | #from chainer.datasets.sub_dataset import SubDataset 47 | #train = SubDataset(train, 0, 500) 48 | #test = SubDataset(train, 0, 500) 49 | 50 | mnist.add_trainset(train) 51 | mnist.add_testset(test) 52 | 53 | mnist.set_searchspace( 54 | nfilters_embeded_last=[16], 55 | nfilters_embeded=[16], 56 | nlayers_embeded=[1], 57 | #nfilters_edge=[16], 58 | #nlayers_edge=[2], 59 | nfilters_cloud=[16], 60 | nlayers_cloud=[2], 61 | lr=[1e-3], 62 | branchweight=[1], 63 | ent_T=[0.8] 64 | ) 65 | 66 | #mnist.set_searchspace( 67 | # nfilters_embeded_last=[8], 68 | # nfilters_embeded=[8], 69 | # nlayers_embeded=[2], 70 | # nfilters_edge=[16], 71 | # nlayers_edge=[1], 72 | # nfilters_cloud=[16], 73 | # nlayers_cloud=[2], 74 | # lr=[1e-3], 75 | # branchweight=[1], 76 | # ent_T=[0.8] 77 | #) 78 | 79 | # mnist.set_searchspace( 80 | # nfilters_embeded=[64, 128], 81 | # nlayers_embeded=[1, 2], 82 | # nfilters_cloud=[128], 83 | # nlayers_cloud=[2,4], 84 | # lr=[1e-3,1e-4, 1e-5], 85 | # branchweight=[.1], 86 | # ent_T=[0.0, 0.01, 0.1] 87 | # ) 88 | 89 | def constraintfn(**kwargs): 90 | #TODO: change to memory cost 91 | return True 92 | 93 | mnist.set_constraints(constraintfn) 94 | 95 | # switch chooser 96 | mnist.set_chooser(deepopt.chooser.GridChooser()) 97 | 98 | # currently optimize based on the validation accuracy of the main model 99 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 100 | #idx, acc = max_acc(traces) 101 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 102 | 103 | # visualize.min_error(traces, args.save_dir) 104 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 105 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 106 | 107 | # generate c 108 | # mnist.generate_c((1,28,28)) 109 | 110 | # generate container 111 | # mnist.generate_container() 112 | 113 | # get traces for the collection 114 | # mnist = Collection('simple_hybrid', save_dir) 115 | # traces = mnist.get_do_traces() 116 | -------------------------------------------------------------------------------- /exp/exp_individual.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | suffix = "_".join([str(i) for i in args.ncams]) 34 | mnist = Collection('binary_multiinput_individual_mpcc_1_nolocal{}'.format(suffix), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 35 | 36 | ncams = args.ncams 37 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function="max_pool_concat") 38 | 39 | train, test = get_mvmc_flatten_eval(ncams) 40 | #print("train",len(train)) 41 | #print("test",len(test)) 42 | #import sys 43 | #sys.exit(0) 44 | #train, test = chainer.datasets.get_cifar10(ndim=3) 45 | #print(train[1]) 46 | #print(len(train[0])) 47 | #from chainer.datasets.sub_dataset import SubDataset 48 | #train = SubDataset(train, 0, 500) 49 | #test = SubDataset(train, 0, 500) 50 | 51 | mnist.add_trainset(train) 52 | mnist.add_testset(test) 53 | 54 | mnist.set_searchspace( 55 | nfilters_embeded_last=[1], 56 | nfilters_embeded=[1], 57 | nlayers_embeded=[1], 58 | nfilters_edge=[8], 59 | nlayers_edge=[1], 60 | nfilters_cloud=[16], 61 | nlayers_cloud=[1], 62 | lr=[1e-3], 63 | branchweight=[1], 64 | ent_T=[0.8] 65 | ) 66 | 67 | # mnist.set_searchspace( 68 | # nfilters_embeded=[64, 128], 69 | # nlayers_embeded=[1, 2], 70 | # nfilters_cloud=[128], 71 | # nlayers_cloud=[2,4], 72 | # lr=[1e-3,1e-4, 1e-5], 73 | # branchweight=[.1], 74 | # ent_T=[0.0, 0.01, 0.1] 75 | # ) 76 | 77 | def constraintfn(**kwargs): 78 | #TODO: change to memory cost 79 | return True 80 | 81 | mnist.set_constraints(constraintfn) 82 | 83 | # switch chooser 84 | mnist.set_chooser(deepopt.chooser.GridChooser()) 85 | 86 | # currently optimize based on the validation accuracy of the main model 87 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 88 | #idx, acc = max_acc(traces) 89 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 90 | 91 | # visualize.min_error(traces, args.save_dir) 92 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 93 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 94 | 95 | # generate c 96 | # mnist.generate_c((1,28,28)) 97 | 98 | # generate container 99 | # mnist.generate_container() 100 | 101 | # get traces for the collection 102 | # mnist = Collection('simple_hybrid', save_dir) 103 | # traces = mnist.get_do_traces() 104 | -------------------------------------------------------------------------------- /exp/exp_individual_single.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.single_input import SingleInputFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="5") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | suffix = "_".join([str(i) for i in args.ncams]) 34 | mnist = Collection('binary_multiinput_individual_single_cccc_1_{}'.format(suffix), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 35 | 36 | ncams = args.ncams 37 | mnist.set_model_family(SingleInputFamily,batchsize=700) 38 | 39 | train, test = get_mvmc_flatten_eval(ncams) 40 | #print("train",len(train)) 41 | #print("test",len(test)) 42 | #import sys 43 | #sys.exit(0) 44 | #train, test = chainer.datasets.get_cifar10(ndim=3) 45 | #print(train[1]) 46 | #print(len(train[0])) 47 | #from chainer.datasets.sub_dataset import SubDataset 48 | #train = SubDataset(train, 0, 500) 49 | #test = SubDataset(train, 0, 500) 50 | 51 | mnist.add_trainset(train) 52 | mnist.add_testset(test) 53 | 54 | mnist.set_searchspace( 55 | nfilters_embeded_last=[1], 56 | nfilters_embeded=[1], 57 | nlayers_embeded=[1], 58 | nfilters_edge=[8], 59 | nlayers_edge=[1], 60 | nfilters_cloud=[16], 61 | nlayers_cloud=[1], 62 | lr=[1e-3], 63 | branchweight=[1], 64 | ent_T=[0.8] 65 | ) 66 | 67 | # mnist.set_searchspace( 68 | # nfilters_embeded=[64, 128], 69 | # nlayers_embeded=[1, 2], 70 | # nfilters_cloud=[128], 71 | # nlayers_cloud=[2,4], 72 | # lr=[1e-3,1e-4, 1e-5], 73 | # branchweight=[.1], 74 | # ent_T=[0.0, 0.01, 0.1] 75 | # ) 76 | 77 | def constraintfn(**kwargs): 78 | #TODO: change to memory cost 79 | return True 80 | 81 | mnist.set_constraints(constraintfn) 82 | 83 | # switch chooser 84 | mnist.set_chooser(deepopt.chooser.GridChooser()) 85 | 86 | # currently optimize based on the validation accuracy of the main model 87 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 88 | #idx, acc = max_acc(traces) 89 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 90 | 91 | # visualize.min_error(traces, args.save_dir) 92 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 93 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 94 | 95 | # generate c 96 | # mnist.generate_c((1,28,28)) 97 | 98 | # generate container 99 | # mnist.generate_container() 100 | 101 | # get traces for the collection 102 | # mnist = Collection('simple_hybrid', save_dir) 103 | # traces = mnist.get_do_traces() 104 | -------------------------------------------------------------------------------- /exp/exp_mem.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0,1,2,3,4,5") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | mnist = Collection('exp_mem_percent_{}'.format(reduce(lambda x,y: str(x)+str(y), args.ncams)), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 34 | 35 | ncams = args.ncams 36 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function="max_pool_concat") 37 | 38 | train, test = get_mvmc_flatten_eval(ncams) 39 | #print("train",len(train)) 40 | #print("test",len(test)) 41 | #import sys 42 | #sys.exit(0) 43 | #train, test = chainer.datasets.get_cifar10(ndim=3) 44 | #print(train[1]) 45 | #print(len(train[0])) 46 | #from chainer.datasets.sub_dataset import SubDataset 47 | #train = SubDataset(train, 0, 500) 48 | #test = SubDataset(train, 0, 500) 49 | 50 | mnist.add_trainset(train) 51 | mnist.add_testset(test) 52 | 53 | mnist.set_searchspace( 54 | target_branch0exit=[0.60], 55 | nfilters_embeded_last=[2], 56 | nfilters_embeded=[1,2,4,8], 57 | nlayers_embeded=[2], 58 | #nfilters_edge=[16], 59 | #nlayers_edge=[2], 60 | nfilters_cloud=[16], 61 | nlayers_cloud=[2], 62 | lr=[1e-3], 63 | branchweight=[1], 64 | ent_T=[0.85] 65 | ) 66 | 67 | # mnist.set_searchspace( 68 | # nfilters_embeded=[64, 128], 69 | # nlayers_embeded=[1, 2], 70 | # nfilters_cloud=[128], 71 | # nlayers_cloud=[2,4], 72 | # lr=[1e-3,1e-4, 1e-5], 73 | # branchweight=[.1], 74 | # ent_T=[0.0, 0.01, 0.1] 75 | # ) 76 | 77 | def constraintfn(**kwargs): 78 | #TODO: change to memory cost 79 | return True 80 | 81 | mnist.set_constraints(constraintfn) 82 | 83 | # switch chooser 84 | mnist.set_chooser(deepopt.chooser.GridChooser()) 85 | 86 | # currently optimize based on the validation accuracy of the main model 87 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 88 | #idx, acc = max_acc(traces) 89 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 90 | 91 | # visualize.min_error(traces, args.save_dir) 92 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 93 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 94 | 95 | # generate c 96 | # mnist.generate_c((1,28,28)) 97 | 98 | # generate container 99 | # mnist.generate_container() 100 | 101 | # get traces for the collection 102 | # mnist = Collection('simple_hybrid', save_dir) 103 | # traces = mnist.get_do_traces() 104 | -------------------------------------------------------------------------------- /exp/exp_model_size.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0,1,2,3,4,5") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | mnist = Collection('exp_model_size_{}'.format(reduce(lambda x,y: str(x)+str(y), args.ncams)), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 34 | 35 | ncams = args.ncams 36 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function="max_pool_concat") 37 | 38 | train, test = get_mvmc_flatten_eval(ncams) 39 | #print("train",len(train)) 40 | #print("test",len(test)) 41 | #import sys 42 | #sys.exit(0) 43 | #train, test = chainer.datasets.get_cifar10(ndim=3) 44 | #print(train[1]) 45 | #print(len(train[0])) 46 | #from chainer.datasets.sub_dataset import SubDataset 47 | #train = SubDataset(train, 0, 500) 48 | #test = SubDataset(train, 0, 500) 49 | 50 | mnist.add_trainset(train) 51 | mnist.add_testset(test) 52 | 53 | mnist.set_searchspace( 54 | nfilters_embeded_last=[1], 55 | nfilters_embeded=[1,2,3,4,8,16], 56 | nlayers_embeded=[1], 57 | #nfilters_edge=[16], 58 | #nlayers_edge=[2], 59 | nfilters_cloud=[16], 60 | nlayers_cloud=[2], 61 | lr=[1e-3], 62 | branchweight=[1], 63 | ent_T=[0.8] 64 | ) 65 | 66 | #mnist.set_searchspace( 67 | # nfilters_embeded_last=[8], 68 | # nfilters_embeded=[8], 69 | # nlayers_embeded=[2], 70 | # nfilters_edge=[16], 71 | # nlayers_edge=[1], 72 | # nfilters_cloud=[16], 73 | # nlayers_cloud=[2], 74 | # lr=[1e-3], 75 | # branchweight=[1], 76 | # ent_T=[0.8] 77 | #) 78 | 79 | # mnist.set_searchspace( 80 | # nfilters_embeded=[64, 128], 81 | # nlayers_embeded=[1, 2], 82 | # nfilters_cloud=[128], 83 | # nlayers_cloud=[2,4], 84 | # lr=[1e-3,1e-4, 1e-5], 85 | # branchweight=[.1], 86 | # ent_T=[0.0, 0.01, 0.1] 87 | # ) 88 | 89 | def constraintfn(**kwargs): 90 | #TODO: change to memory cost 91 | return True 92 | 93 | mnist.set_constraints(constraintfn) 94 | 95 | # switch chooser 96 | mnist.set_chooser(deepopt.chooser.GridChooser()) 97 | 98 | # currently optimize based on the validation accuracy of the main model 99 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 100 | #idx, acc = max_acc(traces) 101 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 102 | 103 | # visualize.min_error(traces, args.save_dir) 104 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 105 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 106 | 107 | # generate c 108 | # mnist.generate_c((1,28,28)) 109 | 110 | # generate container 111 | # mnist.generate_container() 112 | 113 | # get traces for the collection 114 | # mnist = Collection('simple_hybrid', save_dir) 115 | # traces = mnist.get_do_traces() 116 | -------------------------------------------------------------------------------- /exp/exp_threshold.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0,1,2,3,4,5") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | mnist = Collection('exp_threshold_{}'.format(reduce(lambda x,y: str(x)+str(y), args.ncams)), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 34 | 35 | ncams = args.ncams 36 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function="max_pool_concat") 37 | 38 | train, test = get_mvmc_flatten_eval(ncams) 39 | #print("train",len(train)) 40 | #print("test",len(test)) 41 | #import sys 42 | #sys.exit(0) 43 | #train, test = chainer.datasets.get_cifar10(ndim=3) 44 | #print(train[1]) 45 | #print(len(train[0])) 46 | #from chainer.datasets.sub_dataset import SubDataset 47 | #train = SubDataset(train, 0, 500) 48 | #test = SubDataset(train, 0, 500) 49 | 50 | mnist.add_trainset(train) 51 | mnist.add_testset(test) 52 | 53 | mnist.set_searchspace( 54 | nfilters_embeded_last=[1], 55 | nfilters_embeded=[4], 56 | nlayers_embeded=[1], 57 | #nfilters_edge=[16], 58 | #nlayers_edge=[2], 59 | nfilters_cloud=[16], 60 | nlayers_cloud=[2], 61 | lr=[1e-3], 62 | branchweight=[1], 63 | ent_T=[0.1,0.3,0.5,0.6,0.7,0.8,0.9,1] 64 | ) 65 | 66 | # mnist.set_searchspace( 67 | # nfilters_embeded=[64, 128], 68 | # nlayers_embeded=[1, 2], 69 | # nfilters_cloud=[128], 70 | # nlayers_cloud=[2,4], 71 | # lr=[1e-3,1e-4, 1e-5], 72 | # branchweight=[.1], 73 | # ent_T=[0.0, 0.01, 0.1] 74 | # ) 75 | 76 | def constraintfn(**kwargs): 77 | #TODO: change to memory cost 78 | return True 79 | 80 | mnist.set_constraints(constraintfn) 81 | 82 | # switch chooser 83 | mnist.set_chooser(deepopt.chooser.GridChooser()) 84 | 85 | # currently optimize based on the validation accuracy of the main model 86 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 87 | #idx, acc = max_acc(traces) 88 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 89 | 90 | # visualize.min_error(traces, args.save_dir) 91 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 92 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 93 | 94 | # generate c 95 | # mnist.generate_c((1,28,28)) 96 | 97 | # generate container 98 | # mnist.generate_container() 99 | 100 | # get traces for the collection 101 | # mnist = Collection('simple_hybrid', save_dir) 102 | # traces = mnist.get_do_traces() 103 | -------------------------------------------------------------------------------- /exp/exp_weights.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import chainer 7 | from datasets.datasets import get_mvmc, get_mvmc_flatten_eval 8 | import deepopt.chooser 9 | from elaas.elaas import Collection 10 | from elaas.family.multi_input_edge import MultiInputEdgeFamily 11 | from visualize import visualize 12 | 13 | def max_acc(trace): 14 | acc = 0 15 | best_idx = 0 16 | for i, t in enumerate(trace): 17 | if t['action'] == 'add_point': 18 | acc = max(acc, t['y']) 19 | best_idx = i 20 | return acc, best_idx 21 | 22 | 23 | parser = argparse.ArgumentParser(description='Hybrid Example') 24 | parser.add_argument('-s', '--save_dir', default='_models') 25 | parser.add_argument('--iters', type=int, default=100) 26 | parser.add_argument('-e', '--epochs', type=int, default=100) 27 | parser.add_argument('-b', '--bootstrap_epochs', type=int, default=100) 28 | parser.add_argument('-v', '--verbose', action='store_true') 29 | parser.add_argument('-n', '--ncams', type=str, default="0,1,2,3,4,5") 30 | args = parser.parse_args() 31 | 32 | args.ncams = [int(i) for i in args.ncams.split(",")] 33 | mnist = Collection('binary_multiinput_weights_mpcc_{}'.format(args.ncams[-1]), args.save_dir, input_dims=3, nepochs=args.epochs, verbose=args.verbose) 34 | 35 | ncams = args.ncams 36 | mnist.set_model_family(MultiInputEdgeFamily,ninputs=len(ncams),batchsize=700,merge_function="max_pool_concat") 37 | 38 | train, test = get_mvmc_flatten_eval(ncams) 39 | #print("train",len(train)) 40 | #print("test",len(test)) 41 | #import sys 42 | #sys.exit(0) 43 | #train, test = chainer.datasets.get_cifar10(ndim=3) 44 | #print(train[1]) 45 | #print(len(train[0])) 46 | #from chainer.datasets.sub_dataset import SubDataset 47 | #train = SubDataset(train, 0, 500) 48 | #test = SubDataset(train, 0, 500) 49 | 50 | mnist.add_trainset(train) 51 | mnist.add_testset(test) 52 | 53 | mnist.set_searchspace( 54 | nfilters_embeded_last=[4], 55 | nfilters_embeded=[4], 56 | nlayers_embeded=[1], 57 | #nfilters_edge=[16], 58 | #nlayers_edge=[2], 59 | nfilters_cloud=[16], 60 | nlayers_cloud=[2], 61 | lr=[1e-3], 62 | branchweight=[1], 63 | ent_T=[0.8] 64 | ) 65 | 66 | # mnist.set_searchspace( 67 | # nfilters_embeded=[64, 128], 68 | # nlayers_embeded=[1, 2], 69 | # nfilters_cloud=[128], 70 | # nlayers_cloud=[2,4], 71 | # lr=[1e-3,1e-4, 1e-5], 72 | # branchweight=[.1], 73 | # ent_T=[0.0, 0.01, 0.1] 74 | # ) 75 | 76 | def constraintfn(**kwargs): 77 | #TODO: change to memory cost 78 | return True 79 | 80 | mnist.set_constraints(constraintfn) 81 | 82 | # switch chooser 83 | mnist.set_chooser(deepopt.chooser.GridChooser()) 84 | 85 | # currently optimize based on the validation accuracy of the main model 86 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 87 | #idx, acc = max_acc(traces) 88 | #print('Best Binary Acc: {:2.4f}'.format(acc*100.)) 89 | 90 | # visualize.min_error(traces, args.save_dir) 91 | # visualize.embed_memory_err(mnist.do, traces, args.save_dir) 92 | # visualize.embed_transmit_err(mnist.do, traces, args.save_dir) 93 | 94 | # generate c 95 | # mnist.generate_c((1,28,28)) 96 | 97 | # generate container 98 | # mnist.generate_container() 99 | 100 | # get traces for the collection 101 | # mnist = Collection('simple_hybrid', save_dir) 102 | # traces = mnist.get_do_traces() 103 | -------------------------------------------------------------------------------- /exp/run_all.sh: -------------------------------------------------------------------------------- 1 | python exp_combining_concat.py 2 | python exp_combining_avg_pool.py 3 | python exp_combining_max_pool.py 4 | python exp_combining_avg_pool_concat.py 5 | python exp_combining_max_pool_concat.py -------------------------------------------------------------------------------- /exp/run_combining.sh: -------------------------------------------------------------------------------- 1 | python exp_combining.py -m max_pool 2 | python exp_combining.py -m max_pool_concat 3 | python exp_combining.py -m avg_pool 4 | python exp_combining.py -m avg_pool_concat 5 | python exp_combining.py -m concat 6 | python exp_combining.py -m avg_pool_max_pool 7 | python exp_combining.py -m max_pool_avg_pool 8 | python exp_combining.py -m concat_max_pool 9 | python exp_combining.py -m concat_avg_pool 10 | -------------------------------------------------------------------------------- /exp/run_increasing.sh: -------------------------------------------------------------------------------- 1 | python exp_increasing.py -n 0 2 | python exp_increasing.py -n 0,1 3 | python exp_increasing.py -n 0,1,2 4 | python exp_increasing.py -n 0,1,2,3 5 | python exp_increasing.py -n 0,1,2,3,5 6 | python exp_increasing.py -n 0,1,2,3,5,4 7 | -------------------------------------------------------------------------------- /exp/run_individual.sh: -------------------------------------------------------------------------------- 1 | python exp_individual.py -n 0 2 | python exp_individual.py -n 1 3 | python exp_individual.py -n 2 4 | python exp_individual.py -n 3 5 | python exp_individual.py -n 4 6 | python exp_individual.py -n 5 -------------------------------------------------------------------------------- /exp/run_individual_single.sh: -------------------------------------------------------------------------------- 1 | python exp_individual_single.py -n 0 2 | python exp_individual_single.py -n 1 3 | python exp_individual_single.py -n 2 4 | python exp_individual_single.py -n 3 5 | python exp_individual_single.py -n 4 6 | python exp_individual_single.py -n 5 -------------------------------------------------------------------------------- /exp/run_missing.py: -------------------------------------------------------------------------------- 1 | python exp_increasing.py -n 1,2,3,4,5 2 | python exp_increasing.py -n 0,2,3,4,5 3 | python exp_increasing.py -n 0,1,3,4,5 4 | python exp_increasing.py -n 0,1,2,4,5 5 | python exp_increasing.py -n 0,1,2,3,5 6 | python exp_increasing.py -n 0,1,2,3,4 7 | -------------------------------------------------------------------------------- /exp/run_threshod.sh: -------------------------------------------------------------------------------- 1 | python exp_threshold.py 2 | python exp_threshold.py 3 | python exp_threshold.py 4 | python exp_threshold.py 5 | python exp_threshold.py 6 | python exp_threshold.py 7 | -------------------------------------------------------------------------------- /experiments/mnist_dropout.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import random 4 | random.seed(9001) 5 | 6 | import os 7 | import sys 8 | sys.path.append('..') 9 | import argparse 10 | 11 | import chainer 12 | #from datasets.datasets import get_mvmc, get_mvmc_flatten 13 | from datasets.mnist import get_mnist 14 | 15 | import deepopt.chooser 16 | from elaas.elaas import Collection 17 | from elaas.family.multi_input_edge_with_dropout import MultiInputEdgeDropoutFamily 18 | from visualize import visualize 19 | import fire 20 | 21 | class AttrDict(dict): 22 | def __init__(self, *args, **kwargs): 23 | super(AttrDict, self).__init__(*args, **kwargs) 24 | self.__dict__ = self 25 | 26 | class MNISTDropout(object): 27 | def run(dtrain, dtest): 28 | args = AttrDict(**dict( 29 | save_dir="_models", 30 | iters=1, 31 | epochs=2, 32 | bootstrap_epochs=2, 33 | ncams=1, 34 | verbose=True 35 | )) 36 | 37 | mnist = Collection('multiinput_edge_dropout_mpcc_{}'.format(args.ncams), args.save_dir, nepochs=args.epochs, verbose=args.verbose) 38 | ncams = args.ncams 39 | mnist.set_model_family(MultiInputEdgeDropoutFamily, ninputs=ncams, resume=False, 40 | merge_function="max_pool_concat", drop_comm_train=dtrain, 41 | drop_comm_test=dtest, input_dims=1, output_dims=10) 42 | train, test = get_mnist() 43 | mnist.add_trainset(train) 44 | mnist.add_testset(test) 45 | mnist.set_searchspace( 46 | nfilters_embeded=[4], 47 | nlayers_embeded=[2], 48 | nfilters_cloud=[4], 49 | nlayers_cloud=[2], 50 | lr=[1e-3], 51 | branchweight=[.1], 52 | ent_T=[100] 53 | ) 54 | 55 | # switch chooser 56 | # mnist.set_chooser(deepopt.chooser.EpochChooser(k=5)) 57 | 58 | # currently optimize based on the validation accuracy of the main model 59 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 60 | return traces.y 61 | 62 | 63 | if __name__ == '__main__': 64 | fire.Fire(MNISTDropout) 65 | -------------------------------------------------------------------------------- /experiments/mnist_dropout_nofire.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | 4 | import random 5 | random.seed(9001) 6 | 7 | import os 8 | import sys 9 | sys.path.append('..') 10 | import argparse 11 | 12 | import chainer 13 | import numpy as np 14 | #from datasets.datasets import get_mvmc, get_mvmc_flatten 15 | from datasets.mnist import get_mnist 16 | 17 | import deepopt.chooser 18 | from elaas.elaas import Collection 19 | from elaas.family.multi_input_edge_with_dropout import MultiInputEdgeDropoutFamily 20 | from visualize import visualize 21 | import matplotlib 22 | matplotlib.rcParams['font.size'] = 20.0 23 | import matplotlib.pyplot as plt 24 | 25 | class AttrDict(dict): 26 | def __init__(self, *args, **kwargs): 27 | super(AttrDict, self).__init__(*args, **kwargs) 28 | self.__dict__ = self 29 | 30 | def run(dtrain, dtest, epochs=10, verbose=False): 31 | args = AttrDict(**dict( 32 | save_dir="_models", 33 | iters=epochs, 34 | epochs=epochs, 35 | bootstrap_epochs=1, 36 | ncams=1, 37 | verbose=verbose 38 | )) 39 | 40 | mnist = Collection('multiinput_edge_dropout_mpcc_{}'.format(args.ncams), args.save_dir, nepochs=args.epochs, verbose=args.verbose) 41 | ncams = args.ncams 42 | mnist.set_model_family(MultiInputEdgeDropoutFamily, ninputs=ncams, resume=False, 43 | merge_function="max_pool_concat", drop_comm_train=dtrain, 44 | drop_comm_test=dtest, input_dims=1, output_dims=10) 45 | train, test = get_mnist() 46 | mnist.add_trainset(train) 47 | mnist.add_testset(test) 48 | mnist.set_searchspace( 49 | nfilters_embeded=[3], 50 | nlayers_embeded=[2], 51 | nfilters_cloud=[3], 52 | nlayers_cloud=[2], 53 | lr=[1e-3], 54 | branchweight=[.1], 55 | ent_T=[100] 56 | ) 57 | 58 | # currently optimize based on the validation accuracy of the main model 59 | traces = mnist.train(niters=args.iters, bootstrap_nepochs=args.bootstrap_epochs) 60 | return traces[-1]['y'] 61 | 62 | dtrains = { 63 | 'no_d': 0.0, 64 | '50_d': 0.5 65 | } 66 | dtests = [0.0, 0.1, 0.25, 0.5, 0.75] 67 | 68 | acc_dict = {} 69 | for key, dtrain in dtrains.iteritems(): 70 | acc_dict[key] = [] 71 | for dtest in dtests: 72 | print(dtrain, dtest) 73 | y = run(dtrain, dtest, 10) 74 | print(y) 75 | acc_dict[key].append(y*100.) 76 | 77 | 78 | #plot code 79 | names = ['no_d', '50_d'] 80 | linewidth = 4 81 | ms = 8 82 | colors = {'no_d': '#F93943', '50_d': '#445E93'} 83 | styles = {'no_d': '-o', '50_d': '--o'} 84 | labels = {'no_d': 'No Dropout', '50_d': 'Dropout (50%)'} 85 | plt.figure(figsize=(8, 6.5)) 86 | comms = (1-np.array(dtests))*100. 87 | for name in names: 88 | plt.plot(comms, acc_dict[name], styles[name], 89 | linewidth=linewidth, ms=ms, color=colors[name], 90 | label=labels[name]) 91 | plt.xlabel('Communication (%)') 92 | plt.ylabel('Classification Accuracy (%)') 93 | plt.legend(loc=0, prop={'size': 14}) 94 | plt.tight_layout() 95 | plt.grid() 96 | plt.savefig("dropout.png") 97 | plt.clf() 98 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | chainer 2 | scikit-learn -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup, find_packages 4 | 5 | setup(name='hybrid', 6 | version='1', 7 | description='Embedded Learning as a Service', 8 | author='', 9 | author_email='', 10 | packages=find_packages(), 11 | ) 12 | -------------------------------------------------------------------------------- /tests/conv.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import numpy as np 7 | 8 | import chainer_sequential.binary.utils.binary_util as bu 9 | from chainer_sequential.binary.links.link_binary_convolution import BinaryConvolution2D 10 | from chainer_sequential.binary.links.link_batch_normalization import BatchNormalization 11 | from chainer.functions import max_pooling_2d 12 | 13 | # x = np.random.random((2,2,5,5)).astype(np.float32) 14 | # bconv = BinaryConvolution2D(2, 2, ksize=3, stride=1, pad=1) 15 | # bconv.b.data = np.array([0.004, 0.006], dtype=np.float32) 16 | # bn = BatchNormalization(2) 17 | # bn.beta.data = np.array([-0.06, 0.01], dtype=np.float32) 18 | # bn.gamma.data = np.array([1.02, 1.2], dtype=np.float32) 19 | # bn(bconv(x)) #init mean and var 20 | # res = bn(bconv(x), test=True) 21 | # W = bconv.W.data 22 | # print bu.np_to_floatC(x.astype(np.float16), 'A_in', 'row_major') 23 | # print bu.np_to_uint8C(bu.binarize_real(W.reshape(2, -1)), 'F_in', 'row_major', pad='1') 24 | # print bu.np_to_packed_uint8C(bu.binarize_real(res.data.flatten()), 'C_actual', 'row_major', pad='0') 25 | # print bu.np_to_floatC(bconv.b.data.astype(np.float16), 'Bias', 'row_major') 26 | # print bu.np_to_floatC(bn.beta.data.astype(np.float16), 'Beta', 'row_major') 27 | # print bu.np_to_floatC(bn.gamma.data.astype(np.float16), 'Gamma', 'row_major') 28 | # print bu.np_to_floatC(bn.avg_mean.astype(np.float16), 'Mean', 'row_major') 29 | # print bu.np_to_floatC(np.sqrt(bn.avg_var).astype(np.float16), 'Std', 'row_major') 30 | 31 | 32 | print 'Binary (no BN)\n\n' 33 | x = np.random.random((1,2,6,6)).astype(np.float32) - 0.9 34 | x = bu.binarize(x) 35 | bconv = BinaryConvolution2D(2, 2, ksize=3, stride=1, pad=1) 36 | inter_res = bconv(x) 37 | res = max_pooling_2d(inter_res, 3, 1, 1) 38 | W = bconv.W.data 39 | #for binary 40 | print bu.np_to_packed_uint8C(bu.binarize_real(x).flatten(), 'A_in', 'row_major') 41 | print bu.np_to_uint8C(bu.binarize_real(W.reshape(4, -1)), 'F_in', 'row_major', pad='1') 42 | #for float 43 | # print bu.np_to_floatC(x.flatten(), 'A_in', 'row_major') 44 | # print bu.np_to_uint8C(bu.binarize_real(W.reshape(2, -1)), 'F_in', 'row_major', pad='1') 45 | print bu.np_to_packed_uint8C(bu.binarize_real(res.data.flatten()), 'C_actual', 'row_major', pad='0') 46 | assert False 47 | 48 | print 'Binary (BN)\n\n' 49 | x = np.random.random((2,2,5,5)).astype(np.float32) - 0.5 50 | x = bu.binarize(x) 51 | bconv = BinaryConvolution2D(2, 2, ksize=3, stride=1, pad=1) 52 | bconv.b.data = np.array([0.004, 0.006], dtype=np.float32) 53 | bn = BatchNormalization(2) 54 | bn.beta.data = np.array([-0.06, 0.01], dtype=np.float32) 55 | bn.gamma.data = np.array([1.02, 1.2], dtype=np.float32) 56 | bn(bconv(x)) #init mean and var 57 | res = bn(bconv(x), test=True) 58 | W = bconv.W.data 59 | print bu.np_to_packed_uint8C(bu.binarize_real(x), 'A_in', 'row_major') 60 | print bu.np_to_uint8C(bu.binarize_real(W.reshape(4, -1)), 'F_in', 'row_major', pad='1') 61 | print bu.np_to_packed_uint8C(bu.binarize_real(res.data.flatten()), 'C_actual', 'row_major', pad='0') 62 | print bu.np_to_floatC(bconv.b.data.astype(np.float16), 'Bias', 'row_major') 63 | print bu.np_to_floatC(bn.beta.data.astype(np.float16), 'Beta', 'row_major') 64 | print bu.np_to_floatC(bn.gamma.data.astype(np.float16), 'Gamma', 'row_major') 65 | print bu.np_to_floatC(bn.avg_mean.astype(np.float16), 'Mean', 'row_major') 66 | print bu.np_to_floatC(np.sqrt(bn.avg_var).astype(np.float16), 'Std', 'row_major') 67 | -------------------------------------------------------------------------------- /tests/linear.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import numpy as np 7 | 8 | import chainer_sequential.binary.utils.binary_util as bu 9 | from chainer_sequential.binary.links.link_binary_linear import BinaryLinear 10 | from chainer_sequential.binary.links.link_batch_normalization import BatchNormalization 11 | 12 | x = np.random.random((10,127)).astype(np.float32)-0.5 13 | x = bu.binarize(x).astype(np.float32) 14 | bl = BinaryLinear(127, 2) 15 | bl.b.data = np.array([0.004, 0.006], dtype=np.float32) 16 | bn = BatchNormalization(2) 17 | bn.beta.data = np.array([-0.06, 0.01], dtype=np.float32) 18 | bn.gamma.data = np.array([1.02, 1.2], dtype=np.float32) 19 | bn(bl(x)) #init mean and var 20 | res = bn(bl(x), test=True) 21 | W = bl.W.data 22 | print bu.np_to_uint8C(bu.binarize_real(x), 'A_in', 'row_major') 23 | print bu.np_to_uint8C(bu.binarize_real(W.reshape(2, -1)), 'F_in', 'row_major', pad='1') 24 | print bu.np_to_packed_uint8C(bu.binarize_real(res.data.flatten()), 'C_actual', 'row_major', pad='0') 25 | 26 | print bu.np_to_floatC(bl.b.data.astype(np.float16), 'Bias', 'row_major') 27 | print bu.np_to_floatC(bn.beta.data.astype(np.float16), 'Beta', 'row_major') 28 | print bu.np_to_floatC(bn.gamma.data.astype(np.float16), 'Gamma', 'row_major') 29 | print bu.np_to_floatC(bn.avg_mean.astype(np.float16), 'Mean', 'row_major') 30 | print bu.np_to_floatC(np.sqrt(bn.avg_var).astype(np.float16), 'Std', 'row_major') 31 | -------------------------------------------------------------------------------- /tests/slice.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | sys.path.append('..') 4 | import argparse 5 | 6 | import numpy as np 7 | 8 | import chainer_sequential.binary.utils.binary_util as bu 9 | 10 | 11 | x = np.random.random((3, 7)).astype(np.float32)-0.5 12 | print bu.np_to_packed_uint8C(bu.binarize_real(x), 'A_in', 'row_major') 13 | print bu.np_to_packed_uint8C(bu.binarize_real(x[:1, 3:6]), 'C_actual', 'row_major') 14 | 15 | print "====" 16 | 17 | 18 | 19 | x = np.random.random((2, 2, 5, 5)).astype(np.float32)-0.5 20 | print bu.np_to_packed_uint8C(bu.binarize_real(x), 'A_in', 'row_major') 21 | print bu.np_to_packed_uint8C(bu.binarize_real(x[1,1,1:4,1:4]), 'C_actual', 'row_major') 22 | 23 | print "====" 24 | 25 | 26 | x = np.random.random((2, 2, 5, 5)).astype(np.float32)-0.5 27 | x = bu.binarize(x) 28 | y = np.random.random((2, 2, 3, 3)).astype(np.float32)-0.5 29 | y = bu.binarize(y) 30 | print bu.np_to_packed_uint8C(bu.binarize_real(x), 'A_in', 'row_major') 31 | print bu.np_to_uint8C(bu.binarize_real(y.reshape(4, -1)), 'B_in', 'row_major', pad='1') 32 | print "res: ", np.dot(x[0, :, 4:, 1:4].flatten(), y[0, :, 2, :].flatten()) 33 | -------------------------------------------------------------------------------- /visualize/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kunglab/ddnn/adbe5c20f2f6b2c3d875af8b651cce22138928ff/visualize/__init__.py -------------------------------------------------------------------------------- /visualize/data/add_cam.csv: -------------------------------------------------------------------------------- 1 | cam local cloud overall 2 | 0.37 0.37 0.55 0.58 3 | 0.37 0.43 0.77 0.80 4 | 0.46 0.75 0.87 0.88 5 | 0.52 0.94 0.95 0.96 6 | 0.60 0.95 0.97 0.99 7 | 0.73 0.97 0.99 0.99 8 | -------------------------------------------------------------------------------- /visualize/data/comm.csv: -------------------------------------------------------------------------------- 1 | filters local cloud overall local_exit comm device_size 2 | 1 0.86 0.90 0.91 70.18 14 432 3 | 2 0.90 0.92 0.95 73.68 16 660 4 | 4 0.90 0.92 0.96 70.76 21 1116 5 | 8 0.92 0.94 0.97 71.35 30 2028 -------------------------------------------------------------------------------- /visualize/data/entropy.csv: -------------------------------------------------------------------------------- 1 | T exit local cloud overall comm 2 | 0.1 0.00 0.92 0.96 0.96 128 3 | 0.3 0.58 0.92 0.96 0.96 126 4 | 0.5 1.75 0.92 0.96 0.96 126 5 | 0.6 2.92 0.92 0.96 0.96 126 6 | 0.7 22.81 0.92 0.96 0.96 126 7 | 0.8 60.82 0.92 0.96 0.97 57 8 | 0.9 83.04 0.92 0.96 0.96 57 9 | 1.0 100.00 0.92 0.96 0.92 12 10 | -------------------------------------------------------------------------------- /visualize/data/mem.csv: -------------------------------------------------------------------------------- 1 | filters acc mem 2 | 1 0.70 525 3 | 2 0.82 570 4 | 4 0.87 660 5 | 8 0.91 840 -------------------------------------------------------------------------------- /visualize/data/missing_cam.csv: -------------------------------------------------------------------------------- 1 | cam local cloud overall 2 | 0.37 0.92 0.98 0.99 3 | 0.37 0.94 0.98 0.99 4 | 0.46 0.97 0.98 0.99 5 | 0.52 0.95 0.97 0.99 6 | 0.60 0.96 0.96 0.96 7 | 0.73 0.94 0.95 0.98 -------------------------------------------------------------------------------- /visualize/visualize.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | 6 | 7 | def get_max_epoch(do, new_point): 8 | X = np.array(do.X) 9 | for i, param in enumerate(do.params): 10 | if param == 'nepochs': 11 | continue 12 | X = X[X[:, i] == new_point[param]] 13 | 14 | return np.max(X[:, do.params.index('nepochs')]) 15 | 16 | 17 | 18 | def embed_transmit_err(do, traces, save_dir): 19 | save_dir = os.path.join(save_dir, 'figures/') 20 | if not os.path.exists(save_dir): 21 | os.makedirs(save_dir) 22 | 23 | nfilters_idx = do.params.index('nfilters_embeded') 24 | nlayers_idx = do.params.index('nlayers_embeded') 25 | 26 | xs, ys = [], [] 27 | model_hashes = [] 28 | for t in traces[::-1]: 29 | if t['action'] != 'add_point': 30 | continue 31 | point = dict(zip(do.params, t['x'])) 32 | point.pop('nepochs', None) 33 | h = hash(frozenset(point.items())) 34 | if h not in model_hashes: 35 | model_hashes.append(h) 36 | xs.append((point['nfilters_embeded']**2)*(9/8)) 37 | ys.append(t['y']) 38 | 39 | 40 | plt.figure(figsize=(8, 6.5)) 41 | plt.plot(xs, ys, 'o') 42 | x_rng = (np.max(xs) - np.min(xs))*0.05 43 | y_rng = (np.max(ys) - np.min(ys))*0.05 44 | plt.xlim((np.min(xs)-x_rng, np.max(xs)+x_rng)) 45 | plt.ylim((np.min(ys)-y_rng, np.max(ys)+y_rng)) 46 | plt.xlabel('transmission size (bytes)') 47 | plt.ylabel('accuracy') 48 | plt.tight_layout() 49 | plt.savefig(save_dir + 'transmit_error.png') 50 | plt.clf() 51 | 52 | 53 | def embed_memory_err(do, traces, save_dir): 54 | save_dir = os.path.join(save_dir, 'figures/') 55 | if not os.path.exists(save_dir): 56 | os.makedirs(save_dir) 57 | 58 | nfilters_idx = do.params.index('nfilters_embeded') 59 | nlayers_idx = do.params.index('nlayers_embeded') 60 | 61 | xs, ys = [], [] 62 | model_hashes = [] 63 | for t in traces[::-1]: 64 | if t['action'] != 'add_point': 65 | continue 66 | point = dict(zip(do.params, t['x'])) 67 | point.pop('nepochs', None) 68 | h = hash(frozenset(point.items())) 69 | if h not in model_hashes: 70 | model_hashes.append(h) 71 | xs.append((point['nfilters_embeded']**2)*point['nlayers_embeded']*(9/8)) 72 | ys.append(t['y']) 73 | 74 | plt.figure(figsize=(8, 6.5)) 75 | plt.plot(xs, ys, 'o') 76 | plt.xlabel('embedded memory size (bytes)') 77 | plt.ylabel('accuracy') 78 | x_rng = (np.max(xs) - np.min(xs))*0.05 79 | y_rng = (np.max(ys) - np.min(ys))*0.05 80 | plt.xlim((np.min(xs)-x_rng, np.max(xs)+x_rng)) 81 | plt.ylim((np.min(ys)-y_rng, np.max(ys)+y_rng)) 82 | plt.tight_layout() 83 | plt.savefig(save_dir + 'memory_error.png') 84 | plt.clf() 85 | 86 | def min_error(traces, save_dir): 87 | save_dir = os.path.join(save_dir, 'figures/') 88 | if not os.path.exists(save_dir): 89 | os.makedirs(save_dir) 90 | 91 | points = [t for t in traces 92 | if t['action'] == 'add_point'] 93 | plt.figure(figsize=(8, 6.5)) 94 | ys = [1-p['y'] for p in points] 95 | plt.plot(np.minimum.accumulate(ys), linewidth=1.5) 96 | plt.xlabel('epoch') 97 | plt.ylabel('minimum error') 98 | plt.tight_layout() 99 | plt.savefig(save_dir + 'gp_min_error.png') 100 | plt.clf() 101 | --------------------------------------------------------------------------------