├── .idea
    ├── vcs.xml
    ├── modules.xml
    ├── nnq_mc_study.iml
    └── workspace.xml
├── LICENSE
├── .gitignore
├── AwesomePapers.md
└── README.md


/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/nnq_mc_study.iml" filepath="$PROJECT_DIR$/.idea/nnq_mc_study.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/nnq_mc_study.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="inheritedJdk" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="PROJECT_TEST_RUNNER" value="Unittests" />
10 |   </component>
11 | </module>


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 ai-robotics-kr
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | *.egg-info/
 24 | .installed.cfg
 25 | *.egg
 26 | MANIFEST
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | .pytest_cache/
 49 | 
 50 | # Translations
 51 | *.mo
 52 | *.pot
 53 | 
 54 | # Django stuff:
 55 | *.log
 56 | local_settings.py
 57 | db.sqlite3
 58 | 
 59 | # Flask stuff:
 60 | instance/
 61 | .webassets-cache
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # Jupyter Notebook
 73 | .ipynb_checkpoints
 74 | 
 75 | # pyenv
 76 | .python-version
 77 | 
 78 | # celery beat schedule file
 79 | celerybeat-schedule
 80 | 
 81 | # SageMath parsed files
 82 | *.sage.py
 83 | 
 84 | # Environments
 85 | .env
 86 | .venv
 87 | env/
 88 | venv/
 89 | ENV/
 90 | env.bak/
 91 | venv.bak/
 92 | 
 93 | # Spyder project settings
 94 | .spyderproject
 95 | .spyproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | 


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project version="4">
  3 |   <component name="ChangeListManager">
  4 |     <list default="true" id="d0bf0f38-8ce5-4edd-967e-86b9cac9ab20" name="Default Changelist" comment="">
  5 |       <change beforePath="$PROJECT_DIR$/AwesomePapers.md" beforeDir="false" afterPath="$PROJECT_DIR$/AwesomePapers.md" afterDir="false" />
  6 |     </list>
  7 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  8 |     <option name="SHOW_DIALOG" value="false" />
  9 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
 10 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
 11 |     <option name="LAST_RESOLUTION" value="IGNORE" />
 12 |   </component>
 13 |   <component name="FileEditorManager">
 14 |     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
 15 |       <file pinned="false" current-in-tab="true">
 16 |         <entry file="file://$PROJECT_DIR$/AwesomePapers.md">
 17 |           <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
 18 |             <state split_layout="SPLIT">
 19 |               <first_editor relative-caret-position="513">
 20 |                 <caret line="42" column="26" lean-forward="true" selection-start-line="42" selection-start-column="26" selection-end-line="42" selection-end-column="26" />
 21 |               </first_editor>
 22 |               <second_editor />
 23 |             </state>
 24 |           </provider>
 25 |         </entry>
 26 |       </file>
 27 |     </leaf>
 28 |   </component>
 29 |   <component name="Git.Settings">
 30 |     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
 31 |   </component>
 32 |   <component name="IdeDocumentHistory">
 33 |     <option name="CHANGED_PATHS">
 34 |       <list>
 35 |         <option value="$PROJECT_DIR$/AwesomePapers.md" />
 36 |       </list>
 37 |     </option>
 38 |   </component>
 39 |   <component name="ProjectFrameBounds">
 40 |     <option name="x" value="170" />
 41 |     <option name="y" value="50" />
 42 |     <option name="width" value="1400" />
 43 |     <option name="height" value="784" />
 44 |   </component>
 45 |   <component name="ProjectView">
 46 |     <navigator proportions="" version="1">
 47 |       <foldersAlwaysOnTop value="true" />
 48 |     </navigator>
 49 |     <panes>
 50 |       <pane id="Scope" />
 51 |       <pane id="ProjectPane">
 52 |         <subPane>
 53 |           <expand>
 54 |             <path>
 55 |               <item name="nnq_mc_study" type="b2602c69:ProjectViewProjectNode" />
 56 |               <item name="nnq_mc_study" type="462c0819:PsiDirectoryNode" />
 57 |             </path>
 58 |           </expand>
 59 |           <select />
 60 |         </subPane>
 61 |       </pane>
 62 |     </panes>
 63 |   </component>
 64 |   <component name="PropertiesComponent">
 65 |     <property name="last_opened_file_path" value="$PROJECT_DIR$" />
 66 |   </component>
 67 |   <component name="RunDashboard">
 68 |     <option name="ruleStates">
 69 |       <list>
 70 |         <RuleState>
 71 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
 72 |         </RuleState>
 73 |         <RuleState>
 74 |           <option name="name" value="StatusDashboardGroupingRule" />
 75 |         </RuleState>
 76 |       </list>
 77 |     </option>
 78 |   </component>
 79 |   <component name="SvnConfiguration">
 80 |     <configuration />
 81 |   </component>
 82 |   <component name="TaskManager">
 83 |     <task active="true" id="Default" summary="Default task">
 84 |       <changelist id="d0bf0f38-8ce5-4edd-967e-86b9cac9ab20" name="Default Changelist" comment="" />
 85 |       <created>1565625657544</created>
 86 |       <option name="number" value="Default" />
 87 |       <option name="presentableId" value="Default" />
 88 |       <updated>1565625657544</updated>
 89 |     </task>
 90 |     <servers />
 91 |   </component>
 92 |   <component name="ToolWindowManager">
 93 |     <frame x="170" y="50" width="1400" height="784" extended-state="0" />
 94 |     <editor active="true" />
 95 |     <layout>
 96 |       <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.0969163" />
 97 |       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
 98 |       <window_info id="Favorites" order="2" side_tool="true" />
 99 |       <window_info anchor="bottom" id="Message" order="0" />
100 |       <window_info anchor="bottom" id="Find" order="1" />
101 |       <window_info anchor="bottom" id="Run" order="2" />
102 |       <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
103 |       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
104 |       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
105 |       <window_info anchor="bottom" id="TODO" order="6" />
106 |       <window_info anchor="bottom" id="Version Control" order="7" />
107 |       <window_info anchor="bottom" id="Terminal" order="8" />
108 |       <window_info anchor="bottom" id="Event Log" order="9" side_tool="true" />
109 |       <window_info anchor="bottom" id="Python Console" order="10" />
110 |       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
111 |       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
112 |       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
113 |     </layout>
114 |   </component>
115 |   <component name="editorHistoryManager">
116 |     <entry file="file://$PROJECT_DIR$/AwesomePapers.md">
117 |       <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
118 |         <state split_layout="SPLIT">
119 |           <first_editor relative-caret-position="513">
120 |             <caret line="42" column="26" lean-forward="true" selection-start-line="42" selection-start-column="26" selection-end-line="42" selection-end-column="26" />
121 |           </first_editor>
122 |           <second_editor />
123 |         </state>
124 |       </provider>
125 |     </entry>
126 |   </component>
127 | </project>


--------------------------------------------------------------------------------
/AwesomePapers.md:
--------------------------------------------------------------------------------
  1 | ## Neural Network Quantization & Compact Networks Design Awesome Papers!
  2 | 
  3 | **1. Overall Presentation**
  4 |  - Naver techtalk : "https://www.slideshare.net/mobile/NaverEngineering/designing-more-efficient-convolution-neural-network-122869307"
  5 |  - Song Han. "Accelerating inference at the edge" : https://www.hotchips.org/hc30/0tutorials/T2_Part_2_Song_Hanv3.pdf
  6 |  
  7 | 
  8 | **0. Binarized Neural Network & Its Training**
  9 |  - *Courbariaux, Matthieu, et al. “Binarized neural networks: Training deep neural networks with weights and activations constrained to+ 1 or-1.” arXiv preprint arXiv:1602.02830 (2016).*
 10 |  - Rastegari, Mohammad, et al. "Xnor-net: Imagenet classification using binary convolutional neural networks." European Conference on Computer Vision. Springer, Cham, 2016.  
 11 |  - *Darabi, Sajad, et al. “BNN+: Improved binary network training.” arXiv preprint arXiv:1812.11800 (2018).*
 12 |  - Galloway, Angus, Graham W. Taylor, and Medhat Moussa. “Attacking binarized neural networks.” arXiv preprint arXiv:1711.00449 (2017).
 13 |  - *Zhou, Shuchang, et al. “Dorefa-net: Training low bitwidth convolutional neural networks with low bitwidth gradients.” arXiv preprint arXiv:1606.06160 (2016).*  
 14 |  - Wang, Ziwei, et al. “Learning Channel-Wise Interactions for Binary Convolutional Neural Networks.” Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2019.
 15 |  - Hou, Lu, Quanming Yao, and James T. Kwok. "Loss-aware binarization of deep networks." Proceedings of the Fifth International Conference on Learning Representations (ICLR), Toulon, France, Apr 2017.
 16 | 
 17 | **1. Hardware for BNN (Edge device)**
 18 |  - *Yonekawa, Haruyoshi, and Hiroki Nakahara. “On-chip memory based binarized convolutional deep neural network applying batch normalization free technique on an fpga.” 2017 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW). IEEE, 2017.* 
 19 |  - *Umuroglu, Yaman, et al. “Finn: A framework for fast, scalable binarized neural network inference.” Proceedings of the 2017 ACM/SIGDA International Symposium on Field-Programmable Gate Arrays. ACM, 2017.*
 20 | 
 21 | **2. Multi-bit Quantization** 
 22 |  - Jung, Sangil, et al. “Learning to quantize deep networks by optimizing quantization intervals with task loss.” Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2019.
 23 |  -  Chen Xu, Jianqiang Yao, et al. “Alternating Multi-bit Quantization for Recurrent Neural Networks” International Conference on Learning Representations (ICLR), 2018
 24 |  - Zhang, Dongqing, et al. “Lq-nets: Learned quantization for highly accurate and compact deep neural networks.” Proceedings of the European Conference on Computer Vision (ECCV). 2018.
 25 |  - Lee, Dongsoo, Parichay Kapoor, and Byeongwook Kim. “Deeptwist: Learning model compression via occasional weight distortion.” arXiv preprint arXiv:1810.12823 (2018).
 26 |  - Hou, Lu, and James T. Kwok. "Loss-aware weight quantization of deep networks." Proceedings of the Sixth International Conference on Learning Representations (ICLR), Vancouver, BC, Canada, Apr 2018.
 27 |  - Lu Hou, Ruiliang Zhang, James T. Kwok. "Analysis of Quantized Models" Proceedings of the Seventh International Conference on Learning Representations (ICLR), New Orleans, USA, May 2019.
 28 | 
 29 | **3. Neural Network Model Compression**
 30 | 
 31 |  Quantization
 32 | - Kuan Wang, Song Han, et al. "HAQ: Hardware-Aware Automated Quantization with Mixed Precision" IEEE Conference on Computer Vision and Pattern Recognition (CVPR), (2019)
 33 | - Jacob, Benoit, et al. "Quantization and training of neural networks for efficient integer-arithmetic-only inference." Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2018.
 34 | - Zhu, Chenzhuo, et al. “Trained ternary quantization.” arXiv preprint arXiv:1612.01064 (2016).
 35 | 
 36 |  Distillation
 37 | - Elliot J. Crowley, et al. "Moonshine: Distilling with Cheap Convolutions" arXiv preprint arXiv:1711.02613 (2019)
 38 | - Polino, Antonio, Razvan Pascanu, and Dan Alistarh. “Model compression via distillation and quantization.” arXiv preprint arXiv:1802.05668 (2018).
 39 |  
 40 |  Prunning
 41 | - Song Han, Huizi Mao, William J. Dally. "Deep Compression :Deep Compression: Compressing Deep Neural Networks with Pruning, Trained Quantization and Huffman Coding"
 42 | - Zhuang, Zhuangwei, et al. “Discrimination-aware channel pruning for deep neural networks.” Advances in Neural Information Processing Systems. 2018.
 43 | 
 44 |  Hashing
 45 | - Han Zhu, Mingsheng Long, et al. "Deep Hashing Network for Efficient Similarity Retrieval"
 46 | - Dayan Wu, Qi Dai, et al. "Deep Incremental Hashing Network for Efficient Image Retrieval"
 47 | 
 48 |  Automl
 49 | - He, Yihui, et al. “Amc: Automl for model compression and acceleration on mobile devices.” Proceedings of the European Conference on Computer Vision (ECCV). 2018.
 50 | 
 51 | 
 52 | **4. Efficient Neural Network Design**
 53 |  
 54 |  Residual Connection
 55 |  - Mark Sandler, Andrew Howard, et al. "MobileNetV2: Inverted Residuals and Linear Bottlenecks"
 56 |  - Andreas Veit, Michael Wilber, et al. "Residual Networks Behave LIke Ensembles of Relatively Shallow Networks"
 57 |  
 58 |  Dilated Convolution
 59 |   - Junho Yim1 Donggyu Joo1 Jihoon Bae2 Junmo Kim1 "A Gift from Knowledge Distillation: Fast Optimization, Network Minimization and Transfer Learning"
 60 |   
 61 |  Point-wise Convolution
 62 |   - Iandola, Forrest N., et al. “SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and< 0.5 MB model size.” arXiv preprint arXiv:1602.07360 (2016).
 63 |   - Gholami, Amir, et al. “Squeezenext: Hardware-aware neural network design.” Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition Workshops. 2018.
 64 |   - Howard, Andrew G., et al. “Mobilenets: Efficient convolutional neural networks for mobile vision applications.” arXiv preprint arXiv:1704.04861 (2017).
 65 |  
 66 |  Grouped Convolution
 67 |  - Xiangyu Zhang, Xinyu Zhou, et al. "ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices"
 68 |  
 69 |  Dense Convolution
 70 |  - Gao Huang, Zhuang Liu, et al. "Densely Connected Convolutional Networks", CVPR (2017)
 71 |  
 72 |  Depth-wise (Seperable) Convolution
 73 |  - François Chollet, "Xception: Deep Learning with Depthwise Separable Convolutions" arXiv preprint arXiv:1610.02357 
 74 |  - (Mobilenets)
 75 |  
 76 |  (Shift)
 77 |  - Weijie Chen, Di Xie, et al. "All You Need is a Few Shifts: Designing Efficient Convolutional Neural Networks for Image Classification" CVPR (2019)
 78 |  - Bichen Wu, Alvin Wan, et al. "Shift: A Zero FLOP, Zero Parameter Alternative to Spatial Convolutions" arXiv preprint arXiv:1711.08141 (2017)
 79 |  
 80 |  Compound Scaling
 81 | - Tan, Mingxing, and Quoc V. Le. “EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks.” arXiv preprint arXiv:1905.11946 (2019). + EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML (https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html)
 82 | 
 83 | 
 84 |   
 85 | **4. Hardware for Quantized Neural Network**
 86 |  - Lee, Jinmook, et al. “Unpu: An energy-efficient deep neural network accelerator with fully variable weight bit precision.” IEEE Journal of Solid-State Circuits 54.1 (2018): 173-185.
 87 |  - Han, Song, Huizi Mao, and William J. Dally. "Deep compression: Compressing deep neural networks with pruning, trained quantization and huffman coding." arXiv preprint arXiv:1510.00149 (2015).
 88 |  - Yu, Jiecao, et al. "Scalpel: Customizing dnn pruning to the underlying hardware parallelism." ACM SIGARCH Computer Architecture News 45.2 (2017): 548-560.
 89 | 
 90 | 
 91 | **5. Hardware awareness for Accelerating**
 92 | 
 93 |  - EIE [Han, 2016] "EIE: Efficient Inference Engine on Compressed Deep Neural Network"
 94 |  - ESE [Han, 2017] "ESE: Efficient Speech Recognition Engine with Sparse LSTM on FPGA"
 95 |  - SCNN [Parashar, 2017] "SCNN: An Accelerator for Compressed-sparse Convolutional Neural Networks"
 96 |  - DLA [NVIDIA, 2017] "DLA" : https://www.hotchips.org/hc30/2conf/2.08_NVidia_DLA_Nvidia_DLA_HotChips_10Aug18.pdf
 97 |  
 98 |  Computation Specialization
 99 |  - Clement Farabet, Berin Martini et al. "NeuFlow: A Runtime Reconfigurable Dataflow Processor for Vision"
100 |  - Norman P. Jouppi, Cliff Young, Google et al. "In-Datacenter Performance Analysis of a Tensor Processing Unit"
101 |   - A Neural Network Accelerator Exploiting Both Inter-and Intra-Neuron Parallelism
102 |   
103 |  Memory centric Specialization
104 |  - Diannao Family [Chen, 2014-2016] 
105 |  - Eyeriss [Chen, 2016]
106 |  
107 |  Special CNN
108 |  - Kiseok Kwon. "Co-Design of Deep Neural Nets and Neural Net Accelerators for Embedded Vision Applications" (2018)
109 |  - DeePhi’s DPU-v2
110 |  - Baohua Sun, Lin Yang, "Ultra Power-Efficient CNN Domain Specific Accelerator with 9.3TOPS/Watt for Mobile and Embedded Applications"
111 |  - ShiftNet Accelerator [Wu, 2018]
112 |  
113 |  **6. Compiler Optimization**
114 |  
115 |  - DLVM- A modern compiler framework for neural network DSLs
116 |  - DLVM- A MODERN COMPILER INFRASTRUCTURE FOR DEEP LEARNING SYSTEMS
117 |  -  A modern compiler infrastructure for deep learning systems with adjoint code generation in a domain-specific IR
118 |  - Compiling machine learning programs via high-level tracing
119 |  - TVM- An Automated End-to-End Optimizing Compiler for Deep Learning
120 |  - Halide- A Language and Compiler for Optimizing Parallelism, Locality, and Recomputation in Image Processing Pipelines
121 |  - Glow: Graph Lowering Compiler Techniques for Neural Networks
122 |  - TensorFlow XLA compiler and the NNVM compiler (논문 아님) 
123 |  - Cambricon: An Instruction Set Architecture for Neural Networks
124 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Neural Network Quantization & Compact Networks Design
  2 | 
  3 | This is a repository of Facebook Group *AI Robotics KR*.<br>
  4 | 
  5 | *nnq_cnd_study* stands for Neural Network Quantization &amp; Compact Networks Design Study.
  6 | 
  7 | It will be focusing on paper reviews for deep neural networks, model compression, compact network design, and Quantization.
  8 | 
  9 | Online Study supported by AI Robotics KR group have been ongoing since September 1st.
 10 | 
 11 | ## Prerequisite
 12 | 
 13 | - Basic Understanding for deep learning algorithms like DNN, RNN, CNN is preferred
 14 | - Passion for learning
 15 | - Persistence
 16 | - Motivation
 17 | 
 18 | 
 19 | ## Learning Objectives
 20 | 
 21 | - Deep understanding for Deep Neural Network Quantization & Compact Networks Design Algorithms
 22 | 
 23 | 
 24 | ## How to Study
 25 | 
 26 | - Online Presentation
 27 | - Q & A
 28 | 
 29 | ## Participants:
 30 | 
 31 | **Slack** : @Hwigeon Oh, @Seojin Kim, @DongJunMin, @이경준, @Hyunwoo Kim, @Constant, @임병학, @KimYoungBin, @Sanggun Kim, @martin, @Joh, @김석중, @Yongwoo Kim, @MinSeop Lee, @Woz.D, @inwoong.lee (이인웅), @Hoyeolchoi, @Bochan Kim, @Young Seok Kim, @taehkim, @Seongmock Yoo, @Mike.Oh, @최승호, @Davidlee, @Stella Yang, @sejungkwon, @Jaeyoung Lee, @Hyungjun Kim, @tae-ha, @Jeonghoon.
 32 | 
 33 | ------------------------------------
 34 | 
 35 | ## Contributors:
 36 | 
 37 | **Main Study Learder**: Jeonghoon Kim(GitHub:IntelligenceDatum).
 38 | 
 39 | **Compact Networks Design Leader**: Seo Yeon Stella Yang(GitHub:howtowhy).
 40 | 
 41 | **Presenter**:Jeonghoon Kim, Stella Yang, Sanggun Kim, Hyunwoo Kim, Seojin Kim, Hwigeon Oh, Seojin Kim, Seokjoong Kim, Martin Hwang, Youngbin Kim, Sang-soo Park, Jaeyoung Lee, Yongwoo Kim, Hyungjun Kim, Sejung Kwon, 이경준, Bochan Kim, 이인웅. 
 42 | 
 43 | ------------------------------------
 44 | 
 45 | ## Presentation with Video :
 46 | 
 47 | Neural Network Quantization & Compact Network Design Study
 48 | 
 49 | ### Week1: Introduction of NNQ&CND
 50 | **Title: A Piece of Weight**  
 51 | Presentor: 김정훈 (Jeonghoon Kim)  
 52 | PPT: https://drive.google.com/open?id=1RQAiIFX7wOUMiZXPCIZbXb_6DtLlV38e  
 53 | Video: https://youtu.be/pohMFz-uQJ0  
 54 | 
 55 | **Title: Compact Network Design Overview**  
 56 | Presentor: Stella Yang  
 57 | Video: https://youtu.be/R3pE-pGBbBg  
 58 | PPT: https://drive.google.com/open?id=1bTy68uO1Ta4tJLYDLA7d6GJfRx1YbcM4  
 59 | 
 60 | 
 61 | ### Week2: BNN & MobileNet
 62 | **Paper: Binarized Neural Networks:Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1**  
 63 | Presentor: 김정훈 (Jeonghoon Kim)  
 64 | Video:  https://youtu.be/n89CsZpZcNk  
 65 | PPT:  https://drive.google.com/open?id=1DoeGj-goeI5WMIu5LPTQ6aFZ2czl7eNP  
 66 | 
 67 | **Paper: MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications**  
 68 | Presentor:  김상근 (Sanggun Kim)  
 69 | Video: https://youtu.be/GyQUBLDQEJI  
 70 | PPT: https://drive.google.com/open?id=1oQI8Pv7N66pZHflx0CyMIyahhbA-Dce7  
 71 |   
 72 | ### Week3: FINN (BNNs Hardware) & MobileNetV2  
 73 | **Paper: FINN: A Framework for Fast, Scalable Binarized Neural Network Inference**  
 74 | Presentor: 김현우 (Hyun-Woo Kim)  
 75 | Video:  https://youtu.be/DjS8wvXaE8c  
 76 | PPT:  https://drive.google.com/file/d/11uj-UaLiOEBIxExpo43OV5l6b4MSFTuD/view?usp=sharing  
 77 | 
 78 | **Paper: MobileNetV2: Inverted Residuals and Linear Bottlenecks**   
 79 | Presentor:  김서진  
 80 | Video: not available  
 81 | PPT: https://drive.google.com/file/d/1NTfct371Lpasly8XW7zt7OzOTh87sVLA/view?usp=sharing  
 82 | 
 83 | ### Week4: XNOR-Net & SqueezeNet    
 84 | **Paper: XNOR-Net: ImageNet Classification Using Binary Convolutional Neural Networks**  
 85 | Presentor: 오휘건  
 86 | Video:  https://youtu.be/N6oP-8E5cWA   
 87 | PPT:  https://drive.google.com/open?id=1bz3C-fFVSCrOdnbi-8lf_2NS1yhpGdVO  
 88 | 
 89 | **Paper: SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and < 0.5MB model size**  
 90 | Presentor:  Martin Hwang  
 91 | Video: https://youtu.be/eH5O5nDiFoY  
 92 | PPT: https://drive.google.com/open?id=1HNRhl1lxb7oe0gFsbv9f2fduCr_f_G4O  
 93 | 
 94 | ### Week5: BNN+ & SqueezeNext
 95 | **Paper: BNN+: Improved Binary Network Training**  
 96 | Presentor: 김영빈  
 97 | Video:  https://youtu.be/M7-lBoiFHRI  
 98 | 
 99 | **Paper: SqueezeNext: Hardware-Aware Neural Network Design**  
100 | Presentor:  박상수  
101 | Video: https://youtu.be/sbKl92j9Xrs  
102 | 
103 | ### Week6: Loss-aware Binarization 
104 | **Paper: Loss-aware Binarization of Deep Networks**    
105 | Presentor:  이재영 (Jaeyoung Lee)  
106 | Video: https://youtu.be/Bs3SVcvr5cA  
107 | 
108 | ### Week7: Scalpel, Hardware-aware pruning  
109 | **Paper: Scalpel: Customizing DNN Pruning to the Underlying Hardware Parallelism**  
110 | Presentor: Constant (Sang-Soo) Park  
111 | Video: https://youtu.be/DmCCREJ1zAA  
112 | 
113 | ### WEEK8: DOREFANET && SHUFFLENET  
114 | **Paper: Dorefa-net: Training low bitwidth convolutional neural networks with low bitwidth gradients**  
115 | Presentor: Yongwoo Kim  
116 | Video: https://youtu.be/DEnQKMXzx7o  
117 | 
118 | **Paper: ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices**  
119 | Presentor: Jaeyoung Lee    
120 | Video: https://youtu.be/l-Q06pAfBHw  
121 | 
122 | ### WEEK9: LQ-NETS && BI-REAL NETS   
123 | **Paper: LQ-Nets: Learned Quantization for Highly Accurate and Compact Deep Neural Networks**  
124 | Presentor: Hyungjun Kim  
125 | Video: https://youtu.be/ca_d03MYeJE  
126 | 
127 | **Paper: Bi-Real Net: Enhancing the Performance of 1-bit CNNs With Improved Representational Capability and Advanced Training Algorithm**  
128 | Presentor: YoungBin Kim  
129 | Video: https://youtu.be/pQmvmcPZHmM   
130 | 
131 | ### WEEK10: QUANTIZATION & DISTILLATION  
132 | **Paper: Model Compression via Quantization and Distillation**  
133 | Presentor: Seokjoong Kim (김민성)  
134 | Video: https://youtu.be/xOMuav0UVXg  
135 | 
136 | ### WEEK11: ALTERNATING MULTI-BIT QUANTIZATION & DENSENET !!  
137 | **Paper: Alternating multi-bit quantization for recurrent neural networks**   
138 | Presentor: Eunhui Kim  
139 | Video: https://youtu.be/iibC1NZv0S4  
140 | 
141 | **Paper: DenseNet: Densely Connected Convolutional Networks**  
142 | Presentor: Kyeong-Jun Lee  
143 | Video: https://youtu.be/bhvxLB6Qa60  
144 | 
145 | ### Week12  
146 | 
147 | ### Week13: DEFENSIVE QUANTIZATION & EFFICIENTNET
148 | **Paper: Defensive Quantization: When Efficiency Meets Robustness**  
149 | Presentor: Bochan Kim  
150 | Video: https://youtu.be/7UfmDlLHOFA  
151 | 
152 | **Paper: EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks**  
153 | Presentor: Martin Hwang  
154 | Video: https://youtu.be/58ZxZSLr_bU  
155 | 
156 | ### Week14: QIL & MobileNetV3  
157 | **Paper: Learning to Quantize Deep Networks by Optimizing Quantization Intervals with Task Loss**  
158 | Presentor: 이인웅  
159 | Video: https://youtu.be/VLyhhcPwxWc  
160 | 
161 | **Paper: Searching for MobileNetV3**  
162 | Presentor: Seo Yeon Stella Yang  
163 | Video: https://youtu.be/JPs2Uy9DLO8  
164 |   
165 | ------------------------------------
166 | ## Schedule (Presentation List):
167 | 
168 | | Week         | Subject                                                                                            | Presenter       |
169 | |--------------|----------------------------------------------------------------------------------------------------|-----------------|
170 | | Week 1  |1. Introduction <br /> 2. Introduction. |  1.Jeonghoon Kim<br />2.Stella Yang|
171 | | Week 2  |1. Binarized neural networks: Training deep neural networks with weights and activations constrained to+ 1 or-1.<br /> 2. Mobilenets: Efficient convolutional neural networks for mobile vision applications. |1.Jeonghoon Kim<br />2.Sanggun Kim|
172 | | Week 3  |1.Finn: A framework for fast, scalable binarized neural network inference.<br />2. MobileNetV2: Inverted Residuals and Linear Bottlenecks| 1.Hyunwoo Kim<br />2.Seojin Kim  |
173 | | Week 4  |1.XNOR-Net: ImageNet Classification Using Binary Convolutional Neural Networks.<br />2. SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and< 0.5 MB model size. |1.Hwigeon Oh<br />2.Martin Hwang |
174 | | Week 5  |1.BNN+: Improved binary network training.<br />2. Squeezenext: Hardware-aware neural network design. |1.Youngbin Kim<br /> 2.Sang-soo Park|
175 | | Week 6  |1.Loss-aware binarization of deep networks.<br />	2. Deep compression: Compressing deep neural networks with pruning, trained quantization and huffman coding.|1.Jaeyoung Lee<br />2.	Sanggun Kim|
176 | | Week 7  |1. Loss-aware weight quantization of deep networks.<br />2. Scalpel: Customizing dnn pruning to the underlying hardware parallelism.|1.Youngbin Kim<br />2.Sang-soo Park|
177 | | Week 8  |1.Dorefa-net: Training low bitwidth convolutional neural networks with low bitwidth gradients.<br />2.ShuffleNet: An Extremely Efficient Convolutional Neural Network for Mobile Devices.|1.Yongwoo Kim<br />2.Jaeyoung Lee|
178 | | Week 9  |1.Lq-nets: Learned quantization for highly accurate and compact deep neural networks.<br />2. Model compression via distillation and quantization.|1.Hyungjun Kim<br />2. Seokjoong Kim|
179 | | Week 10  |1. Alternating Multi-bit Quantization for Recurrent Neural Networks.<br />	2. Densely Connected Convolutional Networks.|1.Eunhui Kim<br />2.이경준|
180 | | Week 11  |1.TBD<br />2. All You Need is a Few Shifts: Designing Efficient Convolutional Neural Networks for Image Classification.|1.Sejung Kwon<br />2.Stella Yang|
181 | | Week 12  |1.Analysis of Quantized Models.<br />2.EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks.	|1.Bochan Kim<br />2.	Martin Hwang|    
182 | | Week 13  |1.Learning to quantize deep networks by optimizing quantization intervals with task loss.<br />2. Amc: Automl for model compression and acceleration on mobile devices.|1.이인웅<br />2.Seokjoong Kim|
183 | 
184 | 
185 | ## References
186 | https://github.com/ai-robotics-kr/nnq_cnd_study/blob/master/AwesomePapers.md  
187 | 


--------------------------------------------------------------------------------